@dbos-inc/dbos-sdk 5.1.10 → 5.2.4-preview

package/dist/src/system_database.js

@@ -9,12 +9,13 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.retentionLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
+exports.SystemDatabase = exports.retriablePostgresException = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.retentionLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
 const dbos_executor_1 = require("./dbos-executor");
 const pg_1 = require("pg");
 const error_1 = require("./error");
 const workflow_1 = require("./workflow");
 const utils_1 = require("./utils");
+const context_1 = require("./context");
 const async_hooks_1 = require("async_hooks");
 const crypto_1 = require("crypto");
 const utils_2 = require("./utils");
@@ -67,6 +68,24 @@ function validateObservabilityQueryTimeoutMs(value) {
     }
 }
 exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
+// Bounds how long a frozen or unreachable client can hold system database locks.
+exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
+/** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
+function validateIdleTransactionTimeoutMs(value) {
+    if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
+        throw new Error(`systemDatabaseIdleTransactionTimeoutMs must be a finite number no greater than ${MAX_STATEMENT_TIMEOUT_MS}, got ${value}`);
+    }
+}
+exports.validateIdleTransactionTimeoutMs = validateIdleTransactionTimeoutMs;
+/** Whether the connection string already sets idle_in_transaction_session_timeout through libpq options. */
+function setsIdleTransactionTimeout(databaseUrl) {
+    try {
+        return new URL(databaseUrl).searchParams.get('options')?.includes('idle_in_transaction_session_timeout') ?? false;
+    }
+    catch {
+        return false;
+    }
+}
 // Workflows re-owned per transaction by a rename.
 exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
 // Rows deleted per transaction by garbage collection.
@@ -344,10 +363,12 @@ const RETRY_SQLSTATE_PREFIXES = new Set([
 ]);
 const RETRY_SQLSTATE_CODES = new Set([
     '40003', // statement_completion_unknown
+    '25P03', // idle_in_transaction_session_timeout, when the kill lands on an in-flight query; an idle kill surfaces as a message match below
+    '40P01', // deadlock_detected: the victim's transaction rolled back, so rerunning it is safe
 ]);
 /**
- * Kept out of the sets above: those feed `dbRetry`, which retries forever, and the step-recording
- * path maps 40001 to a workflow conflict. Only bulk maintenance work retries on these.
+ * Bulk maintenance retries these a bounded number of times. 40001 is kept out of the sets above,
+ * which feed `dbRetry` and so retry forever; every other path runs READ COMMITTED and never sees it.
  */
 const SERIALIZATION_SQLSTATE_CODES = new Set([
     '40001', // serialization_failure (MVCC conflict)
@@ -442,6 +463,7 @@ function retriablePostgresException(err) {
     }
     return false;
 }
+exports.retriablePostgresException = retriablePostgresException;
 /** 57014 is query_canceled, which is how statement_timeout cancels a query. */
 function isStatementTimeout(err) {
     return !!err && typeof err === 'object' && err.code === '57014';
@@ -575,6 +597,8 @@ class SystemDatabase {
     notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS;
     // Statement timeout for observability reads, in ms; undefined disables the cap.
     observabilityQueryTimeoutMs;
+    // idle_in_transaction_session_timeout set on each connection of a pool we own, in ms; undefined leaves the server's.
+    idleTransactionTimeoutMs;
     // Coalesced NOTIFY payloads keyed by channel, flushed by the notifier loop; soft-private so tests can drive a flush.
     pendingNotifications = new Map();
     #notifierActive = false;
@@ -588,7 +612,8 @@ class SystemDatabase {
      * client and starve control-plane operations. See {@link #pollWithLimiter}.
      */
     pollLimiter;
-    runningWorkflowMap = new Map(); // Map from workflowID to workflow promise, queue name and partition key
+    /** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
+    runningWorkflowMap = new Map();
     #cancelWatchers = new Map();
     #cancelPollerLoop = undefined;
     #cancelPollerWake = null;
@@ -605,7 +630,7 @@ class SystemDatabase {
     #retentionClients = new Set();
     constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS, 
     // The application this handle acts for; undefined writes unclaimed rows.
-    appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS) {
+    appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS, idleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS) {
         this.systemDatabaseUrl = systemDatabaseUrl;
         this.logger = logger;
         this.serializer = serializer;
@@ -614,6 +639,12 @@ class SystemDatabase {
         this.shouldUseDBNotifications = useListenNotify;
         this.notificationCoalesceMs = notificationCoalesceMs;
         validateObservabilityQueryTimeoutMs(observabilityQueryTimeoutMs);
+        validateIdleTransactionTimeoutMs(idleTransactionTimeoutMs);
+        // Floor at 1ms: PostgreSQL reads 0 as "no timeout".
+        this.idleTransactionTimeoutMs =
+            idleTransactionTimeoutMs > 0 && !systemDatabasePool && !setsIdleTransactionTimeout(systemDatabaseUrl)
+                ? Math.max(1, Math.round(idleTransactionTimeoutMs))
+                : undefined;
         // Floor at 1ms: PostgreSQL reads 0 as "no timeout", the loosest cap rather than the tightest.
         this.observabilityQueryTimeoutMs =
             observabilityQueryTimeoutMs > 0 ? Math.max(1, Math.round(observabilityQueryTimeoutMs)) : undefined;
@@ -641,6 +672,15 @@ class SystemDatabase {
             this.pool.on('error', (err) => {
                 this.logger.warn(`Unexpected error in pool: ${err}`);
             });
+            const timeoutMs = this.idleTransactionTimeoutMs;
+            if (timeoutMs !== undefined) {
+                // Queued ahead of the borrower's first query; outside any transaction, so it persists for the session.
+                this.pool.on('connect', (client) => {
+                    client.query(`SET idle_in_transaction_session_timeout = ${timeoutMs}`).catch((err) => {
+                        this.logger.warn(`Could not set idle_in_transaction_session_timeout on a system database connection: ${err}`);
+                    });
+                });
+            }
         }
     }
     #onClientError = (err) => {
@@ -808,18 +848,18 @@ class SystemDatabase {
     }
     // ==================== Workflow Status ====================
     /** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
-    async initWorkflowStatus(initStatus, ownerXid, client, reusePolicy = 'return-existing') {
+    async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
         if (client !== undefined) {
-            return await this.#initWorkflowStatusInternal(client, initStatus, ownerXid, reusePolicy);
+            return await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
         }
-        return await this.initWorkflowStatusStandalone(initStatus, ownerXid, reusePolicy);
+        return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
     }
-    async initWorkflowStatusStandalone(initStatus, ownerXid, reusePolicy) {
+    async initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy) {
         const client = await this.#connect();
         let shouldCommit = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            const result = await this.#initWorkflowStatusInternal(client, initStatus, ownerXid, reusePolicy);
+            const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
             // If there is an existing DB record and we aren't here to recover it, leave it be.
             shouldCommit = result.shouldExecuteOnThisExecutor;
             return result;
@@ -840,13 +880,17 @@ class SystemDatabase {
         }
     }
     /** Insert a child workflow's row and record it as the parent's step in one transaction, so a crash leaves both or neither. */
-    async initChildWorkflowStatus(initStatus, ownerXid, parentWorkflowID, parentFunctionID, startTimeEpochMs, endTimeEpochMs, reusePolicy = 'return-existing') {
+    async initChildWorkflowStatus(initStatus, creatorXid, parentWorkflowID, parentFunctionID, startTimeEpochMs, endTimeEpochMs, reusePolicy = 'return-existing') {
         const client = await this.#connect();
         let shouldCommit = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            const result = await this.#initWorkflowStatusInternal(client, initStatus, ownerXid, reusePolicy);
-            await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs, { childWorkflowID: initStatus.workflowUUID });
+            const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
+            if (parentOwnerXid !== undefined) {
+                await this.#checkOwner(client, parentWorkflowID, parentOwnerXid);
+            }
+            const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
+            await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs, { childWorkflowID: initStatus.workflowUUID, ownerChecked: true });
             shouldCommit = true;
             return result;
         }
@@ -865,9 +909,9 @@ class SystemDatabase {
             }
         }
     }
-    async #initWorkflowStatusInternal(client, initStatus, ownerXid, reusePolicy) {
-        const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid);
-        if (reusePolicy === 'reject' && (ownerXid === null || resRow.owner_xid !== ownerXid)) {
+    async #initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
+        const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
+        if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
             throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
         }
         if (resRow.name !== initStatus.workflowName) {
@@ -889,7 +933,7 @@ class SystemDatabase {
         // The upsert above already set executor assignment for a row we own.
         return {
             status: resRow.status,
-            shouldExecuteOnThisExecutor: ownerXid === resRow.owner_xid,
+            shouldExecuteOnThisExecutor: creatorXid === resRow.creator_xid,
             serialization: resRow.serialization,
         };
     }
@@ -899,6 +943,7 @@ class SystemDatabase {
             return;
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
        SET status = $1,
+           owner_xid = NULL,
            deduplication_id = NULL,
            started_at_epoch_ms = NULL,
            queue_name = NULL,
@@ -1007,7 +1052,7 @@ class SystemDatabase {
             'queue_partition_key',
             'parent_workflow_id',
             'serialization',
-            'owner_xid',
+            'creator_xid',
             'delay_until_epoch_ms',
             'attributes',
             'schedule_name',
@@ -1065,42 +1110,42 @@ class SystemDatabase {
         }
         return inserted;
     }
-    async recordWorkflowOutput(workflowID, status) {
+    async recordWorkflowOutput(workflowID, status, ownerXid) {
         const client = await this.#connect();
         try {
-            return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output });
+            return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
         }
         finally {
             client.release();
         }
     }
-    async recordWorkflowError(workflowID, status) {
+    async recordWorkflowError(workflowID, status, ownerXid) {
         const client = await this.#connect();
         try {
-            return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error });
+            return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
         }
         finally {
             client.release();
         }
     }
     // Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
-    // the write landed. The write applies only to a PENDING row: a run owns its
-    // workflow's outcome exactly as long as the row says that run is what the
-    // workflow is doing. (Note: this does not prevent a write when another
-    // concurrent execution is already running and the status is PENDING. However,
-    // both executions should be deterministic and idempotent.)
+    // the write landed. The write applies only to a PENDING row still owned by
+    // ownerXid, which defaults to the calling execution's token; a caller
+    // outside the workflow's context passes it explicitly or goes unchecked.
     //
     // Returning false means the row was CANCELLED, dead-lettered, already
-    // terminal, handed to another execution (ENQUEUED/DELAYED, e.g. by a
-    // concurrent resume), or deleted; the caller resolves which by awaiting the
-    // recorded outcome.
-    async #recordWorkflowOutcome(client, workflowID, status, outcome) {
+    // terminal, handed to another execution (e.g. by a concurrent resume or
+    // recovery), or deleted; the caller resolves which by awaiting the recorded
+    // outcome.
+    async #recordWorkflowOutcome(client, workflowID, status, outcome, ownerXid) {
+        ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
         let committed = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             const rowCount = await this.updateWorkflowStatus(client, workflowID, status, {
-                update: { resetDeduplicationID: true, setCompletedAt: true },
-                where: { status: workflow_1.StatusString.PENDING },
+                // A finished workflow has no owner.
+                update: { resetDeduplicationID: true, setCompletedAt: true, ownerXid: null },
+                where: { status: workflow_1.StatusString.PENDING, ownerXid },
                 throwOnFailure: false,
             });
             if (rowCount === 0) {
@@ -1142,6 +1187,7 @@ class SystemDatabase {
         const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
        SET started_at_epoch_ms = NULL,
            status = $1,
+           owner_xid = NULL,
            updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            queue_name = COALESCE(queue_name, $2)
        WHERE status = $3
@@ -1161,7 +1207,7 @@ class SystemDatabase {
             const client = await this.#connect();
             try {
                 // Check if the operation has been done before for OAOO (only do this inside a workflow).
-                const json = await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus);
+                const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus));
                 return parseStatus(json);
             }
             finally {
@@ -1202,6 +1248,7 @@ class SystemDatabase {
                     resetNameTo: internalOptions?.updateName,
                     queueName: internalOptions?.queueName,
                     resetStartedAtEpochMs: internalOptions?.resetStartedAtEpochMs,
+                    ownerXid: internalOptions?.ownerXid,
                 },
             });
         }
@@ -1236,7 +1283,7 @@ class SystemDatabase {
     async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
         const client = await this.#connect();
         try {
-            await this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options);
+            await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
         }
         finally {
             client.release();
@@ -1274,31 +1321,38 @@ class SystemDatabase {
         if (functionID === undefined)
             throw new TypeError('functionID must be defined');
         patchName = `DBOS.patch-${patchName}`;
-        const { rows } = await this.pool.query(`SELECT function_name
-       FROM "${this.schemaName}".operation_outputs
-      WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
+        const readCheckpointName = async (queryable) => {
+            const { rows } = await queryable.query(`SELECT function_name
+         FROM "${this.schemaName}".operation_outputs
+        WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
+            return rows.length === 0 ? undefined : rows[0].function_name;
+        };
         if (deprecated) {
             // Deprecated does not write anything.  We skip any existing matching patch marker if it matches
-            if (rows.length === 0) {
+            const checkpointName = await readCheckpointName(this.pool);
+            if (checkpointName === undefined) {
                 return { isPatched: true, hasEntry: false };
             }
-            return { isPatched: true, hasEntry: rows[0].function_name === patchName };
+            return { isPatched: true, hasEntry: checkpointName === patchName };
         }
         // Nondeprecated - skip matching entry, unpatched if nonmatching entry,
-        //  If there is no entry, we insert one that indicates it is patched.
-        if (rows.length !== 0) {
-            if (rows[0].function_name === patchName) {
+        //  If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
+        const client = await this.#connect();
+        try {
+            return await this.#inTransaction(client, async () => {
+                const checkpointName = await readCheckpointName(client);
+                if (checkpointName !== undefined) {
+                    const matches = checkpointName === patchName;
+                    return { isPatched: matches, hasEntry: matches };
+                }
+                const dn = Date.now();
+                await this.recordOperationResultInternal(client, workflowID, functionID, patchName, true, dn, dn);
                 return { isPatched: true, hasEntry: true };
-            }
-            return { isPatched: false, hasEntry: false };
+            });
+        }
+        finally {
+            client.release();
         }
-        // Insert a patchmarker
-        const dn = Date.now();
-        await this.pool.query(`INSERT INTO "${this.schemaName}".operation_outputs
-       (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
-       ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn, this.appName ?? null]);
-        return { isPatched: true, hasEntry: true };
     }
     // ==================== Workflow Management ====================
     async cancelWorkflows(workflowIDs, cancelChildren = false) {
@@ -1320,7 +1374,7 @@ class SystemDatabase {
     }
     async #cancelWorkflows(workflowIDs) {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET status = $1, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
+       SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
            updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
        WHERE workflow_uuid = ANY($2)
@@ -1331,7 +1385,7 @@ class SystemDatabase {
     }
     async resumeWorkflows(workflowIDs, queueName) {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET status = $1, queue_name = $2, recovery_attempts = 0,
+       SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
            workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
            started_at_epoch_ms = NULL,
            updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
@@ -1583,7 +1637,7 @@ class SystemDatabase {
                 params.push(options.applicationVersion);
             }
             const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
-         SET status = $1, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
+         SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
              workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
              completed_at = NULL, output = NULL, error = NULL,
              updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
@@ -1832,9 +1886,9 @@ class SystemDatabase {
             for (const wfID of workflowIDs) {
                 // Export workflow_status
                 const statusResult = await client.query(
-                // owner_xid is intentionally omitted: it is a transient transaction-ownership
-                // token, not logical workflow state, and a source database's xid is
-                // meaningless in the target.
+                // creator_xid and owner_xid are intentionally omitted: they are transient
+                // tokens, not logical workflow state, and a source database's
+                // tokens are meaningless in the target.
                 `SELECT
             ws.workflow_uuid, ws.status, ws.name, ws.authenticated_user, ws.assumed_role,
             ws.authenticated_roles,
@@ -2004,43 +2058,63 @@ class SystemDatabase {
         }
     }
     // ==================== Awaiting Workflows ====================
-    registerRunningWorkflow(workflowID, workflowPromise, onSettled, queueName, queuePartitionKey) {
+    /** Register one execution; the returned release is idempotent, so it can run both before a park and once the run settles. */
+    registerRunningWorkflow(workflowID, queueName, queuePartitionKey) {
+        const entry = {
+            promise: Promise.resolve(),
+            queueName,
+            queuePartitionKey,
+            release: () => {
+                const entries = this.runningWorkflowMap.get(workflowID);
+                const index = entries?.indexOf(entry) ?? -1;
+                if (index < 0)
+                    return;
+                entries.splice(index, 1);
+                if (entries.length === 0)
+                    this.runningWorkflowMap.delete(workflowID);
+            },
+        };
+        const entries = this.runningWorkflowMap.get(workflowID);
+        if (entries)
+            entries.push(entry);
+        else
+            this.runningWorkflowMap.set(workflowID, [entry]);
+        return entry;
+    }
+    /** Track an execution's promise for shutdown, releasing its entry when it settles. */
+    trackRunningWorkflow(workflowID, entry, workflowPromise) {
         // Need to await for the workflow and capture errors.
-        const awaitWorkflowPromise = workflowPromise
+        entry.promise = workflowPromise
             .catch((error) => {
             const outcome = this.#destroyed ? 'was abandoned by shutdown' : 'failed';
             this.logger.debug(`Workflow ${workflowID} ${outcome}: ${error}`);
         })
             .finally(() => {
-            onSettled();
-        });
-        this.runningWorkflowMap.set(workflowID, {
-            promise: awaitWorkflowPromise,
-            queueName,
-            queuePartitionKey,
+            entry.release();
         });
     }
     checkForRunningWorkflow(workflowID) {
         return this.runningWorkflowMap.has(workflowID);
     }
-    clearRunningWorkflow(workflowID) {
-        this.runningWorkflowMap.delete(workflowID);
-    }
     /** Workflows this worker is running for a queue, across every partition of it. */
     countRunningWorkflowsForQueue(queueName) {
         let count = 0;
-        for (const entry of this.runningWorkflowMap.values()) {
-            if (entry.queueName === queueName)
-                count++;
+        for (const entries of this.runningWorkflowMap.values()) {
+            for (const entry of entries) {
+                if (entry.queueName === queueName)
+                    count++;
+            }
         }
         return count;
     }
     /** Workflows this worker is running for one partition of a queue. */
     countRunningWorkflowsForPartition(queueName, queuePartitionKey) {
         let count = 0;
-        for (const entry of this.runningWorkflowMap.values()) {
-            if (entry.queueName === queueName && entry.queuePartitionKey === queuePartitionKey)
-                count++;
+        for (const entries of this.runningWorkflowMap.values()) {
+            for (const entry of entries) {
+                if (entry.queueName === queueName && entry.queuePartitionKey === queuePartitionKey)
+                    count++;
+            }
         }
         return count;
     }
@@ -2054,7 +2128,10 @@ class SystemDatabase {
             // Each pass picks up workflows a draining workflow started, and awaits any given run only once.
             const awaited = new Set();
             for (;;) {
-                const pending = Array.from(this.runningWorkflowMap.values(), (entry) => entry.promise).filter((promise) => !awaited.has(promise));
+                const pending = Array.from(this.runningWorkflowMap.values())
+                    .flat()
+                    .map((entry) => entry.promise)
+                    .filter((promise) => !awaited.has(promise));
                 if (pending.length === 0)
                     break;
                 for (const promise of pending)
@@ -2074,12 +2151,6 @@ class SystemDatabase {
         if (this.runningWorkflowMap.size > 0) {
             this.logger.warn(`Shutting down while ${this.runningWorkflowMap.size} workflows are still running: ${Array.from(this.runningWorkflowMap.keys()).join(', ')}`);
         }
-        if (this.workflowEventsMap.map.size > 0) {
-            this.logger.warn('Workflow events map is not empty - shutdown is not clean.');
-        }
-        if (this.notificationsMap.map.size > 0) {
-            this.logger.warn('Message notification map is not empty - shutdown is not clean.');
-        }
     }
     /**
      * Run a DB-backed polling read under the polling concurrency limiter, so that
@@ -2306,6 +2377,23 @@ class SystemDatabase {
         }
         return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
     }
+    /** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
+    async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
+        const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
+        const client = await this.#connect();
+        try {
+            await this.#inTransaction(client, async () => {
+                await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
+                // After the insert, in the order the recorded send locks, so they cannot deadlock.
+                if (ownerXid !== undefined) {
+                    await this.#checkOwner(client, workflowID, ownerXid);
+                }
+            });
+        }
+        finally {
+            client.release();
+        }
+    }
     async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
         return await this.#sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey);
     }
@@ -2527,27 +2615,41 @@ class SystemDatabase {
     }
     // ==================== Streams ====================
     async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
-        while (true) {
-            try {
-                // Derives the first unused offset inside the insert; two writers can still pick the same one.
-                await this.pool.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
-           SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
-           FROM "${this.schemaName}".streams s
-           WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
-            }
-            catch (e) {
-                // Only an offset conflict resolves on retry; anything else would spin forever.
-                if (e instanceof pg_1.DatabaseError && e.code === '23505') {
-                    this.logger.warn(`Stream offset conflict for workflow ${workflowID}, key ${key}; retrying`);
-                    await (0, utils_1.sleepms)(100);
-                    continue;
+        const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
+        const client = await this.#connect();
+        try {
+            while (true) {
+                try {
+                    await this.#inTransaction(client, async () => {
+                        // Derives the first unused offset inside the insert; two writers can still pick the same one.
+                        await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
+               SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
+               FROM "${this.schemaName}".streams s
+               WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
+                        // After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
+                        if (ownerXid !== undefined) {
+                            await this.#checkOwner(client, workflowID, ownerXid);
+                        }
+                    });
                 }
-                this.logger.error(e);
-                throw e;
+                catch (e) {
+                    // Only an offset conflict resolves on retry; anything else would spin forever.
+                    if (e instanceof pg_1.DatabaseError && e.code === '23505') {
+                        this.logger.warn(`Stream offset conflict for workflow ${workflowID}, key ${key}; retrying`);
+                        await (0, utils_1.sleepms)(100);
+                        continue;
+                    }
+                    if (!(e instanceof error_1.DBOSWorkflowConflictError))
+                        this.logger.error(e);
+                    throw e;
+                }
+                // Notify only after commit, so a woken reader sees the value.
+                this.#signalNotification(exports.DBOS_STREAMS_CHANNEL, `${workflowID}::${key}`);
+                return;
             }
-            // Notify only after commit, so a woken reader sees the value.
-            this.#signalNotification(exports.DBOS_STREAMS_CHANNEL, `${workflowID}::${key}`);
-            return;
+        }
+        finally {
+            client.release();
         }
     }
     async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
@@ -2763,7 +2865,7 @@ class SystemDatabase {
         // Literal statuses let the planner match idx_workflow_status_deadline under any plan mode.
         // SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
         const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET status = $1, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
+       SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
            updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
        WHERE workflow_uuid IN (
@@ -2805,7 +2907,9 @@ class SystemDatabase {
        SELECT pk FROM partitions WHERE pk IS NOT NULL`, params);
         return rows.map((row) => row.pk);
     }
-    async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0) {
+    async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, 
+    // Written to every claimed row: the dispatches of this claim own their executions.
+    ownerXid = null) {
         const claimedIDs = [];
         const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
         // Shares a concurrency or rate limit budget with other executors.
@@ -2938,12 +3042,14 @@ class SystemDatabase {
                     workflow_1.StatusString.ENQUEUED,
                     // Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
                     this.appName ?? null,
+                    ownerXid,
                 ];
                 // Re-check ownership alongside status, as the partitioned claim guard does.
                 const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
                 // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
                 const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
            SET status = $1,
+               owner_xid = $8,
                executor_id = $2,
                application_version = $3,
                started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
@@ -2979,7 +3085,9 @@ class SystemDatabase {
     /** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
     partitionedDequeueSweepCap = 8192;
     /** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
-    async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity) {
+    async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity, 
+    // Written to every claimed row: the dispatches of this claim own their executions.
+    ownerXid = null) {
         if (queue.partitionConcurrency !== 1 ||
             queue.concurrency !== undefined ||
             queue.rateLimit !== undefined ||
@@ -3085,11 +3193,13 @@ class SystemDatabase {
                 appVersion,
                 // Claim the row, as the unpartitioned dequeue does.
                 this.appName ?? null,
+                ownerXid,
             ];
             const flipScope = this.#appNameFilter('application_name', this.appName, flipParams);
             // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
             const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
          SET status = $1,
+             owner_xid = $8,
              executor_id = $2,
              application_version = $6,
              started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
@@ -4429,7 +4539,7 @@ class SystemDatabase {
         return { queues, schedules, versions, workflows: inFlight + terminal, steps };
     }
     // ==================== Internal ====================
-    async insertWorkflowStatus(client, initStatus, ownerXid) {
+    async insertWorkflowStatus(client, initStatus, creatorXid) {
         try {
             const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
           workflow_uuid,
@@ -4454,18 +4564,19 @@ class SystemDatabase {
           forked_from,
           parent_workflow_id,
           serialization,
-          owner_xid,
+          creator_xid,
           delay_until_epoch_ms,
           attributes,
           schedule_name,
           debounce_deadline_epoch_ms,
           is_debounced,
-          application_name
-        ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)
+          application_name,
+          owner_xid
+        ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
         ON CONFLICT (workflow_uuid)
           -- A no-op update, so an existing row comes back unchanged for the caller to inspect.
-          DO UPDATE SET owner_xid = workflow_status.owner_xid
-          RETURNING status, name, class_name, config_name, queue_name, executor_id, owner_xid, serialization`, [
+          DO UPDATE SET creator_xid = workflow_status.creator_xid
+          RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, [
                 initStatus.workflowUUID,
                 initStatus.status,
                 initStatus.workflowName,
@@ -4490,13 +4601,15 @@ class SystemDatabase {
                 initStatus.forkedFrom ?? null,
                 initStatus.parentWorkflowID ?? null,
                 initStatus.serialization,
-                ownerXid,
+                creatorXid,
                 initStatus.delayUntilEpochMS ?? null,
                 initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
                 initStatus.scheduleName ?? null,
                 initStatus.debounceDeadlineEpochMS ?? null,
                 initStatus.isDebounced ?? false,
                 initStatus.applicationName ?? null,
+                // A direct start runs at once, so its creator token also owns the execution.
+                initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
             ]);
             if (rows.length === 0) {
                 throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
@@ -4561,6 +4674,10 @@ class SystemDatabase {
         else if (update.clearCompletedAt) {
             setClause += `, completed_at = NULL`;
         }
+        if (update.ownerXid !== undefined) {
+            const param = args.push(update.ownerXid);
+            setClause += `, owner_xid=$${param}`;
+        }
         const where = options.where ?? {};
         if (where.status) {
             const param = args.push(where.status);
@@ -4570,6 +4687,10 @@ class SystemDatabase {
             const param = args.push(where.notStatus);
             whereClause += ` AND status!=$${param}`;
         }
+        if (where.ownerXid !== undefined) {
+            const param = args.push(where.ownerXid);
+            whereClause += ` AND owner_xid=$${param}`;
+        }
         const result = await client.query(`UPDATE "${this.schemaName}".workflow_status ${setClause} ${whereClause}`, args);
         const throwOnFailure = options.throwOnFailure ?? true;
         if (throwOnFailure && result.rowCount !== 1) {
@@ -4578,6 +4699,10 @@ class SystemDatabase {
         return result.rowCount ?? 0;
     }
     async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
+        const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
+        if (ownerXid !== undefined) {
+            await this.#checkOwner(client, workflowID, ownerXid);
+        }
         try {
             const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
          (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name, retention_timestamp)
@@ -4597,30 +4722,47 @@ class SystemDatabase {
                 // Mirrors the parent: only the running application records its steps.
                 this.appName ?? null,
             ]);
+            // A row already there with another completion time is this execution's own doing
+            // (a retry carries the same time), so it is nondeterminism, not a duplicate.
             if (checkConflict &&
                 (out?.rowCount ?? 0) > 0 &&
                 Number(out?.rows?.[0]?.completed_at_epoch_ms) !== endTimeEpochMs) {
-                dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Step output for ${workflowID}(${functionID}):${functionName} already recorded`);
-                throw new error_1.DBOSWorkflowConflictError(workflowID);
-            }
-            if (Number(out?.rows?.[0]?.completed_at_epoch_ms) === endTimeEpochMs) {
-                // Winning the checkpoint proves this executor is advancing the workflow:
-                // claim the executor_id marker (skips the write when already ours).
-                await client.query(`UPDATE "${this.schemaName}".workflow_status
-             SET executor_id = $1
-           WHERE workflow_uuid = $2 AND executor_id IS DISTINCT FROM $1`, [utils_1.globalParams.executorID, workflowID]);
+                throw new error_1.DBOSStepNondeterminismError(workflowID, functionID);
             }
         }
         catch (error) {
-            const err = error;
-            if (err.code === '40001' || err.code === '23505') {
-                // Serialization and primary key conflict (Postgres).
-                throw new error_1.DBOSWorkflowConflictError(workflowID);
-            }
-            else {
-                throw err;
+            if (error.code === '23505') {
+                throw new error_1.DBOSStepNondeterminismError(workflowID, functionID);
             }
+            throw error;
+        }
+    }
+    /** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
+    async #inTransaction(client, fn) {
+        await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+        let result;
+        try {
+            result = await fn();
+        }
+        catch (e) {
+            // Swallowed: a rollback failure must not mask why we are rolling back.
+            await client.query('ROLLBACK').catch(() => undefined);
+            throw e;
         }
+        await client.query('COMMIT');
+        return result;
+    }
+    /** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
+    async #checkOwner(client, workflowID, ownerXid) {
+        const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1 FOR NO KEY UPDATE`, [workflowID]);
+        if (rows[0]?.owner_xid !== ownerXid) {
+            throw new error_1.DBOSWorkflowConflictError(workflowID);
+        }
+    }
+    /** The workflow's current ownership token; null if unowned or missing. */
+    async getWorkflowOwner(workflowID) {
+        const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
+        return rows[0]?.owner_xid ?? null;
     }
     async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
         await this.#checkIfCanceled(client, workflowID);
@@ -4682,10 +4824,10 @@ class SystemDatabase {
                 }
                 return JSON.parse(res.output);
             }
-            await this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
+            await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
                 output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
                 serialization: serialization_1.DBOSPortableJSON.name(),
-            });
+            }));
             return endTimeMs;
         }
         finally {
@@ -4838,13 +4980,13 @@ __decorate([
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, Object]),
+    __metadata("design:paramtypes", [String, Object, String]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "recordWorkflowOutput", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, Object]),
+    __metadata("design:paramtypes", [String, Object, String]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "recordWorkflowError", null);
 __decorate([
@@ -4871,6 +5013,12 @@ __decorate([
     __metadata("design:paramtypes", [String, Number, String, Boolean, Number, Number, Object]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "recordOperationResult", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, Number, String, Function]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "runTransactionalStep", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
@@ -4931,6 +5079,12 @@ __decorate([
     __metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "send", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "sendFromStep", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
@@ -5009,4 +5163,10 @@ __decorate([
     __metadata("design:paramtypes", [String]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "getLatestApplicationVersion", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "getWorkflowOwner", null);
 //# sourceMappingURL=system_database.js.map