@dbos-inc/dbos-sdk 2.1.5-preview.g539c9d794d → 2.1.12-preview

package/dist/src/dbos-executor.js

@@ -20,10 +20,10 @@ const decorators_1 = require("./decorators");
 const api_1 = require("@opentelemetry/api");
 const knex_1 = __importDefault(require("knex"));
 const context_1 = require("./context");
-const debug_workflow_1 = require("./debugger/debug_workflow");
 const serialize_error_1 = require("serialize-error");
 const utils_1 = require("./utils");
 const node_path_1 = __importDefault(require("node:path"));
+const procedure_1 = require("./procedure");
 const lodash_1 = require("lodash");
 const wfqueue_1 = require("./wfqueue");
 const debugpoint_1 = require("./debugpoint");
@@ -279,7 +279,6 @@ class DBOSExecutor {
                 }
                 this.logger.debug(`Loaded ${length} ORM entities`);
             }
-            await ((0, user_database_1.createDBIfDoesNotExist)(this.config.poolConfig, this.logger));
             this.configureDbClient();
             if (!this.userDatabase) {
                 this.logger.error("No user database configured!");
@@ -354,24 +353,6 @@ class DBOSExecutor {
                 this.logger.error(`Unknown notice severity: ${msg.severity} - ${msg.message}`);
         }
     }
-    async callProcedure(proc, args) {
-        const client = await this.procedurePool.connect();
-        const log = (msg) => this.#logNotice(msg);
-        const procClassName = this.getProcedureClassName(proc);
-        const plainProcName = `${procClassName}_${proc.name}_p`;
-        const procName = this.config.appVersion
-            ? `v${this.config.appVersion}_${plainProcName}`
-            : plainProcName;
-        const sql = `CALL "${procName}"(${args.map((_v, i) => `$${i + 1}`).join()});`;
-        try {
-            client.on('notice', log);
-            return await client.query(sql, args).then(value => value.rows);
-        }
-        finally {
-            client.off('notice', log);
-            client.release();
-        }
-    }
     async destroy() {
         if (this.pendingWorkflowMap.size > 0) {
             this.logger.info("Waiting for pending workflows to finish.");
@@ -501,9 +482,6 @@ class DBOSExecutor {
     }
     // TODO: getProcedureInfoByNames??
     async workflow(wf, params, ...args) {
-        if (this.debugMode) {
-            return this.debugWorkflow(wf, params, undefined, undefined, ...args);
-        }
         return this.internalWorkflow(wf, params, undefined, undefined, ...args);
     }
     // If callerUUID and functionID are set, it means the workflow is invoked from within a workflow.
@@ -547,35 +525,57 @@ class DBOSExecutor {
         if ((wCtxt.tempWfOperationType !== TempWorkflowType.transaction
             && wCtxt.tempWfOperationType !== TempWorkflowType.procedure)
             || params.queueName !== undefined) {
-            // TODO: Make this transactional (and with the queue step below)
-            const ires = await this.systemDatabase.initWorkflowStatus(internalStatus, args);
-            args = ires.args;
-            status = ires.status;
-            await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_WORKFLOW_ENQUEUE);
+            if (this.debugMode) {
+                const wfStatus = await this.systemDatabase.getWorkflowStatus(workflowUUID);
+                const wfInputs = await this.systemDatabase.getWorkflowInputs(workflowUUID);
+                if (!wfStatus || !wfInputs) {
+                    throw new error_1.DBOSDebuggerError(`Failed to find inputs for workflow UUID ${workflowUUID}`);
+                }
+                // Make sure we use the same input.
+                if (utils_1.DBOSJSON.stringify(args) !== utils_1.DBOSJSON.stringify(wfInputs)) {
+                    throw new error_1.DBOSDebuggerError(`Detected different inputs for workflow UUID ${workflowUUID}.\n Received: ${utils_1.DBOSJSON.stringify(args)}\n Original: ${utils_1.DBOSJSON.stringify(wfInputs)}`);
+                }
+                status = wfStatus.status;
+            }
+            else {
+                // TODO: Make this transactional (and with the queue step below)
+                const ires = await this.systemDatabase.initWorkflowStatus(internalStatus, args);
+                args = ires.args;
+                status = ires.status;
+                await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_WORKFLOW_ENQUEUE);
+            }
         }
         const runWorkflow = async () => {
             let result;
             // Execute the workflow.
             try {
-                let cresult;
-                await (0, context_1.runWithWorkflowContext)(wCtxt, async () => {
-                    if (passContext) {
-                        cresult = await wf.call(params.configuredInstance, wCtxt, ...args);
-                    }
-                    else {
-                        cresult = await wf.call(params.configuredInstance, ...args);
-                    }
+                const callResult = await (0, context_1.runWithWorkflowContext)(wCtxt, async () => {
+                    const callPromise = passContext
+                        ? wf.call(params.configuredInstance, wCtxt, ...args)
+                        : wf.call(params.configuredInstance, ...args);
+                    return await callPromise;
                 });
-                result = cresult;
+                if (this.debugMode) {
+                    const recordedResult = await this.systemDatabase.getWorkflowResult(workflowUUID);
+                    if (utils_1.DBOSJSON.stringify(callResult) !== utils_1.DBOSJSON.stringify(recordedResult)) {
+                        this.logger.error(`Detect different output for the workflow UUID ${workflowUUID}!\n Received: ${utils_1.DBOSJSON.stringify(callResult)}\n Original: ${utils_1.DBOSJSON.stringify(recordedResult)}`);
+                    }
+                    result = recordedResult;
+                }
+                else {
+                    result = callResult;
+                }
                 internalStatus.output = result;
                 internalStatus.status = workflow_1.StatusString.SUCCESS;
-                if (internalStatus.queueName) {
+                if (internalStatus.queueName && !this.debugMode) {
                     // Now... the workflow isn't certainly done.
                     //  But waiting this long is for concurrency control anyway,
                     //   so it is probably done enough.
                     await this.systemDatabase.dequeueWorkflow(workflowUUID, this.#getQueueByName(internalStatus.queueName));
                 }
-                this.systemDatabase.bufferWorkflowOutput(workflowUUID, internalStatus);
+                if (!this.debugMode) {
+                    this.systemDatabase.bufferWorkflowOutput(workflowUUID, internalStatus);
+                }
                 wCtxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
             }
             catch (err) {
@@ -596,10 +596,12 @@ class DBOSExecutor {
                     }
                     internalStatus.error = utils_1.DBOSJSON.stringify((0, serialize_error_1.serializeError)(e));
                     internalStatus.status = workflow_1.StatusString.ERROR;
-                    if (internalStatus.queueName) {
+                    if (internalStatus.queueName && !this.debugMode) {
                         await this.systemDatabase.dequeueWorkflow(workflowUUID, this.#getQueueByName(internalStatus.queueName));
                     }
-                    await this.systemDatabase.recordWorkflowError(workflowUUID, internalStatus);
+                    if (!this.debugMode) {
+                        await this.systemDatabase.recordWorkflowError(workflowUUID, internalStatus);
+                    }
                     // TODO: Log errors, but not in the tests when they're expected.
                     wCtxt.span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: e.message });
                     throw err;
@@ -611,7 +613,9 @@ class DBOSExecutor {
                     || wCtxt.tempWfOperationType === TempWorkflowType.procedure) {
                     // For single-transaction workflows, asynchronously record inputs.
                     // We must buffer inputs after workflow status is buffered/flushed because workflow_inputs table has a foreign key reference to the workflow_status table.
-                    this.systemDatabase.bufferWorkflowInputs(workflowUUID, args);
+                    if (!this.debugMode) {
+                        this.systemDatabase.bufferWorkflowInputs(workflowUUID, args);
+                    }
                 }
             }
             // Asynchronously flush the result buffer.
@@ -620,7 +624,7 @@ class DBOSExecutor {
             }
             return result;
         };
-        if (status !== 'SUCCESS' && status !== 'ERROR' && (params.queueName === undefined || params.executeWorkflow)) {
+        if (this.debugMode || (status !== 'SUCCESS' && status !== 'ERROR' && (params.queueName === undefined || params.executeWorkflow))) {
             const workflowPromise = runWorkflow();
             // Need to await for the workflow and capture errors.
             const awaitWorkflowPromise = workflowPromise
@@ -636,7 +640,7 @@ class DBOSExecutor {
             return new workflow_1.InvokedHandle(this.systemDatabase, workflowPromise, workflowUUID, wf.name, callerUUID, callerFunctionID);
         }
         else {
-            if (params.queueName && status === 'ENQUEUED') {
+            if (params.queueName && status === 'ENQUEUED' && !this.debugMode) {
                 await this.systemDatabase.enqueueWorkflow(workflowUUID, this.#getQueueByName(params.queueName));
             }
             return new workflow_1.RetrievedHandle(this.systemDatabase, workflowUUID, callerUUID, callerFunctionID);
@@ -649,45 +653,135 @@ class DBOSExecutor {
         return q;
     }
     /**
-     * DEBUG MODE workflow execution, skipping all the recording
+     * Retrieve the transaction snapshot information of the current transaction
+     */
+    static async #retrieveSnapshot(query) {
+        const rows = await query("SELECT pg_current_snapshot()::text as txn_snapshot;", []);
+        return rows[0].txn_snapshot;
+    }
+    /**
+     * Check if an operation has already executed in a workflow.
+     * If it previously executed successfully, return its output.
+     * If it previously executed and threw an error, throw that error.
+     * Otherwise, return DBOSNull.
+     * Also return the transaction snapshot information of this current transaction.
      */
-    async debugWorkflow(wf, params, callerUUID, callerFunctionID, ...args) {
-        // In debug mode, we must have a specific workflow UUID.
-        if (!params.workflowUUID) {
-            throw new error_1.DBOSDebuggerError("Workflow UUID not found!");
+    async #checkExecution(query, workflowUUID, funcID) {
+        // Note: we read the current snapshot, not the recorded one!
+        const rows = await query("(SELECT output, error, txn_snapshot, true as recorded FROM dbos.transaction_outputs WHERE workflow_uuid=$1 AND function_id=$2 UNION ALL SELECT null as output, null as error, pg_current_snapshot()::text as txn_snapshot, false as recorded) ORDER BY recorded", [workflowUUID, funcID]);
+        if (rows.length === 0 || rows.length > 2) {
+            this.logger.error("Unexpected! This should never happen. Returned rows: " + rows.toString());
+            throw new error_1.DBOSError("This should never happen. Returned rows: " + rows.toString());
+        }
+        const res = {
+            output: exports.dbosNull,
+            txn_snapshot: ""
+        };
+        // recorded=false row will be first because we used ORDER BY.
+        res.txn_snapshot = rows[0].txn_snapshot;
+        if (rows.length === 2) {
+            if (utils_1.DBOSJSON.parse(rows[1].error) !== null) {
+                throw (0, serialize_error_1.deserializeError)(utils_1.DBOSJSON.parse(rows[1].error));
+            }
+            else {
+                res.output = utils_1.DBOSJSON.parse(rows[1].output);
+            }
         }
-        const workflowUUID = params.workflowUUID;
-        const wInfo = this.getWorkflowInfo(wf);
-        if (wInfo === undefined) {
-            throw new error_1.DBOSDebuggerError("Workflow unregistered! " + wf.name);
+        return res;
+    }
+    /**
+     * Write a operation's output to the database.
+     */
+    async #recordOutput(query, workflowUUID, funcID, txnSnapshot, output, isKeyConflict) {
+        if (this.debugMode) {
+            throw new error_1.DBOSDebuggerError("Cannot record output in debug mode.");
         }
-        const wConfig = wInfo.config;
-        const wCtxt = new debug_workflow_1.WorkflowContextDebug(this, params.parentCtx, workflowUUID, wConfig, wf.name);
-        // A workflow must have run before.
-        const wfStatus = await this.systemDatabase.getWorkflowStatus(workflowUUID);
-        const recordedInputs = await this.systemDatabase.getWorkflowInputs(workflowUUID);
-        if (!wfStatus || !recordedInputs) {
-            throw new error_1.DBOSDebuggerError("Workflow status or inputs not found! UUID: " + workflowUUID);
-        }
-        // Make sure we use the same input.
-        if (utils_1.DBOSJSON.stringify(args) !== utils_1.DBOSJSON.stringify(recordedInputs)) {
-            throw new error_1.DBOSDebuggerError(`Detect different input for the workflow UUID ${workflowUUID}!\n Received: ${utils_1.DBOSJSON.stringify(args)}\n Original: ${utils_1.DBOSJSON.stringify(recordedInputs)}`);
-        }
-        const workflowPromise = (0, context_1.runWithWorkflowContext)(wCtxt, async () => {
-            return await wf.call(params.configuredInstance, wCtxt, ...args)
-                .then(async (result) => {
-                // Check if the result is the same.
-                const recordedResult = await this.systemDatabase.getWorkflowResult(workflowUUID);
-                if (result === undefined && !recordedResult) {
-                    return result;
-                }
-                if (utils_1.DBOSJSON.stringify(result) !== utils_1.DBOSJSON.stringify(recordedResult)) {
-                    this.logger.error(`Detect different output for the workflow UUID ${workflowUUID}!\n Received: ${utils_1.DBOSJSON.stringify(result)}\n Original: ${utils_1.DBOSJSON.stringify(recordedResult)}`);
+        try {
+            const serialOutput = utils_1.DBOSJSON.stringify(output);
+            const rows = await query("INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, (select pg_current_xact_id_if_assigned()::text), $4, $5) RETURNING txn_id;", [workflowUUID, funcID, serialOutput, txnSnapshot, Date.now()]);
+            return rows[0].txn_id;
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictUUIDError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    /**
+     * Record an error in an operation to the database.
+     */
+    async #recordError(query, workflowUUID, funcID, txnSnapshot, err, isKeyConflict) {
+        if (this.debugMode) {
+            throw new error_1.DBOSDebuggerError("Cannot record error in debug mode.");
+        }
+        try {
+            const serialErr = utils_1.DBOSJSON.stringify((0, serialize_error_1.serializeError)(err));
+            await query("INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, error, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, null, $4, $5) RETURNING txn_id;", [workflowUUID, funcID, serialErr, txnSnapshot, Date.now()]);
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictUUIDError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    /**
+     * Write all entries in the workflow result buffer to the database.
+     * If it encounters a primary key error, this indicates a concurrent execution with the same UUID, so throw an DBOSError.
+     */
+    async #flushResultBuffer(query, resultBuffer, workflowUUID, isKeyConflict) {
+        const funcIDs = Array.from(resultBuffer.keys());
+        if (funcIDs.length === 0) {
+            return;
+        }
+        if (this.debugMode) {
+            throw new error_1.DBOSDebuggerError("Cannot flush result buffer in debug mode.");
+        }
+        funcIDs.sort();
+        try {
+            let sqlStmt = "INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, error, txn_id, txn_snapshot, created_at) VALUES ";
+            let paramCnt = 1;
+            const values = [];
+            for (const funcID of funcIDs) {
+                // Capture output and also transaction snapshot information.
+                // Initially, no txn_id because no queries executed.
+                const recorded = resultBuffer.get(funcID);
+                const output = recorded.output;
+                const txnSnapshot = recorded.txn_snapshot;
+                const createdAt = recorded.created_at;
+                if (paramCnt > 1) {
+                    sqlStmt += ", ";
                 }
-                return recordedResult; // Always return the recorded result.
-            });
-        });
-        return new workflow_1.InvokedHandle(this.systemDatabase, workflowPromise, workflowUUID, wf.name, callerUUID, callerFunctionID);
+                sqlStmt += `($${paramCnt++}, $${paramCnt++}, $${paramCnt++}, $${paramCnt++}, null, $${paramCnt++}, $${paramCnt++})`;
+                values.push(workflowUUID, funcID, utils_1.DBOSJSON.stringify(output), utils_1.DBOSJSON.stringify(null), txnSnapshot, createdAt);
+            }
+            this.logger.debug(sqlStmt);
+            await query(sqlStmt, values);
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictUUIDError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    flushResultBuffer(client, resultBuffer, workflowUUID) {
+        const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+        return this.#flushResultBuffer(func, resultBuffer, workflowUUID, (error) => this.userDatabase.isKeyConflictError(error));
+    }
+    #flushResultBufferProc(client, resultBuffer, workflowUUID) {
+        const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+        return this.#flushResultBuffer(func, resultBuffer, workflowUUID, user_database_1.pgNodeIsKeyConflictError);
     }
     async transaction(txn, params, ...args) {
         // Create a workflow and call transaction.
@@ -729,7 +823,8 @@ class DBOSExecutor {
                 // If the UUID is preset, it is possible this execution previously happened. Check, and return its original result if it did.
                 // Note: It is possible to retrieve a generated ID from a workflow handle, run a concurrent execution, and cause trouble for yourself. We recommend against this.
                 if (wfCtx.presetUUID) {
-                    const check = await wfCtx.checkTxExecution(client, funcId);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    const check = await this.#checkExecution(func, workflowUUID, funcId);
                     txn_snapshot = check.txn_snapshot;
                     if (check.output !== exports.dbosNull) {
                         tCtxt.span.setAttribute("cached", true);
@@ -740,11 +835,15 @@ class DBOSExecutor {
                 }
                 else {
                     // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
-                    txn_snapshot = await wfCtx.retrieveTxSnapshot(client);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(func);
+                }
+                if (this.debugMode) {
+                    throw new error_1.DBOSDebuggerError(`Failed to find inputs for workflow UUID ${workflowUUID}`);
                 }
                 // For non-read-only transactions, flush the result buffer.
                 if (!readOnly) {
-                    await wfCtx.flushResultBuffer(client);
+                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
                 }
                 // Execute the user's transaction.
                 let cresult;
@@ -773,7 +872,8 @@ class DBOSExecutor {
                 else {
                     try {
                         // Synchronously record the output of write transactions and obtain the transaction ID.
-                        const pg_txn_id = await wfCtx.recordOutputTx(client, funcId, txn_snapshot, result);
+                        const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                        const pg_txn_id = await this.#recordOutput(func, wfCtx.workflowUUID, funcId, txn_snapshot, result, (error) => this.userDatabase.isKeyConflictError(error));
                         tCtxt.span.setAttribute("pg_txn_id", pg_txn_id);
                         wfCtx.resultBuffer.clear();
                     }
@@ -796,6 +896,9 @@ class DBOSExecutor {
                 return result;
             }
             catch (err) {
+                if (this.debugMode) {
+                    throw err;
+                }
                 if (this.userDatabase.isRetriableTransactionError(err)) {
                     // serialization_failure in PostgreSQL
                     span.addEvent("TXN SERIALIZATION FAILURE", { "retryWaitMillis": retryWaitMillis }, performance.now());
@@ -808,8 +911,9 @@ class DBOSExecutor {
                 // Record and throw other errors.
                 const e = err;
                 await this.userDatabase.transaction(async (client) => {
-                    await wfCtx.flushResultBuffer(client);
-                    await wfCtx.recordErrorTx(client, funcId, txn_snapshot, e);
+                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
                 }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
                 wfCtx.resultBuffer.clear();
                 span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: e.message });
@@ -822,15 +926,215 @@ class DBOSExecutor {
         // Create a workflow and call procedure.
         const temp_workflow = async (ctxt, ...args) => {
             const ctxtImpl = ctxt;
-            return await ctxtImpl.procedure(proc, ...args);
+            return this.callProcedureFunction(proc, ctxtImpl, ...args);
         };
-        return (await this.workflow(temp_workflow, { ...params,
+        return await (await this.workflow(temp_workflow, {
+            ...params,
             tempWfType: TempWorkflowType.procedure,
             tempWfName: (0, decorators_1.getRegisteredMethodName)(proc),
             tempWfClass: (0, decorators_1.getRegisteredMethodClassName)(proc),
         }, ...args)).getResult();
     }
-    async executeProcedure(func, config) {
+    async callProcedureFunction(proc, wfCtx, ...args) {
+        const procInfo = this.getProcedureInfo(proc);
+        if (procInfo === undefined) {
+            throw new error_1.DBOSNotRegisteredError(proc.name);
+        }
+        const executeLocally = this.debugMode || (procInfo.config.executeLocally ?? false);
+        const funcId = wfCtx.functionIDGetIncrement();
+        const span = this.tracer.startSpan(proc.name, {
+            operationUUID: wfCtx.workflowUUID,
+            operationType: exports.OperationType.PROCEDURE,
+            authenticatedUser: wfCtx.authenticatedUser,
+            assumedRole: wfCtx.assumedRole,
+            authenticatedRoles: wfCtx.authenticatedRoles,
+            readOnly: procInfo.config.readOnly ?? false,
+            isolationLevel: procInfo.config.isolationLevel,
+            executeLocally,
+        }, wfCtx.span);
+        try {
+            const result = executeLocally
+                ? await this.#callProcedureFunctionLocal(proc, args, wfCtx, span, procInfo, funcId)
+                : await this.#callProcedureFunctionRemote(proc, args, wfCtx, span, procInfo.config, funcId);
+            span.setStatus({ code: api_1.SpanStatusCode.OK });
+            return result;
+        }
+        catch (e) {
+            const { message } = e;
+            span.setStatus({ code: api_1.SpanStatusCode.ERROR, message });
+            throw e;
+        }
+        finally {
+            this.tracer.endSpan(span);
+        }
+    }
+    async #callProcedureFunctionLocal(proc, args, wfCtx, span, procInfo, funcId) {
+        let retryWaitMillis = 1;
+        const backoffFactor = 1.5;
+        const maxRetryWaitMs = 2000; // Maximum wait 2 seconds.
+        const readOnly = procInfo.config.readOnly ?? false;
+        while (true) {
+            let txn_snapshot = "invalid";
+            const wrappedProcedure = async (client) => {
+                const ctxt = new procedure_1.StoredProcedureContextImpl(client, wfCtx, span, this.logger, funcId, proc.name);
+                if (wfCtx.presetUUID) {
+                    const func = (sql, args) => this.procedurePool.query(sql, args).then(v => v.rows);
+                    const check = await this.#checkExecution(func, wfCtx.workflowUUID, funcId);
+                    txn_snapshot = check.txn_snapshot;
+                    if (check.output !== exports.dbosNull) {
+                        ctxt.span.setAttribute("cached", true);
+                        ctxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
+                        this.tracer.endSpan(ctxt.span);
+                        return check.output;
+                    }
+                }
+                else {
+                    // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
+                    const func = (sql, args) => this.procedurePool.query(sql, args).then(v => v.rows);
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(func);
+                }
+                if (this.debugMode) {
+                    throw new error_1.DBOSDebuggerError(`Failed to find inputs for workflow UUID ${wfCtx.workflowUUID}`);
+                }
+                // For non-read-only transactions, flush the result buffer.
+                if (!readOnly) {
+                    await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                }
+                let cresult;
+                if (procInfo.registration.passContext) {
+                    await (0, context_1.runWithStoredProcContext)(ctxt, async () => {
+                        cresult = await proc(ctxt, ...args);
+                    });
+                }
+                else {
+                    await (0, context_1.runWithStoredProcContext)(ctxt, async () => {
+                        const pf = proc;
+                        cresult = await pf(...args);
+                    });
+                }
+                const result = cresult;
+                if (readOnly) {
+                    // Buffer the output of read-only transactions instead of synchronously writing it.
+                    const readOutput = {
+                        output: result,
+                        txn_snapshot: txn_snapshot,
+                        created_at: Date.now(),
+                    };
+                    wfCtx.resultBuffer.set(funcId, readOutput);
+                }
+                else {
+                    // Synchronously record the output of write transactions and obtain the transaction ID.
+                    const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+                    const pg_txn_id = await this.#recordOutput(func, wfCtx.workflowUUID, funcId, txn_snapshot, result, user_database_1.pgNodeIsKeyConflictError);
+                    // const pg_txn_id = await wfCtx.recordOutputProc<R>(client, funcId, txn_snapshot, result);
+                    ctxt.span.setAttribute("pg_txn_id", pg_txn_id);
+                    wfCtx.resultBuffer.clear();
+                }
+                return result;
+            };
+            try {
+                const result = await this.invokeStoredProcFunction(wrappedProcedure, { isolationLevel: procInfo.config.isolationLevel });
+                span.setStatus({ code: api_1.SpanStatusCode.OK });
+                return result;
+            }
+            catch (err) {
+                if (this.userDatabase.isRetriableTransactionError(err)) {
+                    // serialization_failure in PostgreSQL
+                    span.addEvent("TXN SERIALIZATION FAILURE", { "retryWaitMillis": retryWaitMillis }, performance.now());
+                    // Retry serialization failures.
+                    await (0, utils_1.sleepms)(retryWaitMillis);
+                    retryWaitMillis *= backoffFactor;
+                    retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
+                    continue;
+                }
+                // Record and throw other errors.
+                const e = err;
+                await this.invokeStoredProcFunction(async (client) => {
+                    await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                    const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+                    await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, user_database_1.pgNodeIsKeyConflictError);
+                }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                await this.userDatabase.transaction(async (client) => {
+                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
+                }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                wfCtx.resultBuffer.clear();
+                throw err;
+            }
+        }
+    }
+    async #callProcedureFunctionRemote(proc, args, wfCtx, span, config, funcId) {
+        if (this.debugMode) {
+            throw new error_1.DBOSDebuggerError("Can't invoke stored procedure in debug mode.");
+        }
+        const readOnly = config.readOnly ?? false;
+        const $jsonCtx = {
+            request: wfCtx.request,
+            authenticatedUser: wfCtx.authenticatedUser,
+            authenticatedRoles: wfCtx.authenticatedRoles,
+            assumedRole: wfCtx.assumedRole,
+        };
+        // Note, node-pg converts JS arrays to postgres array literals, so must call JSON.strigify on
+        // args and bufferedResults before being passed to #invokeStoredProc
+        const $args = [wfCtx.workflowUUID, funcId, wfCtx.presetUUID, $jsonCtx, null, JSON.stringify(args)];
+        if (!readOnly) {
+            // function_id, output, txn_snapshot, created_at
+            const bufferedResults = new Array();
+            for (const [functionID, { output, txn_snapshot, created_at }] of wfCtx.resultBuffer.entries()) {
+                bufferedResults.push([functionID, output, txn_snapshot, created_at]);
+            }
+            // sort by function ID
+            bufferedResults.sort((a, b) => a[0] - b[0]);
+            $args.unshift(bufferedResults.length > 0 ? JSON.stringify(bufferedResults) : null);
+        }
+        const [{ return_value }] = await this.#invokeStoredProc(proc, $args);
+        const { error, output, txn_snapshot, txn_id, created_at } = return_value;
+        // buffered results are persisted in r/w stored procs, even if it returns an error
+        if (!readOnly) {
+            wfCtx.resultBuffer.clear();
+        }
+        // if the stored proc returns an error, deserialize and throw it.
+        // stored proc saves the error in tx_output before returning
+        if (error) {
+            throw (0, serialize_error_1.deserializeError)(error);
+        }
+        // if txn_snapshot is provided, the output needs to be buffered
+        if (readOnly && txn_snapshot) {
+            wfCtx.resultBuffer.set(funcId, {
+                output,
+                txn_snapshot,
+                created_at: created_at ?? Date.now(),
+            });
+        }
+        if (!readOnly) {
+            wfCtx.resultBuffer.clear();
+        }
+        if (txn_id) {
+            span.setAttribute("pg_txn_id", txn_id);
+        }
+        span.setStatus({ code: api_1.SpanStatusCode.OK });
+        return output;
+    }
+    async #invokeStoredProc(proc, args) {
+        const client = await this.procedurePool.connect();
+        const log = (msg) => this.#logNotice(msg);
+        const procClassName = this.getProcedureClassName(proc);
+        const plainProcName = `${procClassName}_${proc.name}_p`;
+        const procName = this.config.appVersion
+            ? `v${this.config.appVersion}_${plainProcName}`
+            : plainProcName;
+        const sql = `CALL "${procName}"(${args.map((_v, i) => `$${i + 1}`).join()});`;
+        try {
+            client.on('notice', log);
+            return await client.query(sql, args).then(value => value.rows);
+        }
+        finally {
+            client.off('notice', log);
+            client.release();
+        }
+    }
+    async invokeStoredProcFunction(func, config) {
         const client = await this.procedurePool.connect();
         try {
             const readOnly = config.readOnly ?? false;
@@ -889,7 +1193,7 @@ class DBOSExecutor {
         }, wfCtx.span);
         const ctxt = new step_1.StepContextImpl(wfCtx, funcID, span, this.logger, commInfo.config, stepFn.name);
         await this.userDatabase.transaction(async (client) => {
-            await wfCtx.flushResultBuffer(client);
+            await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
         }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
         wfCtx.resultBuffer.clear();
         // Check if this execution previously happened, returning its original result if it did.
@@ -900,6 +1204,9 @@ class DBOSExecutor {
             this.tracer.endSpan(ctxt.span);
             return check;
         }
+        if (this.debugMode) {
+            throw new error_1.DBOSDebuggerError(`Failed to find recorded output for workflow UUID: ${wfCtx.workflowUUID}`);
+        }
         // Execute the step function.  If it throws an exception, retry with exponential backoff.
         // After reaching the maximum number of retries, throw an DBOSError.
         let result = exports.dbosNull;
@@ -1045,6 +1352,9 @@ class DBOSExecutor {
      * It runs to completion all pending workflows that were executing when the previous executor failed.
      */
     async recoverPendingWorkflows(executorIDs = ["local"]) {
+        if (this.debugMode) {
+            throw new error_1.DBOSDebuggerError("Cannot recover pending workflows in debug mode.");
+        }
         const pendingWorkflows = [];
         for (const execID of executorIDs) {
             if (execID === "local" && process.env.DBOS__VMID) {
@@ -1196,13 +1506,16 @@ class DBOSExecutor {
      * Periodically flush the workflow output buffer to the system database.
      */
     async flushWorkflowBuffers() {
-        if (this.initialized) {
+        if (this.initialized && !this.debugMode) {
             await this.flushWorkflowResultBuffer();
             await this.systemDatabase.flushWorkflowSystemBuffers();
         }
         this.isFlushingBuffers = false;
     }
     async flushWorkflowResultBuffer() {
+        if (this.debugMode) {
+            throw new error_1.DBOSDebuggerError(`Cannot flush workflow result buffer in debug mode.`);
+        }
         const localBuffer = new Map(this.workflowResultBuffer);
         this.workflowResultBuffer.clear();
         const totalSize = localBuffer.size;