@dbos-inc/dbos-sdk 2.6.2-preview → 2.6.11-preview

package/dist/src/dbos-executor.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DBOSExecutor = exports.OperationType = exports.dbosNull = void 0;
+exports.DBOSExecutor = exports.OperationType = exports.DebugMode = exports.dbosNull = void 0;
 const error_1 = require("./error");
 const workflow_1 = require("./workflow");
 const transaction_1 = require("./transaction");
@@ -52,6 +52,12 @@ const wfqueue_1 = require("./wfqueue");
 const debugpoint_1 = require("./debugpoint");
 const crypto = __importStar(require("crypto"));
 exports.dbosNull = {};
+var DebugMode;
+(function (DebugMode) {
+    DebugMode[DebugMode["DISABLED"] = 0] = "DISABLED";
+    DebugMode[DebugMode["ENABLED"] = 1] = "ENABLED";
+    DebugMode[DebugMode["TIME_TRAVEL"] = 2] = "TIME_TRAVEL";
+})(DebugMode || (exports.DebugMode = DebugMode = {}));
 exports.OperationType = {
     HANDLER: 'handler',
     WORKFLOW: 'workflow',
@@ -105,6 +111,20 @@ class DBOSExecutor {
     isFlushingBuffers = false;
     static defaultNotificationTimeoutSec = 60;
     debugMode;
+    get isDebugging() {
+        switch (this.debugMode) {
+            case DebugMode.DISABLED:
+                return false;
+            case DebugMode.ENABLED:
+            case DebugMode.TIME_TRAVEL:
+                return true;
+            default: {
+                const _never = this.debugMode;
+                // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+                throw new Error(`Unexpected DBOS debug mode: ${this.debugMode}`);
+            }
+        }
+    }
     static systemDBSchemaName = 'dbos';
     logger;
     tracer;
@@ -117,9 +137,9 @@ class DBOSExecutor {
     executorID = utils_1.globalParams.executorID;
     static globalInstance = undefined;
     /* WORKFLOW EXECUTOR LIFE CYCLE MANAGEMENT */
-    constructor(config, systemDatabase) {
+    constructor(config, { systemDatabase, debugMode } = {}) {
         this.config = config;
-        this.debugMode = config.debugMode ?? false;
+        this.debugMode = debugMode ?? DebugMode.DISABLED;
         // Set configured environment variables
         if (config.env) {
             for (const [key, value] of Object.entries(config.env)) {
@@ -141,7 +161,7 @@ class DBOSExecutor {
         }
         this.logger = new logs_1.GlobalLogger(this.telemetryCollector, this.config.telemetry?.logs);
         this.tracer = new traces_1.Tracer(this.telemetryCollector);
-        if (this.debugMode) {
+        if (this.isDebugging) {
             this.logger.info('Running in debug mode!');
         }
         this.procedurePool = new pg_1.Pool(this.config.poolConfig);
@@ -154,7 +174,7 @@ class DBOSExecutor {
             this.systemDatabase = new system_database_1.PostgresSystemDatabase(this.config.poolConfig, this.config.system_database, this.logger);
         }
         this.flushBufferID = setInterval(() => {
-            if (!this.debugMode && !this.isFlushingBuffers) {
+            if (!this.isDebugging && !this.isFlushingBuffers) {
                 this.isFlushingBuffers = true;
                 void this.flushWorkflowBuffers();
             }
@@ -295,7 +315,7 @@ class DBOSExecutor {
                 }
                 this.logger.debug(`Loaded ${length} ORM entities`);
             }
-            if (!this.debugMode) {
+            if (!this.isDebugging) {
                 await (0, user_database_1.createDBIfDoesNotExist)(this.config.poolConfig, this.logger);
             }
             this.configureDbClient();
@@ -307,8 +327,8 @@ class DBOSExecutor {
                 this.#registerClass(cls);
             }
             // Debug mode doesn't need to initialize the DBs. Everything should appear to be read-only.
-            await this.userDatabase.init(this.debugMode);
-            if (!this.debugMode) {
+            await this.userDatabase.init(this.isDebugging);
+            if (!this.isDebugging) {
                 await this.systemDatabase.init();
             }
         }
@@ -331,7 +351,7 @@ class DBOSExecutor {
         }
         this.initialized = true;
         // Only execute init code if under non-debug mode
-        if (!this.debugMode) {
+        if (!this.isDebugging) {
             for (const cls of classes) {
                 // Init its configurations
                 const creg = (0, decorators_1.getOrCreateClassRegistration)(cls);
@@ -384,7 +404,7 @@ class DBOSExecutor {
             await Promise.allSettled(this.pendingWorkflowMap.values());
         }
         clearInterval(this.flushBufferID);
-        if (!this.debugMode && !this.isFlushingBuffers) {
+        if (!this.isDebugging && !this.isFlushingBuffers) {
             // Don't flush the buffers if we're already flushing them in the background.
             await this.flushWorkflowBuffers();
         }
@@ -522,9 +542,9 @@ class DBOSExecutor {
         const internalStatus = {
             workflowUUID: workflowUUID,
             status: params.queueName !== undefined ? workflow_1.StatusString.ENQUEUED : workflow_1.StatusString.PENDING,
-            name: wf.name,
-            className: wCtxt.isTempWorkflow ? '' : (0, decorators_1.getRegisteredMethodClassName)(wf),
-            configName: params.configuredInstance?.name || '',
+            workflowName: wf.name,
+            workflowClassName: wCtxt.isTempWorkflow ? '' : (0, decorators_1.getRegisteredMethodClassName)(wf),
+            workflowConfigName: params.configuredInstance?.name || '',
             queueName: params.queueName,
             authenticatedUser: wCtxt.authenticatedUser,
             output: undefined,
@@ -532,15 +552,15 @@ class DBOSExecutor {
             assumedRole: wCtxt.assumedRole,
             authenticatedRoles: wCtxt.authenticatedRoles,
             request: wCtxt.request,
-            executorID: wCtxt.executorID,
+            executorId: wCtxt.executorID,
             applicationVersion: utils_1.globalParams.appVersion,
             applicationID: wCtxt.applicationID,
             createdAt: Date.now(), // Remember the start time of this workflow
             maxRetries: wCtxt.maxRecoveryAttempts,
         };
         if (wCtxt.isTempWorkflow) {
-            internalStatus.name = `${DBOSExecutor.tempWorkflowName}-${wCtxt.tempWfOperationType}-${wCtxt.tempWfOperationName}`;
-            internalStatus.className = params.tempWfClass ?? '';
+            internalStatus.workflowName = `${DBOSExecutor.tempWorkflowName}-${wCtxt.tempWfOperationType}-${wCtxt.tempWfOperationName}`;
+            internalStatus.workflowClassName = params.tempWfClass ?? '';
         }
         let status = undefined;
         // Synchronously set the workflow's status to PENDING and record workflow inputs (for non single-transaction workflows).
@@ -548,7 +568,7 @@ class DBOSExecutor {
         if ((wCtxt.tempWfOperationType !== TempWorkflowType.transaction &&
             wCtxt.tempWfOperationType !== TempWorkflowType.procedure) ||
             params.queueName !== undefined) {
-            if (this.debugMode) {
+            if (this.isDebugging) {
                 const wfStatus = await this.systemDatabase.getWorkflowStatus(workflowUUID);
                 const wfInputs = await this.systemDatabase.getWorkflowInputs(workflowUUID);
                 if (!wfStatus || !wfInputs) {
@@ -578,7 +598,7 @@ class DBOSExecutor {
                         : wf.call(params.configuredInstance, ...args);
                     return await callPromise;
                 });
-                if (this.debugMode) {
+                if (this.isDebugging) {
                     const recordedResult = await this.systemDatabase.getWorkflowResult(workflowUUID);
                     if (!resultsMatch(recordedResult, callResult)) {
                         this.logger.error(`Detect different output for the workflow UUID ${workflowUUID}!\n Received: ${utils_1.DBOSJSON.stringify(callResult)}\n Original: ${utils_1.DBOSJSON.stringify(recordedResult)}`);
@@ -596,13 +616,13 @@ class DBOSExecutor {
                 }
                 internalStatus.output = result;
                 internalStatus.status = workflow_1.StatusString.SUCCESS;
-                if (internalStatus.queueName && !this.debugMode) {
+                if (internalStatus.queueName && !this.isDebugging) {
                     // Now... the workflow isn't certainly done.
                     //  But waiting this long is for concurrency control anyway,
                     //   so it is probably done enough.
                     await this.systemDatabase.dequeueWorkflow(workflowUUID, this.#getQueueByName(internalStatus.queueName));
                 }
-                if (!this.debugMode) {
+                if (!this.isDebugging) {
                     this.systemDatabase.bufferWorkflowOutput(workflowUUID, internalStatus);
                 }
                 wCtxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
@@ -618,7 +638,7 @@ class DBOSExecutor {
                 else if (err instanceof error_1.DBOSWorkflowCancelledError) {
                     internalStatus.error = err.message;
                     internalStatus.status = workflow_1.StatusString.CANCELLED;
-                    if (!this.debugMode) {
+                    if (!this.isDebugging) {
                         await this.systemDatabase.setWorkflowStatus(workflowUUID, workflow_1.StatusString.CANCELLED, false);
                     }
                     wCtxt.span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: err.message });
@@ -631,14 +651,14 @@ class DBOSExecutor {
                     this.logger.error(e);
                     e.dbos_already_logged = true;
                     if (wCtxt.isTempWorkflow) {
-                        internalStatus.name = `${DBOSExecutor.tempWorkflowName}-${wCtxt.tempWfOperationType}-${wCtxt.tempWfOperationName}`;
+                        internalStatus.workflowName = `${DBOSExecutor.tempWorkflowName}-${wCtxt.tempWfOperationType}-${wCtxt.tempWfOperationName}`;
                     }
                     internalStatus.error = utils_1.DBOSJSON.stringify((0, serialize_error_1.serializeError)(e));
                     internalStatus.status = workflow_1.StatusString.ERROR;
-                    if (internalStatus.queueName && !this.debugMode) {
+                    if (internalStatus.queueName && !this.isDebugging) {
                         await this.systemDatabase.dequeueWorkflow(workflowUUID, this.#getQueueByName(internalStatus.queueName));
                     }
-                    if (!this.debugMode) {
+                    if (!this.isDebugging) {
                         await this.systemDatabase.recordWorkflowError(workflowUUID, internalStatus);
                     }
                     // TODO: Log errors, but not in the tests when they're expected.
@@ -652,7 +672,7 @@ class DBOSExecutor {
                     wCtxt.tempWfOperationType === TempWorkflowType.procedure) {
                     // For single-transaction workflows, asynchronously record inputs.
                     // We must buffer inputs after workflow status is buffered/flushed because workflow_inputs table has a foreign key reference to the workflow_status table.
-                    if (!this.debugMode) {
+                    if (!this.isDebugging) {
                         this.systemDatabase.bufferWorkflowInputs(workflowUUID, args);
                     }
                 }
@@ -663,7 +683,7 @@ class DBOSExecutor {
             }
             return result;
         };
-        if (this.debugMode ||
+        if (this.isDebugging ||
             (status !== 'SUCCESS' && status !== 'ERROR' && (params.queueName === undefined || params.executeWorkflow))) {
             const workflowPromise = runWorkflow();
             // Need to await for the workflow and capture errors.
@@ -680,7 +700,7 @@ class DBOSExecutor {
             return new workflow_1.InvokedHandle(this.systemDatabase, workflowPromise, workflowUUID, wf.name, callerUUID, callerFunctionID);
         }
         else {
-            if (params.queueName && status === 'ENQUEUED' && !this.debugMode) {
+            if (params.queueName && status === 'ENQUEUED' && !this.isDebugging) {
                 await this.systemDatabase.enqueueWorkflow(workflowUUID, this.#getQueueByName(params.queueName));
             }
             return new workflow_1.RetrievedHandle(this.systemDatabase, workflowUUID, callerUUID, callerFunctionID);
@@ -702,39 +722,37 @@ class DBOSExecutor {
     /**
      * Check if an operation has already executed in a workflow.
      * If it previously executed successfully, return its output.
-     * If it previously executed and threw an error, throw that error.
+     * If it previously executed and threw an error, return that error.
      * Otherwise, return DBOSNull.
-     * Also return the transaction snapshot information of this current transaction.
+     * Also return the transaction snapshot and id information of the original or current transaction.
      */
     async #checkExecution(query, workflowUUID, funcID) {
-        // Note: we read the current snapshot, not the recorded one!
-        const rows = await query('(SELECT output, error, txn_snapshot, true as recorded FROM dbos.transaction_outputs WHERE workflow_uuid=$1 AND function_id=$2 UNION ALL SELECT null as output, null as error, pg_current_snapshot()::text as txn_snapshot, false as recorded) ORDER BY recorded', [workflowUUID, funcID]);
+        const rows = await query('(SELECT output, error, txn_snapshot, txn_id, true as recorded FROM dbos.transaction_outputs WHERE workflow_uuid=$1 AND function_id=$2 UNION ALL SELECT null as output, null as error, pg_current_snapshot()::text as txn_snapshot, null as txn_id, false as recorded) ORDER BY recorded', [workflowUUID, funcID]);
         if (rows.length === 0 || rows.length > 2) {
             const returnedRows = JSON.stringify(rows);
             this.logger.error('Unexpected! This should never happen. Returned rows: ' + returnedRows);
             throw new error_1.DBOSError('This should never happen. Returned rows: ' + returnedRows);
         }
-        const res = {
-            output: exports.dbosNull,
-            txn_snapshot: '',
-        };
-        // recorded=false row will be first because we used ORDER BY.
-        res.txn_snapshot = rows[0].txn_snapshot;
         if (rows.length === 2) {
-            if (utils_1.DBOSJSON.parse(rows[1].error) !== null) {
-                throw (0, serialize_error_1.deserializeError)(utils_1.DBOSJSON.parse(rows[1].error));
+            const { txn_snapshot, txn_id } = rows[1];
+            const error = utils_1.DBOSJSON.parse(rows[1].error);
+            if (error) {
+                return { result: (0, serialize_error_1.deserializeError)(error), txn_snapshot, txn_id: txn_id ?? undefined };
             }
             else {
-                res.output = utils_1.DBOSJSON.parse(rows[1].output);
+                return { result: utils_1.DBOSJSON.parse(rows[1].output), txn_snapshot, txn_id: txn_id ?? undefined };
             }
         }
-        return res;
+        else {
+            const { txn_snapshot } = rows[0];
+            return { result: exports.dbosNull, txn_snapshot, txn_id: undefined };
+        }
     }
     /**
      * Write a operation's output to the database.
      */
     async #recordOutput(query, workflowUUID, funcID, txnSnapshot, output, isKeyConflict) {
-        if (this.debugMode) {
+        if (this.isDebugging) {
             throw new error_1.DBOSDebuggerError('Cannot record output in debug mode.');
         }
         try {
@@ -756,7 +774,7 @@ class DBOSExecutor {
      * Record an error in an operation to the database.
      */
     async #recordError(query, workflowUUID, funcID, txnSnapshot, err, isKeyConflict) {
-        if (this.debugMode) {
+        if (this.isDebugging) {
             throw new error_1.DBOSDebuggerError('Cannot record error in debug mode.');
         }
         try {
@@ -782,7 +800,7 @@ class DBOSExecutor {
         if (funcIDs.length === 0) {
             return;
         }
-        if (this.debugMode) {
+        if (this.isDebugging) {
             throw new error_1.DBOSDebuggerError('Cannot flush result buffer in debug mode.');
         }
         funcIDs.sort();
@@ -869,43 +887,71 @@ class DBOSExecutor {
                 const tCtxt = new transaction_1.TransactionContextImpl(this.userDatabase.getName(), client, wfCtx, span, this.logger, funcId, txn.name);
                 // If the UUID is preset, it is possible this execution previously happened. Check, and return its original result if it did.
                 // Note: It is possible to retrieve a generated ID from a workflow handle, run a concurrent execution, and cause trouble for yourself. We recommend against this.
+                let prevResult = exports.dbosNull;
+                const queryFunc = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
                 if (wfCtx.presetUUID) {
-                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
-                    const check = await this.#checkExecution(func, workflowUUID, funcId);
-                    txn_snapshot = check.txn_snapshot;
-                    if (check.output !== exports.dbosNull) {
+                    const executionResult = await this.#checkExecution(queryFunc, workflowUUID, funcId);
+                    prevResult = executionResult.result;
+                    txn_snapshot = executionResult.txn_snapshot;
+                    if (prevResult !== exports.dbosNull) {
                         tCtxt.span.setAttribute('cached', true);
-                        tCtxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
-                        this.tracer.endSpan(tCtxt.span);
-                        return check.output;
+                        if (this.debugMode === DebugMode.TIME_TRAVEL) {
+                            // for time travel debugging, navigate the proxy to the time of this transaction's snapshot
+                            await queryFunc(`--proxy:${executionResult.txn_id ?? ''}:${txn_snapshot}`, []);
+                        }
+                        else {
+                            // otherwise, return/throw the previous result
+                            if (prevResult instanceof Error) {
+                                throw prevResult;
+                            }
+                            else {
+                                return prevResult;
+                            }
+                        }
                     }
                 }
                 else {
                     // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
-                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
-                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(func);
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(queryFunc);
                 }
-                if (this.debugMode) {
+                if (this.isDebugging && prevResult === exports.dbosNull) {
                     throw new error_1.DBOSDebuggerError(`Failed to find the recorded output for the transaction: workflow UUID ${workflowUUID}, step number ${funcId}`);
                 }
                 // For non-read-only transactions, flush the result buffer.
-                if (!readOnly) {
+                if (!this.isDebugging && !readOnly) {
                     await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
                 }
                 // Execute the user's transaction.
-                let cresult;
-                if (txnInfo.registration.passContext) {
-                    await (0, context_1.runWithTransactionContext)(tCtxt, async () => {
-                        cresult = await txn.call(clsinst, tCtxt, ...args);
-                    });
+                const result = await (async function () {
+                    try {
+                        return await (0, context_1.runWithTransactionContext)(tCtxt, async () => {
+                            if (txnInfo.registration.passContext) {
+                                return await txn.call(clsinst, tCtxt, ...args);
+                            }
+                            else {
+                                const tf = txn;
+                                return await tf.call(clsinst, ...args);
+                            }
+                        });
+                    }
+                    catch (e) {
+                        return e instanceof Error ? e : new Error(`${e}`);
+                    }
+                })();
+                if (this.isDebugging) {
+                    if (prevResult instanceof Error) {
+                        throw prevResult;
+                    }
+                    const prevResultJson = utils_1.DBOSJSON.stringify(prevResult);
+                    const resultJson = utils_1.DBOSJSON.stringify(result);
+                    if (prevResultJson !== resultJson) {
+                        this.logger.error(`Detected different transaction output than the original one!\n Result: ${resultJson}\n Original: ${utils_1.DBOSJSON.stringify(prevResultJson)}`);
+                    }
+                    return prevResult;
                 }
-                else {
-                    await (0, context_1.runWithTransactionContext)(tCtxt, async () => {
-                        const tf = txn;
-                        cresult = await tf.call(clsinst, ...args);
-                    });
+                if (result instanceof Error) {
+                    throw result;
                 }
-                const result = cresult;
                 // Record the execution, commit, and return.
                 if (readOnly) {
                     // Buffer the output of read-only transactions instead of synchronously writing it.
@@ -919,8 +965,7 @@ class DBOSExecutor {
                 else {
                     try {
                         // Synchronously record the output of write transactions and obtain the transaction ID.
-                        const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
-                        const pg_txn_id = await this.#recordOutput(func, wfCtx.workflowUUID, funcId, txn_snapshot, result, (error) => this.userDatabase.isKeyConflictError(error));
+                        const pg_txn_id = await this.#recordOutput(queryFunc, wfCtx.workflowUUID, funcId, txn_snapshot, result, (error) => this.userDatabase.isKeyConflictError(error));
                         tCtxt.span.setAttribute('pg_txn_id', pg_txn_id);
                         wfCtx.resultBuffer.clear();
                     }
@@ -943,26 +988,26 @@ class DBOSExecutor {
                 return result;
             }
             catch (err) {
-                if (this.debugMode) {
-                    throw err;
-                }
-                if (this.userDatabase.isRetriableTransactionError(err)) {
-                    // serialization_failure in PostgreSQL
-                    span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
-                    // Retry serialization failures.
-                    await (0, utils_1.sleepms)(retryWaitMillis);
-                    retryWaitMillis *= backoffFactor;
-                    retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
-                    continue;
-                }
-                // Record and throw other errors.
                 const e = err;
-                await this.userDatabase.transaction(async (client) => {
-                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
-                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
-                    await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
-                }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
-                wfCtx.resultBuffer.clear();
+                if (!this.debugMode) {
+                    if (this.userDatabase.isRetriableTransactionError(err)) {
+                        // serialization_failure in PostgreSQL
+                        span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
+                        // Retry serialization failures.
+                        await (0, utils_1.sleepms)(retryWaitMillis);
+                        retryWaitMillis *= backoffFactor;
+                        retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
+                        continue;
+                    }
+                    // Record and throw other errors.
+                    const e = err;
+                    await this.userDatabase.transaction(async (client) => {
+                        await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                        const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                        await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
+                    }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                    wfCtx.resultBuffer.clear();
+                }
                 span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: e.message });
                 this.tracer.endSpan(span);
                 throw err;
@@ -990,7 +1035,7 @@ class DBOSExecutor {
         if (this.workflowCancellationMap.get(wfCtx.workflowUUID) === true) {
             throw new error_1.DBOSWorkflowCancelledError(wfCtx.workflowUUID);
         }
-        const executeLocally = this.debugMode || (procInfo.config.executeLocally ?? false);
+        const executeLocally = this.isDebugging || (procInfo.config.executeLocally ?? false);
         const funcId = wfCtx.functionIDGetIncrement();
         const span = this.tracer.startSpan(proc.name, {
             operationUUID: wfCtx.workflowUUID,
@@ -1030,42 +1075,71 @@ class DBOSExecutor {
             let txn_snapshot = 'invalid';
             const wrappedProcedure = async (client) => {
                 const ctxt = new procedure_1.StoredProcedureContextImpl(client, wfCtx, span, this.logger, funcId, proc.name);
+                let prevResult = exports.dbosNull;
+                const queryFunc = (sql, args) => this.procedurePool.query(sql, args).then((v) => v.rows);
                 if (wfCtx.presetUUID) {
-                    const func = (sql, args) => this.procedurePool.query(sql, args).then((v) => v.rows);
-                    const check = await this.#checkExecution(func, wfCtx.workflowUUID, funcId);
-                    txn_snapshot = check.txn_snapshot;
-                    if (check.output !== exports.dbosNull) {
+                    const executionResult = await this.#checkExecution(queryFunc, wfCtx.workflowUUID, funcId);
+                    prevResult = executionResult.result;
+                    txn_snapshot = executionResult.txn_snapshot;
+                    if (prevResult !== exports.dbosNull) {
                         ctxt.span.setAttribute('cached', true);
-                        ctxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
-                        this.tracer.endSpan(ctxt.span);
-                        return check.output;
+                        if (this.debugMode === DebugMode.TIME_TRAVEL) {
+                            // for time travel debugging, navigate the proxy to the time of this transaction's snapshot
+                            await queryFunc(`--proxy:${executionResult.txn_id ?? ''}:${txn_snapshot}`, []);
+                        }
+                        else {
+                            // otherwise, return/throw the previous result
+                            if (prevResult instanceof Error) {
+                                throw prevResult;
+                            }
+                            else {
+                                return prevResult;
+                            }
+                        }
                     }
                 }
                 else {
                     // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
-                    const func = (sql, args) => this.procedurePool.query(sql, args).then((v) => v.rows);
-                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(func);
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(queryFunc);
                 }
-                if (this.debugMode) {
+                if (this.isDebugging && prevResult === exports.dbosNull) {
                     throw new error_1.DBOSDebuggerError(`Failed to find the recorded output for the procedure: workflow UUID ${wfCtx.workflowUUID}, step number ${funcId}`);
                 }
                 // For non-read-only transactions, flush the result buffer.
-                if (!readOnly) {
+                if (!this.isDebugging && !readOnly) {
                     await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
                 }
-                let cresult;
-                if (procInfo.registration.passContext) {
-                    await (0, context_1.runWithStoredProcContext)(ctxt, async () => {
-                        cresult = await proc(ctxt, ...args);
-                    });
+                // Execute the user's transaction.
+                const result = await (async function () {
+                    try {
+                        return await (0, context_1.runWithStoredProcContext)(ctxt, async () => {
+                            if (procInfo.registration.passContext) {
+                                return await proc(ctxt, ...args);
+                            }
+                            else {
+                                const pf = proc;
+                                return await pf(...args);
+                            }
+                        });
+                    }
+                    catch (e) {
+                        return e instanceof Error ? e : new Error(`${e}`);
+                    }
+                })();
+                if (this.isDebugging) {
+                    if (prevResult instanceof Error) {
+                        throw prevResult;
+                    }
+                    const prevResultJson = utils_1.DBOSJSON.stringify(prevResult);
+                    const resultJson = utils_1.DBOSJSON.stringify(result);
+                    if (prevResultJson !== resultJson) {
+                        this.logger.error(`Detected different transaction output than the original one!\n Result: ${resultJson}\n Original: ${utils_1.DBOSJSON.stringify(prevResultJson)}`);
+                    }
+                    return prevResult;
                 }
-                else {
-                    await (0, context_1.runWithStoredProcContext)(ctxt, async () => {
-                        const pf = proc;
-                        cresult = await pf(...args);
-                    });
+                if (result instanceof Error) {
+                    throw result;
                 }
-                const result = cresult;
                 if (readOnly) {
                     // Buffer the output of read-only transactions instead of synchronously writing it.
                     const readOutput = {
@@ -1093,34 +1167,36 @@ class DBOSExecutor {
                 return result;
             }
             catch (err) {
-                if (this.userDatabase.isRetriableTransactionError(err)) {
-                    // serialization_failure in PostgreSQL
-                    span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
-                    // Retry serialization failures.
-                    await (0, utils_1.sleepms)(retryWaitMillis);
-                    retryWaitMillis *= backoffFactor;
-                    retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
-                    continue;
+                if (!this.isDebugging) {
+                    if (this.userDatabase.isRetriableTransactionError(err)) {
+                        // serialization_failure in PostgreSQL
+                        span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
+                        // Retry serialization failures.
+                        await (0, utils_1.sleepms)(retryWaitMillis);
+                        retryWaitMillis *= backoffFactor;
+                        retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
+                        continue;
+                    }
+                    // Record and throw other errors.
+                    const e = err;
+                    await this.invokeStoredProcFunction(async (client) => {
+                        await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                        const func = (sql, args) => client.query(sql, args).then((v) => v.rows);
+                        await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, user_database_1.pgNodeIsKeyConflictError);
+                    }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                    await this.userDatabase.transaction(async (client) => {
+                        await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                        const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                        await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
+                    }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                    wfCtx.resultBuffer.clear();
                 }
-                // Record and throw other errors.
-                const e = err;
-                await this.invokeStoredProcFunction(async (client) => {
-                    await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
-                    const func = (sql, args) => client.query(sql, args).then((v) => v.rows);
-                    await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, user_database_1.pgNodeIsKeyConflictError);
-                }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
-                await this.userDatabase.transaction(async (client) => {
-                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
-                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
-                    await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
-                }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
-                wfCtx.resultBuffer.clear();
                 throw err;
             }
         }
     }
     async #callProcedureFunctionRemote(proc, args, wfCtx, span, config, funcId) {
-        if (this.debugMode) {
+        if (this.isDebugging) {
             throw new error_1.DBOSDebuggerError("Can't invoke stored procedure in debug mode.");
         }
         const readOnly = config.readOnly ?? false;
@@ -1263,7 +1339,7 @@ class DBOSExecutor {
             this.tracer.endSpan(ctxt.span);
             return check;
         }
-        if (this.debugMode) {
+        if (this.isDebugging) {
             throw new error_1.DBOSDebuggerError(`Failed to find the recorded output for the step: workflow UUID: ${wfCtx.workflowUUID}, step number: ${funcID}`);
         }
         // Execute the step function.  If it throws an exception, retry with exponential backoff.
@@ -1418,7 +1494,7 @@ class DBOSExecutor {
      * It runs to completion all pending workflows that were executing when the previous executor failed.
      */
     async recoverPendingWorkflows(executorIDs = ['local']) {
-        if (this.debugMode) {
+        if (this.isDebugging) {
             throw new error_1.DBOSDebuggerError('Cannot recover pending workflows in debug mode.');
         }
         const handlerArray = [];
@@ -1602,14 +1678,14 @@ class DBOSExecutor {
      * Periodically flush the workflow output buffer to the system database.
      */
     async flushWorkflowBuffers() {
-        if (this.initialized && !this.debugMode) {
+        if (this.initialized && !this.isDebugging) {
             await this.flushWorkflowResultBuffer();
             await this.systemDatabase.flushWorkflowSystemBuffers();
         }
         this.isFlushingBuffers = false;
     }
     async flushWorkflowResultBuffer() {
-        if (this.debugMode) {
+        if (this.isDebugging) {
             throw new error_1.DBOSDebuggerError(`Cannot flush workflow result buffer in debug mode.`);
         }
         const localBuffer = new Map(this.workflowResultBuffer);