@dbos-inc/dbos-sdk 3.5.44-preview.gc094fdab44 → 3.6.3-preview

package/dist/src/dbos-executor.js

@@ -22,27 +22,38 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DBOSExecutor = exports.TempWorkflowType = exports.OperationType = exports.DBOS_QUEUE_MAX_PRIORITY = exports.DBOS_QUEUE_MIN_PRIORITY = void 0;
 const error_1 = require("./error");
 const workflow_1 = require("./workflow");
+const transaction_1 = require("./transaction");
 const collector_1 = require("./telemetry/collector");
 const traces_1 = require("./telemetry/traces");
 const logs_1 = require("./telemetry/logs");
 const exporters_1 = require("./telemetry/exporters");
+const pg_1 = require("pg");
 const system_database_1 = require("./system_database");
 const node_crypto_1 = require("node:crypto");
+const user_database_1 = require("./user_database");
 const decorators_1 = require("./decorators");
 const api_1 = require("@opentelemetry/api");
+const knex_1 = __importDefault(require("knex"));
 const context_1 = require("./context");
 const serialize_error_1 = require("serialize-error");
 const utils_1 = require("./utils");
+const node_path_1 = __importDefault(require("node:path"));
+const node_fs_1 = __importDefault(require("node:fs"));
+const url_1 = require("url");
 const _1 = require(".");
 const wfqueue_1 = require("./wfqueue");
 const debugpoint_1 = require("./debugpoint");
 const scheduler_1 = require("./scheduler/scheduler");
 const crypto = __importStar(require("crypto"));
-const workflow_management_1 = require("./workflow_management");
+const workflow_management_1 = require("./dbos-runtime/workflow_management");
+const utils_2 = require("./utils");
 const database_utils_1 = require("./database_utils");
 const debouncer_1 = require("./debouncer");
 const dbosNull = {};
@@ -53,14 +64,20 @@ exports.OperationType = {
     WORKFLOW: 'workflow',
     TRANSACTION: 'transaction',
     STEP: 'step',
+    PROCEDURE: 'procedure',
 };
 exports.TempWorkflowType = {
+    transaction: 'transaction',
+    procedure: 'procedure',
     step: 'step',
     send: 'send',
 };
 class DBOSExecutor {
     config;
     initialized;
+    // User Database
+    #userDatabase = undefined;
+    #procedurePool = undefined;
     // System Database
     systemDatabase;
     // Temporary workflows are created by calling transaction/send/recv directly from the executor class
@@ -72,9 +89,42 @@ class DBOSExecutor {
     logger;
     ctxLogger;
     tracer;
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+    #typeormEntities = [];
+    #drizzleEntities = {};
+    #scheduler = new scheduler_1.ScheduledReceiver();
     #wfqEnded = undefined;
     executorID = utils_1.globalParams.executorID;
     static globalInstance = undefined;
+    static async loadClasses(entrypoints) {
+        const allClasses = [];
+        for (const entrypoint of entrypoints) {
+            const operations = node_path_1.default.isAbsolute(entrypoint) ? entrypoint : node_path_1.default.join(process.cwd(), entrypoint);
+            let exports;
+            if (node_fs_1.default.existsSync(operations)) {
+                const operationsURL = (0, url_1.pathToFileURL)(operations).href;
+                exports = (await import(operationsURL));
+            }
+            else {
+                throw new error_1.DBOSFailLoadOperationsError(`Failed to load operations from the entrypoint ${entrypoint}`);
+            }
+            const classes = [];
+            for (const key in exports) {
+                const $export = exports[key];
+                if (isObject($export)) {
+                    classes.push($export);
+                }
+            }
+            allClasses.push(...classes);
+        }
+        if (allClasses.length === 0) {
+            throw new error_1.DBOSFailLoadOperationsError('operations not found');
+        }
+        return allClasses;
+        function isObject(value) {
+            return typeof value === 'function' || (typeof value === 'object' && value !== null);
+        }
+    }
     /* WORKFLOW EXECUTOR LIFE CYCLE MANAGEMENT */
     constructor(config, { systemDatabase, debugMode } = {}) {
         this.config = config;
@@ -93,6 +143,7 @@ class DBOSExecutor {
         if (this.#debugMode) {
             this.logger.info('Running in debug mode!');
         }
+        this.#procedurePool = this.config.userDbClient ? new pg_1.Pool((0, utils_2.getClientConfig)(this.config.databaseUrl)) : undefined;
         if (systemDatabase) {
             this.logger.debug('Using provided system database'); // XXX print the name or something
             this.systemDatabase = systemDatabase;
@@ -101,19 +152,129 @@ class DBOSExecutor {
             this.logger.debug('Using Postgres system database');
             this.systemDatabase = new system_database_1.PostgresSystemDatabase(this.config.systemDatabaseUrl, this.logger, this.config.sysDbPoolSize);
         }
-        new scheduler_1.ScheduledReceiver(); // Create the scheduler, which registers itself.
         this.initialized = false;
         DBOSExecutor.globalInstance = this;
     }
     get appName() {
         return this.config.name;
     }
-    async init() {
+    #configureDbClient() {
+        const userDbClient = this.config.userDbClient;
+        const userDBConfig = (0, utils_2.getClientConfig)(this.config.databaseUrl);
+        userDBConfig.max = this.config.userDbPoolSize ?? 20;
+        if (userDbClient === user_database_1.UserDatabaseName.PRISMA) {
+            // TODO: make Prisma work with debugger proxy.
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
+            const { PrismaClient } = require(node_path_1.default.join(process.cwd(), 'node_modules', '@prisma', 'client')); // Find the prisma client in the node_modules of the current project
+            this.#userDatabase = new user_database_1.PrismaUserDatabase(
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call
+            new PrismaClient({
+                datasources: {
+                    db: {
+                        url: userDBConfig.connectionString,
+                    },
+                },
+            }));
+            this.logger.debug('Loaded Prisma user database');
+        }
+        else if (userDbClient === user_database_1.UserDatabaseName.TYPEORM) {
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
+            const DataSourceExports = require('typeorm');
+            try {
+                this.#userDatabase = new user_database_1.TypeORMDatabase(
+                // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                new DataSourceExports.DataSource({
+                    type: 'postgres',
+                    url: userDBConfig.connectionString,
+                    connectTimeoutMS: userDBConfig.connectionTimeoutMillis,
+                    entities: this.#typeormEntities,
+                    poolSize: userDBConfig.max,
+                }));
+            }
+            catch (s) {
+                s.message = `Error loading TypeORM user database: ${s.message}`;
+                this.logger.error(s);
+            }
+            this.logger.debug('Loaded TypeORM user database');
+        }
+        else if (userDbClient === user_database_1.UserDatabaseName.KNEX) {
+            const knexConfig = {
+                client: 'postgres',
+                connection: (0, utils_2.getClientConfig)(this.config.databaseUrl),
+                pool: {
+                    min: 0,
+                    max: userDBConfig.max,
+                },
+            };
+            this.#userDatabase = new user_database_1.KnexUserDatabase((0, knex_1.default)(knexConfig));
+            this.logger.debug('Loaded Knex user database');
+        }
+        else if (userDbClient === user_database_1.UserDatabaseName.DRIZZLE) {
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
+            const DrizzleExports = require('drizzle-orm/node-postgres');
+            const drizzlePool = new pg_1.Pool(userDBConfig);
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+            const drizzle = DrizzleExports.drizzle(drizzlePool, { schema: this.#drizzleEntities });
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+            this.#userDatabase = new user_database_1.DrizzleUserDatabase(drizzlePool, drizzle);
+            this.logger.debug('Loaded Drizzle user database');
+        }
+        else {
+            this.#userDatabase = new user_database_1.PGNodeUserDatabase(userDBConfig);
+            this.logger.debug('Loaded Postgres user database');
+        }
+    }
+    async init(classes) {
         if (this.initialized) {
             this.logger.error('Workflow executor already initialized!');
             return;
         }
+        let classnames = [];
+        if (!classes || !classes.length) {
+            classnames = (0, decorators_1.getAllRegisteredClassNames)();
+        }
+        else {
+            classnames = classes.map((c) => (0, decorators_1.getNameForClass)(c));
+        }
         try {
+            let length; // Track the length of the array (or number of keys of the object)
+            for (const clsname of classnames) {
+                const reg = (0, decorators_1.getClassRegistrationByName)(clsname);
+                /**
+                 * With TSORM, we take an array of entities (Function[]) and add them to this.entities:
+                 */
+                if (Array.isArray(reg.ormEntities)) {
+                    this.#typeormEntities = this.#typeormEntities.concat(reg.ormEntities);
+                    length = reg.ormEntities.length;
+                }
+                else {
+                    /**
+                     * With Drizzle, we need to take an object of entities, since the object keys are used to access the entities from ctx.client.query:
+                     */
+                    this.#drizzleEntities = { ...this.#drizzleEntities, ...reg.ormEntities };
+                    length = Object.keys(reg.ormEntities).length;
+                }
+                this.logger.debug(`Loaded ${length} ORM entities`);
+            }
+            if (this.config.userDbClient) {
+                if (!this.#debugMode) {
+                    const res = await (0, database_utils_1.ensurePGDatabase)({
+                        urlToEnsure: this.config.databaseUrl,
+                        logger: (msg) => this.logger.debug(msg),
+                    });
+                    if (res.status === 'failed') {
+                        this.logger.warn(`Application database could not be verified / created: ${(0, database_utils_1.maskDatabaseUrl)(this.config.databaseUrl)}: ${res.message} ${res.hint ?? ''}\n  ${res.notes.join('\n')}`);
+                    }
+                }
+                this.#configureDbClient();
+                if (!this.#userDatabase) {
+                    this.logger.error('No user database configured!');
+                    throw new error_1.DBOSInitializationError('No user database configured!');
+                }
+                // Debug mode doesn't need to initialize the DBs. Everything should appear to be read-only.
+                await this.#userDatabase.init(this.#debugMode);
+            }
+            // Debug mode doesn't initialize the sys db
             await this.systemDatabase.init(this.#debugMode);
         }
         catch (err) {
@@ -138,6 +299,20 @@ class DBOSExecutor {
         this.initialized = true;
         // Only execute init code if under non-debug mode
         if (!this.#debugMode) {
+            for (const cls of classnames) {
+                // Init its configurations
+                const creg = (0, decorators_1.getClassRegistrationByName)(cls);
+                for (const [_cfgname, cfg] of creg.configuredInstances) {
+                    await cfg.initialize(new _1.InitContext());
+                }
+            }
+            for (const v of (0, decorators_1.getAllRegisteredFunctions)()) {
+                const m = v;
+                if (m.init === true) {
+                    this.logger.debug('Executing init method: ' + m.name);
+                    await m.origFunction(new _1.InitContext());
+                }
+            }
             // Compute the application version if not provided
             if (utils_1.globalParams.appVersion === '') {
                 utils_1.globalParams.appVersion = this.computeAppVersion();
@@ -151,10 +326,34 @@ class DBOSExecutor {
         }
         this.logger.info('DBOS launched!');
     }
+    #logNotice(msg) {
+        switch (msg.severity) {
+            case 'INFO':
+            case 'LOG':
+            case 'NOTICE':
+                this.logger.info(msg.message);
+                break;
+            case 'WARNING':
+                this.logger.warn(msg.message);
+                break;
+            case 'DEBUG':
+                this.logger.debug(msg.message);
+                break;
+            case 'ERROR':
+            case 'FATAL':
+            case 'PANIC':
+                this.logger.error(msg.message);
+                break;
+            default:
+                this.logger.error(`Unknown notice severity: ${msg.severity} - ${msg.message}`);
+        }
+    }
     async destroy() {
         try {
             await this.systemDatabase.awaitRunningWorkflows();
             await this.systemDatabase.destroy();
+            await this.#userDatabase?.destroy();
+            await this.#procedurePool?.end();
             await this.logger.destroy();
             if (DBOSExecutor.globalInstance === this) {
                 DBOSExecutor.globalInstance = undefined;
@@ -166,6 +365,24 @@ class DBOSExecutor {
             throw err;
         }
     }
+    async createUserSchema() {
+        if (!this.#userDatabase) {
+            throw new Error('User database not enabled.');
+        }
+        await this.#userDatabase.createSchema();
+    }
+    async dropUserSchema() {
+        if (!this.#userDatabase) {
+            throw new Error('User database not enabled.');
+        }
+        await this.#userDatabase.dropSchema();
+    }
+    async queryUserDbFunction(queryFunction, ...params) {
+        if (!this.#userDatabase) {
+            throw new Error('User database not enabled.');
+        }
+        return await this.#userDatabase.queryFunction(queryFunction, ...params);
+    }
     // This could return WF, or the function underlying a temp wf
     #getFunctionInfoFromWFStatus(wf) {
         const methReg = (0, decorators_1.getFunctionRegistrationByName)(wf.workflowClassName, wf.workflowName);
@@ -203,7 +420,7 @@ class DBOSExecutor {
         if (params.queueName) {
             const wfqueue = this.#getQueueByName(params.queueName);
             if (!wfqueue.priorityEnabled && priority !== undefined) {
-                this.logger.warn(`Priority is not enabled for queue ${params.queueName}. Setting priority will not have any effect.`);
+                throw Error(`Priority is not enabled for queue ${params.queueName}. Setting priority will not have any effect.`);
             }
         }
         const pctx = { ...(0, context_1.getCurrentContextStore)() }; // function ID was already incremented...
@@ -229,6 +446,8 @@ class DBOSExecutor {
             assumedRole: pctx?.assumedRole ?? '',
         });
         const isTempWorkflow = DBOSExecutor.#tempWorkflowName === wfname;
+        const funcArgs = (0, utils_1.serializeFunctionInputOutput)(args, [wfname, '<arguments>']);
+        args = funcArgs.deserialized;
         const internalStatus = {
             workflowUUID: workflowID,
             status: params.queueName !== undefined ? workflow_1.StatusString.ENQUEUED : workflow_1.StatusString.PENDING,
@@ -248,7 +467,7 @@ class DBOSExecutor {
             createdAt: Date.now(), // Remember the start time of this workflow,
             timeoutMS: timeoutMS,
             deadlineEpochMS: deadlineEpochMS,
-            input: utils_1.DBOSJSON.stringify(args),
+            input: funcArgs.stringified,
             deduplicationID: params.enqueueOptions?.deduplicationID,
             priority: priority ?? 0,
         };
@@ -266,8 +485,8 @@ class DBOSExecutor {
                 throw new error_1.DBOSDebuggerError(`Failed to find inputs for workflow UUID ${workflowID}`);
             }
             // Make sure we use the same input.
-            if (utils_1.DBOSJSON.stringify(args) !== wfStatus.input) {
-                throw new error_1.DBOSDebuggerError(`Detected different inputs for workflow UUID ${workflowID}.\n Received: ${utils_1.DBOSJSON.stringify(args)}\n Original: ${wfStatus.input}`);
+            if (funcArgs.stringified !== wfStatus.input) {
+                throw new error_1.DBOSDebuggerError(`Detected different inputs for workflow UUID ${workflowID}.\n Received: ${funcArgs.stringified}\n Original: ${wfStatus.input}`);
             }
             status = wfStatus.status;
         }
@@ -356,6 +575,12 @@ class DBOSExecutor {
                     });
                 });
                 if (this.#debugMode) {
+                    function resultsMatch(recordedResult, callResult) {
+                        if (recordedResult === null) {
+                            return callResult === undefined || callResult === null;
+                        }
+                        return utils_1.DBOSJSON.stringify(recordedResult) === utils_1.DBOSJSON.stringify(callResult);
+                    }
                     const recordedResult = DBOSExecutor.reviveResultOrError((await this.systemDatabase.awaitWorkflowResult(workflowID)));
                     if (!resultsMatch(recordedResult, callResult)) {
                         this.logger.error(`Detect different output for the workflow UUID ${workflowID}!\n Received: ${utils_1.DBOSJSON.stringify(callResult)}\n Original: ${utils_1.DBOSJSON.stringify(recordedResult)}`);
@@ -365,13 +590,9 @@ class DBOSExecutor {
                 else {
                     result = callResult;
                 }
-                function resultsMatch(recordedResult, callResult) {
-                    if (recordedResult === null) {
-                        return callResult === undefined || callResult === null;
-                    }
-                    return utils_1.DBOSJSON.stringify(recordedResult) === utils_1.DBOSJSON.stringify(callResult);
-                }
-                internalStatus.output = utils_1.DBOSJSON.stringify(result);
+                const funcResult = (0, utils_1.serializeFunctionInputOutput)(result, [wfname, '<result>']);
+                result = funcResult.deserialized;
+                internalStatus.output = funcResult.stringified;
                 internalStatus.status = workflow_1.StatusString.SUCCESS;
                 if (!this.#debugMode) {
                     await this.systemDatabase.recordWorkflowOutput(workflowID, internalStatus);
@@ -426,6 +647,510 @@ class DBOSExecutor {
             throw new error_1.DBOSNotRegisteredError(name, `Workflow queue '${name}' is not defined.`);
         return q;
     }
+    /**
+     * Retrieve the transaction snapshot information of the current transaction
+     */
+    static async #retrieveSnapshot(query) {
+        const rows = await query('SELECT pg_current_snapshot()::text as txn_snapshot;', []);
+        return rows[0].txn_snapshot;
+    }
+    /**
+     * Check if an operation has already executed in a workflow.
+     * If it previously executed successfully, return its output.
+     * If it previously executed and threw an error, return that error.
+     * Otherwise, return DBOSNull.
+     * Also return the transaction snapshot and id information of the original or current transaction.
+     */
+    async #checkExecution(query, workflowUUID, funcID, funcName) {
+        const rows = await query(`(SELECT output, error, txn_snapshot, txn_id, function_name, true as recorded
+          FROM dbos.transaction_outputs
+          WHERE workflow_uuid=$1 AND function_id=$2
+        UNION ALL
+          SELECT null as output, null as error, pg_current_snapshot()::text as txn_snapshot,
+                 null as txn_id, '' as function_name, false as recorded
+       ) ORDER BY recorded`, [workflowUUID, funcID]);
+        if (rows.length === 0 || rows.length > 2) {
+            const returnedRows = JSON.stringify(rows);
+            this.logger.error('Unexpected! This should never happen. Returned rows: ' + returnedRows);
+            throw new error_1.DBOSError('This should never happen. Returned rows: ' + returnedRows);
+        }
+        if (rows.length === 2) {
+            if (rows[1].function_name !== funcName) {
+                throw new error_1.DBOSUnexpectedStepError(workflowUUID, funcID, funcName, rows[0].function_name);
+            }
+            const { txn_snapshot, txn_id } = rows[1];
+            const error = utils_1.DBOSJSON.parse(rows[1].error);
+            if (error) {
+                return { result: (0, serialize_error_1.deserializeError)(error), txn_snapshot, txn_id: txn_id ?? undefined };
+            }
+            else {
+                return { result: utils_1.DBOSJSON.parse(rows[1].output), txn_snapshot, txn_id: txn_id ?? undefined };
+            }
+        }
+        else {
+            const { txn_snapshot } = rows[0];
+            return { result: dbosNull, txn_snapshot, txn_id: undefined };
+        }
+    }
+    /**
+     * Write a operation's output to the database.
+     */
+    async #recordOutput(query, workflowUUID, funcID, txnSnapshot, serialOutput, isKeyConflict, function_name) {
+        if (this.#debugMode) {
+            throw new error_1.DBOSDebuggerError('Cannot record output in debug mode.');
+        }
+        try {
+            const rows = await query('INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, txn_id, txn_snapshot, created_at, function_name) VALUES ($1, $2, $3, (select pg_current_xact_id_if_assigned()::text), $4, $5, $6) RETURNING txn_id;', [workflowUUID, funcID, serialOutput, txnSnapshot, Date.now(), function_name]);
+            return rows[0].txn_id;
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    /**
+     * Record an error in an operation to the database.
+     */
+    async #recordError(query, workflowUUID, funcID, txnSnapshot, err, isKeyConflict, function_name) {
+        if (this.#debugMode) {
+            throw new error_1.DBOSDebuggerError('Cannot record error in debug mode.');
+        }
+        try {
+            const serialErr = utils_1.DBOSJSON.stringify((0, serialize_error_1.serializeError)(err));
+            await query('INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, error, txn_id, txn_snapshot, created_at, function_name) VALUES ($1, $2, $3, null, $4, $5, $6) RETURNING txn_id;', [workflowUUID, funcID, serialErr, txnSnapshot, Date.now(), function_name]);
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    async getTransactions(workflowUUID) {
+        if (this.#userDatabase) {
+            const rows = await this.#userDatabase.query(`SELECT function_id, function_name, output, error FROM ${DBOSExecutor.systemDBSchemaName}.transaction_outputs WHERE workflow_uuid=$1`, workflowUUID);
+            for (const row of rows) {
+                row.output = row.output !== null ? utils_1.DBOSJSON.parse(row.output) : null;
+                row.error = row.error !== null ? (0, serialize_error_1.deserializeError)(utils_1.DBOSJSON.parse(row.error)) : null;
+            }
+            return rows;
+        }
+        else {
+            return [];
+        }
+    }
+    async runTransactionTempWF(txn, params, ...args) {
+        return await (await this.startTransactionTempWF(txn, params, undefined, undefined, ...args)).getResult();
+    }
+    async startTransactionTempWF(txn, params, callerWFID, callerFunctionID, ...args) {
+        // Create a workflow and call transaction.
+        const temp_workflow = async (...args) => {
+            return await this.callTransactionFunction(txn, params.configuredInstance ?? null, ...args);
+        };
+        return await this.internalWorkflow(temp_workflow, {
+            ...params,
+            tempWfType: exports.TempWorkflowType.transaction,
+            tempWfName: (0, decorators_1.getRegisteredFunctionName)(txn),
+            tempWfClass: (0, decorators_1.getRegisteredFunctionClassName)(txn),
+        }, callerWFID, callerFunctionID, ...args);
+    }
+    async callTransactionFunction(txn, clsinst, ...args) {
+        const userDB = this.#userDatabase;
+        if (!userDB) {
+            throw new Error('No user database configured for transactions.');
+        }
+        const txnReg = (0, decorators_1.getFunctionRegistration)(txn);
+        if (!txnReg || !txnReg.txnConfig) {
+            throw new error_1.DBOSNotRegisteredError(txn.name);
+        }
+        const funcId = (0, context_1.functionIDGetIncrement)();
+        const pctx = { ...(0, context_1.getCurrentContextStore)() };
+        const wfid = pctx.workflowId;
+        await this.systemDatabase.checkIfCanceled(wfid);
+        let retryWaitMillis = 1;
+        const backoffFactor = 1.5;
+        const maxRetryWaitMs = 2000; // Maximum wait 2 seconds.
+        const span = this.tracer.startSpan(txn.name, {
+            operationUUID: wfid,
+            operationType: exports.OperationType.TRANSACTION,
+            operationName: txn.name,
+            authenticatedUser: pctx.authenticatedUser ?? '',
+            assumedRole: pctx.assumedRole ?? '',
+            authenticatedRoles: pctx.authenticatedRoles ?? [],
+            isolationLevel: txnReg.txnConfig.isolationLevel,
+        });
+        while (true) {
+            await this.systemDatabase.checkIfCanceled(wfid);
+            let txn_snapshot = 'invalid';
+            let prevResultFound = false;
+            const wrappedTransaction = async (client) => {
+                // If the UUID is preset, it is possible this execution previously happened. Check, and return its original result if it did.
+                // Note: It is possible to retrieve a generated ID from a workflow handle, run a concurrent execution, and cause trouble for yourself. We recommend against this.
+                let prevResult = dbosNull;
+                const queryFunc = (sql, args) => userDB.queryWithClient(client, sql, ...args);
+                if (pctx.presetID) {
+                    const executionResult = await this.#checkExecution(queryFunc, wfid, funcId, txn.name);
+                    prevResult = executionResult.result;
+                    txn_snapshot = executionResult.txn_snapshot;
+                    if (prevResult !== dbosNull) {
+                        prevResultFound = true;
+                        span.setAttribute('cached', true);
+                        // Return/throw the previous result
+                        if (prevResult instanceof Error) {
+                            throw prevResult;
+                        }
+                        else {
+                            return prevResult;
+                        }
+                    }
+                }
+                else {
+                    // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(queryFunc);
+                }
+                if (this.#debugMode && prevResult === dbosNull) {
+                    throw new error_1.DBOSDebuggerError(`Failed to find the recorded output for the transaction: workflow UUID ${wfid}, step number ${funcId}`);
+                }
+                // Execute the user's transaction.
+                const ctxlog = this.ctxLogger;
+                const result = await (async function () {
+                    try {
+                        return await api_1.context.with(api_1.trace.setSpan(api_1.context.active(), span), async () => {
+                            return await (0, context_1.runWithParentContext)(pctx, {
+                                authenticatedRoles: pctx?.authenticatedRoles,
+                                authenticatedUser: pctx?.authenticatedUser,
+                                workflowId: wfid,
+                                curTxFunctionId: funcId,
+                                parentCtx: pctx,
+                                sqlClient: client,
+                                logger: ctxlog,
+                            }, async () => {
+                                const tf = txn;
+                                return await tf.call(clsinst, ...args);
+                            });
+                        });
+                    }
+                    catch (e) {
+                        return e instanceof Error ? e : new Error(`${e}`);
+                    }
+                })();
+                if (this.#debugMode) {
+                    if (prevResult instanceof Error) {
+                        throw prevResult;
+                    }
+                    const prevResultJson = utils_1.DBOSJSON.stringify(prevResult);
+                    const resultJson = utils_1.DBOSJSON.stringify(result);
+                    if (prevResultJson !== resultJson) {
+                        this.logger.error(`Detected different transaction output than the original one!\n Result: ${resultJson}\n Original: ${utils_1.DBOSJSON.stringify(prevResultJson)}`);
+                    }
+                    return prevResult;
+                }
+                if (result instanceof Error) {
+                    throw result;
+                }
+                // Record the execution, commit, and return.
+                const funcOutput = (0, utils_1.serializeFunctionInputOutput)(result, [txn.name, '<result>']);
+                try {
+                    // Synchronously record the output of write transactions and obtain the transaction ID.
+                    const pg_txn_id = await this.#recordOutput(queryFunc, wfid, funcId, txn_snapshot, funcOutput.stringified, (error) => userDB.isKeyConflictError(error), txn.name);
+                    span.setAttribute('pg_txn_id', pg_txn_id);
+                }
+                catch (error) {
+                    if (userDB.isFailedSqlTransactionError(error)) {
+                        this.logger.error(`Postgres aborted the ${txn.name} @DBOS.transaction of Workflow ${wfid}, but the function did not raise an exception.  Please ensure that the @DBOS.transaction method raises an exception if the database transaction is aborted.`);
+                        throw new error_1.DBOSFailedSqlTransactionError(wfid, txn.name);
+                    }
+                    else {
+                        throw error;
+                    }
+                }
+                return funcOutput.deserialized;
+            };
+            try {
+                const result = await userDB.transaction(wrappedTransaction, txnReg.txnConfig);
+                span.setStatus({ code: api_1.SpanStatusCode.OK });
+                this.tracer.endSpan(span);
+                return result;
+            }
+            catch (err) {
+                const e = err;
+                if (!prevResultFound && !this.#debugMode && !(e instanceof error_1.DBOSUnexpectedStepError)) {
+                    if (userDB.isRetriableTransactionError(err)) {
+                        // serialization_failure in PostgreSQL
+                        span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
+                        // Retry serialization failures.
+                        await (0, utils_1.sleepms)(retryWaitMillis);
+                        retryWaitMillis *= backoffFactor;
+                        retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
+                        continue;
+                    }
+                    // Record and throw other errors.
+                    const e = err;
+                    await userDB.transaction(async (client) => {
+                        const func = (sql, args) => userDB.queryWithClient(client, sql, ...args);
+                        await this.#recordError(func, wfid, funcId, txn_snapshot, e, (error) => userDB.isKeyConflictError(error), txn.name);
+                    }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                }
+                span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: e.message });
+                this.tracer.endSpan(span);
+                throw err;
+            }
+        }
+    }
+    async runProcedureTempWF(proc, params, ...args) {
+        // Create a workflow and call procedure.
+        const temp_workflow = async (...args) => {
+            return this.callProcedureFunction(proc, ...args);
+        };
+        return await (await this.workflow(temp_workflow, {
+            ...params,
+            tempWfType: exports.TempWorkflowType.procedure,
+            tempWfName: (0, decorators_1.getRegisteredFunctionName)(proc),
+            tempWfClass: (0, decorators_1.getRegisteredFunctionClassName)(proc),
+        }, ...args)).getResult();
+    }
+    async callProcedureFunction(proc, ...args) {
+        const procInfo = (0, decorators_1.getFunctionRegistration)(proc);
+        if (!procInfo || !procInfo.procConfig) {
+            throw new error_1.DBOSNotRegisteredError(proc.name);
+        }
+        const procConfig = procInfo.procConfig;
+        const pctx = (0, context_1.getCurrentContextStore)();
+        const wfid = pctx.workflowId;
+        await this.systemDatabase.checkIfCanceled(wfid);
+        const executeLocally = this.#debugMode || (procConfig.executeLocally ?? false);
+        const funcId = (0, context_1.functionIDGetIncrement)();
+        const span = this.tracer.startSpan(proc.name, {
+            operationUUID: wfid,
+            operationType: exports.OperationType.PROCEDURE,
+            operationName: proc.name,
+            authenticatedUser: pctx.authenticatedUser ?? '',
+            assumedRole: pctx.assumedRole ?? '',
+            authenticatedRoles: pctx.authenticatedRoles ?? [],
+            isolationLevel: procInfo.procConfig.isolationLevel,
+            executeLocally,
+        });
+        try {
+            const result = executeLocally
+                ? await this.#callProcedureFunctionLocal(proc, args, span, procInfo, funcId)
+                : await this.#callProcedureFunctionRemote(proc, args, span, procConfig, funcId);
+            span.setStatus({ code: api_1.SpanStatusCode.OK });
+            return result;
+        }
+        catch (e) {
+            const { message } = e;
+            span.setStatus({ code: api_1.SpanStatusCode.ERROR, message });
+            throw e;
+        }
+        finally {
+            this.tracer.endSpan(span);
+        }
+    }
+    async #callProcedureFunctionLocal(proc, args, span, procInfo, funcId) {
+        const procPool = this.#procedurePool;
+        const userDB = this.#userDatabase;
+        if (!procPool || !userDB) {
+            throw new Error('User database not enabled.');
+        }
+        let retryWaitMillis = 1;
+        const backoffFactor = 1.5;
+        const maxRetryWaitMs = 2000; // Maximum wait 2 seconds.
+        const pctx = { ...(0, context_1.getCurrentContextStore)() };
+        const wfid = pctx.workflowId;
+        while (true) {
+            await this.systemDatabase.checkIfCanceled(wfid);
+            let txn_snapshot = 'invalid';
+            const wrappedProcedure = async (client) => {
+                let prevResult = dbosNull;
+                const queryFunc = (sql, args) => procPool.query(sql, args).then((v) => v.rows);
+                if (pctx.presetID) {
+                    const executionResult = await this.#checkExecution(queryFunc, wfid, funcId, proc.name);
+                    prevResult = executionResult.result;
+                    txn_snapshot = executionResult.txn_snapshot;
+                    if (prevResult !== dbosNull) {
+                        span.setAttribute('cached', true);
+                        // Return/throw the previous result
+                        if (prevResult instanceof Error) {
+                            throw prevResult;
+                        }
+                        else {
+                            return prevResult;
+                        }
+                    }
+                }
+                else {
+                    // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(queryFunc);
+                }
+                if (this.#debugMode && prevResult === dbosNull) {
+                    throw new error_1.DBOSDebuggerError(`Failed to find the recorded output for the procedure: workflow UUID ${wfid}, step number ${funcId}`);
+                }
+                // Execute the user's transaction.
+                const ctxlog = this.ctxLogger;
+                const result = await (async function () {
+                    try {
+                        // Check we are in a workflow context and not in a step / transaction already
+                        if (!pctx)
+                            throw new error_1.DBOSInvalidWorkflowTransitionError();
+                        if (!(0, context_1.isInWorkflowCtx)(pctx))
+                            throw new error_1.DBOSInvalidWorkflowTransitionError();
+                        return await api_1.context.with(api_1.trace.setSpan(api_1.context.active(), span), async () => {
+                            return await (0, context_1.runWithParentContext)(pctx, {
+                                curTxFunctionId: funcId,
+                                parentCtx: pctx,
+                                isInStoredProc: true,
+                                sqlClient: client,
+                                logger: ctxlog,
+                            }, async () => {
+                                const pf = proc;
+                                return await pf(...args);
+                            });
+                        });
+                    }
+                    catch (e) {
+                        return e instanceof Error ? e : new Error(`${e}`);
+                    }
+                })();
+                if (this.#debugMode) {
+                    if (prevResult instanceof Error) {
+                        throw prevResult;
+                    }
+                    const prevResultJson = utils_1.DBOSJSON.stringify(prevResult);
+                    const resultJson = utils_1.DBOSJSON.stringify(result);
+                    if (prevResultJson !== resultJson) {
+                        this.logger.error(`Detected different transaction output than the original one!\n Result: ${resultJson}\n Original: ${utils_1.DBOSJSON.stringify(prevResultJson)}`);
+                    }
+                    return prevResult;
+                }
+                if (result instanceof Error) {
+                    throw result;
+                }
+                // Synchronously record the output of write transactions and obtain the transaction ID.
+                const func = (sql, args) => client.query(sql, args).then((v) => v.rows);
+                const funcResult = (0, utils_1.serializeFunctionInputOutput)(result, [proc.name, '<result>']);
+                const pg_txn_id = await this.#recordOutput(func, wfid, funcId, txn_snapshot, funcResult.stringified, user_database_1.pgNodeIsKeyConflictError, proc.name);
+                // const pg_txn_id = await wfCtx.recordOutputProc<R>(client, funcId, txn_snapshot, result);
+                span.setAttribute('pg_txn_id', pg_txn_id);
+                return funcResult.deserialized;
+            };
+            try {
+                const result = await this.invokeStoredProcFunction(wrappedProcedure, {
+                    isolationLevel: procInfo.procConfig.isolationLevel,
+                });
+                span.setStatus({ code: api_1.SpanStatusCode.OK });
+                return result;
+            }
+            catch (err) {
+                if (!this.#debugMode) {
+                    if (userDB.isRetriableTransactionError(err)) {
+                        // serialization_failure in PostgreSQL
+                        span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
+                        // Retry serialization failures.
+                        await (0, utils_1.sleepms)(retryWaitMillis);
+                        retryWaitMillis *= backoffFactor;
+                        retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
+                        continue;
+                    }
+                    // Record and throw other errors.
+                    const e = err;
+                    await this.invokeStoredProcFunction(async (client) => {
+                        const func = (sql, args) => client.query(sql, args).then((v) => v.rows);
+                        await this.#recordError(func, wfid, funcId, txn_snapshot, e, user_database_1.pgNodeIsKeyConflictError, proc.name);
+                    }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                    await userDB.transaction(async (client) => {
+                        const func = (sql, args) => userDB.queryWithClient(client, sql, ...args);
+                        await this.#recordError(func, wfid, funcId, txn_snapshot, e, (error) => userDB.isKeyConflictError(error), proc.name);
+                    }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                }
+                throw err;
+            }
+        }
+    }
+    async #callProcedureFunctionRemote(proc, args, span, config, funcId) {
+        if (this.#debugMode) {
+            throw new error_1.DBOSDebuggerError("Can't invoke stored procedure in debug mode.");
+        }
+        const pctx = (0, context_1.getCurrentContextStore)();
+        const wfid = pctx.workflowId;
+        await this.systemDatabase.checkIfCanceled(wfid);
+        const $jsonCtx = {
+            request: pctx.request,
+            authenticatedUser: pctx.authenticatedUser,
+            authenticatedRoles: pctx.authenticatedRoles,
+            assumedRole: pctx.assumedRole,
+        };
+        // TODO (Qian/Harry): remove this unshift when we remove the resultBuffer argument
+        // Note, node-pg converts JS arrays to postgres array literals, so must call JSON.strigify on
+        // args and bufferedResults before being passed to #invokeStoredProc
+        const $args = [wfid, funcId, pctx.presetID, $jsonCtx, null, JSON.stringify(args)];
+        const readonly = config.readOnly ?? false;
+        if (!readonly) {
+            $args.unshift(null);
+        }
+        const [{ return_value }] = await this.#invokeStoredProc(proc, $args);
+        const { error, output, txn_id } = return_value;
+        // if the stored proc returns an error, deserialize and throw it.
+        // stored proc saves the error in tx_output before returning
+        if (error) {
+            throw (0, serialize_error_1.deserializeError)(error);
+        }
+        if (txn_id) {
+            span.setAttribute('pg_txn_id', txn_id);
+        }
+        span.setStatus({ code: api_1.SpanStatusCode.OK });
+        return output;
+    }
+    async #invokeStoredProc(proc, args) {
+        if (!this.#procedurePool) {
+            throw new Error('User Database not enabled.');
+        }
+        const client = await this.#procedurePool.connect();
+        const log = (msg) => this.#logNotice(msg);
+        const procname = (0, decorators_1.getRegisteredFunctionFullName)(proc);
+        const plainProcName = `${procname.className}_${procname.name}_p`;
+        const procName = utils_1.globalParams.wasComputed ? plainProcName : `v${utils_1.globalParams.appVersion}_${plainProcName}`;
+        const sql = `CALL "${procName}"(${args.map((_v, i) => `$${i + 1}`).join()});`;
+        try {
+            client.on('notice', log);
+            return await client.query(sql, args).then((value) => value.rows);
+        }
+        finally {
+            client.off('notice', log);
+            client.release();
+        }
+    }
+    async invokeStoredProcFunction(func, config) {
+        if (!this.#procedurePool) {
+            throw new Error('User Database not enabled.');
+        }
+        const client = await this.#procedurePool.connect();
+        try {
+            const readOnly = config.readOnly ?? false;
+            const isolationLevel = config.isolationLevel ?? transaction_1.IsolationLevel.Serializable;
+            await client.query(`BEGIN ISOLATION LEVEL ${isolationLevel}`);
+            if (readOnly) {
+                await client.query(`SET TRANSACTION READ ONLY`);
+            }
+            const result = await func(client);
+            await client.query(`COMMIT`);
+            return result;
+        }
+        catch (err) {
+            await client.query(`ROLLBACK`);
+            throw err;
+        }
+        finally {
+            client.release();
+        }
+    }
     async runStepTempWF(stepFn, params, ...args) {
         return await (await this.startStepTempWF(stepFn, params, undefined, undefined, ...args)).getResult();
     }
@@ -553,12 +1278,13 @@ class DBOSExecutor {
         }
         else {
             // Record the execution and return.
+            const funcResult = (0, utils_1.serializeFunctionInputOutput)(result, [stepFnName, '<result>']);
             await this.systemDatabase.recordOperationResult(wfid, funcID, stepFnName, true, {
-                output: utils_1.DBOSJSON.stringify(result),
+                output: funcResult.stringified,
             });
             span.setStatus({ code: api_1.SpanStatusCode.OK });
             this.tracer.endSpan(span);
-            return result;
+            return funcResult.deserialized;
         }
     }
     async runSendTempWF(destinationId, message, topic, idempotencyKey) {
@@ -587,7 +1313,7 @@ class DBOSExecutor {
      */
     forkWorkflow(workflowID, startStep, options = {}) {
         const newWorkflowID = options.newWorkflowID ?? (0, context_1.getNextWFID)(undefined);
-        return (0, workflow_management_1.forkWorkflow)(this.systemDatabase, workflowID, startStep, { ...options, newWorkflowID });
+        return (0, workflow_management_1.forkWorkflow)(this.systemDatabase, this.#userDatabase, workflowID, startStep, { ...options, newWorkflowID });
     }
     /**
      * Retrieve a handle for a workflow UUID.
@@ -605,11 +1331,12 @@ class DBOSExecutor {
         }
         try {
             const output = await callback();
+            const funcOutput = (0, utils_1.serializeFunctionInputOutput)(output, [functionName, '<result>']);
             await this.systemDatabase.recordOperationResult(workflowID, functionID, functionName, true, {
-                output: utils_1.DBOSJSON.stringify(output),
+                output: funcOutput.stringified,
                 childWorkflowID: childWfId,
             });
-            return output;
+            return funcOutput.deserialized;
         }
         catch (e) {
             await this.systemDatabase.recordOperationResult(workflowID, functionID, functionName, false, {
@@ -631,7 +1358,18 @@ class DBOSExecutor {
         return (0, workflow_management_1.listQueuedWorkflows)(this.systemDatabase, input);
     }
     async listWorkflowSteps(workflowID) {
-        return (0, workflow_management_1.listWorkflowSteps)(this.systemDatabase, workflowID);
+        return (0, workflow_management_1.listWorkflowSteps)(this.systemDatabase, this.#userDatabase, workflowID);
+    }
+    async queryUserDB(sql, params) {
+        if (!this.#userDatabase) {
+            throw new Error('User database not enabled.');
+        }
+        if (params !== undefined) {
+            return await this.#userDatabase.query(sql, ...params);
+        }
+        else {
+            return await this.#userDatabase.query(sql);
+        }
     }
     /* INTERNAL HELPERS */
     /**
@@ -737,7 +1475,22 @@ class DBOSExecutor {
         if (!nameArr[0].startsWith(DBOSExecutor.#tempWorkflowName)) {
             throw new error_1.DBOSError(`Cannot find workflow function for a non-temporary workflow, ID ${workflowID}, class '${wfStatus.workflowClassName}', function '${wfName}'; did you change your code?`);
         }
-        if (nameArr[1] === exports.TempWorkflowType.step) {
+        if (nameArr[1] === exports.TempWorkflowType.transaction) {
+            const txnReg = (0, decorators_1.getFunctionRegistrationByName)(wfStatus.workflowClassName, nameArr[2]);
+            if (!txnReg?.txnConfig) {
+                this.logger.error(`Cannot find transaction info for ID ${workflowID}, name ${nameArr[2]}`);
+                throw new error_1.DBOSNotRegisteredError(nameArr[2]);
+            }
+            return await (0, context_1.runWithTopContext)(recoverCtx, async () => {
+                return await this.startTransactionTempWF(txnReg.registeredFunction, {
+                    workflowUUID: workflowStartID,
+                    configuredInstance: configuredInst,
+                    queueName: wfStatus.queueName,
+                    executeWorkflow: true,
+                }, undefined, undefined, ...inputs);
+            });
+        }
+        else if (nameArr[1] === exports.TempWorkflowType.step) {
             const stepReg = (0, decorators_1.getFunctionRegistrationByName)(wfStatus.workflowClassName, nameArr[2]);
             if (!stepReg?.stepConfig) {
                 this.logger.error(`Cannot find step info for ID ${workflowID}, name ${nameArr[2]}`);
@@ -807,6 +1560,19 @@ class DBOSExecutor {
     async resumeWorkflow(workflowID) {
         await this.systemDatabase.resumeWorkflow(workflowID);
     }
+    logRegisteredHTTPUrls() {
+        this.logger.info('HTTP endpoints supported:');
+        (0, decorators_1.getAllRegisteredFunctions)().forEach((registeredOperation) => {
+            const ro = registeredOperation;
+            if (ro.apiURL) {
+                this.logger.info('    ' + ro.apiType.padEnd(6) + '  :  ' + ro.apiURL);
+                const roles = ro.getRequiredRoles();
+                if (roles.length > 0) {
+                    this.logger.info('        Required Roles: ' + utils_1.DBOSJSON.stringify(roles));
+                }
+            }
+        });
+    }
     /**
       An application's version is computed from a hash of the source of its workflows.
       This is guaranteed to be stable given identical source code because it uses an MD5 hash