@dbos-inc/dbos-sdk 2.2.10-preview.g90e74a1e32 → 2.3.9-preview

package/dist/src/dbos-executor.js

@@ -29,17 +29,17 @@ const wfqueue_1 = require("./wfqueue");
 const debugpoint_1 = require("./debugpoint");
 exports.dbosNull = {};
 exports.OperationType = {
-    HANDLER: "handler",
-    WORKFLOW: "workflow",
-    TRANSACTION: "transaction",
-    COMMUNICATOR: "communicator",
-    PROCEDURE: "procedure",
+    HANDLER: 'handler',
+    WORKFLOW: 'workflow',
+    TRANSACTION: 'transaction',
+    COMMUNICATOR: 'communicator',
+    PROCEDURE: 'procedure',
 };
 const TempWorkflowType = {
-    transaction: "transaction",
-    procedure: "procedure",
-    external: "external",
-    send: "send",
+    transaction: 'transaction',
+    procedure: 'procedure',
+    external: 'external',
+    send: 'send',
 };
 class DBOSExecutor {
     config;
@@ -50,14 +50,14 @@ class DBOSExecutor {
     systemDatabase;
     procedurePool;
     // Temporary workflows are created by calling transaction/send/recv directly from the executor class
-    static tempWorkflowName = "temp_workflow";
+    static tempWorkflowName = 'temp_workflow';
     workflowInfoMap = new Map([
         // We initialize the map with an entry for temporary workflows.
         [
             DBOSExecutor.tempWorkflowName,
             {
                 workflow: async () => {
-                    this.logger.error("UNREACHABLE: Indirect invoke of temp workflow");
+                    this.logger.error('UNREACHABLE: Indirect invoke of temp workflow');
                     return Promise.resolve();
                 },
                 config: {},
@@ -76,8 +76,7 @@ class DBOSExecutor {
     isFlushingBuffers = false;
     static defaultNotificationTimeoutSec = 60;
     debugMode;
-    debugProxy;
-    static systemDBSchemaName = "dbos";
+    static systemDBSchemaName = 'dbos';
     logger;
     tracer;
     // eslint-disable-next-line @typescript-eslint/ban-types
@@ -86,17 +85,16 @@ class DBOSExecutor {
     eventReceivers = [];
     scheduler = undefined;
     wfqEnded = undefined;
-    executorID = process.env.DBOS__VMID || "local";
+    executorID = process.env.DBOS__VMID || 'local';
     static globalInstance = undefined;
     /* WORKFLOW EXECUTOR LIFE CYCLE MANAGEMENT */
     constructor(config, systemDatabase) {
         this.config = config;
         this.debugMode = config.debugMode ?? false;
-        this.debugProxy = config.debugProxy;
         // Set configured environment variables
         if (config.env) {
             for (const [key, value] of Object.entries(config.env)) {
-                if (typeof value === "string") {
+                if (typeof value === 'string') {
                     process.env[key] = value;
                 }
                 else {
@@ -115,27 +113,15 @@ class DBOSExecutor {
         this.logger = new logs_1.GlobalLogger(this.telemetryCollector, this.config.telemetry?.logs);
         this.tracer = new traces_1.Tracer(this.telemetryCollector);
         if (this.debugMode) {
-            this.logger.info("Running in debug mode!");
-            if (this.debugProxy) {
-                try {
-                    const url = new URL(this.config.debugProxy);
-                    this.config.poolConfig.host = url.hostname;
-                    this.config.poolConfig.port = parseInt(url.port, 10);
-                    this.logger.info(`Debugging mode proxy: ${this.config.poolConfig.host}:${this.config.poolConfig.port}`);
-                }
-                catch (err) {
-                    this.logger.error(err);
-                    throw err;
-                }
-            }
+            this.logger.info('Running in debug mode!');
         }
         this.procedurePool = new pg_1.Pool(this.config.poolConfig);
         if (systemDatabase) {
-            this.logger.debug("Using provided system database"); // XXX print the name or something
+            this.logger.debug('Using provided system database'); // XXX print the name or something
             this.systemDatabase = systemDatabase;
         }
         else {
-            this.logger.debug("Using Postgres system database");
+            this.logger.debug('Using Postgres system database');
             this.systemDatabase = new system_database_1.PostgresSystemDatabase(this.config.poolConfig, this.config.system_database, this.logger);
         }
         this.flushBufferID = setInterval(() => {
@@ -144,7 +130,7 @@ class DBOSExecutor {
                 void this.flushWorkflowBuffers();
             }
         }, this.flushBufferIntervalMs);
-        this.logger.debug("Started workflow status buffer worker");
+        this.logger.debug('Started workflow status buffer worker');
         this.initialized = false;
         DBOSExecutor.globalInstance = this;
     }
@@ -154,25 +140,26 @@ class DBOSExecutor {
         if (userDbClient === user_database_1.UserDatabaseName.PRISMA) {
             // TODO: make Prisma work with debugger proxy.
             // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
-            const { PrismaClient } = require(node_path_1.default.join(process.cwd(), "node_modules", "@prisma", "client")); // Find the prisma client in the node_modules of the current project
+            const { PrismaClient } = require(node_path_1.default.join(process.cwd(), 'node_modules', '@prisma', 'client')); // Find the prisma client in the node_modules of the current project
+            this.userDatabase = new user_database_1.PrismaUserDatabase(
             // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call
-            this.userDatabase = new user_database_1.PrismaUserDatabase(new PrismaClient({
+            new PrismaClient({
                 datasources: {
                     db: {
                         url: `postgresql://${userDBConfig.user}:${userDBConfig.password}@${userDBConfig.host}:${userDBConfig.port}/${userDBConfig.database}`,
                     },
-                }
+                },
             }));
-            this.logger.debug("Loaded Prisma user database");
+            this.logger.debug('Loaded Prisma user database');
         }
         else if (userDbClient === user_database_1.UserDatabaseName.TYPEORM) {
             // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
-            const DataSourceExports = require("typeorm");
+            const DataSourceExports = require('typeorm');
             try {
                 this.userDatabase = new user_database_1.TypeORMDatabase(
                 // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
                 new DataSourceExports.DataSource({
-                    type: "postgres", // perhaps should move to config file
+                    type: 'postgres', // perhaps should move to config file
                     host: userDBConfig.host,
                     port: userDBConfig.port,
                     username: userDBConfig.user,
@@ -186,11 +173,11 @@ class DBOSExecutor {
                 s.message = `Error loading TypeORM user database: ${s.message}`;
                 this.logger.error(s);
             }
-            this.logger.debug("Loaded TypeORM user database");
+            this.logger.debug('Loaded TypeORM user database');
         }
         else if (userDbClient === user_database_1.UserDatabaseName.KNEX) {
             const knexConfig = {
-                client: "postgres",
+                client: 'postgres',
                 connection: {
                     host: userDBConfig.host,
                     port: userDBConfig.port,
@@ -201,21 +188,21 @@ class DBOSExecutor {
                 },
             };
             this.userDatabase = new user_database_1.KnexUserDatabase((0, knex_1.default)(knexConfig));
-            this.logger.debug("Loaded Knex user database");
+            this.logger.debug('Loaded Knex user database');
         }
         else if (userDbClient === user_database_1.UserDatabaseName.DRIZZLE) {
             // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
-            const DrizzleExports = require("drizzle-orm/node-postgres");
+            const DrizzleExports = require('drizzle-orm/node-postgres');
             const drizzlePool = new pg_1.Pool(userDBConfig);
             // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
             const drizzle = DrizzleExports.drizzle(drizzlePool, { schema: this.drizzleEntities });
             // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
             this.userDatabase = new user_database_1.DrizzleUserDatabase(drizzlePool, drizzle);
-            this.logger.debug("Loaded Drizzle user database");
+            this.logger.debug('Loaded Drizzle user database');
         }
         else {
             this.userDatabase = new user_database_1.PGNodeUserDatabase(userDBConfig);
-            this.logger.debug("Loaded Postgres user database");
+            this.logger.debug('Loaded Postgres user database');
         }
     }
     #registerClass(cls) {
@@ -253,7 +240,7 @@ class DBOSExecutor {
     }
     async init(classes) {
         if (this.initialized) {
-            this.logger.error("Workflow executor already initialized!");
+            this.logger.error('Workflow executor already initialized!');
             return;
         }
         if (!classes || !classes.length) {
@@ -267,7 +254,7 @@ class DBOSExecutor {
                  * With TSORM, we take an array of entities (Function[]) and add them to this.entities:
                  */
                 if (Array.isArray(reg.ormEntities)) {
-                    this.typeormEntities = (this.typeormEntities).concat(reg.ormEntities);
+                    this.typeormEntities = this.typeormEntities.concat(reg.ormEntities);
                     length = reg.ormEntities.length;
                 }
                 else {
@@ -279,11 +266,13 @@ class DBOSExecutor {
                 }
                 this.logger.debug(`Loaded ${length} ORM entities`);
             }
-            await (0, user_database_1.createDBIfDoesNotExist)(this.config.poolConfig, this.logger);
+            if (!this.debugMode) {
+                await (0, user_database_1.createDBIfDoesNotExist)(this.config.poolConfig, this.logger);
+            }
             this.configureDbClient();
             if (!this.userDatabase) {
-                this.logger.error("No user database configured!");
-                throw new error_1.DBOSInitializationError("No user database configured!");
+                this.logger.error('No user database configured!');
+                throw new error_1.DBOSInitializationError('No user database configured!');
             }
             for (const cls of classes) {
                 this.#registerClass(cls);
@@ -324,30 +313,30 @@ class DBOSExecutor {
             for (const v of this.registeredOperations) {
                 const m = v;
                 if (m.init === true) {
-                    this.logger.debug("Executing init method: " + m.name);
+                    this.logger.debug('Executing init method: ' + m.name);
                     await m.origFunction(new context_1.InitContext(this));
                 }
             }
             await this.recoverPendingWorkflows();
         }
-        this.logger.info("Workflow executor initialized");
+        this.logger.info('Workflow executor initialized');
     }
     #logNotice(msg) {
         switch (msg.severity) {
-            case "INFO":
-            case "LOG":
-            case "NOTICE":
+            case 'INFO':
+            case 'LOG':
+            case 'NOTICE':
                 this.logger.info(msg.message);
                 break;
-            case "WARNING":
+            case 'WARNING':
                 this.logger.warn(msg.message);
                 break;
-            case "DEBUG":
+            case 'DEBUG':
                 this.logger.debug(msg.message);
                 break;
-            case "ERROR":
-            case "FATAL":
-            case "PANIC":
+            case 'ERROR':
+            case 'FATAL':
+            case 'PANIC':
                 this.logger.error(msg.message);
                 break;
             default:
@@ -356,7 +345,7 @@ class DBOSExecutor {
     }
     async destroy() {
         if (this.pendingWorkflowMap.size > 0) {
-            this.logger.info("Waiting for pending workflows to finish.");
+            this.logger.info('Waiting for pending workflows to finish.');
             await Promise.allSettled(this.pendingWorkflowMap.values());
         }
         clearInterval(this.flushBufferID);
@@ -365,7 +354,7 @@ class DBOSExecutor {
             await this.flushWorkflowBuffers();
         }
         while (this.isFlushingBuffers) {
-            this.logger.info("Waiting for result buffers to be exported.");
+            this.logger.info('Waiting for result buffers to be exported.');
             await (0, utils_1.sleepms)(1000);
         }
         await this.systemDatabase.destroy();
@@ -439,9 +428,7 @@ class DBOSExecutor {
         this.logger.debug(`Registered stored proc ${cfn}`);
     }
     getWorkflowInfo(wf) {
-        const wfname = (wf.name === DBOSExecutor.tempWorkflowName)
-            ? wf.name
-            : (0, decorators_1.getRegisteredMethodClassName)(wf) + '.' + wf.name;
+        const wfname = wf.name === DBOSExecutor.tempWorkflowName ? wf.name : (0, decorators_1.getRegisteredMethodClassName)(wf) + '.' + wf.name;
         return this.workflowInfoMap.get(wfname);
     }
     getWorkflowInfoByStatus(wf) {
@@ -498,14 +485,14 @@ class DBOSExecutor {
         const wCtxt = new workflow_1.WorkflowContextImpl(this, params.parentCtx, workflowUUID, wConfig, wf.name, presetUUID, params.tempWfType, params.tempWfName);
         const internalStatus = {
             workflowUUID: workflowUUID,
-            status: (params.queueName !== undefined) ? workflow_1.StatusString.ENQUEUED : workflow_1.StatusString.PENDING,
+            status: params.queueName !== undefined ? workflow_1.StatusString.ENQUEUED : workflow_1.StatusString.PENDING,
             name: wf.name,
-            className: wCtxt.isTempWorkflow ? "" : (0, decorators_1.getRegisteredMethodClassName)(wf),
-            configName: params.configuredInstance?.name || "",
+            className: wCtxt.isTempWorkflow ? '' : (0, decorators_1.getRegisteredMethodClassName)(wf),
+            configName: params.configuredInstance?.name || '',
             queueName: params.queueName,
             authenticatedUser: wCtxt.authenticatedUser,
             output: undefined,
-            error: "",
+            error: '',
             assumedRole: wCtxt.assumedRole,
             authenticatedRoles: wCtxt.authenticatedRoles,
             request: wCtxt.request,
@@ -514,18 +501,17 @@ class DBOSExecutor {
             applicationID: wCtxt.applicationID,
             createdAt: Date.now(), // Remember the start time of this workflow
             maxRetries: wCtxt.maxRecoveryAttempts,
-            recovery: params.recovery === true,
         };
         if (wCtxt.isTempWorkflow) {
             internalStatus.name = `${DBOSExecutor.tempWorkflowName}-${wCtxt.tempWfOperationType}-${wCtxt.tempWfOperationName}`;
-            internalStatus.className = params.tempWfClass ?? "";
+            internalStatus.className = params.tempWfClass ?? '';
         }
         let status = undefined;
         // Synchronously set the workflow's status to PENDING and record workflow inputs (for non single-transaction workflows).
         // We have to do it for all types of workflows because operation_outputs table has a foreign key constraint on workflow status table.
-        if ((wCtxt.tempWfOperationType !== TempWorkflowType.transaction
-            && wCtxt.tempWfOperationType !== TempWorkflowType.procedure)
-            || params.queueName !== undefined) {
+        if ((wCtxt.tempWfOperationType !== TempWorkflowType.transaction &&
+            wCtxt.tempWfOperationType !== TempWorkflowType.procedure) ||
+            params.queueName !== undefined) {
             if (this.debugMode) {
                 const wfStatus = await this.systemDatabase.getWorkflowStatus(workflowUUID);
                 const wfInputs = await this.systemDatabase.getWorkflowInputs(workflowUUID);
@@ -590,7 +576,7 @@ class DBOSExecutor {
                     // Retrieve the handle and wait for the result.
                     const retrievedHandle = this.retrieveWorkflow(workflowUUID);
                     result = await retrievedHandle.getResult();
-                    wCtxt.span.setAttribute("cached", true);
+                    wCtxt.span.setAttribute('cached', true);
                     wCtxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
                 }
                 else {
@@ -616,8 +602,8 @@ class DBOSExecutor {
             }
             finally {
                 this.tracer.endSpan(wCtxt.span);
-                if (wCtxt.tempWfOperationType === TempWorkflowType.transaction
-                    || wCtxt.tempWfOperationType === TempWorkflowType.procedure) {
+                if (wCtxt.tempWfOperationType === TempWorkflowType.transaction ||
+                    wCtxt.tempWfOperationType === TempWorkflowType.procedure) {
                     // For single-transaction workflows, asynchronously record inputs.
                     // We must buffer inputs after workflow status is buffered/flushed because workflow_inputs table has a foreign key reference to the workflow_status table.
                     if (!this.debugMode) {
@@ -631,12 +617,13 @@ class DBOSExecutor {
             }
             return result;
         };
-        if (this.debugMode || (status !== 'SUCCESS' && status !== 'ERROR' && (params.queueName === undefined || params.executeWorkflow))) {
+        if (this.debugMode ||
+            (status !== 'SUCCESS' && status !== 'ERROR' && (params.queueName === undefined || params.executeWorkflow))) {
             const workflowPromise = runWorkflow();
             // Need to await for the workflow and capture errors.
             const awaitWorkflowPromise = workflowPromise
                 .catch((error) => {
-                this.logger.debug("Captured error in awaitWorkflowPromise: " + error);
+                this.logger.debug('Captured error in awaitWorkflowPromise: ' + error);
             })
                 .finally(() => {
                 // Remove itself from pending workflow map.
@@ -663,7 +650,7 @@ class DBOSExecutor {
      * Retrieve the transaction snapshot information of the current transaction
      */
     static async #retrieveSnapshot(query) {
-        const rows = await query("SELECT pg_current_snapshot()::text as txn_snapshot;", []);
+        const rows = await query('SELECT pg_current_snapshot()::text as txn_snapshot;', []);
         return rows[0].txn_snapshot;
     }
     /**
@@ -675,14 +662,14 @@ class DBOSExecutor {
      */
     async #checkExecution(query, workflowUUID, funcID) {
         // Note: we read the current snapshot, not the recorded one!
-        const rows = await query("(SELECT output, error, txn_snapshot, true as recorded FROM dbos.transaction_outputs WHERE workflow_uuid=$1 AND function_id=$2 UNION ALL SELECT null as output, null as error, pg_current_snapshot()::text as txn_snapshot, false as recorded) ORDER BY recorded", [workflowUUID, funcID]);
+        const rows = await query('(SELECT output, error, txn_snapshot, true as recorded FROM dbos.transaction_outputs WHERE workflow_uuid=$1 AND function_id=$2 UNION ALL SELECT null as output, null as error, pg_current_snapshot()::text as txn_snapshot, false as recorded) ORDER BY recorded', [workflowUUID, funcID]);
         if (rows.length === 0 || rows.length > 2) {
-            this.logger.error("Unexpected! This should never happen. Returned rows: " + rows.toString());
-            throw new error_1.DBOSError("This should never happen. Returned rows: " + rows.toString());
+            this.logger.error('Unexpected! This should never happen. Returned rows: ' + rows.toString());
+            throw new error_1.DBOSError('This should never happen. Returned rows: ' + rows.toString());
         }
         const res = {
             output: exports.dbosNull,
-            txn_snapshot: ""
+            txn_snapshot: '',
         };
         // recorded=false row will be first because we used ORDER BY.
         res.txn_snapshot = rows[0].txn_snapshot;
@@ -701,11 +688,11 @@ class DBOSExecutor {
      */
     async #recordOutput(query, workflowUUID, funcID, txnSnapshot, output, isKeyConflict) {
         if (this.debugMode) {
-            throw new error_1.DBOSDebuggerError("Cannot record output in debug mode.");
+            throw new error_1.DBOSDebuggerError('Cannot record output in debug mode.');
         }
         try {
             const serialOutput = utils_1.DBOSJSON.stringify(output);
-            const rows = await query("INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, (select pg_current_xact_id_if_assigned()::text), $4, $5) RETURNING txn_id;", [workflowUUID, funcID, serialOutput, txnSnapshot, Date.now()]);
+            const rows = await query('INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, (select pg_current_xact_id_if_assigned()::text), $4, $5) RETURNING txn_id;', [workflowUUID, funcID, serialOutput, txnSnapshot, Date.now()]);
             return rows[0].txn_id;
         }
         catch (error) {
@@ -723,11 +710,11 @@ class DBOSExecutor {
      */
     async #recordError(query, workflowUUID, funcID, txnSnapshot, err, isKeyConflict) {
         if (this.debugMode) {
-            throw new error_1.DBOSDebuggerError("Cannot record error in debug mode.");
+            throw new error_1.DBOSDebuggerError('Cannot record error in debug mode.');
         }
         try {
             const serialErr = utils_1.DBOSJSON.stringify((0, serialize_error_1.serializeError)(err));
-            await query("INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, error, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, null, $4, $5) RETURNING txn_id;", [workflowUUID, funcID, serialErr, txnSnapshot, Date.now()]);
+            await query('INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, error, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, null, $4, $5) RETURNING txn_id;', [workflowUUID, funcID, serialErr, txnSnapshot, Date.now()]);
         }
         catch (error) {
             if (isKeyConflict(error)) {
@@ -749,11 +736,11 @@ class DBOSExecutor {
             return;
         }
         if (this.debugMode) {
-            throw new error_1.DBOSDebuggerError("Cannot flush result buffer in debug mode.");
+            throw new error_1.DBOSDebuggerError('Cannot flush result buffer in debug mode.');
         }
         funcIDs.sort();
         try {
-            let sqlStmt = "INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, error, txn_id, txn_snapshot, created_at) VALUES ";
+            let sqlStmt = 'INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, error, txn_id, txn_snapshot, created_at) VALUES ';
             let paramCnt = 1;
             const values = [];
             for (const funcID of funcIDs) {
@@ -764,7 +751,7 @@ class DBOSExecutor {
                 const txnSnapshot = recorded.txn_snapshot;
                 const createdAt = recorded.created_at;
                 if (paramCnt > 1) {
-                    sqlStmt += ", ";
+                    sqlStmt += ', ';
                 }
                 sqlStmt += `($${paramCnt++}, $${paramCnt++}, $${paramCnt++}, $${paramCnt++}, null, $${paramCnt++}, $${paramCnt++})`;
                 values.push(workflowUUID, funcID, utils_1.DBOSJSON.stringify(output), utils_1.DBOSJSON.stringify(null), txnSnapshot, createdAt);
@@ -787,7 +774,7 @@ class DBOSExecutor {
         return this.#flushResultBuffer(func, resultBuffer, workflowUUID, (error) => this.userDatabase.isKeyConflictError(error));
     }
     #flushResultBufferProc(client, resultBuffer, workflowUUID) {
-        const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+        const func = (sql, args) => client.query(sql, args).then((v) => v.rows);
         return this.#flushResultBuffer(func, resultBuffer, workflowUUID, user_database_1.pgNodeIsKeyConflictError);
     }
     async transaction(txn, params, ...args) {
@@ -823,7 +810,7 @@ class DBOSExecutor {
             isolationLevel: txnInfo.config.isolationLevel,
         }, wfCtx.span);
         while (true) {
-            let txn_snapshot = "invalid";
+            let txn_snapshot = 'invalid';
             const workflowUUID = wfCtx.workflowUUID;
             const wrappedTransaction = async (client) => {
                 const tCtxt = new transaction_1.TransactionContextImpl(this.userDatabase.getName(), client, wfCtx, span, this.logger, funcId, txn.name);
@@ -834,7 +821,7 @@ class DBOSExecutor {
                     const check = await this.#checkExecution(func, workflowUUID, funcId);
                     txn_snapshot = check.txn_snapshot;
                     if (check.output !== exports.dbosNull) {
-                        tCtxt.span.setAttribute("cached", true);
+                        tCtxt.span.setAttribute('cached', true);
                         tCtxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
                         this.tracer.endSpan(tCtxt.span);
                         return check.output;
@@ -881,7 +868,7 @@ class DBOSExecutor {
                         // Synchronously record the output of write transactions and obtain the transaction ID.
                         const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
                         const pg_txn_id = await this.#recordOutput(func, wfCtx.workflowUUID, funcId, txn_snapshot, result, (error) => this.userDatabase.isKeyConflictError(error));
-                        tCtxt.span.setAttribute("pg_txn_id", pg_txn_id);
+                        tCtxt.span.setAttribute('pg_txn_id', pg_txn_id);
                         wfCtx.resultBuffer.clear();
                     }
                     catch (error) {
@@ -908,7 +895,7 @@ class DBOSExecutor {
                 }
                 if (this.userDatabase.isRetriableTransactionError(err)) {
                     // serialization_failure in PostgreSQL
-                    span.addEvent("TXN SERIALIZATION FAILURE", { "retryWaitMillis": retryWaitMillis }, performance.now());
+                    span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
                     // Retry serialization failures.
                     await (0, utils_1.sleepms)(retryWaitMillis);
                     retryWaitMillis *= backoffFactor;
@@ -981,15 +968,15 @@ class DBOSExecutor {
         const maxRetryWaitMs = 2000; // Maximum wait 2 seconds.
         const readOnly = procInfo.config.readOnly ?? false;
         while (true) {
-            let txn_snapshot = "invalid";
+            let txn_snapshot = 'invalid';
             const wrappedProcedure = async (client) => {
                 const ctxt = new procedure_1.StoredProcedureContextImpl(client, wfCtx, span, this.logger, funcId, proc.name);
                 if (wfCtx.presetUUID) {
-                    const func = (sql, args) => this.procedurePool.query(sql, args).then(v => v.rows);
+                    const func = (sql, args) => this.procedurePool.query(sql, args).then((v) => v.rows);
                     const check = await this.#checkExecution(func, wfCtx.workflowUUID, funcId);
                     txn_snapshot = check.txn_snapshot;
                     if (check.output !== exports.dbosNull) {
-                        ctxt.span.setAttribute("cached", true);
+                        ctxt.span.setAttribute('cached', true);
                         ctxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
                         this.tracer.endSpan(ctxt.span);
                         return check.output;
@@ -997,7 +984,7 @@ class DBOSExecutor {
                 }
                 else {
                     // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
-                    const func = (sql, args) => this.procedurePool.query(sql, args).then(v => v.rows);
+                    const func = (sql, args) => this.procedurePool.query(sql, args).then((v) => v.rows);
                     txn_snapshot = await DBOSExecutor.#retrieveSnapshot(func);
                 }
                 if (this.debugMode) {
@@ -1031,23 +1018,25 @@ class DBOSExecutor {
                 }
                 else {
                     // Synchronously record the output of write transactions and obtain the transaction ID.
-                    const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+                    const func = (sql, args) => client.query(sql, args).then((v) => v.rows);
                     const pg_txn_id = await this.#recordOutput(func, wfCtx.workflowUUID, funcId, txn_snapshot, result, user_database_1.pgNodeIsKeyConflictError);
                     // const pg_txn_id = await wfCtx.recordOutputProc<R>(client, funcId, txn_snapshot, result);
-                    ctxt.span.setAttribute("pg_txn_id", pg_txn_id);
+                    ctxt.span.setAttribute('pg_txn_id', pg_txn_id);
                     wfCtx.resultBuffer.clear();
                 }
                 return result;
             };
             try {
-                const result = await this.invokeStoredProcFunction(wrappedProcedure, { isolationLevel: procInfo.config.isolationLevel });
+                const result = await this.invokeStoredProcFunction(wrappedProcedure, {
+                    isolationLevel: procInfo.config.isolationLevel,
+                });
                 span.setStatus({ code: api_1.SpanStatusCode.OK });
                 return result;
             }
             catch (err) {
                 if (this.userDatabase.isRetriableTransactionError(err)) {
                     // serialization_failure in PostgreSQL
-                    span.addEvent("TXN SERIALIZATION FAILURE", { "retryWaitMillis": retryWaitMillis }, performance.now());
+                    span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
                     // Retry serialization failures.
                     await (0, utils_1.sleepms)(retryWaitMillis);
                     retryWaitMillis *= backoffFactor;
@@ -1058,7 +1047,7 @@ class DBOSExecutor {
                 const e = err;
                 await this.invokeStoredProcFunction(async (client) => {
                     await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
-                    const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+                    const func = (sql, args) => client.query(sql, args).then((v) => v.rows);
                     await this.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, user_database_1.pgNodeIsKeyConflictError);
                 }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
                 await this.userDatabase.transaction(async (client) => {
@@ -1118,7 +1107,7 @@ class DBOSExecutor {
             wfCtx.resultBuffer.clear();
         }
         if (txn_id) {
-            span.setAttribute("pg_txn_id", txn_id);
+            span.setAttribute('pg_txn_id', txn_id);
         }
         span.setStatus({ code: api_1.SpanStatusCode.OK });
         return output;
@@ -1128,13 +1117,11 @@ class DBOSExecutor {
         const log = (msg) => this.#logNotice(msg);
         const procClassName = this.getProcedureClassName(proc);
         const plainProcName = `${procClassName}_${proc.name}_p`;
-        const procName = this.config.appVersion
-            ? `v${this.config.appVersion}_${plainProcName}`
-            : plainProcName;
+        const procName = this.config.appVersion ? `v${this.config.appVersion}_${plainProcName}` : plainProcName;
         const sql = `CALL "${procName}"(${args.map((_v, i) => `$${i + 1}`).join()});`;
         try {
             client.on('notice', log);
-            return await client.query(sql, args).then(value => value.rows);
+            return await client.query(sql, args).then((value) => value.rows);
         }
         finally {
             client.off('notice', log);
@@ -1206,7 +1193,7 @@ class DBOSExecutor {
         // Check if this execution previously happened, returning its original result if it did.
         const check = await this.systemDatabase.checkOperationOutput(wfCtx.workflowUUID, ctxt.functionID);
         if (check !== exports.dbosNull) {
-            ctxt.span.setAttribute("cached", true);
+            ctxt.span.setAttribute('cached', true);
             ctxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
             this.tracer.endSpan(ctxt.span);
             return check;
@@ -1245,7 +1232,7 @@ class DBOSExecutor {
                     const e = error;
                     errors.push(e);
                     this.logger.warn(`Error in step being automatically retried. Attempt ${numAttempts} of ${ctxt.maxAttempts}. ${e.stack}`);
-                    span.addEvent(`Step attempt ${numAttempts + 1} failed`, { "retryIntervalSeconds": intervalSeconds, "error": error.message }, performance.now());
+                    span.addEvent(`Step attempt ${numAttempts + 1} failed`, { retryIntervalSeconds: intervalSeconds, error: error.message }, performance.now());
                     if (numAttempts < ctxt.maxAttempts) {
                         // Sleep for an interval, then increase the interval by backoffRate.
                         // Cap at the maximum allowed retry interval.
@@ -1300,7 +1287,9 @@ class DBOSExecutor {
         };
         const workflowUUID = idempotencyKey ? destinationUUID + idempotencyKey : undefined;
         return (await this.workflow(temp_workflow, {
-            workflowUUID: workflowUUID, tempWfType: TempWorkflowType.send, configuredInstance: null,
+            workflowUUID: workflowUUID,
+            tempWfType: TempWorkflowType.send,
+            configuredInstance: null,
         }, destinationUUID, message, topic)).getResult();
     }
     /**
@@ -1337,7 +1326,7 @@ class DBOSExecutor {
         for (const nname of channels) {
             await notificationsClient.query(`LISTEN ${nname};`);
         }
-        notificationsClient.on("notification", callback);
+        notificationsClient.on('notification', callback);
         return {
             close: async () => {
                 for (const nname of channels) {
@@ -1349,7 +1338,7 @@ class DBOSExecutor {
                     }
                     notificationsClient.release();
                 }
-            }
+            },
         };
     }
     /* INTERNAL HELPERS */
@@ -1360,33 +1349,39 @@ class DBOSExecutor {
      * A recovery process that by default runs during executor init time.
      * It runs to completion all pending workflows that were executing when the previous executor failed.
      */
-    async recoverPendingWorkflows(executorIDs = ["local"]) {
+    async recoverPendingWorkflows(executorIDs = ['local']) {
         if (this.debugMode) {
-            throw new error_1.DBOSDebuggerError("Cannot recover pending workflows in debug mode.");
+            throw new error_1.DBOSDebuggerError('Cannot recover pending workflows in debug mode.');
         }
-        const pendingWorkflows = [];
+        const handlerArray = [];
         for (const execID of executorIDs) {
-            if (execID === "local" && process.env.DBOS__VMID) {
+            if (execID === 'local' && process.env.DBOS__VMID) {
                 this.logger.debug(`Skip local recovery because it's running in a VM: ${process.env.DBOS__VMID}`);
                 continue;
             }
-            this.logger.debug(`Recovering workflows of executor: ${execID}`);
-            const wIDs = await this.systemDatabase.getPendingWorkflows(execID);
-            pendingWorkflows.push(...wIDs);
-        }
-        const handlerArray = [];
-        for (const workflowUUID of pendingWorkflows) {
-            try {
-                handlerArray.push(await this.executeWorkflowUUID(workflowUUID));
-            }
-            catch (e) {
-                this.logger.warn(`Recovery of workflow ${workflowUUID} failed: ${e.message}`);
+            this.logger.debug(`Recovering workflows assigned to executor: ${execID}`);
+            const pendingWorkflows = await this.systemDatabase.getPendingWorkflows(execID);
+            for (const pendingWorkflow of pendingWorkflows) {
+                this.logger.debug(`Recovering workflow: ${pendingWorkflow.workflowUUID}. Queue name: ${pendingWorkflow.queueName}`);
+                try {
+                    // If the workflow is member of a queue, re-enqueue it.
+                    if (pendingWorkflow.queueName) {
+                        await this.systemDatabase.clearQueueAssignment(pendingWorkflow.workflowUUID);
+                        handlerArray.push(this.retrieveWorkflow(pendingWorkflow.workflowUUID));
+                    }
+                    else {
+                        handlerArray.push(await this.executeWorkflowUUID(pendingWorkflow.workflowUUID));
+                    }
+                }
+                catch (e) {
+                    this.logger.warn(`Recovery of workflow ${pendingWorkflow.workflowUUID} failed: ${e.message}`);
+                }
             }
         }
         return handlerArray;
     }
     async deactivateEventReceivers() {
-        this.logger.info("Deactivating event receivers");
+        this.logger.info('Deactivating event receivers');
         for (const evtRcvr of this.eventReceivers || []) {
             try {
                 await evtRcvr.destroy();
@@ -1425,15 +1420,18 @@ class DBOSExecutor {
         const workflowStartUUID = startNewWorkflow ? undefined : workflowUUID;
         if (wfInfo) {
             return this.workflow(wfInfo.workflow, {
-                workflowUUID: workflowStartUUID, parentCtx: parentCtx, configuredInstance: configuredInst, recovery: true,
-                queueName: wfStatus.queueName, executeWorkflow: true,
+                workflowUUID: workflowStartUUID,
+                parentCtx: parentCtx,
+                configuredInstance: configuredInst,
+                queueName: wfStatus.queueName,
+                executeWorkflow: true,
             }, 
             // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
             ...inputs);
         }
         // Should be temporary workflows. Parse the name of the workflow.
         const wfName = wfStatus.workflowName;
-        const nameArr = wfName.split("-");
+        const nameArr = wfName.split('-');
         if (!nameArr[0].startsWith(DBOSExecutor.tempWorkflowName)) {
             // CB - Doesn't this happen if the user changed the function name in their code?
             throw new error_1.DBOSError(`This should never happen! Cannot find workflow info for a non-temporary workflow! UUID ${workflowUUID}, name ${wfName}`);
@@ -1485,8 +1483,12 @@ class DBOSExecutor {
             throw new error_1.DBOSNotRegisteredError(wfName);
         }
         return this.workflow(temp_workflow, {
-            workflowUUID: workflowStartUUID, parentCtx: parentCtx ?? undefined, configuredInstance: clsinst,
-            recovery: true, tempWfType, tempWfClass, tempWfName,
+            workflowUUID: workflowStartUUID,
+            parentCtx: parentCtx ?? undefined,
+            configuredInstance: clsinst,
+            tempWfType,
+            tempWfClass,
+            tempWfName,
         }, 
         // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
         ...inputs);
@@ -1510,6 +1512,13 @@ class DBOSExecutor {
         oc.workflowUUID = workflowUUID;
         return oc;
     }
+    async cancelWorkflow(workflowID) {
+        await this.systemDatabase.cancelWorkflow(workflowID);
+    }
+    async resumeWorkflow(workflowID) {
+        await this.systemDatabase.resumeWorkflow(workflowID);
+        return await this.executeWorkflowUUID(workflowID, false);
+    }
     /* BACKGROUND PROCESSES */
     /**
      * Periodically flush the workflow output buffer to the system database.
@@ -1532,7 +1541,7 @@ class DBOSExecutor {
         try {
             let finishedCnt = 0;
             while (finishedCnt < totalSize) {
-                let sqlStmt = "INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, error, txn_id, txn_snapshot, created_at) VALUES ";
+                let sqlStmt = 'INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, error, txn_id, txn_snapshot, created_at) VALUES ';
                 let paramCnt = 1;
                 const values = [];
                 const batchUUIDs = [];
@@ -1542,7 +1551,7 @@ class DBOSExecutor {
                         const txnSnapshot = recorded.txn_snapshot;
                         const createdAt = recorded.created_at;
                         if (paramCnt > 1) {
-                            sqlStmt += ", ";
+                            sqlStmt += ', ';
                         }
                         sqlStmt += `($${paramCnt++}, $${paramCnt++}, $${paramCnt++}, $${paramCnt++}, null, $${paramCnt++}, $${paramCnt++})`;
                         values.push(workflowUUID, funcID, utils_1.DBOSJSON.stringify(output), utils_1.DBOSJSON.stringify(null), txnSnapshot, createdAt);
@@ -1558,7 +1567,9 @@ class DBOSExecutor {
                 // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
                 await this.userDatabase.query(sqlStmt, ...values);
                 // Clean up after each batch succeeds
-                batchUUIDs.forEach((value) => { localBuffer.delete(value); });
+                batchUUIDs.forEach((value) => {
+                    localBuffer.delete(value);
+                });
             }
         }
         catch (error) {
@@ -1573,14 +1584,14 @@ class DBOSExecutor {
         }
     }
     logRegisteredHTTPUrls() {
-        this.logger.info("HTTP endpoints supported:");
+        this.logger.info('HTTP endpoints supported:');
         this.registeredOperations.forEach((registeredOperation) => {
             const ro = registeredOperation;
             if (ro.apiURL) {
-                this.logger.info("    " + ro.apiType.padEnd(6) + "  :  " + ro.apiURL);
+                this.logger.info('    ' + ro.apiType.padEnd(6) + '  :  ' + ro.apiURL);
                 const roles = ro.getRequiredRoles();
                 if (roles.length > 0) {
-                    this.logger.info("        Required Roles: " + utils_1.DBOSJSON.stringify(roles));
+                    this.logger.info('        Required Roles: ' + utils_1.DBOSJSON.stringify(roles));
                 }
             }
         });