@dbos-inc/dbos-sdk 2.1.2-preview → 2.1.9-preview

package/dist/src/dbos-executor.js

@@ -24,6 +24,7 @@ const debug_workflow_1 = require("./debugger/debug_workflow");
 const serialize_error_1 = require("serialize-error");
 const utils_1 = require("./utils");
 const node_path_1 = __importDefault(require("node:path"));
+const procedure_1 = require("./procedure");
 const lodash_1 = require("lodash");
 const wfqueue_1 = require("./wfqueue");
 const debugpoint_1 = require("./debugpoint");
@@ -279,7 +280,6 @@ class DBOSExecutor {
                 }
                 this.logger.debug(`Loaded ${length} ORM entities`);
             }
-            await ((0, user_database_1.createDBIfDoesNotExist)(this.config.poolConfig, this.logger));
             this.configureDbClient();
             if (!this.userDatabase) {
                 this.logger.error("No user database configured!");
@@ -354,24 +354,6 @@ class DBOSExecutor {
                 this.logger.error(`Unknown notice severity: ${msg.severity} - ${msg.message}`);
         }
     }
-    async callProcedure(proc, args) {
-        const client = await this.procedurePool.connect();
-        const log = (msg) => this.#logNotice(msg);
-        const procClassName = this.getProcedureClassName(proc);
-        const plainProcName = `${procClassName}_${proc.name}_p`;
-        const procName = this.config.appVersion
-            ? `v${this.config.appVersion}_${plainProcName}`
-            : plainProcName;
-        const sql = `CALL "${procName}"(${args.map((_v, i) => `$${i + 1}`).join()});`;
-        try {
-            client.on('notice', log);
-            return await client.query(sql, args).then(value => value.rows);
-        }
-        finally {
-            client.off('notice', log);
-            client.release();
-        }
-    }
     async destroy() {
         if (this.pendingWorkflowMap.size > 0) {
             this.logger.info("Waiting for pending workflows to finish.");
@@ -541,13 +523,16 @@ class DBOSExecutor {
             internalStatus.name = `${DBOSExecutor.tempWorkflowName}-${wCtxt.tempWfOperationType}-${wCtxt.tempWfOperationName}`;
             internalStatus.className = params.tempWfClass ?? "";
         }
+        let status = undefined;
         // Synchronously set the workflow's status to PENDING and record workflow inputs (for non single-transaction workflows).
         // We have to do it for all types of workflows because operation_outputs table has a foreign key constraint on workflow status table.
         if ((wCtxt.tempWfOperationType !== TempWorkflowType.transaction
             && wCtxt.tempWfOperationType !== TempWorkflowType.procedure)
             || params.queueName !== undefined) {
             // TODO: Make this transactional (and with the queue step below)
-            args = await this.systemDatabase.initWorkflowStatus(internalStatus, args);
+            const ires = await this.systemDatabase.initWorkflowStatus(internalStatus, args);
+            args = ires.args;
+            status = ires.status;
             await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_WORKFLOW_ENQUEUE);
         }
         const runWorkflow = async () => {
@@ -617,7 +602,7 @@ class DBOSExecutor {
             }
             return result;
         };
-        if (params.queueName === undefined || params.executeWorkflow) {
+        if (status !== 'SUCCESS' && status !== 'ERROR' && (params.queueName === undefined || params.executeWorkflow)) {
             const workflowPromise = runWorkflow();
             // Need to await for the workflow and capture errors.
             const awaitWorkflowPromise = workflowPromise
@@ -633,7 +618,9 @@ class DBOSExecutor {
             return new workflow_1.InvokedHandle(this.systemDatabase, workflowPromise, workflowUUID, wf.name, callerUUID, callerFunctionID);
         }
         else {
-            await this.systemDatabase.enqueueWorkflow(workflowUUID, this.#getQueueByName(params.queueName));
+            if (params.queueName && status === 'ENQUEUED') {
+                await this.systemDatabase.enqueueWorkflow(workflowUUID, this.#getQueueByName(params.queueName));
+            }
             return new workflow_1.RetrievedHandle(this.systemDatabase, workflowUUID, callerUUID, callerFunctionID);
         }
     }
@@ -684,6 +671,128 @@ class DBOSExecutor {
         });
         return new workflow_1.InvokedHandle(this.systemDatabase, workflowPromise, workflowUUID, wf.name, callerUUID, callerFunctionID);
     }
+    /**
+     * Retrieve the transaction snapshot information of the current transaction
+     */
+    static async #retrieveSnapshot(query) {
+        const rows = await query("SELECT pg_current_snapshot()::text as txn_snapshot;", []);
+        return rows[0].txn_snapshot;
+    }
+    /**
+     * Check if an operation has already executed in a workflow.
+     * If it previously executed successfully, return its output.
+     * If it previously executed and threw an error, throw that error.
+     * Otherwise, return DBOSNull.
+     * Also return the transaction snapshot information of this current transaction.
+     */
+    async #checkExecution(query, workflowUUID, funcID) {
+        // Note: we read the current snapshot, not the recorded one!
+        const rows = await query("(SELECT output, error, txn_snapshot, true as recorded FROM dbos.transaction_outputs WHERE workflow_uuid=$1 AND function_id=$2 UNION ALL SELECT null as output, null as error, pg_current_snapshot()::text as txn_snapshot, false as recorded) ORDER BY recorded", [workflowUUID, funcID]);
+        if (rows.length === 0 || rows.length > 2) {
+            this.logger.error("Unexpected! This should never happen. Returned rows: " + rows.toString());
+            throw new error_1.DBOSError("This should never happen. Returned rows: " + rows.toString());
+        }
+        const res = {
+            output: exports.dbosNull,
+            txn_snapshot: ""
+        };
+        // recorded=false row will be first because we used ORDER BY.
+        res.txn_snapshot = rows[0].txn_snapshot;
+        if (rows.length === 2) {
+            if (utils_1.DBOSJSON.parse(rows[1].error) !== null) {
+                throw (0, serialize_error_1.deserializeError)(utils_1.DBOSJSON.parse(rows[1].error));
+            }
+            else {
+                res.output = utils_1.DBOSJSON.parse(rows[1].output);
+            }
+        }
+        return res;
+    }
+    /**
+     * Write a operation's output to the database.
+     */
+    static async #recordOutput(query, workflowUUID, funcID, txnSnapshot, output, isKeyConflict) {
+        try {
+            const serialOutput = utils_1.DBOSJSON.stringify(output);
+            const rows = await query("INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, (select pg_current_xact_id_if_assigned()::text), $4, $5) RETURNING txn_id;", [workflowUUID, funcID, serialOutput, txnSnapshot, Date.now()]);
+            return rows[0].txn_id;
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictUUIDError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    /**
+     * Record an error in an operation to the database.
+     */
+    static async #recordError(query, workflowUUID, funcID, txnSnapshot, err, isKeyConflict) {
+        try {
+            const serialErr = utils_1.DBOSJSON.stringify((0, serialize_error_1.serializeError)(err));
+            await query("INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, error, txn_id, txn_snapshot, created_at) VALUES ($1, $2, $3, null, $4, $5) RETURNING txn_id;", [workflowUUID, funcID, serialErr, txnSnapshot, Date.now()]);
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictUUIDError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    /**
+     * Write all entries in the workflow result buffer to the database.
+     * If it encounters a primary key error, this indicates a concurrent execution with the same UUID, so throw an DBOSError.
+     */
+    async #flushResultBuffer(query, resultBuffer, workflowUUID, isKeyConflict) {
+        const funcIDs = Array.from(resultBuffer.keys());
+        if (funcIDs.length === 0) {
+            return;
+        }
+        funcIDs.sort();
+        try {
+            let sqlStmt = "INSERT INTO dbos.transaction_outputs (workflow_uuid, function_id, output, error, txn_id, txn_snapshot, created_at) VALUES ";
+            let paramCnt = 1;
+            const values = [];
+            for (const funcID of funcIDs) {
+                // Capture output and also transaction snapshot information.
+                // Initially, no txn_id because no queries executed.
+                const recorded = resultBuffer.get(funcID);
+                const output = recorded.output;
+                const txnSnapshot = recorded.txn_snapshot;
+                const createdAt = recorded.created_at;
+                if (paramCnt > 1) {
+                    sqlStmt += ", ";
+                }
+                sqlStmt += `($${paramCnt++}, $${paramCnt++}, $${paramCnt++}, $${paramCnt++}, null, $${paramCnt++}, $${paramCnt++})`;
+                values.push(workflowUUID, funcID, utils_1.DBOSJSON.stringify(output), utils_1.DBOSJSON.stringify(null), txnSnapshot, createdAt);
+            }
+            this.logger.debug(sqlStmt);
+            await query(sqlStmt, values);
+        }
+        catch (error) {
+            if (isKeyConflict(error)) {
+                // Serialization and primary key conflict (Postgres).
+                throw new error_1.DBOSWorkflowConflictUUIDError(workflowUUID);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    flushResultBuffer(client, resultBuffer, workflowUUID) {
+        const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+        return this.#flushResultBuffer(func, resultBuffer, workflowUUID, (error) => this.userDatabase.isKeyConflictError(error));
+    }
+    #flushResultBufferProc(client, resultBuffer, workflowUUID) {
+        const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+        return this.#flushResultBuffer(func, resultBuffer, workflowUUID, user_database_1.pgNodeIsKeyConflictError);
+    }
     async transaction(txn, params, ...args) {
         // Create a workflow and call transaction.
         const temp_workflow = async (ctxt, ...args) => {
@@ -724,7 +833,8 @@ class DBOSExecutor {
                 // If the UUID is preset, it is possible this execution previously happened. Check, and return its original result if it did.
                 // Note: It is possible to retrieve a generated ID from a workflow handle, run a concurrent execution, and cause trouble for yourself. We recommend against this.
                 if (wfCtx.presetUUID) {
-                    const check = await wfCtx.checkTxExecution(client, funcId);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    const check = await this.#checkExecution(func, workflowUUID, funcId);
                     txn_snapshot = check.txn_snapshot;
                     if (check.output !== exports.dbosNull) {
                         tCtxt.span.setAttribute("cached", true);
@@ -735,11 +845,12 @@ class DBOSExecutor {
                 }
                 else {
                     // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
-                    txn_snapshot = await wfCtx.retrieveTxSnapshot(client);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(func);
                 }
                 // For non-read-only transactions, flush the result buffer.
                 if (!readOnly) {
-                    await wfCtx.flushResultBuffer(client);
+                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
                 }
                 // Execute the user's transaction.
                 let cresult;
@@ -768,7 +879,8 @@ class DBOSExecutor {
                 else {
                     try {
                         // Synchronously record the output of write transactions and obtain the transaction ID.
-                        const pg_txn_id = await wfCtx.recordOutputTx(client, funcId, txn_snapshot, result);
+                        const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                        const pg_txn_id = await DBOSExecutor.#recordOutput(func, wfCtx.workflowUUID, funcId, txn_snapshot, result, (error) => this.userDatabase.isKeyConflictError(error));
                         tCtxt.span.setAttribute("pg_txn_id", pg_txn_id);
                         wfCtx.resultBuffer.clear();
                     }
@@ -803,8 +915,9 @@ class DBOSExecutor {
                 // Record and throw other errors.
                 const e = err;
                 await this.userDatabase.transaction(async (client) => {
-                    await wfCtx.flushResultBuffer(client);
-                    await wfCtx.recordErrorTx(client, funcId, txn_snapshot, e);
+                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    await DBOSExecutor.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
                 }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
                 wfCtx.resultBuffer.clear();
                 span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: e.message });
@@ -817,15 +930,209 @@ class DBOSExecutor {
         // Create a workflow and call procedure.
         const temp_workflow = async (ctxt, ...args) => {
             const ctxtImpl = ctxt;
-            return await ctxtImpl.procedure(proc, ...args);
+            return this.callProcedureFunction(proc, ctxtImpl, ...args);
         };
-        return (await this.workflow(temp_workflow, { ...params,
+        return await (await this.workflow(temp_workflow, {
+            ...params,
             tempWfType: TempWorkflowType.procedure,
             tempWfName: (0, decorators_1.getRegisteredMethodName)(proc),
             tempWfClass: (0, decorators_1.getRegisteredMethodClassName)(proc),
         }, ...args)).getResult();
     }
-    async executeProcedure(func, config) {
+    async callProcedureFunction(proc, wfCtx, ...args) {
+        const procInfo = this.getProcedureInfo(proc);
+        if (procInfo === undefined) {
+            throw new error_1.DBOSNotRegisteredError(proc.name);
+        }
+        const executeLocally = procInfo.config.executeLocally ?? false;
+        const funcId = wfCtx.functionIDGetIncrement();
+        const span = this.tracer.startSpan(proc.name, {
+            operationUUID: wfCtx.workflowUUID,
+            operationType: exports.OperationType.PROCEDURE,
+            authenticatedUser: wfCtx.authenticatedUser,
+            assumedRole: wfCtx.assumedRole,
+            authenticatedRoles: wfCtx.authenticatedRoles,
+            readOnly: procInfo.config.readOnly ?? false,
+            isolationLevel: procInfo.config.isolationLevel,
+            executeLocally,
+        }, wfCtx.span);
+        try {
+            const result = executeLocally
+                ? await this.#callProcedureFunctionLocal(proc, args, wfCtx, span, procInfo, funcId)
+                : await this.#callProcedureFunctionRemote(proc, args, wfCtx, span, procInfo.config, funcId);
+            span.setStatus({ code: api_1.SpanStatusCode.OK });
+            return result;
+        }
+        catch (e) {
+            const { message } = e;
+            span.setStatus({ code: api_1.SpanStatusCode.ERROR, message });
+            throw e;
+        }
+        finally {
+            this.tracer.endSpan(span);
+        }
+    }
+    async #callProcedureFunctionLocal(proc, args, wfCtx, span, procInfo, funcId) {
+        let retryWaitMillis = 1;
+        const backoffFactor = 1.5;
+        const maxRetryWaitMs = 2000; // Maximum wait 2 seconds.
+        const readOnly = procInfo.config.readOnly ?? false;
+        while (true) {
+            let txn_snapshot = "invalid";
+            const wrappedProcedure = async (client) => {
+                const ctxt = new procedure_1.StoredProcedureContextImpl(client, wfCtx, span, this.logger, funcId, proc.name);
+                if (wfCtx.presetUUID) {
+                    const func = (sql, args) => this.procedurePool.query(sql, args).then(v => v.rows);
+                    const check = await this.#checkExecution(func, wfCtx.workflowUUID, funcId);
+                    txn_snapshot = check.txn_snapshot;
+                    if (check.output !== exports.dbosNull) {
+                        ctxt.span.setAttribute("cached", true);
+                        ctxt.span.setStatus({ code: api_1.SpanStatusCode.OK });
+                        this.tracer.endSpan(ctxt.span);
+                        return check.output;
+                    }
+                }
+                else {
+                    // Collect snapshot information for read-only transactions and non-preset UUID transactions, if not already collected above
+                    const func = (sql, args) => this.procedurePool.query(sql, args).then(v => v.rows);
+                    txn_snapshot = await DBOSExecutor.#retrieveSnapshot(func);
+                }
+                // For non-read-only transactions, flush the result buffer.
+                if (!readOnly) {
+                    await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                }
+                let cresult;
+                if (procInfo.registration.passContext) {
+                    await (0, context_1.runWithStoredProcContext)(ctxt, async () => {
+                        cresult = await proc(ctxt, ...args);
+                    });
+                }
+                else {
+                    await (0, context_1.runWithStoredProcContext)(ctxt, async () => {
+                        const pf = proc;
+                        cresult = await pf(...args);
+                    });
+                }
+                const result = cresult;
+                if (readOnly) {
+                    // Buffer the output of read-only transactions instead of synchronously writing it.
+                    const readOutput = {
+                        output: result,
+                        txn_snapshot: txn_snapshot,
+                        created_at: Date.now(),
+                    };
+                    wfCtx.resultBuffer.set(funcId, readOutput);
+                }
+                else {
+                    // Synchronously record the output of write transactions and obtain the transaction ID.
+                    const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+                    const pg_txn_id = await DBOSExecutor.#recordOutput(func, wfCtx.workflowUUID, funcId, txn_snapshot, result, user_database_1.pgNodeIsKeyConflictError);
+                    // const pg_txn_id = await wfCtx.recordOutputProc<R>(client, funcId, txn_snapshot, result);
+                    ctxt.span.setAttribute("pg_txn_id", pg_txn_id);
+                    wfCtx.resultBuffer.clear();
+                }
+                return result;
+            };
+            try {
+                const result = await this.invokeStoredProcFunction(wrappedProcedure, { isolationLevel: procInfo.config.isolationLevel });
+                span.setStatus({ code: api_1.SpanStatusCode.OK });
+                return result;
+            }
+            catch (err) {
+                if (this.userDatabase.isRetriableTransactionError(err)) {
+                    // serialization_failure in PostgreSQL
+                    span.addEvent("TXN SERIALIZATION FAILURE", { "retryWaitMillis": retryWaitMillis }, performance.now());
+                    // Retry serialization failures.
+                    await (0, utils_1.sleepms)(retryWaitMillis);
+                    retryWaitMillis *= backoffFactor;
+                    retryWaitMillis = retryWaitMillis < maxRetryWaitMs ? retryWaitMillis : maxRetryWaitMs;
+                    continue;
+                }
+                // Record and throw other errors.
+                const e = err;
+                await this.invokeStoredProcFunction(async (client) => {
+                    await this.#flushResultBufferProc(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                    const func = (sql, args) => client.query(sql, args).then(v => v.rows);
+                    await DBOSExecutor.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, user_database_1.pgNodeIsKeyConflictError);
+                }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                await this.userDatabase.transaction(async (client) => {
+                    await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
+                    const func = (sql, args) => this.userDatabase.queryWithClient(client, sql, ...args);
+                    await DBOSExecutor.#recordError(func, wfCtx.workflowUUID, funcId, txn_snapshot, e, (error) => this.userDatabase.isKeyConflictError(error));
+                }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
+                wfCtx.resultBuffer.clear();
+                throw err;
+            }
+        }
+    }
+    async #callProcedureFunctionRemote(proc, args, wfCtx, span, config, funcId) {
+        const readOnly = config.readOnly ?? false;
+        const $jsonCtx = {
+            request: wfCtx.request,
+            authenticatedUser: wfCtx.authenticatedUser,
+            authenticatedRoles: wfCtx.authenticatedRoles,
+            assumedRole: wfCtx.assumedRole,
+        };
+        // Note, node-pg converts JS arrays to postgres array literals, so must call JSON.strigify on
+        // args and bufferedResults before being passed to #invokeStoredProc
+        const $args = [wfCtx.workflowUUID, funcId, wfCtx.presetUUID, $jsonCtx, null, JSON.stringify(args)];
+        if (!readOnly) {
+            // function_id, output, txn_snapshot, created_at
+            const bufferedResults = new Array();
+            for (const [functionID, { output, txn_snapshot, created_at }] of wfCtx.resultBuffer.entries()) {
+                bufferedResults.push([functionID, output, txn_snapshot, created_at]);
+            }
+            // sort by function ID
+            bufferedResults.sort((a, b) => a[0] - b[0]);
+            $args.unshift(bufferedResults.length > 0 ? JSON.stringify(bufferedResults) : null);
+        }
+        const [{ return_value }] = await this.#invokeStoredProc(proc, $args);
+        const { error, output, txn_snapshot, txn_id, created_at } = return_value;
+        // buffered results are persisted in r/w stored procs, even if it returns an error
+        if (!readOnly) {
+            wfCtx.resultBuffer.clear();
+        }
+        // if the stored proc returns an error, deserialize and throw it.
+        // stored proc saves the error in tx_output before returning
+        if (error) {
+            throw (0, serialize_error_1.deserializeError)(error);
+        }
+        // if txn_snapshot is provided, the output needs to be buffered
+        if (readOnly && txn_snapshot) {
+            wfCtx.resultBuffer.set(funcId, {
+                output,
+                txn_snapshot,
+                created_at: created_at ?? Date.now(),
+            });
+        }
+        if (!readOnly) {
+            wfCtx.resultBuffer.clear();
+        }
+        if (txn_id) {
+            span.setAttribute("pg_txn_id", txn_id);
+        }
+        span.setStatus({ code: api_1.SpanStatusCode.OK });
+        return output;
+    }
+    async #invokeStoredProc(proc, args) {
+        const client = await this.procedurePool.connect();
+        const log = (msg) => this.#logNotice(msg);
+        const procClassName = this.getProcedureClassName(proc);
+        const plainProcName = `${procClassName}_${proc.name}_p`;
+        const procName = this.config.appVersion
+            ? `v${this.config.appVersion}_${plainProcName}`
+            : plainProcName;
+        const sql = `CALL "${procName}"(${args.map((_v, i) => `$${i + 1}`).join()});`;
+        try {
+            client.on('notice', log);
+            return await client.query(sql, args).then(value => value.rows);
+        }
+        finally {
+            client.off('notice', log);
+            client.release();
+        }
+    }
+    async invokeStoredProcFunction(func, config) {
         const client = await this.procedurePool.connect();
         try {
             const readOnly = config.readOnly ?? false;
@@ -884,7 +1191,7 @@ class DBOSExecutor {
         }, wfCtx.span);
         const ctxt = new step_1.StepContextImpl(wfCtx, funcID, span, this.logger, commInfo.config, stepFn.name);
         await this.userDatabase.transaction(async (client) => {
-            await wfCtx.flushResultBuffer(client);
+            await this.flushResultBuffer(client, wfCtx.resultBuffer, wfCtx.workflowUUID);
         }, { isolationLevel: transaction_1.IsolationLevel.ReadCommitted });
         wfCtx.resultBuffer.clear();
         // Check if this execution previously happened, returning its original result if it did.
@@ -1064,11 +1371,29 @@ class DBOSExecutor {
     async deactivateEventReceivers() {
         this.logger.info("Deactivating event receivers");
         for (const evtRcvr of this.eventReceivers || []) {
-            await evtRcvr.destroy();
+            try {
+                await evtRcvr.destroy();
+            }
+            catch (err) {
+                const e = err;
+                this.logger.warn(`Error destroying event receiver: ${e.message}`);
+            }
+        }
+        try {
+            await this.scheduler?.destroyScheduler();
+        }
+        catch (err) {
+            const e = err;
+            this.logger.warn(`Error destroying scheduler: ${e.message}`);
+        }
+        try {
+            wfqueue_1.wfQueueRunner.stop();
+            await this.wfqEnded;
+        }
+        catch (err) {
+            const e = err;
+            this.logger.warn(`Error destroying wf queue runner: ${e.message}`);
         }
-        await this.scheduler?.destroyScheduler();
-        wfqueue_1.wfQueueRunner.stop();
-        await this.wfqEnded;
     }
     async executeWorkflowUUID(workflowUUID, startNewWorkflow = false) {
         const wfStatus = await this.systemDatabase.getWorkflowStatus(workflowUUID);