@dbos-inc/dbos-sdk 2.8.19-preview → 2.8.46-preview.g30b0e27ed1

package/dist/src/system_database.js

@@ -1,5 +1,4 @@
 "use strict";
-/* eslint-disable @typescript-eslint/no-explicit-any */
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -40,6 +39,37 @@ async function migrateSystemDatabase(systemPoolConfig, logger) {
     }
 }
 exports.migrateSystemDatabase = migrateSystemDatabase;
+class NotificationMap {
+    map = new Map();
+    curCK = 0;
+    registerCallback(key, cb) {
+        if (!this.map.has(key)) {
+            this.map.set(key, new Map());
+        }
+        const ck = this.curCK++;
+        this.map.get(key).set(ck, cb);
+        return { key, ck };
+    }
+    deregisterCallback(k) {
+        if (!this.map.has(k.key))
+            return;
+        const sm = this.map.get(k.key);
+        if (!sm.has(k.ck))
+            return;
+        sm.delete(k.ck);
+        if (sm.size === 0) {
+            this.map.delete(k.key);
+        }
+    }
+    callCallbacks(key, event) {
+        if (!this.map.has(key))
+            return;
+        const sm = this.map.get(key);
+        for (const cb of sm.values()) {
+            cb(event);
+        }
+    }
+}
 class PostgresSystemDatabase {
     pgPoolConfig;
     systemDatabaseName;
@@ -48,9 +78,32 @@ class PostgresSystemDatabase {
     pool;
     systemPoolConfig;
     knexDB;
+    /*
+     * Generally, notifications are asynchronous.  One should:
+     *  Subscribe to updates
+     *  Read the database item in question
+     *  In response to updates, re-read the database item
+     *  Unsubscribe at the end
+     * The notification mechanism is reliable in the sense that it will eventually deliver updates
+     *  or the DB connection will get dropped.  The right thing to do if you lose connectivity to
+     *  the system DB is to exit the process and go through recovery... system DB writes, notifications,
+     *  etc may not have completed correctly, and recovery is the way to rebuild in-memory state.
+     *
+     * NOTE:
+     * PG Notifications are not fully reliable.
+     *   Dropped connections are recoverable - you just need to restart and scan everything.
+     *      (The whole VM being the logical choice, so workflows can recover from any write failures.)
+     *   The real problem is, if the pipes out of the server are full... then notifications can be
+     *     dropped, and only the PG server log may note it.  For those reasons, we do occasional polling
+     */
     notificationsClient = null;
-    notificationsMap = {};
-    workflowEventsMap = {};
+    dbPollingIntervalMs = 1000;
+    shouldUseDBNotifications = true;
+    notificationsMap = new NotificationMap();
+    workflowEventsMap = new NotificationMap();
+    cancelWakeupMap = new NotificationMap();
+    runningWorkflowMap = new Map(); // Map from workflowID to workflow promise
+    workflowCancellationMap = new Map(); // Map from workflowID to its cancellation status.
     constructor(pgPoolConfig, systemDatabaseName, logger, sysDbPoolSize) {
         this.pgPoolConfig = pgPoolConfig;
         this.systemDatabaseName = systemDatabaseName;
@@ -61,6 +114,7 @@ class PostgresSystemDatabase {
         systemDbConnectionString.pathname = `/${systemDatabaseName}`;
         this.systemPoolConfig = {
             connectionString: systemDbConnectionString.toString(),
+            connectionTimeoutMillis: pgPoolConfig.connectionTimeoutMillis,
             // This sets the application_name column in pg_stat_activity
             application_name: `dbos_transact_${utils_1.globalParams.executorID}_${utils_1.globalParams.appVersion}`,
         };
@@ -99,7 +153,9 @@ class PostgresSystemDatabase {
         finally {
             await pgSystemClient.end();
         }
-        await this.listenForNotifications();
+        if (this.shouldUseDBNotifications) {
+            await this.listenForNotifications();
+        }
     }
     async destroy() {
         await this.knexDB.destroy();
@@ -122,7 +178,7 @@ class PostgresSystemDatabase {
         await pgSystemClient.query(`DROP DATABASE IF EXISTS ${dbosConfig.system_database};`);
         await pgSystemClient.end();
     }
-    async initWorkflowStatus(initStatus, args) {
+    async initWorkflowStatus(initStatus, serializedInputs) {
         const result = await this.pool.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status (
         workflow_uuid,
         status,
@@ -156,8 +212,8 @@ class PostgresSystemDatabase {
             initStatus.queueName,
             initStatus.authenticatedUser,
             initStatus.assumedRole,
-            utils_1.DBOSJSON.stringify(initStatus.authenticatedRoles),
-            utils_1.DBOSJSON.stringify(initStatus.request),
+            JSON.stringify(initStatus.authenticatedRoles),
+            JSON.stringify(initStatus.request),
             null,
             initStatus.executorId,
             initStatus.applicationVersion,
@@ -199,12 +255,11 @@ class PostgresSystemDatabase {
         }
         this.logger.debug(`Workflow ${initStatus.workflowUUID} attempt number: ${attempts}.`);
         const status = resRow.status;
-        const serializedInputs = utils_1.DBOSJSON.stringify(args);
         const { rows } = await this.pool.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_inputs (workflow_uuid, inputs) VALUES($1, $2) ON CONFLICT (workflow_uuid) DO UPDATE SET workflow_uuid = excluded.workflow_uuid  RETURNING inputs`, [initStatus.workflowUUID, serializedInputs]);
         if (serializedInputs !== rows[0].inputs) {
             this.logger.warn(`Workflow inputs for ${initStatus.workflowUUID} changed since the first call! Use the original inputs.`);
         }
-        return { args: utils_1.DBOSJSON.parse(rows[0].inputs), status };
+        return { serializedInputs: rows[0].inputs, status };
     }
     async recordWorkflowStatusChange(workflowID, status, update, client) {
         let rec = '';
@@ -217,7 +272,7 @@ class PostgresSystemDatabase {
         const wRes = await (client ?? this.pool).query(`UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status
        SET ${rec} status=$2, output=$3, error=$4, updated_at=$5 WHERE workflow_uuid=$1`, [workflowID, status, update.output, update.error, Date.now()]);
         if (wRes.rowCount !== 1) {
-            throw new error_1.DBOSWorkflowConflictUUIDError(`Attempt to record transition of nonexistent workflow ${workflowID}`);
+            throw new error_1.DBOSWorkflowConflictError(`Attempt to record transition of nonexistent workflow ${workflowID}`);
         }
     }
     async recordWorkflowOutput(workflowID, status) {
@@ -238,9 +293,10 @@ class PostgresSystemDatabase {
         if (rows.length === 0) {
             return null;
         }
-        return utils_1.DBOSJSON.parse(rows[0].inputs);
+        return rows[0].inputs;
     }
     async getOperationResult(workflowID, functionID, client) {
+        await this.checkIfCanceled(workflowID);
         const { rows } = await (client ?? this.pool).query(`SELECT output, error, child_workflow_id, function_name
        FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs
       WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
@@ -281,7 +337,7 @@ class PostgresSystemDatabase {
             const err = error;
             if (err.code === '40001' || err.code === '23505') {
                 // Serialization and primary key conflict (Postgres).
-                throw new error_1.DBOSWorkflowConflictUUIDError(workflowID);
+                throw new error_1.DBOSWorkflowConflictError(workflowID);
             }
             else {
                 throw err;
@@ -306,6 +362,30 @@ class PostgresSystemDatabase {
         return serialOutput;
     }
     async durableSleepms(workflowID, functionID, durationMS) {
+        let resolveNotification;
+        const cancelPromise = new Promise((resolve) => {
+            resolveNotification = resolve;
+        });
+        const cbr = this.cancelWakeupMap.registerCallback(workflowID, resolveNotification);
+        try {
+            let timeoutPromise = Promise.resolve();
+            const { promise, cancel: timeoutCancel } = await this.durableSleepmsInternal(workflowID, functionID, durationMS);
+            timeoutPromise = promise;
+            try {
+                await Promise.race([cancelPromise, timeoutPromise]);
+            }
+            finally {
+                timeoutCancel();
+            }
+        }
+        finally {
+            this.cancelWakeupMap.deregisterCallback(cbr);
+        }
+        await this.checkIfCanceled(workflowID);
+    }
+    async durableSleepmsInternal(workflowID, functionID, durationMS, maxSleepPerIteration) {
+        if (maxSleepPerIteration === undefined)
+            maxSleepPerIteration = durationMS;
         const curTime = Date.now();
         let endTimeMs = curTime + durationMS;
         const res = await this.getOperationResult(workflowID, functionID);
@@ -318,7 +398,10 @@ class PostgresSystemDatabase {
         else {
             await this.recordOperationResult(workflowID, functionID, { serialOutput: JSON.stringify(endTimeMs), functionName: exports.DBOS_FUNCNAME_SLEEP }, false);
         }
-        return (0, utils_1.cancellableSleep)(Math.max(endTimeMs - curTime, 0));
+        return {
+            ...(0, utils_1.cancellableSleep)(Math.max(Math.min(maxSleepPerIteration, endTimeMs - curTime), 0)),
+            endTime: endTimeMs,
+        };
     }
     nullTopic = '__null__topic__';
     async send(workflowID, functionID, destinationID, message, topic) {
@@ -357,37 +440,56 @@ class PostgresSystemDatabase {
             }
             return res.res.res;
         }
-        // Check if the key is already in the DB, then wait for the notification if it isn't.
-        const initRecvRows = (await this.pool.query(`SELECT topic FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.notifications WHERE destination_uuid=$1 AND topic=$2;`, [workflowID, topic])).rows;
-        if (initRecvRows.length === 0) {
-            // Then, register the key with the global notifications listener.
+        const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
+        let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
+        while (true) {
+            // register the key with the global notifications listener.
             let resolveNotification;
             const messagePromise = new Promise((resolve) => {
                 resolveNotification = resolve;
             });
             const payload = `${workflowID}::${topic}`;
-            this.notificationsMap[payload] = resolveNotification; // The resolver assignment in the Promise definition runs synchronously.
-            let timeoutPromise = Promise.resolve();
-            let timeoutCancel = () => { };
-            try {
-                const { promise, cancel } = await this.durableSleepms(workflowID, timeoutFunctionID, timeoutSeconds * 1000);
-                timeoutPromise = promise;
-                timeoutCancel = cancel;
-            }
-            catch (e) {
-                this.logger.error(e);
-                delete this.notificationsMap[payload];
-                timeoutCancel();
-                throw new Error('durable sleepms failed');
-            }
+            const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
+            const crh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
+                resolveNotification();
+            });
             try {
-                await Promise.race([messagePromise, timeoutPromise]);
+                await this.checkIfCanceled(workflowID);
+                // Check if the key is already in the DB, then wait for the notification if it isn't.
+                const initRecvRows = (await this.pool.query(`SELECT topic FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.notifications WHERE destination_uuid=$1 AND topic=$2;`, [workflowID, topic])).rows;
+                if (initRecvRows.length !== 0)
+                    break;
+                const ct = Date.now();
+                if (finishTime && ct > finishTime)
+                    break; // Time's up
+                let timeoutPromise = Promise.resolve();
+                let timeoutCancel = () => { };
+                if (timeoutms) {
+                    const { promise, cancel, endTime } = await this.durableSleepmsInternal(workflowID, timeoutFunctionID, timeoutms, this.dbPollingIntervalMs);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                    finishTime = endTime;
+                }
+                else {
+                    let poll = finishTime ? finishTime - ct : this.dbPollingIntervalMs;
+                    poll = Math.min(this.dbPollingIntervalMs, poll);
+                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                }
+                try {
+                    await Promise.race([messagePromise, timeoutPromise]);
+                }
+                finally {
+                    timeoutCancel();
+                }
             }
             finally {
-                timeoutCancel();
-                delete this.notificationsMap[payload];
+                this.notificationsMap.deregisterCallback(cbr);
+                this.cancelWakeupMap.deregisterCallback(crh);
             }
         }
+        await this.checkIfCanceled(workflowID);
         // Transactionally consume and return the message if it's in the DB, otherwise return null.
         let message = null;
         const client = await this.pool.connect();
@@ -461,40 +563,49 @@ class PostgresSystemDatabase {
         // Get the return the value. if it's in the DB, otherwise return null.
         let value = null;
         const payloadKey = `${workflowID}::${key}`;
+        const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
+        let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
         // Register the key with the global notifications listener first... we do not want to look in the DB first
         //  or that would cause a timing hole.
-        let resolveNotification;
-        const valuePromise = new Promise((resolve) => {
-            resolveNotification = resolve;
-        });
-        this.workflowEventsMap[payloadKey] = resolveNotification; // The resolver assignment in the Promise definition runs synchronously.
-        try {
-            // Check if the key is already in the DB, then wait for the notification if it isn't.
-            const initRecvRows = (await this.pool.query(`
-        SELECT key, value
-        FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_events
-        WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
-            if (initRecvRows.length > 0) {
-                value = initRecvRows[0].value;
-            }
-            else {
+        while (true) {
+            let resolveNotification;
+            const valuePromise = new Promise((resolve) => {
+                resolveNotification = resolve;
+            });
+            const cbr = this.workflowEventsMap.registerCallback(payloadKey, resolveNotification);
+            const crh = callerWorkflow?.workflowID
+                ? this.cancelWakeupMap.registerCallback(callerWorkflow.workflowID, (_res) => {
+                    resolveNotification();
+                })
+                : undefined;
+            try {
+                if (callerWorkflow?.workflowID)
+                    await this.checkIfCanceled(callerWorkflow?.workflowID);
+                // Check if the key is already in the DB, then wait for the notification if it isn't.
+                const initRecvRows = (await this.pool.query(`
+          SELECT key, value
+          FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_events
+          WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
+                if (initRecvRows.length > 0) {
+                    value = initRecvRows[0].value;
+                    break;
+                }
+                const ct = Date.now();
+                if (finishTime && ct > finishTime)
+                    break; // Time's up
                 // If we have a callerWorkflow, we want a durable sleep, otherwise, not
                 let timeoutPromise = Promise.resolve();
                 let timeoutCancel = () => { };
-                if (callerWorkflow) {
-                    try {
-                        const { promise, cancel } = await this.durableSleepms(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutSeconds * 1000);
-                        timeoutPromise = promise;
-                        timeoutCancel = cancel;
-                    }
-                    catch (e) {
-                        this.logger.error(e);
-                        delete this.workflowEventsMap[payloadKey];
-                        throw new Error('durable sleepms failed');
-                    }
+                if (callerWorkflow && timeoutms) {
+                    const { promise, cancel, endTime } = await this.durableSleepmsInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms, this.dbPollingIntervalMs);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                    finishTime = endTime;
                 }
                 else {
-                    const { promise, cancel } = (0, utils_1.cancellableSleep)(timeoutSeconds * 1000);
+                    let poll = finishTime ? finishTime - ct : this.dbPollingIntervalMs;
+                    poll = Math.min(this.dbPollingIntervalMs, poll);
+                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
                     timeoutPromise = promise;
                     timeoutCancel = cancel;
                 }
@@ -504,17 +615,12 @@ class PostgresSystemDatabase {
                 finally {
                     timeoutCancel();
                 }
-                const finalRecvRows = (await this.pool.query(`
-            SELECT value
-            FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_events
-            WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
-                if (finalRecvRows.length > 0) {
-                    value = finalRecvRows[0].value;
-                }
             }
-        }
-        finally {
-            delete this.workflowEventsMap[payloadKey];
+            finally {
+                this.workflowEventsMap.deregisterCallback(cbr);
+                if (crh)
+                    this.cancelWakeupMap.deregisterCallback(crh);
+            }
         }
         // Record the output if it is inside a workflow.
         if (callerWorkflow) {
@@ -528,10 +634,23 @@ class PostgresSystemDatabase {
     async setWorkflowStatus(workflowID, status, resetRecoveryAttempts) {
         await this.recordWorkflowStatusChange(workflowID, status, { resetRecoveryAttempts });
     }
+    setWFCancelMap(workflowID) {
+        if (this.runningWorkflowMap.has(workflowID)) {
+            this.workflowCancellationMap.set(workflowID, true);
+        }
+        this.cancelWakeupMap.callCallbacks(workflowID);
+    }
+    clearWFCancelMap(workflowID) {
+        if (this.workflowCancellationMap.has(workflowID)) {
+            this.workflowCancellationMap.delete(workflowID);
+        }
+    }
     async cancelWorkflow(workflowID) {
         const client = await this.pool.connect();
         try {
             await client.query('BEGIN');
+            await client.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_cancel(workflow_id)
+         VALUES ($1)`, [workflowID]);
             // Remove workflow from queues table
             await client.query(`DELETE FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue 
          WHERE workflow_uuid = $1`, [workflowID]);
@@ -540,12 +659,20 @@ class PostgresSystemDatabase {
             await client.query('COMMIT');
         }
         catch (error) {
+            this.logger.error(error);
             await client.query('ROLLBACK');
             throw error;
         }
         finally {
             client.release();
         }
+        this.setWFCancelMap(workflowID);
+    }
+    async checkIfCanceled(workflowID) {
+        if (this.workflowCancellationMap.get(workflowID) === true) {
+            throw new error_1.DBOSWorkflowCancelledError(workflowID);
+        }
+        return Promise.resolve();
     }
     async resumeWorkflow(workflowID) {
         const client = await this.pool.connect();
@@ -563,17 +690,48 @@ class PostgresSystemDatabase {
             // Remove the workflow from the queues table so resume can safely be called on an ENQUEUED workflow
             await client.query(`DELETE FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue 
          WHERE workflow_uuid = $1`, [workflowID]);
+            await client.query(`DELETE FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_cancel 
+         WHERE workflow_id = $1`, [workflowID]);
             // Update status to pending and reset recovery attempts
             await this.recordWorkflowStatusChange(workflowID, workflow_1.StatusString.PENDING, { resetRecoveryAttempts: true }, client);
             await client.query('COMMIT');
         }
         catch (error) {
+            this.logger.error(error);
             await client.query('ROLLBACK');
             throw error;
         }
         finally {
             client.release();
         }
+        this.clearWFCancelMap(workflowID);
+    }
+    registerRunningWorkflow(workflowID, workflowPromise) {
+        // Need to await for the workflow and capture errors.
+        const awaitWorkflowPromise = workflowPromise
+            .catch((error) => {
+            this.logger.debug('Captured error in awaitWorkflowPromise: ' + error);
+        })
+            .finally(() => {
+            // Remove itself from pending workflow map.
+            this.runningWorkflowMap.delete(workflowID);
+            this.workflowCancellationMap.delete(workflowID);
+        });
+        this.runningWorkflowMap.set(workflowID, awaitWorkflowPromise);
+    }
+    async awaitRunningWorkflows() {
+        if (this.runningWorkflowMap.size > 0) {
+            this.logger.info('Waiting for pending workflows to finish.');
+            await Promise.allSettled(this.runningWorkflowMap.values());
+        }
+        if (this.workflowEventsMap.map.size > 0) {
+            this.logger.warn('Workflow events map is not empty - shutdown is not clean.');
+            //throw new Error('Workflow events map is not empty - shutdown is not clean.');
+        }
+        if (this.notificationsMap.map.size > 0) {
+            this.logger.warn('Message notification map is not empty - shutdown is not clean.');
+            //throw new Error('Message notification map is not empty - shutdown is not clean.');
+        }
     }
     async getWorkflowStatus(workflowID, callerID, callerFN) {
         const internalStatus = await this.getWorkflowStatusInternal(workflowID, callerID, callerFN);
@@ -610,8 +768,8 @@ class PostgresSystemDatabase {
                     queueName: rows[0].queue_name || undefined,
                     authenticatedUser: rows[0].authenticated_user,
                     assumedRole: rows[0].assumed_role,
-                    authenticatedRoles: utils_1.DBOSJSON.parse(rows[0].authenticated_roles),
-                    request: utils_1.DBOSJSON.parse(rows[0].request),
+                    authenticatedRoles: JSON.parse(rows[0].authenticated_roles),
+                    request: JSON.parse(rows[0].request),
                     executorId: rows[0].executor_id,
                     createdAt: Number(rows[0].created_at),
                     updatedAt: Number(rows[0].updated_at),
@@ -625,34 +783,79 @@ class PostgresSystemDatabase {
         }, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN);
         return sv ? JSON.parse(sv) : null;
     }
-    async awaitWorkflowResult(workflowID, timeoutms) {
-        const pollingIntervalMs = 1000;
-        const et = timeoutms !== undefined ? new Date().getTime() + timeoutms : undefined;
+    async awaitWorkflowResult(workflowID, timeoutSeconds, callerID, timerFuncID) {
+        const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
+        let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
         while (true) {
-            const { rows } = await this.pool.query(`SELECT status, output, error FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status WHERE workflow_uuid=$1`, [workflowID]);
-            if (rows.length > 0) {
-                const status = rows[0].status;
-                if (status === workflow_1.StatusString.SUCCESS) {
-                    return { res: rows[0].output };
+            let resolveNotification;
+            const statusPromise = new Promise((resolve) => {
+                resolveNotification = resolve;
+            });
+            const irh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
+                resolveNotification();
+            });
+            const crh = callerID
+                ? this.cancelWakeupMap.registerCallback(callerID, (_res) => {
+                    resolveNotification();
+                })
+                : undefined;
+            try {
+                if (callerID)
+                    await this.checkIfCanceled(callerID);
+                try {
+                    const { rows } = await this.pool.query(`SELECT status, output, error FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status WHERE workflow_uuid=$1`, [workflowID]);
+                    if (rows.length > 0) {
+                        const status = rows[0].status;
+                        if (status === workflow_1.StatusString.SUCCESS) {
+                            return { res: rows[0].output };
+                        }
+                        else if (status === workflow_1.StatusString.ERROR) {
+                            return { err: rows[0].error };
+                        }
+                        else if (status === workflow_1.StatusString.CANCELLED) {
+                            return { cancelled: true };
+                        }
+                        else {
+                            // Status is not actionable
+                        }
+                    }
                 }
-                else if (status === workflow_1.StatusString.ERROR) {
-                    return { err: rows[0].error };
+                catch (e) {
+                    const err = e;
+                    this.logger.error(`Exception from system database: ${err}`);
+                    throw err;
                 }
-            }
-            if (et !== undefined) {
-                const ct = new Date().getTime();
-                if (et > ct) {
-                    await (0, utils_1.sleepms)(Math.min(pollingIntervalMs, et - ct));
+                const ct = Date.now();
+                if (finishTime && ct > finishTime)
+                    return undefined; // Time's up
+                let timeoutPromise = Promise.resolve();
+                let timeoutCancel = () => { };
+                if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
+                    const { promise, cancel, endTime } = await this.durableSleepmsInternal(callerID, timerFuncID, timeoutms, this.dbPollingIntervalMs);
+                    finishTime = endTime;
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
                 }
                 else {
-                    break;
+                    let poll = finishTime ? finishTime - ct : this.dbPollingIntervalMs;
+                    poll = Math.min(this.dbPollingIntervalMs, poll);
+                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                }
+                try {
+                    await Promise.race([statusPromise, timeoutPromise]);
+                }
+                finally {
+                    timeoutCancel();
                 }
             }
-            else {
-                await (0, utils_1.sleepms)(pollingIntervalMs);
+            finally {
+                this.cancelWakeupMap.deregisterCallback(irh);
+                if (crh)
+                    this.cancelWakeupMap.deregisterCallback(crh);
             }
         }
-        return undefined;
     }
     /* BACKGROUND PROCESSES */
     /**
@@ -663,15 +866,29 @@ class PostgresSystemDatabase {
         this.notificationsClient = await this.pool.connect();
         await this.notificationsClient.query('LISTEN dbos_notifications_channel;');
         await this.notificationsClient.query('LISTEN dbos_workflow_events_channel;');
+        await this.notificationsClient.query('LISTEN dbos_workflow_cancel_channel;');
         const handler = (msg) => {
+            if (!this.shouldUseDBNotifications)
+                return; // Testing parameter
             if (msg.channel === 'dbos_notifications_channel') {
-                if (msg.payload && msg.payload in this.notificationsMap) {
-                    this.notificationsMap[msg.payload]();
+                if (msg.payload) {
+                    this.notificationsMap.callCallbacks(msg.payload);
                 }
             }
-            else {
-                if (msg.payload && msg.payload in this.workflowEventsMap) {
-                    this.workflowEventsMap[msg.payload]();
+            else if (msg.channel === 'dbos_workflow_events_channel') {
+                if (msg.payload) {
+                    this.workflowEventsMap.callCallbacks(msg.payload);
+                }
+            }
+            else if (msg.channel === 'dbos_workflow_cancel_channel') {
+                if (msg.payload) {
+                    const notif = JSON.parse(msg.payload);
+                    if (notif.cancelled === 't') {
+                        this.setWFCancelMap(notif.wfid);
+                    }
+                    else {
+                        this.clearWFCancelMap(notif.wfid);
+                    }
                 }
             }
         };
@@ -868,7 +1085,7 @@ class PostgresSystemDatabase {
     }
     async dequeueWorkflow(workflowId, queue) {
         if (queue.rateLimit) {
-            const time = new Date().getTime();
+            const time = Date.now();
             await this.pool.query(`
         UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue
         SET completed_at_epoch_ms = $2
@@ -883,7 +1100,7 @@ class PostgresSystemDatabase {
         }
     }
     async findAndMarkStartableWorkflows(queue, executorID, appVersion) {
-        const startTimeMs = new Date().getTime();
+        const startTimeMs = Date.now();
         const limiterPeriodMS = queue.rateLimit ? queue.rateLimit.periodSec * 1000 : 0;
         const claimedIDs = [];
         await this.knexDB.transaction(async (trx) => {