@dbos-inc/dbos-sdk 2.9.9-preview → 2.9.17-preview

package/dist/src/system_database.js

@@ -1,5 +1,4 @@
 "use strict";
-/* eslint-disable @typescript-eslint/no-explicit-any */
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -12,6 +11,7 @@ const workflow_1 = require("./workflow");
 const utils_1 = require("./utils");
 const knex_1 = __importDefault(require("knex"));
 const path_1 = __importDefault(require("path"));
+const crypto_1 = require("crypto");
 exports.DBOS_FUNCNAME_SEND = 'DBOS.send';
 exports.DBOS_FUNCNAME_RECV = 'DBOS.recv';
 exports.DBOS_FUNCNAME_SETEVENT = 'DBOS.setEvent';
@@ -40,6 +40,186 @@ async function migrateSystemDatabase(systemPoolConfig, logger) {
     }
 }
 exports.migrateSystemDatabase = migrateSystemDatabase;
+class NotificationMap {
+    map = new Map();
+    curCK = 0;
+    registerCallback(key, cb) {
+        if (!this.map.has(key)) {
+            this.map.set(key, new Map());
+        }
+        const ck = this.curCK++;
+        this.map.get(key).set(ck, cb);
+        return { key, ck };
+    }
+    deregisterCallback(k) {
+        if (!this.map.has(k.key))
+            return;
+        const sm = this.map.get(k.key);
+        if (!sm.has(k.ck))
+            return;
+        sm.delete(k.ck);
+        if (sm.size === 0) {
+            this.map.delete(k.key);
+        }
+    }
+    callCallbacks(key, event) {
+        if (!this.map.has(key))
+            return;
+        const sm = this.map.get(key);
+        for (const cb of sm.values()) {
+            cb(event);
+        }
+    }
+}
+async function insertWorkflowStatus(client, initStatus) {
+    const { rows } = await client.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status (
+      workflow_uuid,
+      status,
+      name,
+      class_name,
+      config_name,
+      queue_name,
+      authenticated_user,
+      assumed_role,
+      authenticated_roles,
+      request,
+      executor_id,
+      application_version,
+      application_id,
+      created_at,
+      recovery_attempts,
+      updated_at
+    ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
+     ON CONFLICT (workflow_uuid)
+      DO UPDATE SET
+        recovery_attempts = workflow_status.recovery_attempts + 1,
+        updated_at = EXCLUDED.updated_at,
+        executor_id = EXCLUDED.executor_id 
+      RETURNING recovery_attempts, status, name, class_name, config_name, queue_name`, [
+        initStatus.workflowUUID,
+        initStatus.status,
+        initStatus.workflowName,
+        initStatus.workflowClassName,
+        initStatus.workflowConfigName,
+        initStatus.queueName ?? null,
+        initStatus.authenticatedUser,
+        initStatus.assumedRole,
+        JSON.stringify(initStatus.authenticatedRoles),
+        JSON.stringify(initStatus.request),
+        initStatus.executorId,
+        initStatus.applicationVersion ?? null,
+        initStatus.applicationID,
+        initStatus.createdAt,
+        initStatus.status === workflow_1.StatusString.ENQUEUED ? 0 : 1,
+        initStatus.updatedAt ?? Date.now(),
+    ]);
+    if (rows.length === 0) {
+        throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
+    }
+    return rows[0];
+}
+async function insertWorkflowInputs(client, workflowID, serializedInputs) {
+    const { rows } = await client.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_inputs 
+     (workflow_uuid, inputs) VALUES($1, $2) 
+     ON CONFLICT (workflow_uuid) DO UPDATE SET workflow_uuid = excluded.workflow_uuid  
+     RETURNING inputs`, [workflowID, serializedInputs]);
+    if (rows.length === 0) {
+        throw new Error(`Attempt to insert workflow ${workflowID} inputs failed`);
+    }
+    return rows[0].inputs;
+}
+async function enqueueWorkflow(client, workflowID, queueName) {
+    await client.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue (workflow_uuid, queue_name) VALUES ($1, $2)
+     ON CONFLICT (workflow_uuid) DO NOTHING;`, [workflowID, queueName]);
+}
+async function deleteQueuedWorkflows(client, workflowID) {
+    await client.query(`DELETE FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue  WHERE workflow_uuid = $1`, [
+        workflowID,
+    ]);
+}
+async function getWorkflowStatusValue(client, workflowID) {
+    const { rows } = await client.query(`SELECT status FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status WHERE workflow_uuid=$1`, [workflowID]);
+    return rows.length === 0 ? undefined : rows[0].status;
+}
+async function updateWorkflowStatus(client, workflowID, status, options = {}) {
+    let setClause = `SET status=$2, updated_at=$3`;
+    let whereClause = `WHERE workflow_uuid=$1`;
+    const args = [workflowID, status, Date.now()];
+    const update = options.update ?? {};
+    if (update.output) {
+        const param = args.push(update.output);
+        setClause += `, output=$${param}`;
+    }
+    if (update.error) {
+        const param = args.push(update.error);
+        setClause += `, error=$${param}`;
+    }
+    if (update.resetRecoveryAttempts) {
+        setClause += `, recovery_attempts = 0`;
+    }
+    if (update.queueName) {
+        const param = args.push(update.queueName);
+        setClause += `, queue_name=$${param}`;
+    }
+    const where = options.where ?? {};
+    if (where.status) {
+        const param = args.push(where.status);
+        whereClause += ` AND status=$${param}`;
+    }
+    const result = await client.query(`UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status ${setClause} ${whereClause}`, args);
+    const throwOnFailure = options.throwOnFailure ?? true;
+    if (throwOnFailure && result.rowCount !== 1) {
+        throw new error_1.DBOSWorkflowConflictError(`Attempt to record transition of nonexistent workflow ${workflowID}`);
+    }
+}
+async function recordOperationResult(client, workflowID, functionID, functionName, checkConflict, options = {}) {
+    try {
+        await client.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs
+       (workflow_uuid, function_id, output, error, function_name, child_workflow_id)
+       VALUES ($1, $2, $3, $4, $5, $6)
+       ${checkConflict ? '' : ' ON CONFLICT DO NOTHING'};`, [
+            workflowID,
+            functionID,
+            options.output ?? null,
+            options.error ?? null,
+            functionName,
+            options.childWorkflowID ?? null,
+        ]);
+    }
+    catch (error) {
+        const err = error;
+        if (err.code === '40001' || err.code === '23505') {
+            // Serialization and primary key conflict (Postgres).
+            throw new error_1.DBOSWorkflowConflictError(workflowID);
+        }
+        else {
+            throw err;
+        }
+    }
+}
+function mapWorkflowStatus(row) {
+    return {
+        workflowUUID: row.workflow_uuid,
+        status: row.status,
+        workflowName: row.name,
+        output: row.output ? row.output : null,
+        error: row.error ? row.error : null,
+        workflowClassName: row.class_name ?? '',
+        workflowConfigName: row.config_name ?? '',
+        queueName: row.queue_name,
+        authenticatedUser: row.authenticated_user,
+        assumedRole: row.assumed_role,
+        authenticatedRoles: JSON.parse(row.authenticated_roles),
+        request: JSON.parse(row.request),
+        executorId: row.executor_id,
+        createdAt: Number(row.created_at),
+        updatedAt: Number(row.updated_at),
+        applicationVersion: row.application_version,
+        applicationID: row.application_id,
+        recoveryAttempts: Number(row.recovery_attempts),
+        input: row.inputs,
+    };
+}
 class PostgresSystemDatabase {
     pgPoolConfig;
     systemDatabaseName;
@@ -47,10 +227,35 @@ class PostgresSystemDatabase {
     sysDbPoolSize;
     pool;
     systemPoolConfig;
+    // TODO: remove Knex connection in favor of just using Pool
     knexDB;
+    /*
+     * Generally, notifications are asynchronous.  One should:
+     *  Subscribe to updates
+     *  Read the database item in question
+     *  In response to updates, re-read the database item
+     *  Unsubscribe at the end
+     * The notification mechanism is reliable in the sense that it will eventually deliver updates
+     *  or the DB connection will get dropped.  The right thing to do if you lose connectivity to
+     *  the system DB is to exit the process and go through recovery... system DB writes, notifications,
+     *  etc may not have completed correctly, and recovery is the way to rebuild in-memory state.
+     *
+     * NOTE:
+     * PG Notifications are not fully reliable.
+     *   Dropped connections are recoverable - you just need to restart and scan everything.
+     *      (The whole VM being the logical choice, so workflows can recover from any write failures.)
+     *   The real problem is, if the pipes out of the server are full... then notifications can be
+     *     dropped, and only the PG server log may note it.  For those reasons, we do occasional polling
+     */
     notificationsClient = null;
-    notificationsMap = {};
-    workflowEventsMap = {};
+    dbPollingIntervalResultMs = 1000;
+    dbPollingIntervalEventMs = 10000;
+    shouldUseDBNotifications = true;
+    notificationsMap = new NotificationMap();
+    workflowEventsMap = new NotificationMap();
+    cancelWakeupMap = new NotificationMap();
+    runningWorkflowMap = new Map(); // Map from workflowID to workflow promise
+    workflowCancellationMap = new Map(); // Map from workflowID to its cancellation status.
     constructor(pgPoolConfig, systemDatabaseName, logger, sysDbPoolSize) {
         this.pgPoolConfig = pgPoolConfig;
         this.systemDatabaseName = systemDatabaseName;
@@ -100,7 +305,9 @@ class PostgresSystemDatabase {
         finally {
             await pgSystemClient.end();
         }
-        await this.listenForNotifications();
+        if (this.shouldUseDBNotifications) {
+            await this.#listenForNotifications();
+        }
     }
     async destroy() {
         await this.knexDB.destroy();
@@ -123,258 +330,174 @@ class PostgresSystemDatabase {
         await pgSystemClient.query(`DROP DATABASE IF EXISTS ${dbosConfig.system_database};`);
         await pgSystemClient.end();
     }
-    async initWorkflowStatus(initStatus, args, maxRetries) {
-        const result = await this.pool.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status (
-        workflow_uuid,
-        status,
-        name,
-        class_name,
-        config_name,
-        queue_name,
-        authenticated_user,
-        assumed_role,
-        authenticated_roles,
-        request,
-        executor_id,
-        application_version,
-        application_id,
-        created_at,
-        recovery_attempts,
-        updated_at
-      ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
-       ON CONFLICT (workflow_uuid)
-        DO UPDATE SET
-          recovery_attempts = workflow_status.recovery_attempts + 1,
-          updated_at = EXCLUDED.updated_at,
-          executor_id = EXCLUDED.executor_id 
-        RETURNING recovery_attempts, status, name, class_name, config_name, queue_name`, [
-            initStatus.workflowUUID,
-            initStatus.status,
-            initStatus.workflowName,
-            initStatus.workflowClassName,
-            initStatus.workflowConfigName,
-            initStatus.queueName,
-            initStatus.authenticatedUser,
-            initStatus.assumedRole,
-            utils_1.DBOSJSON.stringify(initStatus.authenticatedRoles),
-            utils_1.DBOSJSON.stringify(initStatus.request),
-            initStatus.executorId,
-            initStatus.applicationVersion,
-            initStatus.applicationID,
-            initStatus.createdAt,
-            initStatus.status === workflow_1.StatusString.ENQUEUED ? 0 : 1,
-            Date.now(),
-        ]);
-        // Check the started workflow matches the expected name, class_name, config_name, and queue_name
-        // A mismatch indicates a workflow starting with the same UUID but different functions, which should not be allowed.
-        const resRow = result.rows[0];
-        initStatus.workflowConfigName = initStatus.workflowConfigName || '';
-        resRow.config_name = resRow.config_name || '';
-        resRow.queue_name = resRow.queue_name === null ? undefined : resRow.queue_name; // Convert null in SQL to undefined
-        let msg = '';
-        if (resRow.name !== initStatus.workflowName) {
-            msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
-        }
-        else if (resRow.class_name !== initStatus.workflowClassName) {
-            msg = `Workflow already exists with a different class name: ${resRow.class_name}, but the provided class name is: ${initStatus.workflowClassName}`;
-        }
-        else if (resRow.config_name !== initStatus.workflowConfigName) {
-            msg = `Workflow already exists with a different class configuration: ${resRow.config_name}, but the provided class configuration is: ${initStatus.workflowConfigName}`;
-        }
-        else if (resRow.queue_name !== initStatus.queueName) {
-            // This is a warning because a different queue name is not necessarily an error.
-            this.logger.warn(`Workflow (${initStatus.workflowUUID}) already exists in queue: ${resRow.queue_name}, but the provided queue name is: ${initStatus.queueName}. The queue is not updated. ${new Error().stack}`);
-        }
-        if (msg !== '') {
-            throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
-        }
-        // recovery_attempt means "attempts" (we kept the name for backward compatibility). It's default value is 1.
-        // Every time we init the status, we increment `recovery_attempts` by 1.
-        // Thus, when this number becomes equal to `maxRetries + 1`, we should mark the workflow as `RETRIES_EXCEEDED`.
-        const attempts = resRow.recovery_attempts;
-        if (maxRetries && attempts > maxRetries + 1) {
-            await this.pool.query(`UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status SET status=$1 WHERE workflow_uuid=$2 AND status=$3`, [workflow_1.StatusString.RETRIES_EXCEEDED, initStatus.workflowUUID, workflow_1.StatusString.PENDING]);
-            throw new error_1.DBOSDeadLetterQueueError(initStatus.workflowUUID, maxRetries);
-        }
-        this.logger.debug(`Workflow ${initStatus.workflowUUID} attempt number: ${attempts}.`);
-        const status = resRow.status;
-        const serializedInputs = utils_1.DBOSJSON.stringify(args);
-        const { rows } = await this.pool.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_inputs (workflow_uuid, inputs) VALUES($1, $2) ON CONFLICT (workflow_uuid) DO UPDATE SET workflow_uuid = excluded.workflow_uuid  RETURNING inputs`, [initStatus.workflowUUID, serializedInputs]);
-        if (serializedInputs !== rows[0].inputs) {
-            this.logger.warn(`Workflow inputs for ${initStatus.workflowUUID} changed since the first call! Use the original inputs.`);
-        }
-        return { args: utils_1.DBOSJSON.parse(rows[0].inputs), status };
-    }
-    async recordWorkflowStatusChange(workflowID, status, update, client) {
-        let rec = '';
-        if (update.resetRecoveryAttempts) {
-            rec = ' recovery_attempts = 0, ';
-        }
-        if (update.incrementRecoveryAttempts) {
-            rec = ' recovery_attempts = recovery_attempts + 1';
+    async initWorkflowStatus(initStatus, serializedInputs, maxRetries) {
+        const client = await this.pool.connect();
+        try {
+            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+            const resRow = await insertWorkflowStatus(client, initStatus);
+            if (resRow.name !== initStatus.workflowName) {
+                const msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
+                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+            }
+            else if (resRow.class_name !== initStatus.workflowClassName) {
+                const msg = `Workflow already exists with a different class name: ${resRow.class_name}, but the provided class name is: ${initStatus.workflowClassName}`;
+                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+            }
+            else if ((resRow.config_name || '') !== (initStatus.workflowConfigName || '')) {
+                const msg = `Workflow already exists with a different class configuration: ${resRow.config_name}, but the provided class configuration is: ${initStatus.workflowConfigName}`;
+                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+            }
+            else if ((resRow.queue_name ?? undefined) !== initStatus.queueName) {
+                // This is a warning because a different queue name is not necessarily an error.
+                this.logger.warn(`Workflow (${initStatus.workflowUUID}) already exists in queue: ${resRow.queue_name}, but the provided queue name is: ${initStatus.queueName}. The queue is not updated. ${new Error().stack}`);
+            }
+            // recovery_attempt means "attempts" (we kept the name for backward compatibility). It's default value is 1.
+            // Every time we init the status, we increment `recovery_attempts` by 1.
+            // Thus, when this number becomes equal to `maxRetries + 1`, we should mark the workflow as `RETRIES_EXCEEDED`.
+            const attempts = resRow.recovery_attempts;
+            if (maxRetries && attempts > maxRetries + 1) {
+                await updateWorkflowStatus(client, initStatus.workflowUUID, workflow_1.StatusString.RETRIES_EXCEEDED, {
+                    where: { status: workflow_1.StatusString.PENDING },
+                    throwOnFailure: false,
+                });
+                throw new error_1.DBOSDeadLetterQueueError(initStatus.workflowUUID, maxRetries);
+            }
+            this.logger.debug(`Workflow ${initStatus.workflowUUID} attempt number: ${attempts}.`);
+            const status = resRow.status;
+            const inputResult = await insertWorkflowInputs(client, initStatus.workflowUUID, serializedInputs);
+            if (serializedInputs !== inputResult) {
+                this.logger.warn(`Workflow inputs for ${initStatus.workflowUUID} changed since the first call! Use the original inputs.`);
+            }
+            return { serializedInputs: inputResult, status };
         }
-        const wRes = await (client ?? this.pool).query(`UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status
-       SET ${rec} status=$2, output=$3, error=$4, updated_at=$5 WHERE workflow_uuid=$1`, [workflowID, status, update.output, update.error, Date.now()]);
-        if (wRes.rowCount !== 1) {
-            throw new error_1.DBOSWorkflowConflictUUIDError(`Attempt to record transition of nonexistent workflow ${workflowID}`);
+        finally {
+            await client.query('COMMIT');
+            client.release();
         }
     }
     async recordWorkflowOutput(workflowID, status) {
-        await this.recordWorkflowStatusChange(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output });
+        const client = await this.pool.connect();
+        try {
+            await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.SUCCESS, { update: { output: status.output } });
+        }
+        finally {
+            client.release();
+        }
     }
     async recordWorkflowError(workflowID, status) {
-        await this.recordWorkflowStatusChange(workflowID, workflow_1.StatusString.ERROR, { error: status.error });
+        const client = await this.pool.connect();
+        try {
+            await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.ERROR, { update: { error: status.error } });
+        }
+        finally {
+            client.release();
+        }
     }
     async getPendingWorkflows(executorID, appVersion) {
-        const getWorkflows = await this.pool.query(`SELECT workflow_uuid, queue_name FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status WHERE status=$1 AND executor_id=$2 AND application_version=$3`, [workflow_1.StatusString.PENDING, executorID, appVersion]);
+        const getWorkflows = await this.pool.query(`SELECT workflow_uuid, queue_name 
+       FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status 
+       WHERE status=$1 AND executor_id=$2 AND application_version=$3`, [workflow_1.StatusString.PENDING, executorID, appVersion]);
         return getWorkflows.rows.map((i) => ({
             workflowUUID: i.workflow_uuid,
             queueName: i.queue_name,
         }));
     }
     async getWorkflowInputs(workflowID) {
-        const { rows } = await this.pool.query(`SELECT inputs FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_inputs WHERE workflow_uuid=$1`, [workflowID]);
+        const { rows } = await this.pool.query(`SELECT inputs FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_inputs 
+      WHERE workflow_uuid=$1`, [workflowID]);
         if (rows.length === 0) {
             return null;
         }
-        return utils_1.DBOSJSON.parse(rows[0].inputs);
+        return rows[0].inputs;
     }
-    async getOperationResult(workflowID, functionID, client) {
-        const { rows } = await (client ?? this.pool).query(`SELECT output, error, child_workflow_id, function_name
+    async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
+        await this.#checkIfCanceled(client, workflowID);
+        const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name
        FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs
       WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
         if (rows.length === 0) {
-            return {};
+            return undefined;
         }
         else {
             return {
-                res: {
-                    res: rows[0].output,
-                    err: rows[0].error,
-                    child: rows[0].child_workflow_id,
-                    functionName: rows[0].function_name,
-                },
+                output: rows[0].output,
+                error: rows[0].error,
+                childWorkflowID: rows[0].child_workflow_id,
+                functionName: rows[0].function_name,
             };
         }
     }
+    async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
+        const client = await this.pool.connect();
+        try {
+            return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
+        }
+        finally {
+            client.release();
+        }
+    }
     async getAllOperationResults(workflowID) {
         const { rows } = await this.pool.query(`SELECT * FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs WHERE workflow_uuid=$1`, [workflowID]);
         return rows;
     }
-    async recordOperationResult(workflowID, functionID, rec, checkConflict, client) {
+    async recordOperationResult(workflowID, functionID, functionName, checkConflict, options = {}) {
+        const client = await this.pool.connect();
         try {
-            await (client ?? this.pool).query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs
-          (workflow_uuid, function_id, output, error, function_name, child_workflow_id)
-          VALUES ($1, $2, $3, $4, $5, $6)
-          ${checkConflict ? '' : ' ON CONFLICT DO NOTHING'}
-          ;`, [
-                workflowID,
-                functionID,
-                rec.serialOutput ?? null,
-                rec.serialError ?? null,
-                rec.functionName,
-                rec.childWfId ?? null,
-            ]);
+            await recordOperationResult(client, workflowID, functionID, functionName, checkConflict, options);
         }
-        catch (error) {
-            const err = error;
-            if (err.code === '40001' || err.code === '23505') {
-                // Serialization and primary key conflict (Postgres).
-                throw new error_1.DBOSWorkflowConflictUUIDError(workflowID);
-            }
-            else {
-                throw err;
-            }
+        finally {
+            client.release();
         }
     }
     async getMaxFunctionID(workflowID) {
         const { rows } = await this.pool.query(`SELECT max(function_id) as max_function_id FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs WHERE workflow_uuid=$1`, [workflowID]);
         return rows.length === 0 ? 0 : rows[0].max_function_id;
     }
-    async forkWorkflow(originalWorkflowID, forkedWorkflowId, startStep = 0) {
-        const workflowStatus = await this.getWorkflowStatus(originalWorkflowID);
+    async forkWorkflow(workflowID, startStep, options = {}) {
+        const newWorkflowID = options.newWorkflowID ?? (0, crypto_1.randomUUID)();
+        const workflowStatus = await this.getWorkflowStatus(workflowID);
         if (workflowStatus === null) {
-            throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${originalWorkflowID} does not exist`);
+            throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
+        }
+        if (!workflowStatus.input) {
+            throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} has no input`);
         }
         const client = await this.pool.connect();
         try {
-            await client.query('BEGIN');
-            const query = `
-        INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status (
-        workflow_uuid,
-        status,
-        name,
-        class_name,
-        config_name,
-        queue_name,
-        authenticated_user,
-        assumed_role,
-        authenticated_roles,
-        request,
-        output,
-        executor_id,
-        application_version,
-        application_id,
-        created_at,
-        recovery_attempts,
-        updated_at
-      ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
-        `;
-            await client.query(query, [
-                forkedWorkflowId,
-                workflow_1.StatusString.ENQUEUED,
-                workflowStatus.workflowName,
-                workflowStatus.workflowClassName,
-                workflowStatus.workflowConfigName,
-                utils_1.INTERNAL_QUEUE_NAME,
-                workflowStatus.authenticatedUser,
-                workflowStatus.assumedRole,
-                utils_1.DBOSJSON.stringify(workflowStatus.authenticatedRoles),
-                utils_1.DBOSJSON.stringify(workflowStatus.request),
-                null,
-                null,
-                workflowStatus.applicationVersion,
-                workflowStatus.applicationID,
-                Date.now(),
-                0,
-                Date.now(),
-            ]);
-            // Copy the inputs to the new workflow
-            const inputQuery = `INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_inputs (workflow_uuid, inputs)
-                            SELECT $2, inputs
-                            FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_inputs
-                            WHERE workflow_uuid = $1;`;
-            await client.query(inputQuery, [originalWorkflowID, forkedWorkflowId]);
+            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+            const now = Date.now();
+            await insertWorkflowStatus(client, {
+                workflowUUID: newWorkflowID,
+                status: workflow_1.StatusString.ENQUEUED,
+                workflowName: workflowStatus.workflowName,
+                workflowClassName: workflowStatus.workflowClassName,
+                workflowConfigName: workflowStatus.workflowConfigName,
+                queueName: utils_1.INTERNAL_QUEUE_NAME,
+                authenticatedUser: workflowStatus.authenticatedUser,
+                assumedRole: workflowStatus.assumedRole,
+                authenticatedRoles: workflowStatus.authenticatedRoles,
+                output: null,
+                error: null,
+                request: workflowStatus.request,
+                executorId: utils_1.globalParams.executorID,
+                applicationVersion: options.applicationVersion ?? workflowStatus.applicationVersion,
+                applicationID: workflowStatus.applicationID,
+                createdAt: now,
+                recoveryAttempts: 0,
+                updatedAt: now,
+            });
+            await insertWorkflowInputs(client, newWorkflowID, workflowStatus.input);
             if (startStep > 0) {
-                const query = `
-          INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs (
-          workflow_uuid,
-          function_id,
-          output,
-          error,
-          function_name,
-          child_workflow_id
-        )
-        SELECT
-          $1 AS workflow_uuid,
-          function_id,
-          output,
-          error,
-          function_name,
-          child_workflow_id
+                const query = `INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs 
+          (workflow_uuid, function_id, output, error, function_name, child_workflow_id )
+          SELECT $1 AS workflow_uuid, function_id, output, error, function_name, child_workflow_id
           FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.operation_outputs
-          WHERE workflow_uuid = $2 AND function_id < $3
-        `;
-                await client.query(query, [forkedWorkflowId, originalWorkflowID, startStep]);
+          WHERE workflow_uuid = $2 AND function_id < $3`;
+                await client.query(query, [newWorkflowID, workflowID, startStep]);
             }
-            await client.query(`
-      INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue (workflow_uuid, queue_name)
-      VALUES ($1, $2)
-      ON CONFLICT (workflow_uuid)
-      DO NOTHING;
-      `, [forkedWorkflowId, utils_1.INTERNAL_QUEUE_NAME]);
+            await enqueueWorkflow(client, newWorkflowID, utils_1.INTERNAL_QUEUE_NAME);
             await client.query('COMMIT');
-            return forkedWorkflowId;
+            return newWorkflowID;
         }
         catch (error) {
             await client.query('ROLLBACK');
@@ -384,37 +507,67 @@ class PostgresSystemDatabase {
             client.release();
         }
     }
-    async runAsStep(callback, functionName, workflowID, functionID, client) {
-        if (workflowID !== undefined && functionID !== undefined) {
-            const res = await this.getOperationResult(workflowID, functionID, client);
-            if (res.res !== undefined) {
-                if (res.res.functionName !== functionName) {
-                    throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, functionName, res.res.functionName);
-                }
-                await client?.query('ROLLBACK');
-                return res.res.res;
+    async #runAndRecordResult(client, functionName, workflowID, functionID, func) {
+        const result = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
+        if (result !== undefined) {
+            if (result.functionName !== functionName) {
+                throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, functionName, result.functionName);
             }
+            return result.output;
         }
-        const serialOutput = await callback();
-        if (workflowID !== undefined && functionID !== undefined) {
-            await this.recordOperationResult(workflowID, functionID, { serialOutput, functionName }, true, client);
-        }
-        return serialOutput;
+        const output = await func();
+        await recordOperationResult(client, workflowID, functionID, functionName, true, { output });
+        return output;
     }
     async durableSleepms(workflowID, functionID, durationMS) {
+        let resolveNotification;
+        const cancelPromise = new Promise((resolve) => {
+            resolveNotification = resolve;
+        });
+        const cbr = this.cancelWakeupMap.registerCallback(workflowID, resolveNotification);
+        try {
+            let timeoutPromise = Promise.resolve();
+            const { promise, cancel: timeoutCancel } = await this.#durableSleep(workflowID, functionID, durationMS);
+            timeoutPromise = promise;
+            try {
+                await Promise.race([cancelPromise, timeoutPromise]);
+            }
+            finally {
+                timeoutCancel();
+            }
+        }
+        finally {
+            this.cancelWakeupMap.deregisterCallback(cbr);
+        }
+        await this.checkIfCanceled(workflowID);
+    }
+    async #durableSleep(workflowID, functionID, durationMS, maxSleepPerIteration) {
+        if (maxSleepPerIteration === undefined)
+            maxSleepPerIteration = durationMS;
         const curTime = Date.now();
         let endTimeMs = curTime + durationMS;
-        const res = await this.getOperationResult(workflowID, functionID);
-        if (res.res !== undefined) {
-            if (res.res.functionName !== exports.DBOS_FUNCNAME_SLEEP) {
-                throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, res.res.functionName);
+        const client = await this.pool.connect();
+        try {
+            const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
+            if (res) {
+                if (res.functionName !== exports.DBOS_FUNCNAME_SLEEP) {
+                    throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, res.functionName);
+                }
+                endTimeMs = JSON.parse(res.output);
+            }
+            else {
+                await recordOperationResult(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, {
+                    output: JSON.stringify(endTimeMs),
+                });
             }
-            endTimeMs = JSON.parse(res.res.res);
+            return {
+                ...(0, utils_1.cancellableSleep)(Math.max(Math.min(maxSleepPerIteration, endTimeMs - curTime), 0)),
+                endTime: endTimeMs,
+            };
         }
-        else {
-            await this.recordOperationResult(workflowID, functionID, { serialOutput: JSON.stringify(endTimeMs), functionName: exports.DBOS_FUNCNAME_SLEEP }, false);
+        finally {
+            client.release();
         }
-        return (0, utils_1.cancellableSleep)(Math.max(endTimeMs - curTime, 0));
     }
     nullTopic = '__null__topic__';
     async send(workflowID, functionID, destinationID, message, topic) {
@@ -422,11 +575,11 @@ class PostgresSystemDatabase {
         const client = await this.pool.connect();
         await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
         try {
-            await this.runAsStep(async () => {
+            await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
                 await client.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.notifications (destination_uuid, topic, message) VALUES ($1, $2, $3);`, [destinationID, topic, message]);
                 await client.query('COMMIT');
                 return undefined;
-            }, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, client);
+            });
         }
         catch (error) {
             await client.query('ROLLBACK');
@@ -446,44 +599,63 @@ class PostgresSystemDatabase {
     async recv(workflowID, functionID, timeoutFunctionID, topic, timeoutSeconds = dbos_executor_1.DBOSExecutor.defaultNotificationTimeoutSec) {
         topic = topic ?? this.nullTopic;
         // First, check for previous executions.
-        const res = await this.getOperationResult(workflowID, functionID);
-        if (res.res) {
-            if (res.res.functionName !== exports.DBOS_FUNCNAME_RECV) {
-                throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.res.functionName);
+        const res = await this.getOperationResultAndThrowIfCancelled(workflowID, functionID);
+        if (res) {
+            if (res.functionName !== exports.DBOS_FUNCNAME_RECV) {
+                throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.functionName);
             }
-            return res.res.res;
+            return res.output;
         }
-        // Check if the key is already in the DB, then wait for the notification if it isn't.
-        const initRecvRows = (await this.pool.query(`SELECT topic FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.notifications WHERE destination_uuid=$1 AND topic=$2;`, [workflowID, topic])).rows;
-        if (initRecvRows.length === 0) {
-            // Then, register the key with the global notifications listener.
+        const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
+        let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
+        while (true) {
+            // register the key with the global notifications listener.
             let resolveNotification;
             const messagePromise = new Promise((resolve) => {
                 resolveNotification = resolve;
             });
             const payload = `${workflowID}::${topic}`;
-            this.notificationsMap[payload] = resolveNotification; // The resolver assignment in the Promise definition runs synchronously.
-            let timeoutPromise = Promise.resolve();
-            let timeoutCancel = () => { };
-            try {
-                const { promise, cancel } = await this.durableSleepms(workflowID, timeoutFunctionID, timeoutSeconds * 1000);
-                timeoutPromise = promise;
-                timeoutCancel = cancel;
-            }
-            catch (e) {
-                this.logger.error(e);
-                delete this.notificationsMap[payload];
-                timeoutCancel();
-                throw new Error('durable sleepms failed');
-            }
+            const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
+            const crh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
+                resolveNotification();
+            });
             try {
-                await Promise.race([messagePromise, timeoutPromise]);
+                await this.checkIfCanceled(workflowID);
+                // Check if the key is already in the DB, then wait for the notification if it isn't.
+                const initRecvRows = (await this.pool.query(`SELECT topic FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.notifications WHERE destination_uuid=$1 AND topic=$2;`, [workflowID, topic])).rows;
+                if (initRecvRows.length !== 0)
+                    break;
+                const ct = Date.now();
+                if (finishTime && ct > finishTime)
+                    break; // Time's up
+                let timeoutPromise = Promise.resolve();
+                let timeoutCancel = () => { };
+                if (timeoutms) {
+                    const { promise, cancel, endTime } = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms, this.dbPollingIntervalEventMs);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                    finishTime = endTime;
+                }
+                else {
+                    let poll = finishTime ? finishTime - ct : this.dbPollingIntervalEventMs;
+                    poll = Math.min(this.dbPollingIntervalEventMs, poll);
+                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                }
+                try {
+                    await Promise.race([messagePromise, timeoutPromise]);
+                }
+                finally {
+                    timeoutCancel();
+                }
             }
             finally {
-                timeoutCancel();
-                delete this.notificationsMap[payload];
+                this.notificationsMap.deregisterCallback(cbr);
+                this.cancelWakeupMap.deregisterCallback(crh);
             }
         }
+        await this.checkIfCanceled(workflowID);
         // Transactionally consume and return the message if it's in the DB, otherwise return null.
         let message = null;
         const client = await this.pool.connect();
@@ -507,7 +679,7 @@ class PostgresSystemDatabase {
             if (finalRecvRows.length > 0) {
                 message = finalRecvRows[0].message;
             }
-            await this.recordOperationResult(workflowID, functionID, { serialOutput: message, functionName: exports.DBOS_FUNCNAME_RECV }, true, client);
+            await recordOperationResult(client, workflowID, functionID, exports.DBOS_FUNCNAME_RECV, true, { output: message });
             await client.query(`COMMIT`);
         }
         catch (e) {
@@ -524,7 +696,7 @@ class PostgresSystemDatabase {
         const client = await this.pool.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            await this.runAsStep(async () => {
+            await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
                 await client.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_events (workflow_uuid, key, value)
              VALUES ($1, $2, $3)
              ON CONFLICT (workflow_uuid, key)
@@ -532,7 +704,7 @@ class PostgresSystemDatabase {
              RETURNING workflow_uuid;`, [workflowID, key, message]);
                 await client.query('COMMIT');
                 return undefined;
-            }, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, client);
+            });
         }
         catch (e) {
             this.logger.error(e);
@@ -546,51 +718,59 @@ class PostgresSystemDatabase {
     async getEvent(workflowID, key, timeoutSeconds, callerWorkflow) {
         // Check if the operation has been done before for OAOO (only do this inside a workflow).
         if (callerWorkflow) {
-            const res = await this.getOperationResult(callerWorkflow.workflowID, callerWorkflow.functionID);
-            if (res.res !== undefined) {
-                if (res.res.functionName !== exports.DBOS_FUNCNAME_GETEVENT) {
-                    throw new error_1.DBOSUnexpectedStepError(callerWorkflow.workflowID, callerWorkflow.functionID, exports.DBOS_FUNCNAME_GETEVENT, res.res.functionName);
+            const res = await this.getOperationResultAndThrowIfCancelled(callerWorkflow.workflowID, callerWorkflow.functionID);
+            if (res) {
+                if (res.functionName !== exports.DBOS_FUNCNAME_GETEVENT) {
+                    throw new error_1.DBOSUnexpectedStepError(callerWorkflow.workflowID, callerWorkflow.functionID, exports.DBOS_FUNCNAME_GETEVENT, res.functionName);
                 }
-                return res.res.res;
+                return res.output;
             }
         }
         // Get the return the value. if it's in the DB, otherwise return null.
         let value = null;
         const payloadKey = `${workflowID}::${key}`;
+        const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
+        let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
         // Register the key with the global notifications listener first... we do not want to look in the DB first
         //  or that would cause a timing hole.
-        let resolveNotification;
-        const valuePromise = new Promise((resolve) => {
-            resolveNotification = resolve;
-        });
-        this.workflowEventsMap[payloadKey] = resolveNotification; // The resolver assignment in the Promise definition runs synchronously.
-        try {
-            // Check if the key is already in the DB, then wait for the notification if it isn't.
-            const initRecvRows = (await this.pool.query(`
-        SELECT key, value
-        FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_events
-        WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
-            if (initRecvRows.length > 0) {
-                value = initRecvRows[0].value;
-            }
-            else {
+        while (true) {
+            let resolveNotification;
+            const valuePromise = new Promise((resolve) => {
+                resolveNotification = resolve;
+            });
+            const cbr = this.workflowEventsMap.registerCallback(payloadKey, resolveNotification);
+            const crh = callerWorkflow?.workflowID
+                ? this.cancelWakeupMap.registerCallback(callerWorkflow.workflowID, (_res) => {
+                    resolveNotification();
+                })
+                : undefined;
+            try {
+                if (callerWorkflow?.workflowID)
+                    await this.checkIfCanceled(callerWorkflow?.workflowID);
+                // Check if the key is already in the DB, then wait for the notification if it isn't.
+                const initRecvRows = (await this.pool.query(`SELECT key, value
+             FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_events
+             WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
+                if (initRecvRows.length > 0) {
+                    value = initRecvRows[0].value;
+                    break;
+                }
+                const ct = Date.now();
+                if (finishTime && ct > finishTime)
+                    break; // Time's up
                 // If we have a callerWorkflow, we want a durable sleep, otherwise, not
                 let timeoutPromise = Promise.resolve();
                 let timeoutCancel = () => { };
-                if (callerWorkflow) {
-                    try {
-                        const { promise, cancel } = await this.durableSleepms(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutSeconds * 1000);
-                        timeoutPromise = promise;
-                        timeoutCancel = cancel;
-                    }
-                    catch (e) {
-                        this.logger.error(e);
-                        delete this.workflowEventsMap[payloadKey];
-                        throw new Error('durable sleepms failed');
-                    }
+                if (callerWorkflow && timeoutms) {
+                    const { promise, cancel, endTime } = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms, this.dbPollingIntervalEventMs);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                    finishTime = endTime;
                 }
                 else {
-                    const { promise, cancel } = (0, utils_1.cancellableSleep)(timeoutSeconds * 1000);
+                    let poll = finishTime ? finishTime - ct : this.dbPollingIntervalEventMs;
+                    poll = Math.min(this.dbPollingIntervalEventMs, poll);
+                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
                     timeoutPromise = promise;
                     timeoutCancel = cancel;
                 }
@@ -600,78 +780,112 @@ class PostgresSystemDatabase {
                 finally {
                     timeoutCancel();
                 }
-                const finalRecvRows = (await this.pool.query(`
-            SELECT value
-            FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_events
-            WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
-                if (finalRecvRows.length > 0) {
-                    value = finalRecvRows[0].value;
-                }
             }
-        }
-        finally {
-            delete this.workflowEventsMap[payloadKey];
+            finally {
+                this.workflowEventsMap.deregisterCallback(cbr);
+                if (crh)
+                    this.cancelWakeupMap.deregisterCallback(crh);
+            }
         }
         // Record the output if it is inside a workflow.
         if (callerWorkflow) {
-            await this.recordOperationResult(callerWorkflow.workflowID, callerWorkflow.functionID, {
-                serialOutput: value,
-                functionName: exports.DBOS_FUNCNAME_GETEVENT,
-            }, true);
+            await this.recordOperationResult(callerWorkflow.workflowID, callerWorkflow.functionID, exports.DBOS_FUNCNAME_GETEVENT, true, { output: value });
         }
         return value;
     }
     async setWorkflowStatus(workflowID, status, resetRecoveryAttempts) {
-        await this.recordWorkflowStatusChange(workflowID, status, { resetRecoveryAttempts });
+        const client = await this.pool.connect();
+        try {
+            await updateWorkflowStatus(client, workflowID, status, { update: { resetRecoveryAttempts } });
+        }
+        finally {
+            client.release();
+        }
+    }
+    #setWFCancelMap(workflowID) {
+        if (this.runningWorkflowMap.has(workflowID)) {
+            this.workflowCancellationMap.set(workflowID, true);
+        }
+        this.cancelWakeupMap.callCallbacks(workflowID);
     }
+    #clearWFCancelMap(workflowID) {
+        if (this.workflowCancellationMap.has(workflowID)) {
+            this.workflowCancellationMap.delete(workflowID);
+        }
+    }
+    // TODO: make cancel throw an error if the workflow doesn't exist.
     async cancelWorkflow(workflowID) {
         const client = await this.pool.connect();
         try {
-            await client.query('BEGIN');
+            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Remove workflow from queues table
-            await client.query(`DELETE FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue 
-         WHERE workflow_uuid = $1`, [workflowID]);
-            // Should we check if it is incomplete first?
-            await this.recordWorkflowStatusChange(workflowID, workflow_1.StatusString.CANCELLED, {}, client);
+            await deleteQueuedWorkflows(client, workflowID);
+            const statusResult = await getWorkflowStatusValue(client, workflowID);
+            if (!statusResult || statusResult === workflow_1.StatusString.SUCCESS || statusResult === workflow_1.StatusString.ERROR) {
+                await client.query('COMMIT');
+                return;
+            }
+            await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.CANCELLED);
             await client.query('COMMIT');
         }
         catch (error) {
+            this.logger.error(error);
             await client.query('ROLLBACK');
             throw error;
         }
         finally {
             client.release();
         }
+        this.#setWFCancelMap(workflowID);
+    }
+    async #checkIfCanceled(client, workflowID) {
+        if (this.workflowCancellationMap.get(workflowID) === true) {
+            throw new error_1.DBOSWorkflowCancelledError(workflowID);
+        }
+        const statusValue = await getWorkflowStatusValue(client, workflowID);
+        if (statusValue === workflow_1.StatusString.CANCELLED) {
+            throw new error_1.DBOSWorkflowCancelledError(workflowID);
+        }
+    }
+    async checkIfCanceled(workflowID) {
+        const client = await this.pool.connect();
+        try {
+            await this.#checkIfCanceled(client, workflowID);
+        }
+        finally {
+            client.release();
+        }
     }
     async resumeWorkflow(workflowID) {
+        this.#clearWFCancelMap(workflowID);
         const client = await this.pool.connect();
         try {
-            await client.query('BEGIN');
-            await client.query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ');
+            await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ');
             // Check workflow status. If it is complete, do nothing.
-            const statusResult = await client.query(`SELECT status FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status 
-         WHERE workflow_uuid = $1`, [workflowID]);
-            if (statusResult.rows.length === 0 ||
-                statusResult.rows[0].status === workflow_1.StatusString.SUCCESS ||
-                statusResult.rows[0].status === workflow_1.StatusString.ERROR) {
-                await client.query('COMMIT');
+            const statusResult = await getWorkflowStatusValue(client, workflowID);
+            if (!statusResult || statusResult === workflow_1.StatusString.SUCCESS || statusResult === workflow_1.StatusString.ERROR) {
+                await client.query('ROLLBACK');
+                if (!statusResult) {
+                    if (statusResult === undefined) {
+                        throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
+                    }
+                }
                 return;
             }
             // Remove the workflow from the queues table so resume can safely be called on an ENQUEUED workflow
-            await client.query(`DELETE FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue 
-         WHERE workflow_uuid = $1`, [workflowID]);
-            await client.query(`UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status 
-         SET status = $1, queue_name= $2, recovery_attempts = 0 
-         WHERE workflow_uuid = $3`, [workflow_1.StatusString.ENQUEUED, utils_1.INTERNAL_QUEUE_NAME, workflowID]);
-            await client.query(`
-        INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue (workflow_uuid, queue_name)
-        VALUES ($1, $2)
-        ON CONFLICT (workflow_uuid)
-        DO NOTHING;
-      `, [workflowID, utils_1.INTERNAL_QUEUE_NAME]);
+            await deleteQueuedWorkflows(client, workflowID);
+            await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.ENQUEUED, {
+                update: {
+                    queueName: utils_1.INTERNAL_QUEUE_NAME,
+                    resetRecoveryAttempts: true,
+                },
+                throwOnFailure: false,
+            });
+            await enqueueWorkflow(client, workflowID, utils_1.INTERNAL_QUEUE_NAME);
             await client.query('COMMIT');
         }
         catch (error) {
+            this.logger.error(error);
             await client.query('ROLLBACK');
             throw error;
         }
@@ -679,76 +893,166 @@ class PostgresSystemDatabase {
             client.release();
         }
     }
+    registerRunningWorkflow(workflowID, workflowPromise) {
+        // Need to await for the workflow and capture errors.
+        const awaitWorkflowPromise = workflowPromise
+            .catch((error) => {
+            this.logger.debug('Captured error in awaitWorkflowPromise: ' + error);
+        })
+            .finally(() => {
+            // Remove itself from pending workflow map.
+            this.runningWorkflowMap.delete(workflowID);
+            this.workflowCancellationMap.delete(workflowID);
+        });
+        this.runningWorkflowMap.set(workflowID, awaitWorkflowPromise);
+    }
+    async awaitRunningWorkflows() {
+        if (this.runningWorkflowMap.size > 0) {
+            this.logger.info('Waiting for pending workflows to finish.');
+            await Promise.allSettled(this.runningWorkflowMap.values());
+        }
+        if (this.workflowEventsMap.map.size > 0) {
+            this.logger.warn('Workflow events map is not empty - shutdown is not clean.');
+            //throw new Error('Workflow events map is not empty - shutdown is not clean.');
+        }
+        if (this.notificationsMap.map.size > 0) {
+            this.logger.warn('Message notification map is not empty - shutdown is not clean.');
+            //throw new Error('Message notification map is not empty - shutdown is not clean.');
+        }
+    }
     async getWorkflowStatus(workflowID, callerID, callerFN) {
-        // Check if the operation has been done before for OAOO (only do this inside a workflow).
-        const sv = await this.runAsStep(async () => {
+        const funcGetStatus = async () => {
             const statuses = await this.listWorkflows({ workflowIDs: [workflowID] });
             const status = statuses.find((s) => s.workflowUUID === workflowID);
             return status ? JSON.stringify(status) : null;
-        }, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN);
-        return sv ? JSON.parse(sv) : null;
+        };
+        if (callerID && callerFN) {
+            const client = await this.pool.connect();
+            try {
+                // Check if the operation has been done before for OAOO (only do this inside a workflow).
+                const json = await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, async () => {
+                    const statuses = await this.listWorkflows({ workflowIDs: [workflowID] });
+                    const status = statuses.find((s) => s.workflowUUID === workflowID);
+                    return status ? JSON.stringify(status) : null;
+                });
+                return parseStatus(json);
+            }
+            finally {
+                client.release();
+            }
+        }
+        else {
+            const json = await funcGetStatus();
+            return parseStatus(json);
+        }
+        function parseStatus(json) {
+            return json ? JSON.parse(json) : null;
+        }
     }
-    async awaitWorkflowResult(workflowID, timeoutms) {
-        const pollingIntervalMs = 1000;
-        const et = timeoutms !== undefined ? new Date().getTime() + timeoutms : undefined;
+    async awaitWorkflowResult(workflowID, timeoutSeconds, callerID, timerFuncID) {
+        const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
+        let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
         while (true) {
-            const { rows } = await this.pool.query(`SELECT status, output, error FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status WHERE workflow_uuid=$1`, [workflowID]);
-            if (rows.length > 0) {
-                const status = rows[0].status;
-                if (status === workflow_1.StatusString.SUCCESS) {
-                    return { res: rows[0].output };
+            let resolveNotification;
+            const statusPromise = new Promise((resolve) => {
+                resolveNotification = resolve;
+            });
+            const irh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
+                resolveNotification();
+            });
+            const crh = callerID
+                ? this.cancelWakeupMap.registerCallback(callerID, (_res) => {
+                    resolveNotification();
+                })
+                : undefined;
+            try {
+                if (callerID)
+                    await this.checkIfCanceled(callerID);
+                try {
+                    const { rows } = await this.pool.query(`SELECT status, output, error FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_status 
+             WHERE workflow_uuid=$1`, [workflowID]);
+                    if (rows.length > 0) {
+                        const status = rows[0].status;
+                        if (status === workflow_1.StatusString.SUCCESS) {
+                            return { output: rows[0].output };
+                        }
+                        else if (status === workflow_1.StatusString.ERROR) {
+                            return { error: rows[0].error };
+                        }
+                        else if (status === workflow_1.StatusString.CANCELLED) {
+                            return { cancelled: true };
+                        }
+                        else {
+                            // Status is not actionable
+                        }
+                    }
                 }
-                else if (status === workflow_1.StatusString.ERROR) {
-                    return { err: rows[0].error };
+                catch (e) {
+                    const err = e;
+                    this.logger.error(`Exception from system database: ${err}`);
+                    throw err;
                 }
-            }
-            if (et !== undefined) {
-                const ct = new Date().getTime();
-                if (et > ct) {
-                    await (0, utils_1.sleepms)(Math.min(pollingIntervalMs, et - ct));
+                const ct = Date.now();
+                if (finishTime && ct > finishTime)
+                    return undefined; // Time's up
+                let timeoutPromise = Promise.resolve();
+                let timeoutCancel = () => { };
+                if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
+                    const { promise, cancel, endTime } = await this.#durableSleep(callerID, timerFuncID, timeoutms, this.dbPollingIntervalResultMs);
+                    finishTime = endTime;
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
                 }
                 else {
-                    break;
+                    let poll = finishTime ? finishTime - ct : this.dbPollingIntervalResultMs;
+                    poll = Math.min(this.dbPollingIntervalResultMs, poll);
+                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
+                    timeoutPromise = promise;
+                    timeoutCancel = cancel;
+                }
+                try {
+                    await Promise.race([statusPromise, timeoutPromise]);
+                }
+                finally {
+                    timeoutCancel();
                 }
             }
-            else {
-                await (0, utils_1.sleepms)(pollingIntervalMs);
+            finally {
+                this.cancelWakeupMap.deregisterCallback(irh);
+                if (crh)
+                    this.cancelWakeupMap.deregisterCallback(crh);
             }
         }
-        return undefined;
     }
     /* BACKGROUND PROCESSES */
     /**
      * A background process that listens for notifications from Postgres then signals the appropriate
      * workflow listener by resolving its promise.
      */
-    async listenForNotifications() {
+    async #listenForNotifications() {
         this.notificationsClient = await this.pool.connect();
         await this.notificationsClient.query('LISTEN dbos_notifications_channel;');
         await this.notificationsClient.query('LISTEN dbos_workflow_events_channel;');
         const handler = (msg) => {
+            if (!this.shouldUseDBNotifications)
+                return; // Testing parameter
             if (msg.channel === 'dbos_notifications_channel') {
-                if (msg.payload && msg.payload in this.notificationsMap) {
-                    this.notificationsMap[msg.payload]();
+                if (msg.payload) {
+                    this.notificationsMap.callCallbacks(msg.payload);
                 }
             }
-            else {
-                if (msg.payload && msg.payload in this.workflowEventsMap) {
-                    this.workflowEventsMap[msg.payload]();
+            else if (msg.channel === 'dbos_workflow_events_channel') {
+                if (msg.payload) {
+                    this.workflowEventsMap.callCallbacks(msg.payload);
                 }
             }
         };
         this.notificationsClient.on('notification', handler);
     }
     // Event dispatcher queries / updates
-    async getEventDispatchState(svc, wfn, key) {
-        const res = await this.pool.query(`
-      SELECT *
-      FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.event_dispatch_kv
-      WHERE workflow_fn_name = $1
-      AND service_name = $2
-      AND key = $3;
-    `, [wfn, svc, key]);
+    async getEventDispatchState(service, workflowName, key) {
+        const res = await this.pool.query(`SELECT * FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.event_dispatch_kv
+       WHERE workflow_fn_name = $1 AND service_name = $2 AND key = $3;`, [workflowName, service, key]);
         if (res.rows.length === 0)
             return undefined;
         return {
@@ -763,19 +1067,18 @@ class PostgresSystemDatabase {
         };
     }
     async upsertEventDispatchState(state) {
-        const res = await this.pool.query(`
-      INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.event_dispatch_kv (
+        const res = await this.pool.query(`INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.event_dispatch_kv (
         service_name, workflow_fn_name, key, value, update_time, update_seq)
-      VALUES ($1, $2, $3, $4, $5, $6)
-      ON CONFLICT (service_name, workflow_fn_name, key)
-      DO UPDATE SET
-        update_time = GREATEST(EXCLUDED.update_time, event_dispatch_kv.update_time),
-        update_seq =  GREATEST(EXCLUDED.update_seq,  event_dispatch_kv.update_seq),
-        value = CASE WHEN (EXCLUDED.update_time > event_dispatch_kv.update_time OR EXCLUDED.update_seq > event_dispatch_kv.update_seq OR
-                            (event_dispatch_kv.update_time IS NULL and event_dispatch_kv.update_seq IS NULL))
-          THEN EXCLUDED.value ELSE event_dispatch_kv.value END
-      RETURNING value, update_time, update_seq;
-    `, [state.service, state.workflowFnName, state.key, state.value, state.updateTime, state.updateSeq]);
+       VALUES ($1, $2, $3, $4, $5, $6)
+       ON CONFLICT (service_name, workflow_fn_name, key)
+       DO UPDATE SET
+         update_time = GREATEST(EXCLUDED.update_time, event_dispatch_kv.update_time),
+         update_seq =  GREATEST(EXCLUDED.update_seq,  event_dispatch_kv.update_seq),
+         value = CASE WHEN (EXCLUDED.update_time > event_dispatch_kv.update_time 
+            OR EXCLUDED.update_seq > event_dispatch_kv.update_seq 
+            OR (event_dispatch_kv.update_time IS NULL and event_dispatch_kv.update_seq IS NULL)
+         ) THEN EXCLUDED.value ELSE event_dispatch_kv.value END
+       RETURNING value, update_time, update_seq;`, [state.service, state.workflowFnName, state.key, state.value, state.updateTime, state.updateSeq]);
         return {
             service: state.service,
             workflowFnName: state.workflowFnName,
@@ -824,7 +1127,7 @@ class PostgresSystemDatabase {
             query = query.offset(input.offset);
         }
         const rows = await query;
-        return rows.map(PostgresSystemDatabase.#mapWorkflowStatus);
+        return rows.map(mapWorkflowStatus);
     }
     async listQueuedWorkflows(input) {
         const schemaName = dbos_executor_1.DBOSExecutor.systemDBSchemaName;
@@ -855,30 +1158,7 @@ class PostgresSystemDatabase {
             query = query.offset(input.offset);
         }
         const rows = await query;
-        return rows.map(PostgresSystemDatabase.#mapWorkflowStatus);
-    }
-    static #mapWorkflowStatus(row) {
-        return {
-            workflowUUID: row.workflow_uuid,
-            status: row.status,
-            workflowName: row.name,
-            output: row.output ? row.output : null,
-            error: row.error ? row.error : null,
-            workflowClassName: row.class_name ?? '',
-            workflowConfigName: row.config_name ?? '',
-            queueName: row.queue_name,
-            authenticatedUser: row.authenticated_user,
-            assumedRole: row.assumed_role,
-            authenticatedRoles: JSON.parse(row.authenticated_roles),
-            request: JSON.parse(row.request),
-            executorId: row.executor_id,
-            createdAt: Number(row.created_at),
-            updatedAt: Number(row.updated_at),
-            applicationVersion: row.application_version,
-            applicationID: row.application_id,
-            recoveryAttempts: Number(row.recovery_attempts),
-            input: row.inputs,
-        };
+        return rows.map(mapWorkflowStatus);
     }
     async getWorkflowQueue(input) {
         // Create the initial query with a join to workflow_status table to get executor_id
@@ -920,29 +1200,29 @@ class PostgresSystemDatabase {
         return { workflows };
     }
     async enqueueWorkflow(workflowId, queueName) {
-        await this.pool.query(`
-      INSERT INTO ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue (workflow_uuid, queue_name)
-      VALUES ($1, $2)
-      ON CONFLICT (workflow_uuid)
-      DO NOTHING;
-    `, [workflowId, queueName]);
+        const client = await this.pool.connect();
+        try {
+            await enqueueWorkflow(client, workflowId, queueName);
+        }
+        finally {
+            client.release();
+        }
     }
-    async clearQueueAssignment(workflowId) {
+    async clearQueueAssignment(workflowID) {
         const client = await this.pool.connect();
         try {
+            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Reset the start time in the queue to mark it as not started
-            const wqRes = await client.query(`
-        UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue
-        SET started_at_epoch_ms = NULL
-        WHERE workflow_uuid = $1 AND completed_at_epoch_ms IS NULL;
-      `, [workflowId]);
+            const wqRes = await client.query(`UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue
+         SET started_at_epoch_ms = NULL
+         WHERE workflow_uuid = $1 AND completed_at_epoch_ms IS NULL;`, [workflowID]);
             // If no rows were affected, the workflow is not anymore in the queue or was already completed
             if (wqRes.rowCount === 0) {
                 await client.query('ROLLBACK');
                 return false;
             }
             // Reset the status of the task to "ENQUEUED"
-            await this.recordWorkflowStatusChange(workflowId, workflow_1.StatusString.ENQUEUED, {}, client);
+            await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.ENQUEUED);
             await client.query('COMMIT');
             return true;
         }
@@ -954,24 +1234,25 @@ class PostgresSystemDatabase {
             client.release();
         }
     }
-    async dequeueWorkflow(workflowId, queue) {
-        if (queue.rateLimit) {
-            const time = new Date().getTime();
-            await this.pool.query(`
-        UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue
-        SET completed_at_epoch_ms = $2
-        WHERE workflow_uuid = $1;
-      `, [workflowId, time]);
+    async dequeueWorkflow(workflowID, queue) {
+        const client = await this.pool.connect();
+        try {
+            if (queue.rateLimit) {
+                const time = Date.now();
+                await client.query(`UPDATE ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue
+           SET completed_at_epoch_ms = $2
+           WHERE workflow_uuid = $1;`, [workflowID, time]);
+            }
+            else {
+                await deleteQueuedWorkflows(client, workflowID);
+            }
         }
-        else {
-            await this.pool.query(`
-        DELETE FROM ${dbos_executor_1.DBOSExecutor.systemDBSchemaName}.workflow_queue
-        WHERE workflow_uuid = $1;
-      `, [workflowId]);
+        finally {
+            client.release();
         }
     }
     async findAndMarkStartableWorkflows(queue, executorID, appVersion) {
-        const startTimeMs = new Date().getTime();
+        const startTimeMs = Date.now();
         const limiterPeriodMS = queue.rateLimit ? queue.rateLimit.periodSec * 1000 : 0;
         const claimedIDs = [];
         await this.knexDB.transaction(async (trx) => {