@dbos-inc/dbos-enterprise 5.3.1-preview → 5.4.0

package/dist/src/conductor/conductor.js

@@ -0,0 +1,978 @@
+// Copyright (c) 2026 DBOS, Inc. All rights reserved.
+//
+// This file is part of Transact Enterprise, proprietary software licensed under the
+// DBOS Conductor Software License: https://www.dbos.dev/conductor-license
+// You may not use, copy, modify, or distribute this file except as that license permits.
+// Disabling or circumventing the entitlement checks in @dbos-inc/dbos-sdk's src/enterprise.ts
+// that gate it is a material breach of that license.
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConductorWebsocket = void 0;
+const decorators_1 = require("@dbos-inc/dbos-sdk/internal/decorators");
+const utils_1 = require("@dbos-inc/dbos-sdk/internal/utils");
+const ws_1 = __importDefault(require("ws"));
+const protocol = __importStar(require("./protocol"));
+const node_os_1 = require("node:os");
+const node_assert_1 = __importDefault(require("node:assert"));
+const zlib = __importStar(require("node:zlib"));
+const node_util_1 = require("node:util");
+const export_1 = require("../export");
+const fork_1 = require("../fork");
+const observability_1 = require("../observability");
+const retention_1 = require("../retention");
+const scheduler_1 = require("@dbos-inc/dbos-sdk/internal/scheduler/scheduler");
+const shard_keys_1 = require("@dbos-inc/dbos-sdk/internal/shard_keys");
+const gzip = (0, node_util_1.promisify)(zlib.gzip);
+const gunzip = (0, node_util_1.promisify)(zlib.gunzip);
+const DATA_MESSAGE_TYPES = new Set([
+    protocol.MessageType.GET_WORKFLOW_EVENTS,
+    protocol.MessageType.GET_WORKFLOW_NOTIFICATIONS,
+    protocol.MessageType.GET_WORKFLOW_STREAMS,
+    protocol.MessageType.EXPORT_WORKFLOW,
+    protocol.MessageType.IMPORT_WORKFLOW,
+]);
+function positionsByKey(ids, shardKeys) {
+    const groups = new Map();
+    ids.forEach((workflowID, i) => {
+        const key = shardKeys.get(workflowID);
+        const group = groups.get(key) ?? [];
+        group.push(i);
+        groups.set(key, group);
+    });
+    return Array.from(groups.values());
+}
+class ConductorWebsocket {
+    dbosExec;
+    url;
+    websocket = undefined;
+    isShuttingDown = false;
+    pingPeriodMs = 20000;
+    pingTimeoutMs = 15000;
+    pingIntervalTimeout = undefined;
+    reconnectDelayMs = 1000;
+    reconnectTimeout = undefined;
+    retentionRound = undefined;
+    appName;
+    executorMetadata;
+    metadataOnlyMode;
+    constructor(dbosExec, options) {
+        this.dbosExec = dbosExec;
+        this.appName = options.appName;
+        this.executorMetadata = options.executorMetadata;
+        this.metadataOnlyMode = options.metadataOnlyMode ?? false;
+        (0, node_assert_1.default)(this.appName, 'Application name must be set in configuration in order to use DBOS Conductor');
+        const cleanConductorURL = options.conductorURL.replace(/\/+$/, '');
+        this.url = `${cleanConductorURL}/websocket/${this.appName}/${options.conductorKey}`;
+    }
+    start() {
+        this.dispatchLoop();
+    }
+    reportException(context, e) {
+        const errorMsg = `${context}: ${e.message}`;
+        this.dbosExec.logger.error(errorMsg);
+        if (this.metadataOnlyMode) {
+            const exceptionName = e instanceof Error ? e.constructor.name : typeof e;
+            return `${context}: ${exceptionName} (details withheld in metadata-only mode)`;
+        }
+        return errorMsg;
+    }
+    resetWebsocket(currWebsocket, currPing) {
+        clearInterval(currPing?.interval);
+        clearTimeout(currPing?.timeout);
+        if (currWebsocket) {
+            currWebsocket.terminate();
+            if (this.websocket === currWebsocket) {
+                this.websocket = undefined;
+            }
+        }
+        if (this.reconnectTimeout || this.isShuttingDown) {
+            return;
+        }
+        this.dbosExec.logger.debug(`Reconnecting in ${this.reconnectDelayMs / 1000} second`);
+        this.reconnectTimeout = setTimeout(() => {
+            this.dispatchLoop();
+        }, this.reconnectDelayMs);
+    }
+    async #handleMessage(currWebsocket, baseMsg, shardKeys) {
+        const msgType = baseMsg.type;
+        let errorMsg = undefined;
+        if (this.metadataOnlyMode && DATA_MESSAGE_TYPES.has(msgType)) {
+            const refusedResp = new protocol.BaseResponse(msgType, baseMsg.request_id, `${msgType} is not allowed in conductor metadata-only mode`);
+            currWebsocket.send(JSON.stringify(refusedResp));
+            return;
+        }
+        switch (msgType) {
+            case protocol.MessageType.EXECUTOR_INFO:
+                const infoResp = new protocol.ExecutorInfoResponse(baseMsg.request_id, utils_1.globalParams.executorID, utils_1.globalParams.appVersion, (0, node_os_1.hostname)(), 'typescript', utils_1.globalParams.dbosVersion, undefined, this.executorMetadata, this.dbosExec.systemDatabase.sharded);
+                currWebsocket.send(JSON.stringify(infoResp));
+                this.dbosExec.logger.info('Connected to DBOS conductor');
+                break;
+            case protocol.MessageType.RECOVERY:
+                const recoveryMsg = baseMsg;
+                let success = true;
+                try {
+                    await this.dbosExec.recoverPendingWorkflows(recoveryMsg.executor_ids);
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when recovering workflows', e);
+                    success = false;
+                }
+                const recoveryResp = new protocol.RecoveryResponse(baseMsg.request_id, success, errorMsg);
+                currWebsocket.send(JSON.stringify(recoveryResp));
+                break;
+            case protocol.MessageType.CANCEL:
+                const cancelMsg = baseMsg;
+                const cancelIds = cancelMsg.workflow_ids?.length ? cancelMsg.workflow_ids : [cancelMsg.workflow_id];
+                let cancelSuccess = true;
+                try {
+                    for (const group of positionsByKey(cancelIds, shardKeys)) {
+                        await this.dbosExec.systemDatabase.cancelWorkflows(group.map((i) => cancelIds[i]), cancelMsg.cancel_children);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when cancelling workflow(s) ${String(cancelIds)}`, e);
+                    cancelSuccess = false;
+                }
+                const cancelResp = new protocol.CancelResponse(baseMsg.request_id, cancelSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(cancelResp));
+                break;
+            case protocol.MessageType.DELETE:
+                const deleteMsg = baseMsg;
+                const deleteIds = deleteMsg.workflow_ids?.length ? deleteMsg.workflow_ids : [deleteMsg.workflow_id];
+                let deleteSuccess = true;
+                try {
+                    for (const group of positionsByKey(deleteIds, shardKeys)) {
+                        await this.dbosExec.systemDatabase.deleteWorkflows(group.map((i) => deleteIds[i]), deleteMsg.delete_children ?? false);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when deleting workflow(s) ${String(deleteIds)}`, e);
+                    deleteSuccess = false;
+                }
+                const deleteResp = new protocol.DeleteResponse(baseMsg.request_id, deleteSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(deleteResp));
+                break;
+            case protocol.MessageType.RESUME:
+                const resumeMsg = baseMsg;
+                const resumeIds = resumeMsg.workflow_ids?.length ? resumeMsg.workflow_ids : [resumeMsg.workflow_id];
+                let resumeSuccess = true;
+                try {
+                    for (const group of positionsByKey(resumeIds, shardKeys)) {
+                        await this.dbosExec.systemDatabase.resumeWorkflows(group.map((i) => resumeIds[i]), resumeMsg.queue_name);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when resuming workflow(s) ${String(resumeIds)}`, e);
+                    resumeSuccess = false;
+                }
+                const resumeResp = new protocol.ResumeResponse(baseMsg.request_id, resumeSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(resumeResp));
+                break;
+            case protocol.MessageType.REWIND_WORKFLOW:
+                const rewindMsg = baseMsg;
+                const rewindBody = rewindMsg.body;
+                let rewindSuccess = true;
+                try {
+                    await this.dbosExec.rewindWorkflow(rewindBody.workflow_id, rewindBody.start_step ?? 0, {
+                        applicationVersion: rewindBody.application_version,
+                        queueName: rewindBody.queue_name,
+                        queuePartitionKey: rewindBody.queue_partition_key,
+                    });
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when rewinding workflow ${rewindBody.workflow_id} to step ${rewindBody.start_step ?? 0}`, e);
+                    rewindSuccess = false;
+                }
+                const rewindResp = new protocol.RewindWorkflowResponse(baseMsg.request_id, rewindSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(rewindResp));
+                break;
+            case protocol.MessageType.FORK_WORKFLOW:
+                const forkMsg = baseMsg;
+                let newWorkflowID = forkMsg.body.new_workflow_id;
+                try {
+                    newWorkflowID = await this.dbosExec.forkWorkflow(forkMsg.body.workflow_id, forkMsg.body.start_step, {
+                        newWorkflowID: newWorkflowID,
+                        applicationVersion: forkMsg.body.application_version,
+                        queueName: forkMsg.body.queue_name,
+                        queuePartitionKey: forkMsg.body.queue_partition_key,
+                    });
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when forking workflow ${forkMsg.body.workflow_id} to new workflow ${newWorkflowID} on step ${forkMsg.body.start_step}, app version ${forkMsg.body.application_version}`, e);
+                    newWorkflowID = undefined;
+                }
+                const forkResp = new protocol.ForkWorkflowResponse(baseMsg.request_id, newWorkflowID, errorMsg);
+                currWebsocket.send(JSON.stringify(forkResp));
+                break;
+            case protocol.MessageType.FORK_FROM_FAILURE:
+                const forkFromFailureMsg = baseMsg;
+                const forkFromFailureBody = forkFromFailureMsg.body;
+                let forkedWorkflowIDs = undefined;
+                try {
+                    forkedWorkflowIDs = await (0, fork_1.forkFromFailure)(this.dbosExec.systemDatabase, forkFromFailureBody.workflow_ids, {
+                        applicationVersion: forkFromFailureBody.application_version,
+                        queueName: forkFromFailureBody.queue_name,
+                        queuePartitionKey: forkFromFailureBody.queue_partition_key,
+                        fromLastFailure: forkFromFailureBody.from_last_failure ?? false,
+                        fromLastStep: forkFromFailureBody.from_last_step ?? false,
+                        fromStep: forkFromFailureBody.from_step,
+                        fromStepName: forkFromFailureBody.from_step_name,
+                    });
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when bulk forking workflows', e);
+                }
+                const forkFromFailureResp = new protocol.ForkFromFailureResponse(baseMsg.request_id, forkedWorkflowIDs, errorMsg);
+                currWebsocket.send(JSON.stringify(forkFromFailureResp));
+                break;
+            case protocol.MessageType.LIST_WORKFLOWS:
+                const listWFMsg = baseMsg;
+                const body = listWFMsg.body;
+                const listWFReq = {
+                    workflowIDs: body.workflow_uuids,
+                    shardKey: body.shard_key ?? undefined,
+                    parentShardKey: body.parent_shard_key ?? undefined,
+                    workflowName: body.workflow_name,
+                    authenticatedUser: body.authenticated_user,
+                    startTime: body.start_time,
+                    endTime: body.end_time,
+                    completedAfter: body.completed_after,
+                    completedBefore: body.completed_before,
+                    dequeuedAfter: body.dequeued_after,
+                    dequeuedBefore: body.dequeued_before,
+                    status: body.status,
+                    applicationVersion: body.application_version,
+                    forkedFrom: body.forked_from,
+                    parentWorkflowID: body.parent_workflow_id,
+                    queueName: body.queue_name,
+                    limit: body.limit,
+                    offset: body.offset,
+                    sortDesc: body.sort_desc,
+                    workflow_id_prefix: body.workflow_id_prefix,
+                    loadInput: (body.load_input ?? false) && !this.metadataOnlyMode,
+                    loadOutput: (body.load_output ?? false) && !this.metadataOnlyMode,
+                    executorId: body.executor_id,
+                    queuesOnly: body.queues_only,
+                    wasForkedFrom: body.was_forked_from,
+                    isFork: body.is_fork,
+                    hasParent: body.has_parent,
+                    attributes: body.attributes,
+                    scheduleName: body.schedule_name,
+                    applicationName: body.application_name,
+                };
+                let workflowsOutput = [];
+                try {
+                    const workflows = await this.dbosExec.listWorkflows(listWFReq);
+                    workflowsOutput = workflows.map((wf) => new protocol.WorkflowsOutput(wf));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing workflows', e);
+                }
+                const wfsResp = new protocol.ListWorkflowsResponse(listWFMsg.request_id, workflowsOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(wfsResp));
+                break;
+            case protocol.MessageType.LIST_QUEUED_WORKFLOWS:
+                const listQueuedWFMsg = baseMsg;
+                const bodyQueued = listQueuedWFMsg.body;
+                const listQueuedWFReq = {
+                    workflowIDs: bodyQueued.workflow_uuids,
+                    shardKey: bodyQueued.shard_key ?? undefined,
+                    parentShardKey: bodyQueued.parent_shard_key ?? undefined,
+                    workflowName: bodyQueued.workflow_name,
+                    authenticatedUser: bodyQueued.authenticated_user,
+                    startTime: bodyQueued.start_time,
+                    endTime: bodyQueued.end_time,
+                    completedAfter: bodyQueued.completed_after,
+                    completedBefore: bodyQueued.completed_before,
+                    dequeuedAfter: bodyQueued.dequeued_after,
+                    dequeuedBefore: bodyQueued.dequeued_before,
+                    status: bodyQueued.status,
+                    applicationVersion: bodyQueued.application_version,
+                    forkedFrom: bodyQueued.forked_from,
+                    parentWorkflowID: bodyQueued.parent_workflow_id,
+                    queueName: bodyQueued.queue_name,
+                    limit: bodyQueued.limit,
+                    offset: bodyQueued.offset,
+                    sortDesc: bodyQueued.sort_desc,
+                    workflow_id_prefix: bodyQueued.workflow_id_prefix,
+                    loadInput: (bodyQueued.load_input ?? false) && !this.metadataOnlyMode,
+                    loadOutput: (bodyQueued.load_output ?? false) && !this.metadataOnlyMode,
+                    executorId: bodyQueued.executor_id,
+                    wasForkedFrom: bodyQueued.was_forked_from,
+                    isFork: bodyQueued.is_fork,
+                    hasParent: bodyQueued.has_parent,
+                    attributes: bodyQueued.attributes,
+                    scheduleName: bodyQueued.schedule_name,
+                    applicationName: bodyQueued.application_name,
+                };
+                let queuedWFOutput = [];
+                try {
+                    const workflows = await this.dbosExec.listQueuedWorkflows(listQueuedWFReq);
+                    queuedWFOutput = workflows.map((wf) => new protocol.WorkflowsOutput(wf));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing queued workflows', e);
+                }
+                const queuedWfsResp = new protocol.ListQueuedWorkflowsResponse(listQueuedWFMsg.request_id, queuedWFOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(queuedWfsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW:
+                const getWFMsg = baseMsg;
+                let wfOutput = undefined;
+                try {
+                    const workflows = await this.dbosExec.listWorkflows({
+                        workflowIDs: [getWFMsg.workflow_id],
+                        shardKey: this.dbosExec.systemDatabase.keyHint(getWFMsg.workflow_id),
+                        loadInput: (getWFMsg.load_input ?? true) && !this.metadataOnlyMode,
+                        loadOutput: (getWFMsg.load_output ?? true) && !this.metadataOnlyMode,
+                    });
+                    if (workflows.length > 0) {
+                        wfOutput = new protocol.WorkflowsOutput(workflows[0]);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting workflow ${getWFMsg.workflow_id}`, e);
+                }
+                const getWFResp = new protocol.GetWorkflowResponse(getWFMsg.request_id, wfOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(getWFResp));
+                break;
+            case protocol.MessageType.EXIST_PENDING_WORKFLOWS:
+                const existPendingMsg = baseMsg;
+                let hasPendingWFs = false;
+                try {
+                    const pendingWFs = await (0, observability_1.getPendingWorkflows)(this.dbosExec.systemDatabase, existPendingMsg.executor_id, existPendingMsg.application_version);
+                    hasPendingWFs = pendingWFs.length > 0;
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when checking for pending workflows', e);
+                }
+                const existPendingResp = new protocol.ExistPendingWorkflowsResponse(baseMsg.request_id, hasPendingWFs, errorMsg);
+                currWebsocket.send(JSON.stringify(existPendingResp));
+                break;
+            case protocol.MessageType.LIST_STEPS:
+                const listStepsMessage = baseMsg;
+                let workflowSteps = undefined;
+                try {
+                    const stepsInfo = await this.dbosExec.listWorkflowSteps(listStepsMessage.workflow_id, (listStepsMessage.load_output ?? true) && !this.metadataOnlyMode, { limit: listStepsMessage.limit, offset: listStepsMessage.offset });
+                    workflowSteps = stepsInfo?.map((i) => new protocol.WorkflowSteps(i));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when listing steps ${listStepsMessage.workflow_id}`, e);
+                }
+                const listStepsResponse = new protocol.ListStepsResponse(listStepsMessage.request_id, workflowSteps, errorMsg);
+                currWebsocket.send(JSON.stringify(listStepsResponse));
+                break;
+            case protocol.MessageType.RETENTION:
+                const retentionMessage = baseMsg;
+                const retentionBody = retentionMessage.body;
+                let rowsThreshold = retentionBody.gc_rows_threshold;
+                if (rowsThreshold !== undefined && rowsThreshold !== null && this.dbosExec.systemDatabase.sharded) {
+                    this.dbosExec.logger.warn("Ignoring Conductor's rows-based retention: a sharded system database supports only time-based retention.");
+                    rowsThreshold = undefined;
+                }
+                if (this.retentionRound !== undefined) {
+                    this.dbosExec.logger.warn('Skipping retention: the previous round on this executor is still running.');
+                }
+                else {
+                    this.retentionRound = (async () => {
+                        try {
+                            await (0, retention_1.garbageCollect)(this.dbosExec.systemDatabase, retentionBody.gc_cutoff_epoch_ms, rowsThreshold, { batchSize: retentionBody.gc_batch_size || undefined });
+                            if (retentionBody.timeout_cutoff_epoch_ms) {
+                                await (0, retention_1.globalTimeout)(this.dbosExec.systemDatabase, retentionBody.timeout_cutoff_epoch_ms);
+                            }
+                        }
+                        catch (e) {
+                            if (this.isShuttingDown) {
+                                this.dbosExec.logger.debug('Retention interrupted by shutdown');
+                                return;
+                            }
+                            this.dbosExec.logger.error(`Exception encountered during enforcing retention policy: ${e.message}`);
+                        }
+                    })().finally(() => {
+                        this.retentionRound = undefined;
+                    });
+                }
+                currWebsocket.send(JSON.stringify(new protocol.RetentionResponse(retentionMessage.request_id, true)));
+                break;
+            case protocol.MessageType.GET_METRICS:
+                const getMetricsMessage = baseMsg;
+                this.dbosExec.logger.debug(`Received metrics request for time range ${getMetricsMessage.start_time} to ${getMetricsMessage.end_time}`);
+                let metricsData = [];
+                if (getMetricsMessage.metric_class === 'workflow_step_count') {
+                    try {
+                        const sysMetrics = await (0, observability_1.getMetrics)(this.dbosExec.systemDatabase, getMetricsMessage.start_time, getMetricsMessage.end_time, getMetricsMessage.application_name);
+                        metricsData = sysMetrics.map((m) => new protocol.MetricDataOutput(m.metricType, m.metricName, m.value));
+                    }
+                    catch (e) {
+                        errorMsg = this.reportException('Exception encountered when getting metrics', e);
+                    }
+                }
+                else {
+                    errorMsg = `Unexpected metric class: ${getMetricsMessage.metric_class}`;
+                    this.dbosExec.logger.warn(errorMsg);
+                }
+                const getMetricsResponse = new protocol.GetMetricsResponse(getMetricsMessage.request_id, metricsData, errorMsg);
+                currWebsocket.send(JSON.stringify(getMetricsResponse));
+                break;
+            case protocol.MessageType.EXPORT_WORKFLOW:
+                const exportMsg = baseMsg;
+                let serializedWorkflow = null;
+                try {
+                    const exported = await (0, export_1.exportWorkflow)(this.dbosExec.systemDatabase, exportMsg.workflow_id, exportMsg.export_children ?? false);
+                    if (exported.length > 0) {
+                        const jsonStr = JSON.stringify(exported);
+                        const compressed = await gzip(jsonStr);
+                        serializedWorkflow = compressed.toString('base64');
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when exporting workflow ${exportMsg.workflow_id}`, e);
+                }
+                const exportResp = new protocol.ExportWorkflowResponse(baseMsg.request_id, serializedWorkflow, errorMsg);
+                currWebsocket.send(JSON.stringify(exportResp));
+                break;
+            case protocol.MessageType.IMPORT_WORKFLOW:
+                const importMsg = baseMsg;
+                let importSuccess = true;
+                try {
+                    const compressedData = Buffer.from(importMsg.serialized_workflow, 'base64');
+                    const decompressed = await gunzip(compressedData);
+                    const workflows = JSON.parse(decompressed.toString());
+                    await (0, export_1.importWorkflow)(this.dbosExec.systemDatabase, workflows);
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when importing workflow', e);
+                    importSuccess = false;
+                }
+                const importResp = new protocol.ImportWorkflowResponse(baseMsg.request_id, importSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(importResp));
+                break;
+            case protocol.MessageType.ALERT:
+                const alertMsg = baseMsg;
+                let alertSuccess = true;
+                try {
+                    const handler = (0, decorators_1.getAlertHandler)();
+                    if (handler) {
+                        await Promise.resolve(handler(alertMsg.name, alertMsg.message, alertMsg.metadata));
+                    }
+                    else {
+                        this.dbosExec.logger.warn(`Alert received (no handler registered): [${alertMsg.name}] ${alertMsg.message} | Metadata: ${JSON.stringify(alertMsg.metadata)}`);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception in alert handler', e);
+                    alertSuccess = false;
+                }
+                const alertResp = new protocol.AlertResponse(baseMsg.request_id, alertSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(alertResp));
+                break;
+            case protocol.MessageType.LIST_SCHEDULES:
+                const listSchedMsg = baseMsg;
+                const loadContextList = (listSchedMsg.body.load_context ?? true) && !this.metadataOnlyMode;
+                let schedOutput = [];
+                try {
+                    const scheds = await this.dbosExec.systemDatabase.listSchedules({
+                        status: listSchedMsg.body.status,
+                        workflowName: listSchedMsg.body.workflow_name,
+                        scheduleNamePrefix: listSchedMsg.body.schedule_name_prefix,
+                        applicationName: listSchedMsg.body.application_name,
+                    });
+                    schedOutput = await Promise.all(scheds.map(async (s) => ({
+                        schedule_id: s.scheduleId,
+                        schedule_name: s.scheduleName,
+                        workflow_name: s.workflowName,
+                        workflow_class_name: s.workflowClassName || undefined,
+                        schedule: s.schedule,
+                        status: s.status,
+                        context: loadContextList
+                            ? protocol.inspectForConductor(await this.dbosExec.serializer.parse(s.context))
+                            : undefined,
+                        last_fired_at: s.lastFiredAt,
+                        automatic_backfill: s.automaticBackfill,
+                        cron_timezone: s.cronTimezone,
+                        queue_name: s.queueName,
+                        application_name: s.applicationName ?? null,
+                        shard_key: s.shardKey ?? null,
+                    })));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing schedules', e);
+                }
+                const listSchedResp = new protocol.ListSchedulesResponse(listSchedMsg.request_id, schedOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(listSchedResp));
+                break;
+            case protocol.MessageType.GET_SCHEDULE:
+                const getSchedMsg = baseMsg;
+                const loadContextGet = (getSchedMsg.load_context ?? true) && !this.metadataOnlyMode;
+                let getSchedOutput = undefined;
+                try {
+                    const sched = await this.dbosExec.systemDatabase.getSchedule(getSchedMsg.schedule_name);
+                    if (sched) {
+                        getSchedOutput = {
+                            schedule_id: sched.scheduleId,
+                            schedule_name: sched.scheduleName,
+                            workflow_name: sched.workflowName,
+                            workflow_class_name: sched.workflowClassName || undefined,
+                            schedule: sched.schedule,
+                            status: sched.status,
+                            context: loadContextGet
+                                ? protocol.inspectForConductor(await this.dbosExec.serializer.parse(sched.context))
+                                : undefined,
+                            last_fired_at: sched.lastFiredAt,
+                            automatic_backfill: sched.automaticBackfill,
+                            cron_timezone: sched.cronTimezone,
+                            queue_name: sched.queueName,
+                            application_name: sched.applicationName ?? null,
+                            shard_key: sched.shardKey ?? null,
+                        };
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting schedule ${getSchedMsg.schedule_name}`, e);
+                }
+                const getSchedResp = new protocol.GetScheduleResponse(getSchedMsg.request_id, getSchedOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(getSchedResp));
+                break;
+            case protocol.MessageType.PAUSE_SCHEDULE:
+                const pauseSchedMsg = baseMsg;
+                let pauseSuccess = true;
+                try {
+                    await this.dbosExec.systemDatabase.setScheduleStatus(pauseSchedMsg.schedule_name, 'PAUSED');
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when pausing schedule '${pauseSchedMsg.schedule_name}'`, e);
+                    pauseSuccess = false;
+                }
+                const pauseSchedResp = new protocol.PauseScheduleResponse(baseMsg.request_id, pauseSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(pauseSchedResp));
+                break;
+            case protocol.MessageType.RESUME_SCHEDULE:
+                const resumeSchedMsg = baseMsg;
+                let resumeSchedSuccess = true;
+                try {
+                    await this.dbosExec.systemDatabase.setScheduleStatus(resumeSchedMsg.schedule_name, 'ACTIVE');
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when resuming schedule '${resumeSchedMsg.schedule_name}'`, e);
+                    resumeSchedSuccess = false;
+                }
+                const resumeSchedResp = new protocol.ResumeScheduleResponse(baseMsg.request_id, resumeSchedSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(resumeSchedResp));
+                break;
+            case protocol.MessageType.BACKFILL_SCHEDULE:
+                const backfillSchedMsg = baseMsg;
+                let backfillWorkflowIds = [];
+                try {
+                    backfillWorkflowIds = await (0, scheduler_1.backfillSchedule)(this.dbosExec.systemDatabase, this.dbosExec.serializer, backfillSchedMsg.schedule_name, new Date(backfillSchedMsg.start), new Date(backfillSchedMsg.end));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when backfilling schedule '${backfillSchedMsg.schedule_name}'`, e);
+                }
+                const backfillSchedResp = new protocol.BackfillScheduleResponse(baseMsg.request_id, backfillWorkflowIds, errorMsg);
+                currWebsocket.send(JSON.stringify(backfillSchedResp));
+                break;
+            case protocol.MessageType.TRIGGER_SCHEDULE:
+                const triggerSchedMsg = baseMsg;
+                let triggerWorkflowId = undefined;
+                try {
+                    triggerWorkflowId = await (0, scheduler_1.triggerSchedule)(this.dbosExec.systemDatabase, this.dbosExec.serializer, triggerSchedMsg.schedule_name);
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when triggering schedule '${triggerSchedMsg.schedule_name}'`, e);
+                }
+                const triggerSchedResp = new protocol.TriggerScheduleResponse(baseMsg.request_id, triggerWorkflowId, errorMsg);
+                currWebsocket.send(JSON.stringify(triggerSchedResp));
+                break;
+            case protocol.MessageType.LIST_APPLICATION_VERSIONS:
+                let versionsOutput = [];
+                try {
+                    const versions = await this.dbosExec.systemDatabase.listApplicationVersions();
+                    versionsOutput = versions.map((v) => ({
+                        version_id: v.versionId,
+                        version_name: v.versionName,
+                        version_timestamp: v.versionTimestamp,
+                        created_at: v.createdAt,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing application versions', e);
+                }
+                const listVersionsResp = new protocol.ListApplicationVersionsResponse(baseMsg.request_id, versionsOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(listVersionsResp));
+                break;
+            case protocol.MessageType.SET_LATEST_APPLICATION_VERSION:
+                const setVersionMsg = baseMsg;
+                let setVersionSuccess = true;
+                try {
+                    await this.dbosExec.systemDatabase.updateApplicationVersionTimestamp(setVersionMsg.version_name, Date.now());
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when setting latest application version '${setVersionMsg.version_name}'`, e);
+                    setVersionSuccess = false;
+                }
+                const setVersionResp = new protocol.SetLatestApplicationVersionResponse(baseMsg.request_id, setVersionSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(setVersionResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_EVENTS:
+                const eventsMsg = baseMsg;
+                let eventOutputs = undefined;
+                try {
+                    const rawEvents = await (0, observability_1.getAllEvents)(this.dbosExec.systemDatabase, eventsMsg.workflow_id);
+                    eventOutputs = Object.entries(rawEvents).map(([key, value]) => ({
+                        key,
+                        value: protocol.inspectForConductor(value),
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting events for workflow ${eventsMsg.workflow_id}`, e);
+                }
+                const eventsResp = new protocol.GetWorkflowEventsResponse(baseMsg.request_id, eventOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(eventsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_NOTIFICATIONS:
+                const notifsMsg = baseMsg;
+                let notifOutputs = undefined;
+                try {
+                    const rawNotifs = await (0, observability_1.getAllNotifications)(this.dbosExec.systemDatabase, notifsMsg.workflow_id);
+                    notifOutputs = rawNotifs.map((n) => ({
+                        topic: n.topic,
+                        message: protocol.inspectForConductor(n.message),
+                        created_at_epoch_ms: n.createdAtEpochMs,
+                        consumed: n.consumed,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting notifications for workflow ${notifsMsg.workflow_id}`, e);
+                }
+                const notifsResp = new protocol.GetWorkflowNotificationsResponse(baseMsg.request_id, notifOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(notifsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_STREAMS:
+                const streamsMsg = baseMsg;
+                let streamOutputs = undefined;
+                try {
+                    const rawStreams = await (0, observability_1.getAllStreamEntries)(this.dbosExec.systemDatabase, streamsMsg.workflow_id);
+                    streamOutputs = Object.entries(rawStreams).map(([key, values]) => ({
+                        key,
+                        values: values.map((v) => protocol.inspectForConductor(v)),
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting streams for workflow ${streamsMsg.workflow_id}`, e);
+                }
+                const streamsResp = new protocol.GetWorkflowStreamsResponse(baseMsg.request_id, streamOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(streamsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_AGGREGATES:
+                const aggMsg = baseMsg;
+                const aggBody = aggMsg.body;
+                let aggOutput = [];
+                let selectCount = aggBody.select_count ?? false;
+                const selectMinCreatedAt = aggBody.select_min_created_at ?? false;
+                const selectMaxQueueWaitMs = aggBody.select_max_queue_wait_ms ?? false;
+                const selectMaxTotalLatencyMs = aggBody.select_max_total_latency_ms ?? false;
+                if (!(selectCount || selectMinCreatedAt || selectMaxQueueWaitMs || selectMaxTotalLatencyMs)) {
+                    selectCount = true;
+                }
+                try {
+                    const rows = await (0, observability_1.getWorkflowAggregates)(this.dbosExec.systemDatabase, {
+                        groupByStatus: aggBody.group_by_status ?? false,
+                        groupByName: aggBody.group_by_name ?? false,
+                        groupByQueueName: aggBody.group_by_queue_name ?? false,
+                        groupByExecutorId: aggBody.group_by_executor_id ?? false,
+                        groupByApplicationVersion: aggBody.group_by_application_version ?? false,
+                        groupByApplicationName: aggBody.group_by_application_name ?? false,
+                        selectCount,
+                        selectMinCreatedAt,
+                        selectMaxQueueWaitMs,
+                        selectMaxTotalLatencyMs,
+                        timeBucketSizeMs: aggBody.time_bucket_size_ms ?? undefined,
+                        status: aggBody.status,
+                        startTime: aggBody.start_time,
+                        endTime: aggBody.end_time,
+                        completedAfter: aggBody.completed_after,
+                        completedBefore: aggBody.completed_before,
+                        dequeuedAfter: aggBody.dequeued_after,
+                        dequeuedBefore: aggBody.dequeued_before,
+                        name: aggBody.name,
+                        appVersion: aggBody.app_version,
+                        executorId: aggBody.executor_id,
+                        queueName: aggBody.queue_name,
+                        workflowIdPrefix: aggBody.workflow_id_prefix,
+                        workflowIDs: aggBody.workflow_ids,
+                        authenticatedUser: aggBody.user,
+                        forkedFrom: aggBody.forked_from,
+                        parentWorkflowID: aggBody.parent_workflow_id,
+                        scheduleName: aggBody.schedule_name,
+                        applicationName: aggBody.application_name,
+                        wasForkedFrom: aggBody.was_forked_from,
+                        isFork: aggBody.is_fork,
+                        hasParent: aggBody.has_parent,
+                        attributes: aggBody.attributes,
+                    });
+                    aggOutput = rows.map((r) => ({
+                        group: r.group,
+                        count: r.count,
+                        min_created_at: r.minCreatedAt,
+                        max_queue_wait_ms: r.maxQueueWaitMs,
+                        max_total_latency_ms: r.maxTotalLatencyMs,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when getting workflow aggregates', e);
+                }
+                const aggResp = new protocol.GetWorkflowAggregatesResponse(baseMsg.request_id, aggOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(aggResp));
+                break;
+            case protocol.MessageType.GET_STEP_AGGREGATES:
+                const stepAggMsg = baseMsg;
+                const stepAggBody = stepAggMsg.body;
+                let stepAggOutput = [];
+                let stepSelectCount = stepAggBody.select_count ?? false;
+                const stepSelectMaxDurationMs = stepAggBody.select_max_duration_ms ?? false;
+                if (!(stepSelectCount || stepSelectMaxDurationMs)) {
+                    stepSelectCount = true;
+                }
+                try {
+                    const rows = await (0, observability_1.getStepAggregates)(this.dbosExec.systemDatabase, {
+                        groupByFunctionName: stepAggBody.group_by_function_name ?? false,
+                        groupByStatus: stepAggBody.group_by_status ?? false,
+                        selectCount: stepSelectCount,
+                        selectMaxDurationMs: stepSelectMaxDurationMs,
+                        timeBucketSizeMs: stepAggBody.time_bucket_size_ms ?? undefined,
+                        status: stepAggBody.status,
+                        functionName: stepAggBody.function_name,
+                        workflowIdPrefix: stepAggBody.workflow_id_prefix,
+                        completedAfter: stepAggBody.completed_after,
+                        completedBefore: stepAggBody.completed_before,
+                        applicationName: stepAggBody.application_name,
+                    });
+                    stepAggOutput = rows.map((r) => ({
+                        group: r.group,
+                        count: r.count,
+                        max_duration_ms: r.maxDurationMs,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when getting step aggregates', e);
+                }
+                const stepAggResp = new protocol.GetStepAggregatesResponse(baseMsg.request_id, stepAggOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(stepAggResp));
+                break;
+            case protocol.MessageType.LIST_QUEUES:
+                const listQueuesMsg = baseMsg;
+                let queueOutputs = [];
+                try {
+                    const queues = await this.dbosExec.systemDatabase.listQueues(listQueuesMsg.body?.application_name);
+                    queueOutputs = queues.map((q) => ({
+                        name: q.name,
+                        concurrency: q.concurrency,
+                        worker_concurrency: q.workerConcurrency,
+                        rate_limit_max: q.rateLimitMax,
+                        rate_limit_period_sec: q.rateLimitPeriodSec,
+                        priority_enabled: true,
+                        partition_queue: q.partitionQueue,
+                        polling_interval_sec: q.pollingIntervalSec,
+                        application_name: q.applicationName ?? null,
+                        partition_concurrency: q.partitionConcurrency,
+                        partition_worker_concurrency: q.partitionWorkerConcurrency,
+                        partition_rate_limit_max: q.partitionRateLimitMax,
+                        partition_rate_limit_period_sec: q.partitionRateLimitPeriodSec,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing queues', e);
+                }
+                const listQueuesResp = new protocol.ListQueuesResponse(baseMsg.request_id, queueOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(listQueuesResp));
+                break;
+            case protocol.MessageType.GET_QUEUE:
+                const getQueueMsg = baseMsg;
+                let getQueueOutput = null;
+                try {
+                    const queue = await this.dbosExec.systemDatabase.getQueue(getQueueMsg.name);
+                    if (queue !== null) {
+                        getQueueOutput = {
+                            name: queue.name,
+                            concurrency: queue.concurrency,
+                            worker_concurrency: queue.workerConcurrency,
+                            rate_limit_max: queue.rateLimitMax,
+                            rate_limit_period_sec: queue.rateLimitPeriodSec,
+                            priority_enabled: true,
+                            partition_queue: queue.partitionQueue,
+                            polling_interval_sec: queue.pollingIntervalSec,
+                            application_name: queue.applicationName ?? null,
+                            partition_concurrency: queue.partitionConcurrency,
+                            partition_worker_concurrency: queue.partitionWorkerConcurrency,
+                            partition_rate_limit_max: queue.partitionRateLimitMax,
+                            partition_rate_limit_period_sec: queue.partitionRateLimitPeriodSec,
+                        };
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting queue ${getQueueMsg.name}`, e);
+                }
+                const getQueueResp = new protocol.GetQueueResponse(baseMsg.request_id, getQueueOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(getQueueResp));
+                break;
+            default:
+                this.dbosExec.logger.warn(`Unknown message type: ${baseMsg.type}`);
+                const unknownResp = new protocol.BaseResponse(baseMsg.type, baseMsg.request_id, 'Unknown message type');
+                currWebsocket.send(JSON.stringify(unknownResp));
+        }
+    }
+    setPingInterval(currWebsocket, currPing) {
+        clearInterval(currPing.interval);
+        clearTimeout(currPing.timeout);
+        currPing.interval = setInterval(() => {
+            currPing.timeout = setTimeout(() => {
+                if (this.isShuttingDown) {
+                    return;
+                }
+                else if (this.reconnectTimeout === undefined) {
+                    this.dbosExec.logger.warn('Connection to conductor lost. Reconnecting...');
+                    this.resetWebsocket(currWebsocket, currPing);
+                }
+            }, this.pingTimeoutMs);
+            if (currWebsocket.readyState === ws_1.default.OPEN) {
+                this.dbosExec.logger.debug('Sending ping to conductor');
+                currWebsocket.ping();
+            }
+        }, this.pingPeriodMs);
+    }
+    dispatchLoop() {
+        if (this.websocket &&
+            (this.websocket.readyState === ws_1.default.OPEN || this.websocket.readyState === ws_1.default.CONNECTING)) {
+            this.dbosExec.logger.warn('Conductor websocket already exists');
+            return;
+        }
+        clearTimeout(this.pingIntervalTimeout?.timeout);
+        clearInterval(this.pingIntervalTimeout?.interval);
+        clearTimeout(this.reconnectTimeout);
+        this.reconnectTimeout = undefined;
+        if (this.isShuttingDown) {
+            this.dbosExec.logger.debug('Not starting dispatch loop as conductor is shutting down');
+            return;
+        }
+        try {
+            this.dbosExec.logger.debug(`Connecting to conductor at ${this.url}`);
+            const currWebsocket = new ws_1.default(this.url, { handshakeTimeout: 5000 });
+            this.websocket = currWebsocket;
+            const currPing = { interval: undefined, timeout: undefined };
+            this.setPingInterval(currWebsocket, currPing);
+            this.pingIntervalTimeout = currPing;
+            currWebsocket.on('open', () => {
+                this.dbosExec.logger.debug('Opened connection to DBOS conductor');
+                clearTimeout(currPing.timeout);
+            });
+            currWebsocket.on('pong', () => {
+                this.dbosExec.logger.debug('Received pong from conductor');
+                clearTimeout(currPing.timeout);
+            });
+            currWebsocket.on('message', async (data) => {
+                this.dbosExec.logger.debug(`Received message from conductor: ${data}`);
+                const parsed = JSON.parse(data);
+                const baseMsg = parsed;
+                clearTimeout(currPing.timeout);
+                let shardKeys;
+                try {
+                    shardKeys = protocol.messageShardKeys(parsed);
+                }
+                catch (e) {
+                    if (e instanceof protocol.ShardKeyMismatchError) {
+                        currWebsocket.send(JSON.stringify(new protocol.BaseResponse(baseMsg.type, baseMsg.request_id, e.message)));
+                    }
+                    else {
+                        this.dbosExec.logger.warn(`Could not read the shard keys of a message from Conductor: ${String(e)}`);
+                    }
+                    return;
+                }
+                await (0, shard_keys_1.bindShardKeys)(shardKeys, () => this.#handleMessage(currWebsocket, baseMsg, shardKeys));
+            });
+            currWebsocket.on('close', () => {
+                if (this.isShuttingDown) {
+                    this.dbosExec.logger.info('Shutdown Conductor connection');
+                    return;
+                }
+                else if (this.reconnectTimeout === undefined) {
+                    this.dbosExec.logger.warn('Connection to conductor lost. Reconnecting.');
+                    this.resetWebsocket(currWebsocket, currPing);
+                }
+            });
+            currWebsocket.on('unexpected-response', (_, res) => {
+                const chunks = [];
+                res.on('data', (chunk) => chunks.push(chunk));
+                res.on('end', () => {
+                    const body = Buffer.concat(chunks).toString('utf-8');
+                    this.dbosExec.logger.warn(`Unexpected response from conductor: ${res.statusCode} ${res.statusMessage}. Details: ${body}`);
+                    if (this.reconnectTimeout === undefined) {
+                        this.resetWebsocket(currWebsocket, currPing);
+                    }
+                });
+            });
+            currWebsocket.on('error', (err) => {
+                this.dbosExec.logger.warn(`Unexpected exception in connection to conductor. Reconnecting: ${err.message}`);
+                if (this.reconnectTimeout === undefined) {
+                    this.resetWebsocket(currWebsocket, currPing);
+                }
+            });
+        }
+        catch (e) {
+            this.dbosExec.logger.warn(`Error in conductor loop. Reconnecting: ${e.message}`);
+            if (this.reconnectTimeout === undefined) {
+                this.resetWebsocket(this.websocket, this.pingIntervalTimeout);
+            }
+        }
+    }
+    async awaitRetention(timeoutMs = 10000) {
+        const round = this.retentionRound;
+        if (round === undefined) {
+            return;
+        }
+        let timer = undefined;
+        try {
+            await Promise.race([round, new Promise((resolve) => (timer = setTimeout(resolve, timeoutMs)))]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    async stop() {
+        this.isShuttingDown = true;
+        clearInterval(this.pingIntervalTimeout?.interval);
+        clearTimeout(this.pingIntervalTimeout?.timeout);
+        clearTimeout(this.reconnectTimeout);
+        if (this.websocket) {
+            this.websocket.close();
+        }
+        await this.awaitRetention();
+    }
+}
+exports.ConductorWebsocket = ConductorWebsocket;
+//# sourceMappingURL=conductor.js.map