@dbos-inc/dbos-enterprise 5.4.1-preview → 5.5.0

package/dist/src/conductor/conductor.js

@@ -1,3 +1,10 @@
+// Copyright (c) 2026 DBOS, Inc. All rights reserved.
+//
+// This file is part of Transact Enterprise, proprietary software licensed under the
+// DBOS Conductor Software License: https://www.dbos.dev/conductor-license
+// You may not use, copy, modify, or distribute this file except as that license permits.
+// Disabling or circumventing the entitlement checks in @dbos-inc/dbos-sdk's src/enterprise.ts
+// that gate it is a material breach of that license.
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
@@ -40,9 +47,9 @@ const fork_1 = require("../fork");
 const observability_1 = require("../observability");
 const retention_1 = require("../retention");
 const scheduler_1 = require("@dbos-inc/dbos-sdk/internal/scheduler/scheduler");
+const shard_keys_1 = require("@dbos-inc/dbos-sdk/internal/shard_keys");
 const gzip = (0, node_util_1.promisify)(zlib.gzip);
 const gunzip = (0, node_util_1.promisify)(zlib.gunzip);
-// Commands that exist only to move workflow data, refused in metadata-only mode.
 const DATA_MESSAGE_TYPES = new Set([
     protocol.MessageType.GET_WORKFLOW_EVENTS,
     protocol.MessageType.GET_WORKFLOW_NOTIFICATIONS,
@@ -50,17 +57,26 @@ const DATA_MESSAGE_TYPES = new Set([
     protocol.MessageType.EXPORT_WORKFLOW,
     protocol.MessageType.IMPORT_WORKFLOW,
 ]);
+function positionsByKey(ids, shardKeys) {
+    const groups = new Map();
+    ids.forEach((workflowID, i) => {
+        const key = shardKeys.get(workflowID);
+        const group = groups.get(key) ?? [];
+        group.push(i);
+        groups.set(key, group);
+    });
+    return Array.from(groups.values());
+}
 class ConductorWebsocket {
     dbosExec;
     url;
     websocket = undefined;
-    isShuttingDown = false; // Is in the process of shutting down the connection
-    pingPeriodMs = 20000; // Time in milliseconds to wait before sending a ping message to the conductor
-    pingTimeoutMs = 15000; // Time in milliseconds to wait for a response to a ping message before considering the connection dead
-    pingIntervalTimeout = undefined; // Combined interval and timeout for pinging Conductor
+    isShuttingDown = false;
+    pingPeriodMs = 20000;
+    pingTimeoutMs = 15000;
+    pingIntervalTimeout = undefined;
     reconnectDelayMs = 1000;
     reconnectTimeout = undefined;
-    // The retention round running off the message loop, if there is one.
     retentionRound = undefined;
     appName;
     executorMetadata;
@@ -77,7 +93,6 @@ class ConductorWebsocket {
     start() {
         this.dispatchLoop();
     }
-    /** Logs a command's exception in full and returns the error message Conductor may see. */
     reportException(context, e) {
         const errorMsg = `${context}: ${e.message}`;
         this.dbosExec.logger.error(errorMsg);
@@ -91,9 +106,8 @@ class ConductorWebsocket {
         clearInterval(currPing?.interval);
         clearTimeout(currPing?.timeout);
         if (currWebsocket) {
-            currWebsocket.terminate(); // Terminate the existing connection
+            currWebsocket.terminate();
             if (this.websocket === currWebsocket) {
-                // Only clear if it's the same websocket
                 this.websocket = undefined;
             }
         }
@@ -105,23 +119,750 @@ class ConductorWebsocket {
             this.dispatchLoop();
         }, this.reconnectDelayMs);
     }
+    async #handleMessage(currWebsocket, baseMsg, shardKeys) {
+        const msgType = baseMsg.type;
+        let errorMsg = undefined;
+        if (this.metadataOnlyMode && DATA_MESSAGE_TYPES.has(msgType)) {
+            const refusedResp = new protocol.BaseResponse(msgType, baseMsg.request_id, `${msgType} is not allowed in conductor metadata-only mode`);
+            currWebsocket.send(JSON.stringify(refusedResp));
+            return;
+        }
+        switch (msgType) {
+            case protocol.MessageType.EXECUTOR_INFO:
+                const infoResp = new protocol.ExecutorInfoResponse(baseMsg.request_id, utils_1.globalParams.executorID, utils_1.globalParams.appVersion, (0, node_os_1.hostname)(), 'typescript', utils_1.globalParams.dbosVersion, undefined, this.executorMetadata, this.dbosExec.systemDatabase.sharded);
+                currWebsocket.send(JSON.stringify(infoResp));
+                this.dbosExec.logger.info('Connected to DBOS conductor');
+                break;
+            case protocol.MessageType.RECOVERY:
+                const recoveryMsg = baseMsg;
+                let success = true;
+                try {
+                    await this.dbosExec.recoverPendingWorkflows(recoveryMsg.executor_ids);
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when recovering workflows', e);
+                    success = false;
+                }
+                const recoveryResp = new protocol.RecoveryResponse(baseMsg.request_id, success, errorMsg);
+                currWebsocket.send(JSON.stringify(recoveryResp));
+                break;
+            case protocol.MessageType.CANCEL:
+                const cancelMsg = baseMsg;
+                const cancelIds = cancelMsg.workflow_ids?.length ? cancelMsg.workflow_ids : [cancelMsg.workflow_id];
+                let cancelSuccess = true;
+                try {
+                    for (const group of positionsByKey(cancelIds, shardKeys)) {
+                        await this.dbosExec.systemDatabase.cancelWorkflows(group.map((i) => cancelIds[i]), cancelMsg.cancel_children);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when cancelling workflow(s) ${String(cancelIds)}`, e);
+                    cancelSuccess = false;
+                }
+                const cancelResp = new protocol.CancelResponse(baseMsg.request_id, cancelSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(cancelResp));
+                break;
+            case protocol.MessageType.DELETE:
+                const deleteMsg = baseMsg;
+                const deleteIds = deleteMsg.workflow_ids?.length ? deleteMsg.workflow_ids : [deleteMsg.workflow_id];
+                let deleteSuccess = true;
+                try {
+                    for (const group of positionsByKey(deleteIds, shardKeys)) {
+                        await this.dbosExec.systemDatabase.deleteWorkflows(group.map((i) => deleteIds[i]), deleteMsg.delete_children ?? false);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when deleting workflow(s) ${String(deleteIds)}`, e);
+                    deleteSuccess = false;
+                }
+                const deleteResp = new protocol.DeleteResponse(baseMsg.request_id, deleteSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(deleteResp));
+                break;
+            case protocol.MessageType.RESUME:
+                const resumeMsg = baseMsg;
+                const resumeIds = resumeMsg.workflow_ids?.length ? resumeMsg.workflow_ids : [resumeMsg.workflow_id];
+                let resumeSuccess = true;
+                try {
+                    for (const group of positionsByKey(resumeIds, shardKeys)) {
+                        await this.dbosExec.systemDatabase.resumeWorkflows(group.map((i) => resumeIds[i]), resumeMsg.queue_name);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when resuming workflow(s) ${String(resumeIds)}`, e);
+                    resumeSuccess = false;
+                }
+                const resumeResp = new protocol.ResumeResponse(baseMsg.request_id, resumeSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(resumeResp));
+                break;
+            case protocol.MessageType.REWIND_WORKFLOW:
+                const rewindMsg = baseMsg;
+                const rewindBody = rewindMsg.body;
+                let rewindSuccess = true;
+                try {
+                    await this.dbosExec.rewindWorkflow(rewindBody.workflow_id, rewindBody.start_step ?? 0, {
+                        applicationVersion: rewindBody.application_version,
+                        queueName: rewindBody.queue_name,
+                        queuePartitionKey: rewindBody.queue_partition_key,
+                    });
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when rewinding workflow ${rewindBody.workflow_id} to step ${rewindBody.start_step ?? 0}`, e);
+                    rewindSuccess = false;
+                }
+                const rewindResp = new protocol.RewindWorkflowResponse(baseMsg.request_id, rewindSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(rewindResp));
+                break;
+            case protocol.MessageType.FORK_WORKFLOW:
+                const forkMsg = baseMsg;
+                let newWorkflowID = forkMsg.body.new_workflow_id;
+                try {
+                    newWorkflowID = await this.dbosExec.forkWorkflow(forkMsg.body.workflow_id, forkMsg.body.start_step, {
+                        newWorkflowID: newWorkflowID,
+                        applicationVersion: forkMsg.body.application_version,
+                        queueName: forkMsg.body.queue_name,
+                        queuePartitionKey: forkMsg.body.queue_partition_key,
+                    });
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when forking workflow ${forkMsg.body.workflow_id} to new workflow ${newWorkflowID} on step ${forkMsg.body.start_step}, app version ${forkMsg.body.application_version}`, e);
+                    newWorkflowID = undefined;
+                }
+                const forkResp = new protocol.ForkWorkflowResponse(baseMsg.request_id, newWorkflowID, errorMsg);
+                currWebsocket.send(JSON.stringify(forkResp));
+                break;
+            case protocol.MessageType.FORK_FROM_FAILURE:
+                const forkFromFailureMsg = baseMsg;
+                const forkFromFailureBody = forkFromFailureMsg.body;
+                let forkedWorkflowIDs = undefined;
+                try {
+                    forkedWorkflowIDs = await (0, fork_1.forkFromFailure)(this.dbosExec.systemDatabase, forkFromFailureBody.workflow_ids, {
+                        applicationVersion: forkFromFailureBody.application_version,
+                        queueName: forkFromFailureBody.queue_name,
+                        queuePartitionKey: forkFromFailureBody.queue_partition_key,
+                        fromLastFailure: forkFromFailureBody.from_last_failure ?? false,
+                        fromLastStep: forkFromFailureBody.from_last_step ?? false,
+                        fromStep: forkFromFailureBody.from_step,
+                        fromStepName: forkFromFailureBody.from_step_name,
+                    });
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when bulk forking workflows', e);
+                }
+                const forkFromFailureResp = new protocol.ForkFromFailureResponse(baseMsg.request_id, forkedWorkflowIDs, errorMsg);
+                currWebsocket.send(JSON.stringify(forkFromFailureResp));
+                break;
+            case protocol.MessageType.LIST_WORKFLOWS:
+                const listWFMsg = baseMsg;
+                const body = listWFMsg.body;
+                const listWFReq = {
+                    workflowIDs: body.workflow_uuids,
+                    shardKey: body.shard_key ?? undefined,
+                    parentShardKey: body.parent_shard_key ?? undefined,
+                    workflowName: body.workflow_name,
+                    authenticatedUser: body.authenticated_user,
+                    startTime: body.start_time,
+                    endTime: body.end_time,
+                    completedAfter: body.completed_after,
+                    completedBefore: body.completed_before,
+                    dequeuedAfter: body.dequeued_after,
+                    dequeuedBefore: body.dequeued_before,
+                    status: body.status,
+                    applicationVersion: body.application_version,
+                    forkedFrom: body.forked_from,
+                    parentWorkflowID: body.parent_workflow_id,
+                    queueName: body.queue_name,
+                    limit: body.limit,
+                    offset: body.offset,
+                    sortDesc: body.sort_desc,
+                    workflow_id_prefix: body.workflow_id_prefix,
+                    loadInput: (body.load_input ?? false) && !this.metadataOnlyMode,
+                    loadOutput: (body.load_output ?? false) && !this.metadataOnlyMode,
+                    executorId: body.executor_id,
+                    queuesOnly: body.queues_only,
+                    wasForkedFrom: body.was_forked_from,
+                    isFork: body.is_fork,
+                    hasParent: body.has_parent,
+                    attributes: body.attributes,
+                    scheduleName: body.schedule_name,
+                    applicationName: body.application_name,
+                };
+                let workflowsOutput = [];
+                try {
+                    const workflows = await this.dbosExec.listWorkflows(listWFReq);
+                    workflowsOutput = workflows.map((wf) => new protocol.WorkflowsOutput(wf));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing workflows', e);
+                }
+                const wfsResp = new protocol.ListWorkflowsResponse(listWFMsg.request_id, workflowsOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(wfsResp));
+                break;
+            case protocol.MessageType.LIST_QUEUED_WORKFLOWS:
+                const listQueuedWFMsg = baseMsg;
+                const bodyQueued = listQueuedWFMsg.body;
+                const listQueuedWFReq = {
+                    workflowIDs: bodyQueued.workflow_uuids,
+                    shardKey: bodyQueued.shard_key ?? undefined,
+                    parentShardKey: bodyQueued.parent_shard_key ?? undefined,
+                    workflowName: bodyQueued.workflow_name,
+                    authenticatedUser: bodyQueued.authenticated_user,
+                    startTime: bodyQueued.start_time,
+                    endTime: bodyQueued.end_time,
+                    completedAfter: bodyQueued.completed_after,
+                    completedBefore: bodyQueued.completed_before,
+                    dequeuedAfter: bodyQueued.dequeued_after,
+                    dequeuedBefore: bodyQueued.dequeued_before,
+                    status: bodyQueued.status,
+                    applicationVersion: bodyQueued.application_version,
+                    forkedFrom: bodyQueued.forked_from,
+                    parentWorkflowID: bodyQueued.parent_workflow_id,
+                    queueName: bodyQueued.queue_name,
+                    limit: bodyQueued.limit,
+                    offset: bodyQueued.offset,
+                    sortDesc: bodyQueued.sort_desc,
+                    workflow_id_prefix: bodyQueued.workflow_id_prefix,
+                    loadInput: (bodyQueued.load_input ?? false) && !this.metadataOnlyMode,
+                    loadOutput: (bodyQueued.load_output ?? false) && !this.metadataOnlyMode,
+                    executorId: bodyQueued.executor_id,
+                    wasForkedFrom: bodyQueued.was_forked_from,
+                    isFork: bodyQueued.is_fork,
+                    hasParent: bodyQueued.has_parent,
+                    attributes: bodyQueued.attributes,
+                    scheduleName: bodyQueued.schedule_name,
+                    applicationName: bodyQueued.application_name,
+                };
+                let queuedWFOutput = [];
+                try {
+                    const workflows = await this.dbosExec.listQueuedWorkflows(listQueuedWFReq);
+                    queuedWFOutput = workflows.map((wf) => new protocol.WorkflowsOutput(wf));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing queued workflows', e);
+                }
+                const queuedWfsResp = new protocol.ListQueuedWorkflowsResponse(listQueuedWFMsg.request_id, queuedWFOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(queuedWfsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW:
+                const getWFMsg = baseMsg;
+                let wfOutput = undefined;
+                try {
+                    const workflows = await this.dbosExec.listWorkflows({
+                        workflowIDs: [getWFMsg.workflow_id],
+                        shardKey: this.dbosExec.systemDatabase.keyHint(getWFMsg.workflow_id),
+                        loadInput: (getWFMsg.load_input ?? true) && !this.metadataOnlyMode,
+                        loadOutput: (getWFMsg.load_output ?? true) && !this.metadataOnlyMode,
+                    });
+                    if (workflows.length > 0) {
+                        wfOutput = new protocol.WorkflowsOutput(workflows[0]);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting workflow ${getWFMsg.workflow_id}`, e);
+                }
+                const getWFResp = new protocol.GetWorkflowResponse(getWFMsg.request_id, wfOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(getWFResp));
+                break;
+            case protocol.MessageType.EXIST_PENDING_WORKFLOWS:
+                const existPendingMsg = baseMsg;
+                let hasPendingWFs = false;
+                try {
+                    const pendingWFs = await (0, observability_1.getPendingWorkflows)(this.dbosExec.systemDatabase, existPendingMsg.executor_id, existPendingMsg.application_version);
+                    hasPendingWFs = pendingWFs.length > 0;
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when checking for pending workflows', e);
+                }
+                const existPendingResp = new protocol.ExistPendingWorkflowsResponse(baseMsg.request_id, hasPendingWFs, errorMsg);
+                currWebsocket.send(JSON.stringify(existPendingResp));
+                break;
+            case protocol.MessageType.LIST_STEPS:
+                const listStepsMessage = baseMsg;
+                let workflowSteps = undefined;
+                try {
+                    const stepsInfo = await this.dbosExec.listWorkflowSteps(listStepsMessage.workflow_id, (listStepsMessage.load_output ?? true) && !this.metadataOnlyMode, { limit: listStepsMessage.limit, offset: listStepsMessage.offset });
+                    workflowSteps = stepsInfo?.map((i) => new protocol.WorkflowSteps(i));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when listing steps ${listStepsMessage.workflow_id}`, e);
+                }
+                const listStepsResponse = new protocol.ListStepsResponse(listStepsMessage.request_id, workflowSteps, errorMsg);
+                currWebsocket.send(JSON.stringify(listStepsResponse));
+                break;
+            case protocol.MessageType.RETENTION:
+                const retentionMessage = baseMsg;
+                const retentionBody = retentionMessage.body;
+                let rowsThreshold = retentionBody.gc_rows_threshold;
+                if (rowsThreshold !== undefined && rowsThreshold !== null && this.dbosExec.systemDatabase.sharded) {
+                    this.dbosExec.logger.warn("Ignoring Conductor's rows-based retention: a sharded system database supports only time-based retention.");
+                    rowsThreshold = undefined;
+                }
+                if (this.retentionRound !== undefined) {
+                    this.dbosExec.logger.warn('Skipping retention: the previous round on this executor is still running.');
+                }
+                else {
+                    this.retentionRound = (async () => {
+                        try {
+                            await (0, retention_1.garbageCollect)(this.dbosExec.systemDatabase, retentionBody.gc_cutoff_epoch_ms, rowsThreshold, {
+                                batchSize: retentionBody.gc_batch_size || undefined,
+                                externalStores: this.dbosExec.externalStorage?.stores,
+                            });
+                            if (retentionBody.timeout_cutoff_epoch_ms) {
+                                await (0, retention_1.globalTimeout)(this.dbosExec.systemDatabase, retentionBody.timeout_cutoff_epoch_ms);
+                            }
+                        }
+                        catch (e) {
+                            if (this.isShuttingDown) {
+                                this.dbosExec.logger.debug('Retention interrupted by shutdown');
+                                return;
+                            }
+                            this.dbosExec.logger.error(`Exception encountered during enforcing retention policy: ${e.message}`);
+                        }
+                    })().finally(() => {
+                        this.retentionRound = undefined;
+                    });
+                }
+                currWebsocket.send(JSON.stringify(new protocol.RetentionResponse(retentionMessage.request_id, true)));
+                break;
+            case protocol.MessageType.GET_METRICS:
+                const getMetricsMessage = baseMsg;
+                this.dbosExec.logger.debug(`Received metrics request for time range ${getMetricsMessage.start_time} to ${getMetricsMessage.end_time}`);
+                let metricsData = [];
+                if (getMetricsMessage.metric_class === 'workflow_step_count') {
+                    try {
+                        const sysMetrics = await (0, observability_1.getMetrics)(this.dbosExec.systemDatabase, getMetricsMessage.start_time, getMetricsMessage.end_time, getMetricsMessage.application_name);
+                        metricsData = sysMetrics.map((m) => new protocol.MetricDataOutput(m.metricType, m.metricName, m.value));
+                    }
+                    catch (e) {
+                        errorMsg = this.reportException('Exception encountered when getting metrics', e);
+                    }
+                }
+                else {
+                    errorMsg = `Unexpected metric class: ${getMetricsMessage.metric_class}`;
+                    this.dbosExec.logger.warn(errorMsg);
+                }
+                const getMetricsResponse = new protocol.GetMetricsResponse(getMetricsMessage.request_id, metricsData, errorMsg);
+                currWebsocket.send(JSON.stringify(getMetricsResponse));
+                break;
+            case protocol.MessageType.EXPORT_WORKFLOW:
+                const exportMsg = baseMsg;
+                let serializedWorkflow = null;
+                try {
+                    const exported = await (0, export_1.exportWorkflow)(this.dbosExec.systemDatabase, exportMsg.workflow_id, exportMsg.export_children ?? false);
+                    if (exported.length > 0) {
+                        const jsonStr = JSON.stringify(exported);
+                        const compressed = await gzip(jsonStr);
+                        serializedWorkflow = compressed.toString('base64');
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when exporting workflow ${exportMsg.workflow_id}`, e);
+                }
+                const exportResp = new protocol.ExportWorkflowResponse(baseMsg.request_id, serializedWorkflow, errorMsg);
+                currWebsocket.send(JSON.stringify(exportResp));
+                break;
+            case protocol.MessageType.IMPORT_WORKFLOW:
+                const importMsg = baseMsg;
+                let importSuccess = true;
+                try {
+                    const compressedData = Buffer.from(importMsg.serialized_workflow, 'base64');
+                    const decompressed = await gunzip(compressedData);
+                    const workflows = JSON.parse(decompressed.toString());
+                    await (0, export_1.importWorkflow)(this.dbosExec.systemDatabase, workflows);
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when importing workflow', e);
+                    importSuccess = false;
+                }
+                const importResp = new protocol.ImportWorkflowResponse(baseMsg.request_id, importSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(importResp));
+                break;
+            case protocol.MessageType.ALERT:
+                const alertMsg = baseMsg;
+                let alertSuccess = true;
+                try {
+                    const handler = (0, decorators_1.getAlertHandler)();
+                    if (handler) {
+                        await Promise.resolve(handler(alertMsg.name, alertMsg.message, alertMsg.metadata));
+                    }
+                    else {
+                        this.dbosExec.logger.warn(`Alert received (no handler registered): [${alertMsg.name}] ${alertMsg.message} | Metadata: ${JSON.stringify(alertMsg.metadata)}`);
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception in alert handler', e);
+                    alertSuccess = false;
+                }
+                const alertResp = new protocol.AlertResponse(baseMsg.request_id, alertSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(alertResp));
+                break;
+            case protocol.MessageType.LIST_SCHEDULES:
+                const listSchedMsg = baseMsg;
+                const loadContextList = (listSchedMsg.body.load_context ?? true) && !this.metadataOnlyMode;
+                let schedOutput = [];
+                try {
+                    const scheds = await this.dbosExec.systemDatabase.listSchedules({
+                        status: listSchedMsg.body.status,
+                        workflowName: listSchedMsg.body.workflow_name,
+                        scheduleNamePrefix: listSchedMsg.body.schedule_name_prefix,
+                        applicationName: listSchedMsg.body.application_name,
+                    });
+                    schedOutput = await Promise.all(scheds.map(async (s) => ({
+                        schedule_id: s.scheduleId,
+                        schedule_name: s.scheduleName,
+                        workflow_name: s.workflowName,
+                        workflow_class_name: s.workflowClassName || undefined,
+                        schedule: s.schedule,
+                        status: s.status,
+                        context: loadContextList
+                            ? protocol.inspectForConductor(await this.dbosExec.serializer.parse(s.context))
+                            : undefined,
+                        last_fired_at: s.lastFiredAt,
+                        automatic_backfill: s.automaticBackfill,
+                        cron_timezone: s.cronTimezone,
+                        queue_name: s.queueName,
+                        application_name: s.applicationName ?? null,
+                        shard_key: s.shardKey ?? null,
+                    })));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing schedules', e);
+                }
+                const listSchedResp = new protocol.ListSchedulesResponse(listSchedMsg.request_id, schedOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(listSchedResp));
+                break;
+            case protocol.MessageType.GET_SCHEDULE:
+                const getSchedMsg = baseMsg;
+                const loadContextGet = (getSchedMsg.load_context ?? true) && !this.metadataOnlyMode;
+                let getSchedOutput = undefined;
+                try {
+                    const sched = await this.dbosExec.systemDatabase.getSchedule(getSchedMsg.schedule_name);
+                    if (sched) {
+                        getSchedOutput = {
+                            schedule_id: sched.scheduleId,
+                            schedule_name: sched.scheduleName,
+                            workflow_name: sched.workflowName,
+                            workflow_class_name: sched.workflowClassName || undefined,
+                            schedule: sched.schedule,
+                            status: sched.status,
+                            context: loadContextGet
+                                ? protocol.inspectForConductor(await this.dbosExec.serializer.parse(sched.context))
+                                : undefined,
+                            last_fired_at: sched.lastFiredAt,
+                            automatic_backfill: sched.automaticBackfill,
+                            cron_timezone: sched.cronTimezone,
+                            queue_name: sched.queueName,
+                            application_name: sched.applicationName ?? null,
+                            shard_key: sched.shardKey ?? null,
+                        };
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting schedule ${getSchedMsg.schedule_name}`, e);
+                }
+                const getSchedResp = new protocol.GetScheduleResponse(getSchedMsg.request_id, getSchedOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(getSchedResp));
+                break;
+            case protocol.MessageType.PAUSE_SCHEDULE:
+                const pauseSchedMsg = baseMsg;
+                let pauseSuccess = true;
+                try {
+                    await this.dbosExec.systemDatabase.setScheduleStatus(pauseSchedMsg.schedule_name, 'PAUSED');
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when pausing schedule '${pauseSchedMsg.schedule_name}'`, e);
+                    pauseSuccess = false;
+                }
+                const pauseSchedResp = new protocol.PauseScheduleResponse(baseMsg.request_id, pauseSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(pauseSchedResp));
+                break;
+            case protocol.MessageType.RESUME_SCHEDULE:
+                const resumeSchedMsg = baseMsg;
+                let resumeSchedSuccess = true;
+                try {
+                    await this.dbosExec.systemDatabase.setScheduleStatus(resumeSchedMsg.schedule_name, 'ACTIVE');
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when resuming schedule '${resumeSchedMsg.schedule_name}'`, e);
+                    resumeSchedSuccess = false;
+                }
+                const resumeSchedResp = new protocol.ResumeScheduleResponse(baseMsg.request_id, resumeSchedSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(resumeSchedResp));
+                break;
+            case protocol.MessageType.BACKFILL_SCHEDULE:
+                const backfillSchedMsg = baseMsg;
+                let backfillWorkflowIds = [];
+                try {
+                    backfillWorkflowIds = await (0, scheduler_1.backfillSchedule)(this.dbosExec.systemDatabase, this.dbosExec.serializer, backfillSchedMsg.schedule_name, new Date(backfillSchedMsg.start), new Date(backfillSchedMsg.end));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when backfilling schedule '${backfillSchedMsg.schedule_name}'`, e);
+                }
+                const backfillSchedResp = new protocol.BackfillScheduleResponse(baseMsg.request_id, backfillWorkflowIds, errorMsg);
+                currWebsocket.send(JSON.stringify(backfillSchedResp));
+                break;
+            case protocol.MessageType.TRIGGER_SCHEDULE:
+                const triggerSchedMsg = baseMsg;
+                let triggerWorkflowId = undefined;
+                try {
+                    triggerWorkflowId = await (0, scheduler_1.triggerSchedule)(this.dbosExec.systemDatabase, this.dbosExec.serializer, triggerSchedMsg.schedule_name);
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when triggering schedule '${triggerSchedMsg.schedule_name}'`, e);
+                }
+                const triggerSchedResp = new protocol.TriggerScheduleResponse(baseMsg.request_id, triggerWorkflowId, errorMsg);
+                currWebsocket.send(JSON.stringify(triggerSchedResp));
+                break;
+            case protocol.MessageType.LIST_APPLICATION_VERSIONS:
+                let versionsOutput = [];
+                try {
+                    const versions = await this.dbosExec.systemDatabase.listApplicationVersions();
+                    versionsOutput = versions.map((v) => ({
+                        version_id: v.versionId,
+                        version_name: v.versionName,
+                        version_timestamp: v.versionTimestamp,
+                        created_at: v.createdAt,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing application versions', e);
+                }
+                const listVersionsResp = new protocol.ListApplicationVersionsResponse(baseMsg.request_id, versionsOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(listVersionsResp));
+                break;
+            case protocol.MessageType.SET_LATEST_APPLICATION_VERSION:
+                const setVersionMsg = baseMsg;
+                let setVersionSuccess = true;
+                try {
+                    await this.dbosExec.systemDatabase.updateApplicationVersionTimestamp(setVersionMsg.version_name, Date.now());
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when setting latest application version '${setVersionMsg.version_name}'`, e);
+                    setVersionSuccess = false;
+                }
+                const setVersionResp = new protocol.SetLatestApplicationVersionResponse(baseMsg.request_id, setVersionSuccess, errorMsg);
+                currWebsocket.send(JSON.stringify(setVersionResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_EVENTS:
+                const eventsMsg = baseMsg;
+                let eventOutputs = undefined;
+                try {
+                    const rawEvents = await (0, observability_1.getAllEvents)(this.dbosExec.systemDatabase, eventsMsg.workflow_id);
+                    eventOutputs = Object.entries(rawEvents).map(([key, value]) => ({
+                        key,
+                        value: protocol.inspectForConductor(value),
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting events for workflow ${eventsMsg.workflow_id}`, e);
+                }
+                const eventsResp = new protocol.GetWorkflowEventsResponse(baseMsg.request_id, eventOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(eventsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_NOTIFICATIONS:
+                const notifsMsg = baseMsg;
+                let notifOutputs = undefined;
+                try {
+                    const rawNotifs = await (0, observability_1.getAllNotifications)(this.dbosExec.systemDatabase, notifsMsg.workflow_id);
+                    notifOutputs = rawNotifs.map((n) => ({
+                        topic: n.topic,
+                        message: protocol.inspectForConductor(n.message),
+                        created_at_epoch_ms: n.createdAtEpochMs,
+                        consumed: n.consumed,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting notifications for workflow ${notifsMsg.workflow_id}`, e);
+                }
+                const notifsResp = new protocol.GetWorkflowNotificationsResponse(baseMsg.request_id, notifOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(notifsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_STREAMS:
+                const streamsMsg = baseMsg;
+                let streamOutputs = undefined;
+                try {
+                    const rawStreams = await (0, observability_1.getAllStreamEntries)(this.dbosExec.systemDatabase, streamsMsg.workflow_id);
+                    streamOutputs = Object.entries(rawStreams).map(([key, values]) => ({
+                        key,
+                        values: values.map((v) => protocol.inspectForConductor(v)),
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting streams for workflow ${streamsMsg.workflow_id}`, e);
+                }
+                const streamsResp = new protocol.GetWorkflowStreamsResponse(baseMsg.request_id, streamOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(streamsResp));
+                break;
+            case protocol.MessageType.GET_WORKFLOW_AGGREGATES:
+                const aggMsg = baseMsg;
+                const aggBody = aggMsg.body;
+                let aggOutput = [];
+                let selectCount = aggBody.select_count ?? false;
+                const selectMinCreatedAt = aggBody.select_min_created_at ?? false;
+                const selectMaxQueueWaitMs = aggBody.select_max_queue_wait_ms ?? false;
+                const selectMaxTotalLatencyMs = aggBody.select_max_total_latency_ms ?? false;
+                if (!(selectCount || selectMinCreatedAt || selectMaxQueueWaitMs || selectMaxTotalLatencyMs)) {
+                    selectCount = true;
+                }
+                try {
+                    const rows = await (0, observability_1.getWorkflowAggregates)(this.dbosExec.systemDatabase, {
+                        groupByStatus: aggBody.group_by_status ?? false,
+                        groupByName: aggBody.group_by_name ?? false,
+                        groupByQueueName: aggBody.group_by_queue_name ?? false,
+                        groupByExecutorId: aggBody.group_by_executor_id ?? false,
+                        groupByApplicationVersion: aggBody.group_by_application_version ?? false,
+                        groupByApplicationName: aggBody.group_by_application_name ?? false,
+                        selectCount,
+                        selectMinCreatedAt,
+                        selectMaxQueueWaitMs,
+                        selectMaxTotalLatencyMs,
+                        timeBucketSizeMs: aggBody.time_bucket_size_ms ?? undefined,
+                        status: aggBody.status,
+                        startTime: aggBody.start_time,
+                        endTime: aggBody.end_time,
+                        completedAfter: aggBody.completed_after,
+                        completedBefore: aggBody.completed_before,
+                        dequeuedAfter: aggBody.dequeued_after,
+                        dequeuedBefore: aggBody.dequeued_before,
+                        name: aggBody.name,
+                        appVersion: aggBody.app_version,
+                        executorId: aggBody.executor_id,
+                        queueName: aggBody.queue_name,
+                        workflowIdPrefix: aggBody.workflow_id_prefix,
+                        workflowIDs: aggBody.workflow_ids,
+                        authenticatedUser: aggBody.user,
+                        forkedFrom: aggBody.forked_from,
+                        parentWorkflowID: aggBody.parent_workflow_id,
+                        scheduleName: aggBody.schedule_name,
+                        applicationName: aggBody.application_name,
+                        wasForkedFrom: aggBody.was_forked_from,
+                        isFork: aggBody.is_fork,
+                        hasParent: aggBody.has_parent,
+                        attributes: aggBody.attributes,
+                    });
+                    aggOutput = rows.map((r) => ({
+                        group: r.group,
+                        count: r.count,
+                        min_created_at: r.minCreatedAt,
+                        max_queue_wait_ms: r.maxQueueWaitMs,
+                        max_total_latency_ms: r.maxTotalLatencyMs,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when getting workflow aggregates', e);
+                }
+                const aggResp = new protocol.GetWorkflowAggregatesResponse(baseMsg.request_id, aggOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(aggResp));
+                break;
+            case protocol.MessageType.GET_STEP_AGGREGATES:
+                const stepAggMsg = baseMsg;
+                const stepAggBody = stepAggMsg.body;
+                let stepAggOutput = [];
+                let stepSelectCount = stepAggBody.select_count ?? false;
+                const stepSelectMaxDurationMs = stepAggBody.select_max_duration_ms ?? false;
+                if (!(stepSelectCount || stepSelectMaxDurationMs)) {
+                    stepSelectCount = true;
+                }
+                try {
+                    const rows = await (0, observability_1.getStepAggregates)(this.dbosExec.systemDatabase, {
+                        groupByFunctionName: stepAggBody.group_by_function_name ?? false,
+                        groupByStatus: stepAggBody.group_by_status ?? false,
+                        selectCount: stepSelectCount,
+                        selectMaxDurationMs: stepSelectMaxDurationMs,
+                        timeBucketSizeMs: stepAggBody.time_bucket_size_ms ?? undefined,
+                        status: stepAggBody.status,
+                        functionName: stepAggBody.function_name,
+                        workflowIdPrefix: stepAggBody.workflow_id_prefix,
+                        completedAfter: stepAggBody.completed_after,
+                        completedBefore: stepAggBody.completed_before,
+                        applicationName: stepAggBody.application_name,
+                    });
+                    stepAggOutput = rows.map((r) => ({
+                        group: r.group,
+                        count: r.count,
+                        max_duration_ms: r.maxDurationMs,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when getting step aggregates', e);
+                }
+                const stepAggResp = new protocol.GetStepAggregatesResponse(baseMsg.request_id, stepAggOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(stepAggResp));
+                break;
+            case protocol.MessageType.LIST_QUEUES:
+                const listQueuesMsg = baseMsg;
+                let queueOutputs = [];
+                try {
+                    const queues = await this.dbosExec.systemDatabase.listQueues(listQueuesMsg.body?.application_name);
+                    queueOutputs = queues.map((q) => ({
+                        name: q.name,
+                        concurrency: q.concurrency,
+                        worker_concurrency: q.workerConcurrency,
+                        rate_limit_max: q.rateLimitMax,
+                        rate_limit_period_sec: q.rateLimitPeriodSec,
+                        priority_enabled: true,
+                        partition_queue: q.partitionQueue,
+                        polling_interval_sec: q.pollingIntervalSec,
+                        application_name: q.applicationName ?? null,
+                        partition_concurrency: q.partitionConcurrency,
+                        partition_worker_concurrency: q.partitionWorkerConcurrency,
+                        partition_rate_limit_max: q.partitionRateLimitMax,
+                        partition_rate_limit_period_sec: q.partitionRateLimitPeriodSec,
+                    }));
+                }
+                catch (e) {
+                    errorMsg = this.reportException('Exception encountered when listing queues', e);
+                }
+                const listQueuesResp = new protocol.ListQueuesResponse(baseMsg.request_id, queueOutputs, errorMsg);
+                currWebsocket.send(JSON.stringify(listQueuesResp));
+                break;
+            case protocol.MessageType.GET_QUEUE:
+                const getQueueMsg = baseMsg;
+                let getQueueOutput = null;
+                try {
+                    const queue = await this.dbosExec.systemDatabase.getQueue(getQueueMsg.name);
+                    if (queue !== null) {
+                        getQueueOutput = {
+                            name: queue.name,
+                            concurrency: queue.concurrency,
+                            worker_concurrency: queue.workerConcurrency,
+                            rate_limit_max: queue.rateLimitMax,
+                            rate_limit_period_sec: queue.rateLimitPeriodSec,
+                            priority_enabled: true,
+                            partition_queue: queue.partitionQueue,
+                            polling_interval_sec: queue.pollingIntervalSec,
+                            application_name: queue.applicationName ?? null,
+                            partition_concurrency: queue.partitionConcurrency,
+                            partition_worker_concurrency: queue.partitionWorkerConcurrency,
+                            partition_rate_limit_max: queue.partitionRateLimitMax,
+                            partition_rate_limit_period_sec: queue.partitionRateLimitPeriodSec,
+                        };
+                    }
+                }
+                catch (e) {
+                    errorMsg = this.reportException(`Exception encountered when getting queue ${getQueueMsg.name}`, e);
+                }
+                const getQueueResp = new protocol.GetQueueResponse(baseMsg.request_id, getQueueOutput, errorMsg);
+                currWebsocket.send(JSON.stringify(getQueueResp));
+                break;
+            default:
+                this.dbosExec.logger.warn(`Unknown message type: ${baseMsg.type}`);
+                const unknownResp = new protocol.BaseResponse(baseMsg.type, baseMsg.request_id, 'Unknown message type');
+                currWebsocket.send(JSON.stringify(unknownResp));
+        }
+    }
     setPingInterval(currWebsocket, currPing) {
-        // Clear any existing ping interval to avoid multiple intervals being set
         clearInterval(currPing.interval);
         clearTimeout(currPing.timeout);
         currPing.interval = setInterval(() => {
-            // Set ping timeout before sending ping. This prevents the socket from hanging at a non-open state indefinitely
             currPing.timeout = setTimeout(() => {
                 if (this.isShuttingDown) {
                     return;
                 }
                 else if (this.reconnectTimeout === undefined) {
-                    // Otherwise, try to reconnect if we haven't already
                     this.dbosExec.logger.warn('Connection to conductor lost. Reconnecting...');
                     this.resetWebsocket(currWebsocket, currPing);
                 }
             }, this.pingTimeoutMs);
-            // Only ping if the connection is open
             if (currWebsocket.readyState === ws_1.default.OPEN) {
                 this.dbosExec.logger.debug('Sending ping to conductor');
                 currWebsocket.ping();
@@ -144,10 +885,8 @@ class ConductorWebsocket {
         }
         try {
             this.dbosExec.logger.debug(`Connecting to conductor at ${this.url}`);
-            // Start a new websocket connection
             const currWebsocket = new ws_1.default(this.url, { handshakeTimeout: 5000 });
             this.websocket = currWebsocket;
-            // Start ping interval
             const currPing = { interval: undefined, timeout: undefined };
             this.setPingInterval(currWebsocket, currPing);
             this.pingIntervalTimeout = currPing;
@@ -161,724 +900,23 @@ class ConductorWebsocket {
             });
             currWebsocket.on('message', async (data) => {
                 this.dbosExec.logger.debug(`Received message from conductor: ${data}`);
-                const baseMsg = JSON.parse(data);
-                const msgType = baseMsg.type;
-                let errorMsg = undefined;
+                const parsed = JSON.parse(data);
+                const baseMsg = parsed;
                 clearTimeout(currPing.timeout);
-                if (this.metadataOnlyMode && DATA_MESSAGE_TYPES.has(msgType)) {
-                    const refusedResp = new protocol.BaseResponse(msgType, baseMsg.request_id, `${msgType} is not allowed in conductor metadata-only mode`);
-                    currWebsocket.send(JSON.stringify(refusedResp));
-                    return;
+                let shardKeys;
+                try {
+                    shardKeys = protocol.messageShardKeys(parsed);
                 }
-                switch (msgType) {
-                    case protocol.MessageType.EXECUTOR_INFO:
-                        const infoResp = new protocol.ExecutorInfoResponse(baseMsg.request_id, utils_1.globalParams.executorID, utils_1.globalParams.appVersion, (0, node_os_1.hostname)(), 'typescript', utils_1.globalParams.dbosVersion, undefined, this.executorMetadata);
-                        currWebsocket.send(JSON.stringify(infoResp));
-                        this.dbosExec.logger.info('Connected to DBOS conductor');
-                        break;
-                    case protocol.MessageType.RECOVERY:
-                        const recoveryMsg = baseMsg;
-                        let success = true;
-                        try {
-                            await this.dbosExec.recoverPendingWorkflows(recoveryMsg.executor_ids);
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when recovering workflows', e);
-                            success = false;
-                        }
-                        const recoveryResp = new protocol.RecoveryResponse(baseMsg.request_id, success, errorMsg);
-                        currWebsocket.send(JSON.stringify(recoveryResp));
-                        break;
-                    case protocol.MessageType.CANCEL:
-                        const cancelMsg = baseMsg;
-                        const cancelIds = cancelMsg.workflow_ids ?? [cancelMsg.workflow_id];
-                        let cancelSuccess = true;
-                        try {
-                            await this.dbosExec.systemDatabase.cancelWorkflows(cancelIds, cancelMsg.cancel_children);
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when cancelling workflow(s) ${String(cancelIds)}`, e);
-                            cancelSuccess = false;
-                        }
-                        const cancelResp = new protocol.CancelResponse(baseMsg.request_id, cancelSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(cancelResp));
-                        break;
-                    case protocol.MessageType.DELETE:
-                        const deleteMsg = baseMsg;
-                        const deleteIds = deleteMsg.workflow_ids ?? [deleteMsg.workflow_id];
-                        let deleteSuccess = true;
-                        try {
-                            await this.dbosExec.systemDatabase.deleteWorkflows(deleteIds, deleteMsg.delete_children ?? false);
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when deleting workflow(s) ${String(deleteIds)}`, e);
-                            deleteSuccess = false;
-                        }
-                        const deleteResp = new protocol.DeleteResponse(baseMsg.request_id, deleteSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(deleteResp));
-                        break;
-                    case protocol.MessageType.RESUME:
-                        const resumeMsg = baseMsg;
-                        const resumeIds = resumeMsg.workflow_ids ?? [resumeMsg.workflow_id];
-                        let resumeSuccess = true;
-                        try {
-                            await this.dbosExec.systemDatabase.resumeWorkflows(resumeIds, resumeMsg.queue_name);
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when resuming workflow(s) ${String(resumeIds)}`, e);
-                            resumeSuccess = false;
-                        }
-                        const resumeResp = new protocol.ResumeResponse(baseMsg.request_id, resumeSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(resumeResp));
-                        break;
-                    case protocol.MessageType.REWIND_WORKFLOW:
-                        const rewindMsg = baseMsg;
-                        const rewindBody = rewindMsg.body;
-                        let rewindSuccess = true;
-                        try {
-                            await this.dbosExec.rewindWorkflow(rewindBody.workflow_id, rewindBody.start_step ?? 0, {
-                                applicationVersion: rewindBody.application_version,
-                                queueName: rewindBody.queue_name,
-                                queuePartitionKey: rewindBody.queue_partition_key,
-                            });
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when rewinding workflow ${rewindBody.workflow_id} to step ${rewindBody.start_step ?? 0}`, e);
-                            rewindSuccess = false;
-                        }
-                        const rewindResp = new protocol.RewindWorkflowResponse(baseMsg.request_id, rewindSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(rewindResp));
-                        break;
-                    case protocol.MessageType.FORK_WORKFLOW:
-                        const forkMsg = baseMsg;
-                        let newWorkflowID = forkMsg.body.new_workflow_id;
-                        try {
-                            newWorkflowID = await this.dbosExec.forkWorkflow(forkMsg.body.workflow_id, forkMsg.body.start_step, {
-                                newWorkflowID: newWorkflowID,
-                                applicationVersion: forkMsg.body.application_version,
-                                queueName: forkMsg.body.queue_name,
-                                queuePartitionKey: forkMsg.body.queue_partition_key,
-                            });
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when forking workflow ${forkMsg.body.workflow_id} to new workflow ${newWorkflowID} on step ${forkMsg.body.start_step}, app version ${forkMsg.body.application_version}`, e);
-                            newWorkflowID = undefined;
-                        }
-                        const forkResp = new protocol.ForkWorkflowResponse(baseMsg.request_id, newWorkflowID, errorMsg);
-                        currWebsocket.send(JSON.stringify(forkResp));
-                        break;
-                    case protocol.MessageType.FORK_FROM_FAILURE:
-                        const forkFromFailureMsg = baseMsg;
-                        const forkFromFailureBody = forkFromFailureMsg.body;
-                        let forkedWorkflowIDs = undefined;
-                        try {
-                            forkedWorkflowIDs = await (0, fork_1.forkFromFailure)(this.dbosExec.systemDatabase, forkFromFailureBody.workflow_ids, {
-                                applicationVersion: forkFromFailureBody.application_version,
-                                queueName: forkFromFailureBody.queue_name,
-                                queuePartitionKey: forkFromFailureBody.queue_partition_key,
-                                fromLastFailure: forkFromFailureBody.from_last_failure ?? false,
-                                fromLastStep: forkFromFailureBody.from_last_step ?? false,
-                                fromStep: forkFromFailureBody.from_step,
-                                fromStepName: forkFromFailureBody.from_step_name,
-                            });
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when bulk forking workflows', e);
-                        }
-                        const forkFromFailureResp = new protocol.ForkFromFailureResponse(baseMsg.request_id, forkedWorkflowIDs, errorMsg);
-                        currWebsocket.send(JSON.stringify(forkFromFailureResp));
-                        break;
-                    case protocol.MessageType.LIST_WORKFLOWS:
-                        const listWFMsg = baseMsg;
-                        const body = listWFMsg.body;
-                        const listWFReq = {
-                            workflowIDs: body.workflow_uuids,
-                            workflowName: body.workflow_name,
-                            authenticatedUser: body.authenticated_user,
-                            startTime: body.start_time,
-                            endTime: body.end_time,
-                            completedAfter: body.completed_after,
-                            completedBefore: body.completed_before,
-                            dequeuedAfter: body.dequeued_after,
-                            dequeuedBefore: body.dequeued_before,
-                            status: body.status,
-                            applicationVersion: body.application_version,
-                            forkedFrom: body.forked_from,
-                            parentWorkflowID: body.parent_workflow_id,
-                            queueName: body.queue_name,
-                            limit: body.limit,
-                            offset: body.offset,
-                            sortDesc: body.sort_desc,
-                            workflow_id_prefix: body.workflow_id_prefix,
-                            loadInput: (body.load_input ?? false) && !this.metadataOnlyMode, // Default to false if not provided
-                            loadOutput: (body.load_output ?? false) && !this.metadataOnlyMode, // Default to false if not provided
-                            executorId: body.executor_id,
-                            queuesOnly: body.queues_only,
-                            wasForkedFrom: body.was_forked_from,
-                            isFork: body.is_fork,
-                            hasParent: body.has_parent,
-                            attributes: body.attributes,
-                            scheduleName: body.schedule_name,
-                            applicationName: body.application_name,
-                        };
-                        let workflowsOutput = [];
-                        try {
-                            const workflows = await this.dbosExec.listWorkflows(listWFReq);
-                            workflowsOutput = workflows.map((wf) => new protocol.WorkflowsOutput(wf));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when listing workflows', e);
-                        }
-                        const wfsResp = new protocol.ListWorkflowsResponse(listWFMsg.request_id, workflowsOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(wfsResp));
-                        break;
-                    case protocol.MessageType.LIST_QUEUED_WORKFLOWS:
-                        const listQueuedWFMsg = baseMsg;
-                        const bodyQueued = listQueuedWFMsg.body;
-                        const listQueuedWFReq = {
-                            workflowIDs: bodyQueued.workflow_uuids,
-                            workflowName: bodyQueued.workflow_name,
-                            authenticatedUser: bodyQueued.authenticated_user,
-                            startTime: bodyQueued.start_time,
-                            endTime: bodyQueued.end_time,
-                            completedAfter: bodyQueued.completed_after,
-                            completedBefore: bodyQueued.completed_before,
-                            dequeuedAfter: bodyQueued.dequeued_after,
-                            dequeuedBefore: bodyQueued.dequeued_before,
-                            status: bodyQueued.status,
-                            applicationVersion: bodyQueued.application_version,
-                            forkedFrom: bodyQueued.forked_from,
-                            parentWorkflowID: bodyQueued.parent_workflow_id,
-                            queueName: bodyQueued.queue_name,
-                            limit: bodyQueued.limit,
-                            offset: bodyQueued.offset,
-                            sortDesc: bodyQueued.sort_desc,
-                            workflow_id_prefix: bodyQueued.workflow_id_prefix,
-                            loadInput: (bodyQueued.load_input ?? false) && !this.metadataOnlyMode, // Default to false if not provided
-                            loadOutput: (bodyQueued.load_output ?? false) && !this.metadataOnlyMode, // Default to false if not provided
-                            executorId: bodyQueued.executor_id,
-                            wasForkedFrom: bodyQueued.was_forked_from,
-                            isFork: bodyQueued.is_fork,
-                            hasParent: bodyQueued.has_parent,
-                            attributes: bodyQueued.attributes,
-                            scheduleName: bodyQueued.schedule_name,
-                            applicationName: bodyQueued.application_name,
-                        };
-                        let queuedWFOutput = [];
-                        try {
-                            const workflows = await this.dbosExec.listQueuedWorkflows(listQueuedWFReq);
-                            queuedWFOutput = workflows.map((wf) => new protocol.WorkflowsOutput(wf));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when listing queued workflows', e);
-                        }
-                        const queuedWfsResp = new protocol.ListQueuedWorkflowsResponse(listQueuedWFMsg.request_id, queuedWFOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(queuedWfsResp));
-                        break;
-                    case protocol.MessageType.GET_WORKFLOW:
-                        const getWFMsg = baseMsg;
-                        let wfOutput = undefined;
-                        try {
-                            const workflows = await this.dbosExec.listWorkflows({
-                                workflowIDs: [getWFMsg.workflow_id],
-                                loadInput: (getWFMsg.load_input ?? true) && !this.metadataOnlyMode,
-                                loadOutput: (getWFMsg.load_output ?? true) && !this.metadataOnlyMode,
-                            });
-                            if (workflows.length > 0) {
-                                wfOutput = new protocol.WorkflowsOutput(workflows[0]);
-                            }
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when getting workflow ${getWFMsg.workflow_id}`, e);
-                        }
-                        const getWFResp = new protocol.GetWorkflowResponse(getWFMsg.request_id, wfOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(getWFResp));
-                        break;
-                    case protocol.MessageType.EXIST_PENDING_WORKFLOWS:
-                        const existPendingMsg = baseMsg;
-                        let hasPendingWFs = false;
-                        try {
-                            const pendingWFs = await (0, observability_1.getPendingWorkflows)(this.dbosExec.systemDatabase, existPendingMsg.executor_id, existPendingMsg.application_version);
-                            hasPendingWFs = pendingWFs.length > 0;
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when checking for pending workflows', e);
-                        }
-                        const existPendingResp = new protocol.ExistPendingWorkflowsResponse(baseMsg.request_id, hasPendingWFs, errorMsg);
-                        currWebsocket.send(JSON.stringify(existPendingResp));
-                        break;
-                    case protocol.MessageType.LIST_STEPS:
-                        const listStepsMessage = baseMsg;
-                        let workflowSteps = undefined;
-                        try {
-                            const stepsInfo = await this.dbosExec.listWorkflowSteps(listStepsMessage.workflow_id, (listStepsMessage.load_output ?? true) && !this.metadataOnlyMode, { limit: listStepsMessage.limit, offset: listStepsMessage.offset });
-                            workflowSteps = stepsInfo?.map((i) => new protocol.WorkflowSteps(i));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when listing steps ${listStepsMessage.workflow_id}`, e);
-                        }
-                        const listStepsResponse = new protocol.ListStepsResponse(listStepsMessage.request_id, workflowSteps, errorMsg);
-                        currWebsocket.send(JSON.stringify(listStepsResponse));
-                        break;
-                    case protocol.MessageType.RETENTION:
-                        const retentionMessage = baseMsg;
-                        const retentionBody = retentionMessage.body;
-                        // Off the message loop: a round takes minutes, and awaiting it here stops every
-                        // other command and lets the dispatch time out into a second round on another executor.
-                        if (this.retentionRound !== undefined) {
-                            this.dbosExec.logger.warn('Skipping retention: the previous round on this executor is still running.');
-                        }
-                        else {
-                            this.retentionRound = (async () => {
-                                try {
-                                    await (0, retention_1.garbageCollect)(this.dbosExec.systemDatabase, retentionBody.gc_cutoff_epoch_ms, retentionBody.gc_rows_threshold, 
-                                    // An omitted, null, or zero gc_batch_size takes the default.
-                                    { batchSize: retentionBody.gc_batch_size || undefined });
-                                    if (retentionBody.timeout_cutoff_epoch_ms) {
-                                        await (0, retention_1.globalTimeout)(this.dbosExec.systemDatabase, retentionBody.timeout_cutoff_epoch_ms);
-                                    }
-                                }
-                                catch (e) {
-                                    if (this.isShuttingDown) {
-                                        // Shutdown took the system database away mid-round; the next round resumes the work.
-                                        this.dbosExec.logger.debug('Retention interrupted by shutdown');
-                                        return;
-                                    }
-                                    this.dbosExec.logger.error(`Exception encountered during enforcing retention policy: ${e.message}`);
-                                }
-                            })().finally(() => {
-                                this.retentionRound = undefined;
-                            });
-                        }
-                        currWebsocket.send(JSON.stringify(new protocol.RetentionResponse(retentionMessage.request_id, true)));
-                        break;
-                    case protocol.MessageType.GET_METRICS:
-                        const getMetricsMessage = baseMsg;
-                        this.dbosExec.logger.debug(`Received metrics request for time range ${getMetricsMessage.start_time} to ${getMetricsMessage.end_time}`);
-                        let metricsData = [];
-                        if (getMetricsMessage.metric_class === 'workflow_step_count') {
-                            try {
-                                const sysMetrics = await (0, observability_1.getMetrics)(this.dbosExec.systemDatabase, getMetricsMessage.start_time, getMetricsMessage.end_time, getMetricsMessage.application_name);
-                                metricsData = sysMetrics.map((m) => new protocol.MetricDataOutput(m.metricType, m.metricName, m.value));
-                            }
-                            catch (e) {
-                                errorMsg = this.reportException('Exception encountered when getting metrics', e);
-                            }
-                        }
-                        else {
-                            errorMsg = `Unexpected metric class: ${getMetricsMessage.metric_class}`;
-                            this.dbosExec.logger.warn(errorMsg);
-                        }
-                        const getMetricsResponse = new protocol.GetMetricsResponse(getMetricsMessage.request_id, metricsData, errorMsg);
-                        currWebsocket.send(JSON.stringify(getMetricsResponse));
-                        break;
-                    case protocol.MessageType.EXPORT_WORKFLOW:
-                        const exportMsg = baseMsg;
-                        let serializedWorkflow = null;
-                        try {
-                            const exported = await (0, export_1.exportWorkflow)(this.dbosExec.systemDatabase, exportMsg.workflow_id, exportMsg.export_children ?? false);
-                            if (exported.length > 0) {
-                                const jsonStr = JSON.stringify(exported);
-                                const compressed = await gzip(jsonStr);
-                                serializedWorkflow = compressed.toString('base64');
-                            }
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when exporting workflow ${exportMsg.workflow_id}`, e);
-                        }
-                        const exportResp = new protocol.ExportWorkflowResponse(baseMsg.request_id, serializedWorkflow, errorMsg);
-                        currWebsocket.send(JSON.stringify(exportResp));
-                        break;
-                    case protocol.MessageType.IMPORT_WORKFLOW:
-                        const importMsg = baseMsg;
-                        let importSuccess = true;
-                        try {
-                            const compressedData = Buffer.from(importMsg.serialized_workflow, 'base64');
-                            const decompressed = await gunzip(compressedData);
-                            const workflows = JSON.parse(decompressed.toString());
-                            await (0, export_1.importWorkflow)(this.dbosExec.systemDatabase, workflows);
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when importing workflow', e);
-                            importSuccess = false;
-                        }
-                        const importResp = new protocol.ImportWorkflowResponse(baseMsg.request_id, importSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(importResp));
-                        break;
-                    case protocol.MessageType.ALERT:
-                        const alertMsg = baseMsg;
-                        let alertSuccess = true;
-                        try {
-                            const handler = (0, decorators_1.getAlertHandler)();
-                            if (handler) {
-                                await Promise.resolve(handler(alertMsg.name, alertMsg.message, alertMsg.metadata));
-                            }
-                            else {
-                                this.dbosExec.logger.warn(`Alert received (no handler registered): [${alertMsg.name}] ${alertMsg.message} | Metadata: ${JSON.stringify(alertMsg.metadata)}`);
-                            }
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception in alert handler', e);
-                            alertSuccess = false;
-                        }
-                        const alertResp = new protocol.AlertResponse(baseMsg.request_id, alertSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(alertResp));
-                        break;
-                    case protocol.MessageType.LIST_SCHEDULES:
-                        const listSchedMsg = baseMsg;
-                        const loadContextList = (listSchedMsg.body.load_context ?? true) && !this.metadataOnlyMode;
-                        let schedOutput = [];
-                        try {
-                            const scheds = await this.dbosExec.systemDatabase.listSchedules({
-                                status: listSchedMsg.body.status,
-                                workflowName: listSchedMsg.body.workflow_name,
-                                scheduleNamePrefix: listSchedMsg.body.schedule_name_prefix,
-                                applicationName: listSchedMsg.body.application_name,
-                            });
-                            schedOutput = await Promise.all(scheds.map(async (s) => ({
-                                schedule_id: s.scheduleId,
-                                schedule_name: s.scheduleName,
-                                workflow_name: s.workflowName,
-                                workflow_class_name: s.workflowClassName || undefined,
-                                schedule: s.schedule,
-                                status: s.status,
-                                context: loadContextList
-                                    ? protocol.inspectForConductor(await this.dbosExec.serializer.parse(s.context))
-                                    : undefined,
-                                last_fired_at: s.lastFiredAt,
-                                automatic_backfill: s.automaticBackfill,
-                                cron_timezone: s.cronTimezone,
-                                queue_name: s.queueName,
-                                application_name: s.applicationName ?? null,
-                            })));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when listing schedules', e);
-                        }
-                        const listSchedResp = new protocol.ListSchedulesResponse(listSchedMsg.request_id, schedOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(listSchedResp));
-                        break;
-                    case protocol.MessageType.GET_SCHEDULE:
-                        const getSchedMsg = baseMsg;
-                        const loadContextGet = (getSchedMsg.load_context ?? true) && !this.metadataOnlyMode;
-                        let getSchedOutput = undefined;
-                        try {
-                            const sched = await this.dbosExec.systemDatabase.getSchedule(getSchedMsg.schedule_name);
-                            if (sched) {
-                                getSchedOutput = {
-                                    schedule_id: sched.scheduleId,
-                                    schedule_name: sched.scheduleName,
-                                    workflow_name: sched.workflowName,
-                                    workflow_class_name: sched.workflowClassName || undefined,
-                                    schedule: sched.schedule,
-                                    status: sched.status,
-                                    context: loadContextGet
-                                        ? protocol.inspectForConductor(await this.dbosExec.serializer.parse(sched.context))
-                                        : undefined,
-                                    last_fired_at: sched.lastFiredAt,
-                                    automatic_backfill: sched.automaticBackfill,
-                                    cron_timezone: sched.cronTimezone,
-                                    queue_name: sched.queueName,
-                                    application_name: sched.applicationName ?? null,
-                                };
-                            }
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when getting schedule ${getSchedMsg.schedule_name}`, e);
-                        }
-                        const getSchedResp = new protocol.GetScheduleResponse(getSchedMsg.request_id, getSchedOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(getSchedResp));
-                        break;
-                    case protocol.MessageType.PAUSE_SCHEDULE:
-                        const pauseSchedMsg = baseMsg;
-                        let pauseSuccess = true;
-                        try {
-                            await this.dbosExec.systemDatabase.setScheduleStatus(pauseSchedMsg.schedule_name, 'PAUSED');
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when pausing schedule '${pauseSchedMsg.schedule_name}'`, e);
-                            pauseSuccess = false;
-                        }
-                        const pauseSchedResp = new protocol.PauseScheduleResponse(baseMsg.request_id, pauseSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(pauseSchedResp));
-                        break;
-                    case protocol.MessageType.RESUME_SCHEDULE:
-                        const resumeSchedMsg = baseMsg;
-                        let resumeSchedSuccess = true;
-                        try {
-                            await this.dbosExec.systemDatabase.setScheduleStatus(resumeSchedMsg.schedule_name, 'ACTIVE');
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when resuming schedule '${resumeSchedMsg.schedule_name}'`, e);
-                            resumeSchedSuccess = false;
-                        }
-                        const resumeSchedResp = new protocol.ResumeScheduleResponse(baseMsg.request_id, resumeSchedSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(resumeSchedResp));
-                        break;
-                    case protocol.MessageType.BACKFILL_SCHEDULE:
-                        const backfillSchedMsg = baseMsg;
-                        let backfillWorkflowIds = [];
-                        try {
-                            backfillWorkflowIds = await (0, scheduler_1.backfillSchedule)(this.dbosExec.systemDatabase, this.dbosExec.serializer, backfillSchedMsg.schedule_name, new Date(backfillSchedMsg.start), new Date(backfillSchedMsg.end));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when backfilling schedule '${backfillSchedMsg.schedule_name}'`, e);
-                        }
-                        const backfillSchedResp = new protocol.BackfillScheduleResponse(baseMsg.request_id, backfillWorkflowIds, errorMsg);
-                        currWebsocket.send(JSON.stringify(backfillSchedResp));
-                        break;
-                    case protocol.MessageType.TRIGGER_SCHEDULE:
-                        const triggerSchedMsg = baseMsg;
-                        let triggerWorkflowId = undefined;
-                        try {
-                            triggerWorkflowId = await (0, scheduler_1.triggerSchedule)(this.dbosExec.systemDatabase, this.dbosExec.serializer, triggerSchedMsg.schedule_name);
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when triggering schedule '${triggerSchedMsg.schedule_name}'`, e);
-                        }
-                        const triggerSchedResp = new protocol.TriggerScheduleResponse(baseMsg.request_id, triggerWorkflowId, errorMsg);
-                        currWebsocket.send(JSON.stringify(triggerSchedResp));
-                        break;
-                    case protocol.MessageType.LIST_APPLICATION_VERSIONS:
-                        let versionsOutput = [];
-                        try {
-                            const versions = await this.dbosExec.systemDatabase.listApplicationVersions();
-                            versionsOutput = versions.map((v) => ({
-                                version_id: v.versionId,
-                                version_name: v.versionName,
-                                version_timestamp: v.versionTimestamp,
-                                created_at: v.createdAt,
-                            }));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when listing application versions', e);
-                        }
-                        const listVersionsResp = new protocol.ListApplicationVersionsResponse(baseMsg.request_id, versionsOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(listVersionsResp));
-                        break;
-                    case protocol.MessageType.SET_LATEST_APPLICATION_VERSION:
-                        const setVersionMsg = baseMsg;
-                        let setVersionSuccess = true;
-                        try {
-                            await this.dbosExec.systemDatabase.updateApplicationVersionTimestamp(setVersionMsg.version_name, Date.now());
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when setting latest application version '${setVersionMsg.version_name}'`, e);
-                            setVersionSuccess = false;
-                        }
-                        const setVersionResp = new protocol.SetLatestApplicationVersionResponse(baseMsg.request_id, setVersionSuccess, errorMsg);
-                        currWebsocket.send(JSON.stringify(setVersionResp));
-                        break;
-                    case protocol.MessageType.GET_WORKFLOW_EVENTS:
-                        const eventsMsg = baseMsg;
-                        let eventOutputs = undefined;
-                        try {
-                            const rawEvents = await (0, observability_1.getAllEvents)(this.dbosExec.systemDatabase, eventsMsg.workflow_id);
-                            eventOutputs = Object.entries(rawEvents).map(([key, value]) => ({
-                                key,
-                                value: protocol.inspectForConductor(value),
-                            }));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when getting events for workflow ${eventsMsg.workflow_id}`, e);
-                        }
-                        const eventsResp = new protocol.GetWorkflowEventsResponse(baseMsg.request_id, eventOutputs, errorMsg);
-                        currWebsocket.send(JSON.stringify(eventsResp));
-                        break;
-                    case protocol.MessageType.GET_WORKFLOW_NOTIFICATIONS:
-                        const notifsMsg = baseMsg;
-                        let notifOutputs = undefined;
-                        try {
-                            const rawNotifs = await (0, observability_1.getAllNotifications)(this.dbosExec.systemDatabase, notifsMsg.workflow_id);
-                            notifOutputs = rawNotifs.map((n) => ({
-                                topic: n.topic,
-                                message: protocol.inspectForConductor(n.message),
-                                created_at_epoch_ms: n.createdAtEpochMs,
-                                consumed: n.consumed,
-                            }));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when getting notifications for workflow ${notifsMsg.workflow_id}`, e);
-                        }
-                        const notifsResp = new protocol.GetWorkflowNotificationsResponse(baseMsg.request_id, notifOutputs, errorMsg);
-                        currWebsocket.send(JSON.stringify(notifsResp));
-                        break;
-                    case protocol.MessageType.GET_WORKFLOW_STREAMS:
-                        const streamsMsg = baseMsg;
-                        let streamOutputs = undefined;
-                        try {
-                            const rawStreams = await (0, observability_1.getAllStreamEntries)(this.dbosExec.systemDatabase, streamsMsg.workflow_id);
-                            streamOutputs = Object.entries(rawStreams).map(([key, values]) => ({
-                                key,
-                                values: values.map((v) => protocol.inspectForConductor(v)),
-                            }));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when getting streams for workflow ${streamsMsg.workflow_id}`, e);
-                        }
-                        const streamsResp = new protocol.GetWorkflowStreamsResponse(baseMsg.request_id, streamOutputs, errorMsg);
-                        currWebsocket.send(JSON.stringify(streamsResp));
-                        break;
-                    case protocol.MessageType.GET_WORKFLOW_AGGREGATES:
-                        const aggMsg = baseMsg;
-                        const aggBody = aggMsg.body;
-                        let aggOutput = [];
-                        // Backwards compat: older clients send no select_* flags and expect counts.
-                        let selectCount = aggBody.select_count ?? false;
-                        const selectMinCreatedAt = aggBody.select_min_created_at ?? false;
-                        const selectMaxQueueWaitMs = aggBody.select_max_queue_wait_ms ?? false;
-                        const selectMaxTotalLatencyMs = aggBody.select_max_total_latency_ms ?? false;
-                        if (!(selectCount || selectMinCreatedAt || selectMaxQueueWaitMs || selectMaxTotalLatencyMs)) {
-                            selectCount = true;
-                        }
-                        try {
-                            const rows = await (0, observability_1.getWorkflowAggregates)(this.dbosExec.systemDatabase, {
-                                groupByStatus: aggBody.group_by_status ?? false,
-                                groupByName: aggBody.group_by_name ?? false,
-                                groupByQueueName: aggBody.group_by_queue_name ?? false,
-                                groupByExecutorId: aggBody.group_by_executor_id ?? false,
-                                groupByApplicationVersion: aggBody.group_by_application_version ?? false,
-                                groupByApplicationName: aggBody.group_by_application_name ?? false,
-                                selectCount,
-                                selectMinCreatedAt,
-                                selectMaxQueueWaitMs,
-                                selectMaxTotalLatencyMs,
-                                timeBucketSizeMs: aggBody.time_bucket_size_ms ?? undefined,
-                                status: aggBody.status,
-                                startTime: aggBody.start_time,
-                                endTime: aggBody.end_time,
-                                completedAfter: aggBody.completed_after,
-                                completedBefore: aggBody.completed_before,
-                                dequeuedAfter: aggBody.dequeued_after,
-                                dequeuedBefore: aggBody.dequeued_before,
-                                name: aggBody.name,
-                                appVersion: aggBody.app_version,
-                                executorId: aggBody.executor_id,
-                                queueName: aggBody.queue_name,
-                                workflowIdPrefix: aggBody.workflow_id_prefix,
-                                workflowIDs: aggBody.workflow_ids,
-                                authenticatedUser: aggBody.user,
-                                forkedFrom: aggBody.forked_from,
-                                parentWorkflowID: aggBody.parent_workflow_id,
-                                scheduleName: aggBody.schedule_name,
-                                applicationName: aggBody.application_name,
-                                wasForkedFrom: aggBody.was_forked_from,
-                                isFork: aggBody.is_fork,
-                                hasParent: aggBody.has_parent,
-                                attributes: aggBody.attributes,
-                            });
-                            aggOutput = rows.map((r) => ({
-                                group: r.group,
-                                count: r.count,
-                                min_created_at: r.minCreatedAt,
-                                max_queue_wait_ms: r.maxQueueWaitMs,
-                                max_total_latency_ms: r.maxTotalLatencyMs,
-                            }));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when getting workflow aggregates', e);
-                        }
-                        const aggResp = new protocol.GetWorkflowAggregatesResponse(baseMsg.request_id, aggOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(aggResp));
-                        break;
-                    case protocol.MessageType.GET_STEP_AGGREGATES:
-                        const stepAggMsg = baseMsg;
-                        const stepAggBody = stepAggMsg.body;
-                        let stepAggOutput = [];
-                        // Backwards compat: older clients send no select_* flags and expect counts.
-                        let stepSelectCount = stepAggBody.select_count ?? false;
-                        const stepSelectMaxDurationMs = stepAggBody.select_max_duration_ms ?? false;
-                        if (!(stepSelectCount || stepSelectMaxDurationMs)) {
-                            stepSelectCount = true;
-                        }
-                        try {
-                            const rows = await (0, observability_1.getStepAggregates)(this.dbosExec.systemDatabase, {
-                                groupByFunctionName: stepAggBody.group_by_function_name ?? false,
-                                groupByStatus: stepAggBody.group_by_status ?? false,
-                                selectCount: stepSelectCount,
-                                selectMaxDurationMs: stepSelectMaxDurationMs,
-                                timeBucketSizeMs: stepAggBody.time_bucket_size_ms ?? undefined,
-                                status: stepAggBody.status,
-                                functionName: stepAggBody.function_name,
-                                workflowIdPrefix: stepAggBody.workflow_id_prefix,
-                                completedAfter: stepAggBody.completed_after,
-                                completedBefore: stepAggBody.completed_before,
-                                applicationName: stepAggBody.application_name,
-                            });
-                            stepAggOutput = rows.map((r) => ({
-                                group: r.group,
-                                count: r.count,
-                                max_duration_ms: r.maxDurationMs,
-                            }));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when getting step aggregates', e);
-                        }
-                        const stepAggResp = new protocol.GetStepAggregatesResponse(baseMsg.request_id, stepAggOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(stepAggResp));
-                        break;
-                    case protocol.MessageType.LIST_QUEUES:
-                        const listQueuesMsg = baseMsg;
-                        let queueOutputs = [];
-                        try {
-                            const queues = await this.dbosExec.systemDatabase.listQueues(listQueuesMsg.body?.application_name);
-                            queueOutputs = queues.map((q) => ({
-                                name: q.name,
-                                concurrency: q.concurrency,
-                                worker_concurrency: q.workerConcurrency,
-                                rate_limit_max: q.rateLimitMax,
-                                rate_limit_period_sec: q.rateLimitPeriodSec,
-                                priority_enabled: true,
-                                partition_queue: q.partitionQueue,
-                                polling_interval_sec: q.pollingIntervalSec,
-                                application_name: q.applicationName ?? null,
-                                partition_concurrency: q.partitionConcurrency,
-                                partition_worker_concurrency: q.partitionWorkerConcurrency,
-                                partition_rate_limit_max: q.partitionRateLimitMax,
-                                partition_rate_limit_period_sec: q.partitionRateLimitPeriodSec,
-                            }));
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException('Exception encountered when listing queues', e);
-                        }
-                        const listQueuesResp = new protocol.ListQueuesResponse(baseMsg.request_id, queueOutputs, errorMsg);
-                        currWebsocket.send(JSON.stringify(listQueuesResp));
-                        break;
-                    case protocol.MessageType.GET_QUEUE:
-                        const getQueueMsg = baseMsg;
-                        let getQueueOutput = null;
-                        try {
-                            const queue = await this.dbosExec.systemDatabase.getQueue(getQueueMsg.name);
-                            if (queue !== null) {
-                                getQueueOutput = {
-                                    name: queue.name,
-                                    concurrency: queue.concurrency,
-                                    worker_concurrency: queue.workerConcurrency,
-                                    rate_limit_max: queue.rateLimitMax,
-                                    rate_limit_period_sec: queue.rateLimitPeriodSec,
-                                    priority_enabled: true,
-                                    partition_queue: queue.partitionQueue,
-                                    polling_interval_sec: queue.pollingIntervalSec,
-                                    application_name: queue.applicationName ?? null,
-                                    partition_concurrency: queue.partitionConcurrency,
-                                    partition_worker_concurrency: queue.partitionWorkerConcurrency,
-                                    partition_rate_limit_max: queue.partitionRateLimitMax,
-                                    partition_rate_limit_period_sec: queue.partitionRateLimitPeriodSec,
-                                };
-                            }
-                        }
-                        catch (e) {
-                            errorMsg = this.reportException(`Exception encountered when getting queue ${getQueueMsg.name}`, e);
-                        }
-                        const getQueueResp = new protocol.GetQueueResponse(baseMsg.request_id, getQueueOutput, errorMsg);
-                        currWebsocket.send(JSON.stringify(getQueueResp));
-                        break;
-                    default:
-                        this.dbosExec.logger.warn(`Unknown message type: ${baseMsg.type}`);
-                        // Still need to send a response to the conductor
-                        const unknownResp = new protocol.BaseResponse(baseMsg.type, baseMsg.request_id, 'Unknown message type');
-                        currWebsocket.send(JSON.stringify(unknownResp));
+                catch (e) {
+                    if (e instanceof protocol.ShardKeyMismatchError) {
+                        currWebsocket.send(JSON.stringify(new protocol.BaseResponse(baseMsg.type, baseMsg.request_id, e.message)));
+                    }
+                    else {
+                        this.dbosExec.logger.warn(`Could not read the shard keys of a message from Conductor: ${String(e)}`);
+                    }
+                    return;
                 }
+                await (0, shard_keys_1.bindShardKeys)(shardKeys, () => this.#handleMessage(currWebsocket, baseMsg, shardKeys));
             });
             currWebsocket.on('close', () => {
                 if (this.isShuttingDown) {
@@ -915,10 +953,6 @@ class ConductorWebsocket {
             }
         }
     }
-    /**
-     * Give a round in flight a bounded chance to finish before shutdown takes the system
-     * database away from it.
-     */
     async awaitRetention(timeoutMs = 10000) {
         const round = this.retentionRound;
         if (round === undefined) {
@@ -940,7 +974,6 @@ class ConductorWebsocket {
         if (this.websocket) {
             this.websocket.close();
         }
-        // Grace only: a round still running past this is cut when the system database is destroyed.
         await this.awaitRetention();
     }
 }