@webex/internal-plugin-mercury 3.9.0 → 3.10.0-multi-llms.1

package/src/mercury.js

@@ -6,7 +6,7 @@
 import url from 'url';
 
 import {WebexPlugin} from '@webex/webex-core';
-import {deprecated, oneFlight} from '@webex/common';
+import {deprecated} from '@webex/common';
 import {camelCase, get, set} from 'lodash';
 import backoff from 'backoff';
 
@@ -25,6 +25,7 @@ const normalReconnectReasons = ['idle', 'done (forced)', 'pong not received', 'p
 const Mercury = WebexPlugin.extend({
   namespace: 'Mercury',
   lastError: undefined,
+  defaultSessionId: 'mercury-default-session',
 
   session: {
     connected: {
@@ -39,7 +40,18 @@ const Mercury = WebexPlugin.extend({
       default: false,
       type: 'boolean',
     },
-    socket: 'object',
+    sockets: {
+      default: () => new Map(),
+      type: 'object',
+    },
+    backoffCalls: {
+      default: () => new Map(),
+      type: 'object',
+    },
+    _shutdownSwitchoverBackoffCalls: {
+      default: () => new Map(),
+      type: 'object',
+    },
     localClusterServiceUrls: 'object',
     mercuryTimeOffset: {
       default: undefined,
@@ -68,6 +80,121 @@ const Mercury = WebexPlugin.extend({
         this.webex.internal.feature.updateFeature(envelope.data.featureToggle);
       }
     });
+    /*
+     * When Cluster Migrations, notify clients using ActiveClusterStatusEvent via mercury
+     * https://wwwin-github.cisco.com/pages/Webex/crr-docs/techdocs/rr-002.html#wip-notifying-clients-of-cluster-migrations
+     * */
+    this.on('event:ActiveClusterStatusEvent', (envelope) => {
+      if (
+        typeof this.webex.internal.services?.switchActiveClusterIds === 'function' &&
+        envelope &&
+        envelope.data
+      ) {
+        this.webex.internal.services.switchActiveClusterIds(envelope.data?.activeClusters);
+      }
+    });
+    /*
+     * Using cache-invalidation via mercury to instead the method of polling via the new /timestamp endpoint from u2c
+     * https://wwwin-github.cisco.com/pages/Webex/crr-docs/techdocs/rr-005.html#websocket-notifications
+     * */
+    this.on('event:u2c.cache-invalidation', (envelope) => {
+      if (
+        typeof this.webex.internal.services?.invalidateCache === 'function' &&
+        envelope &&
+        envelope.data
+      ) {
+        this.webex.internal.services.invalidateCache(envelope.data?.timestamp);
+      }
+    });
+  },
+
+  /**
+   * Attach event listeners to a socket.
+   * @param {Socket} socket - The socket to attach listeners to
+   * @param {sessionId} sessionId - The socket related session ID
+   * @returns {void}
+   */
+  _attachSocketEventListeners(socket, sessionId) {
+    socket.on('close', (event) => this._onclose(sessionId, event, socket));
+    socket.on('message', (...args) => this._onmessage(sessionId, ...args));
+    socket.on('pong', (...args) => this._setTimeOffset(sessionId, ...args));
+    socket.on('sequence-mismatch', (...args) =>
+      this._emit(sessionId, 'sequence-mismatch', ...args)
+    );
+    socket.on('ping-pong-latency', (...args) =>
+      this._emit(sessionId, 'ping-pong-latency', ...args)
+    );
+  },
+
+  /**
+   * Handle imminent shutdown by establishing a new connection while keeping
+   * the current one alive (make-before-break).
+   * Idempotent: will no-op if already in progress.
+   * @param {string} sessionId - The session ID for which the shutdown is imminent
+   * @returns {void}
+   */
+  _handleImminentShutdown(sessionId) {
+    const oldSocket = this.sockets.get(sessionId);
+
+    try {
+      if (this._shutdownSwitchoverBackoffCalls.get(sessionId)) {
+        this.logger.info(
+          `${this.namespace}: [shutdown] switchover already in progress for ${sessionId}`
+        );
+
+        return;
+      }
+      this._shutdownSwitchoverId = `${Date.now()}`;
+      this.logger.info(
+        `${this.namespace}: [shutdown] switchover start, id=${this._shutdownSwitchoverId} for ${sessionId}`
+      );
+
+      this._connectWithBackoff(undefined, sessionId, {
+        isShutdownSwitchover: true,
+        attemptOptions: {
+          isShutdownSwitchover: true,
+          onSuccess: (newSocket, webSocketUrl) => {
+            this.logger.info(
+              `${this.namespace}: [shutdown] switchover connected, url: ${webSocketUrl} for ${sessionId}`
+            );
+
+            // Atomically switch active socket reference
+            this.socket = this.sockets.get(this.defaultSessionId) || newSocket;
+            this.connected = this.hasConnectedSockets(); // remain connected throughout
+
+            this._emit(sessionId, 'event:mercury_shutdown_switchover_complete', {
+              url: webSocketUrl,
+            });
+
+            if (oldSocket) {
+              this.logger.info(
+                `${this.namespace}: [shutdown] old socket retained; server will close with 4001`
+              );
+            }
+          },
+        },
+      })
+        .then(() => {
+          this.logger.info(
+            `${this.namespace}: [shutdown] switchover completed successfully for ${sessionId}`
+          );
+        })
+        .catch((err) => {
+          this.logger.info(
+            `${this.namespace}: [shutdown] switchover exhausted retries; will fall back to normal reconnection for ${sessionId}: `,
+            err
+          );
+          this._emit(sessionId, 'event:mercury_shutdown_switchover_failed', {reason: err});
+          // Old socket will eventually close with 4001, triggering normal reconnection
+        });
+    } catch (e) {
+      this.logger.error(
+        `${this.namespace}: [shutdown] error during switchover for ${sessionId}`,
+        e
+      );
+      this._shutdownSwitchoverBackoffCalls.delete(sessionId);
+      this._emit(sessionId, 'event:mercury_shutdown_switchover_failed', {reason: e});
+    }
   },
 
   /**
@@ -78,29 +205,96 @@ const Mercury = WebexPlugin.extend({
     return this.lastError;
   },
 
-  @oneFlight
-  connect(webSocketUrl) {
-    if (this.connected) {
-      this.logger.info(`${this.namespace}: already connected, will not connect again`);
+  /**
+   * Get all active socket connections
+   * @returns {Map} Map of sessionId to socket instances
+   */
+  getSockets() {
+    return this.sockets;
+  },
+
+  /**
+   * Get a specific socket by connection ID
+   * @param {string} sessionId - The connection identifier
+   * @returns {Socket|undefined} The socket instance or undefined if not found
+   */
+  getSocket(sessionId = this.defaultSessionId) {
+    return this.sockets.get(sessionId);
+  },
+
+  /**
+   * Check if any sockets are connected
+   * @returns {boolean} True if at least one socket is connected
+   */
+  hasConnectedSockets() {
+    for (const socket of this.sockets.values()) {
+      if (socket && socket.connected) {
+        return true;
+      }
+    }
+
+    return false;
+  },
+
+  /**
+   * Check if any sockets are connecting
+   * @returns {boolean} True if at least one socket is connected
+   */
+  hasConnectingSockets() {
+    for (const socket of this.sockets.values()) {
+      if (socket && socket.connecting) {
+        return true;
+      }
+    }
+
+    return false;
+  },
+
+  // @oneFlight
+  connect(webSocketUrl, sessionId = this.defaultSessionId) {
+    if (!this._connectPromises) this._connectPromises = new Map();
+
+    // First check if there's already a connection promise for this session
+    if (this._connectPromises.has(sessionId)) {
+      this.logger.info(
+        `${this.namespace}: connection ${sessionId} already in progress, returning existing promise`
+      );
+
+      return this._connectPromises.get(sessionId);
+    }
+
+    const sessionSocket = this.sockets.get(sessionId);
+    if (sessionSocket?.connected || sessionSocket?.connecting) {
+      this.logger.info(
+        `${this.namespace}: connection ${sessionId} already connected, will not connect again`
+      );
 
       return Promise.resolve();
     }
 
     this.connecting = true;
 
-    this.logger.info(`${this.namespace}: starting connection attempt`);
+    this.logger.info(`${this.namespace}: starting connection attempt for ${sessionId}`);
     this.logger.info(
       `${this.namespace}: debug_mercury_logging stack: `,
       new Error('debug_mercury_logging').stack
     );
 
-    return Promise.resolve(
+    const connectPromise = Promise.resolve(
       this.webex.internal.device.registered || this.webex.internal.device.register()
-    ).then(() => {
-      this.logger.info(`${this.namespace}: connecting`);
+    )
+      .then(() => {
+        this.logger.info(`${this.namespace}: connecting ${sessionId}`);
 
-      return this._connectWithBackoff(webSocketUrl);
-    });
+        return this._connectWithBackoff(webSocketUrl, sessionId);
+      })
+      .finally(() => {
+        this._connectPromises.delete(sessionId);
+      });
+
+    this._connectPromises.set(sessionId, connectPromise);
+
+    return connectPromise;
   },
 
   logout() {
@@ -110,7 +304,7 @@ const Mercury = WebexPlugin.extend({
       new Error('debug_mercury_logging').stack
     );
 
-    return this.disconnect(
+    return this.disconnectAll(
       this.config.beforeLogoutOptionsCloseReason &&
         !normalReconnectReasons.includes(this.config.beforeLogoutOptionsCloseReason)
         ? {code: 3050, reason: this.config.beforeLogoutOptionsCloseReason}
@@ -118,21 +312,63 @@ const Mercury = WebexPlugin.extend({
     );
   },
 
-  @oneFlight
-  disconnect(options) {
+  // @oneFlight
+  disconnect(options, sessionId = this.defaultSessionId) {
     return new Promise((resolve) => {
-      if (this.backoffCall) {
-        this.logger.info(`${this.namespace}: aborting connection`);
-        this.backoffCall.abort();
+      const backoffCall = this.backoffCalls.get(sessionId);
+      if (backoffCall) {
+        this.logger.info(`${this.namespace}: aborting connection ${sessionId}`);
+        backoffCall.abort();
+        this.backoffCalls.delete(sessionId);
       }
-
-      if (this.socket) {
-        this.socket.removeAllListeners('message');
-        this.once('offline', resolve);
-        resolve(this.socket.close(options || undefined));
+      const shutdownSwitchoverBackoffCalls = this._shutdownSwitchoverBackoffCalls.get(sessionId);
+      if (shutdownSwitchoverBackoffCalls) {
+        this.logger.info(`${this.namespace}: aborting shutdown switchover connection ${sessionId}`);
+        shutdownSwitchoverBackoffCalls.abort();
+        this._shutdownSwitchoverBackoffCalls.delete(sessionId);
       }
+      // Clean up any pending connection promises
+      if (this._connectPromises) {
+        this._connectPromises.delete(sessionId);
+      }
+
+      const sessionSocket = this.sockets.get(sessionId);
+      const suffix = sessionId === this.defaultSessionId ? '' : `:${sessionId}`;
 
+      if (sessionSocket) {
+        sessionSocket.removeAllListeners('message');
+        sessionSocket.connecting = false;
+        sessionSocket.connected = false;
+        this.once(sessionId === this.defaultSessionId ? 'offline' : `offline${suffix}`, resolve);
+        resolve(sessionSocket.close(options || undefined));
+      }
       resolve();
+
+      // Update overall connected status
+      this.connected = this.hasConnectedSockets();
+    });
+  },
+
+  /**
+   * Disconnect all socket connections
+   * @param {object} options - Close options
+   * @returns {Promise} Promise that resolves when all connections are closed
+   */
+  disconnectAll(options) {
+    const disconnectPromises = [];
+
+    for (const sessionId of this.sockets.keys()) {
+      disconnectPromises.push(this.disconnect(options, sessionId));
+    }
+
+    return Promise.all(disconnectPromises).then(() => {
+      this.connected = false;
+      this.sockets.clear();
+      this.backoffCalls.clear();
+      // Clear connection promises to prevent stale promises
+      if (this._connectPromises) {
+        this._connectPromises.clear();
+      }
     });
   },
 
@@ -207,55 +443,69 @@ const Mercury = WebexPlugin.extend({
       });
   },
 
-  _attemptConnection(socketUrl, callback) {
+  _attemptConnection(socketUrl, sessionId, callback, options = {}) {
+    const {isShutdownSwitchover = false, onSuccess = null} = options;
+
     const socket = new Socket();
-    let attemptWSUrl;
+    socket.connecting = true;
+    let newWSUrl;
 
-    socket.on('close', (...args) => this._onclose(...args));
-    socket.on('message', (...args) => this._onmessage(...args));
-    socket.on('pong', (...args) => this._setTimeOffset(...args));
-    socket.on('sequence-mismatch', (...args) => this._emit('sequence-mismatch', ...args));
-    socket.on('ping-pong-latency', (...args) => this._emit('ping-pong-latency', ...args));
+    this._attachSocketEventListeners(socket, sessionId);
 
-    Promise.all([this._prepareUrl(socketUrl), this.webex.credentials.getUserToken()])
-      .then(([webSocketUrl, token]) => {
-        if (!this.backoffCall) {
-          const msg = `${this.namespace}: prevent socket open when backoffCall no longer defined`;
+    const backoffCall = isShutdownSwitchover
+      ? this._shutdownSwitchoverBackoffCalls.get(sessionId)
+      : this.backoffCalls.get(sessionId);
 
-          this.logger.info(msg);
+    // Check appropriate backoff call based on connection type
+    if (isShutdownSwitchover && !backoffCall) {
+      const msg = `${this.namespace}: prevent socket open when switchover backoff call no longer defined for ${sessionId}`;
+      const err = new Error(msg);
 
-          return Promise.reject(new Error(msg));
-        }
+      this.logger.info(msg);
 
-        attemptWSUrl = webSocketUrl;
+      // Call the callback with the error before rejecting
+      callback(err);
 
-        let options = {
-          forceCloseDelay: this.config.forceCloseDelay,
-          pingInterval: this.config.pingInterval,
-          pongTimeout: this.config.pongTimeout,
-          token: token.toString(),
-          trackingId: `${this.webex.sessionId}_${Date.now()}`,
-          logger: this.logger,
-        };
+      return Promise.reject(err);
+    }
 
-        // if the consumer has supplied request options use them
-        if (this.webex.config.defaultMercuryOptions) {
-          this.logger.info(`${this.namespace}: setting custom options`);
-          options = {...options, ...this.webex.config.defaultMercuryOptions};
-        }
+    if (!isShutdownSwitchover && !backoffCall) {
+      const msg = `${this.namespace}: prevent socket open when backoffCall no longer defined for ${sessionId}`;
+      const err = new Error(msg);
 
-        // Set the socket before opening it. This allows a disconnect() to close
-        // the socket if it is in the process of being opened.
-        this.socket = socket;
+      this.logger.info(msg);
 
-        this.logger.info(`${this.namespace} connection url: ${webSocketUrl}`);
+      // Call the callback with the error before rejecting
+      callback(err);
+
+      return Promise.reject(err);
+    }
+
+    // For shutdown switchover, don't set socket yet (make-before-break)
+    // For normal connection, set socket before opening to allow disconnect() to close it
+    if (!isShutdownSwitchover) {
+      this.sockets.set(sessionId, socket);
+    }
+
+    return this._prepareAndOpenSocket(socket, socketUrl, sessionId, isShutdownSwitchover)
+      .then((webSocketUrl) => {
+        newWSUrl = webSocketUrl;
 
-        return socket.open(webSocketUrl, options);
-      })
-      .then(() => {
         this.logger.info(
-          `${this.namespace}: connected to mercury, success, action: connected, url: ${attemptWSUrl}`
+          `${this.namespace}: ${
+            isShutdownSwitchover ? '[shutdown] switchover' : ''
+          } connected to mercury, success, action: connected for ${sessionId}, url: ${newWSUrl}`
         );
+
+        // Custom success handler for shutdown switchover
+        if (onSuccess) {
+          onSuccess(socket, webSocketUrl);
+          callback();
+
+          return Promise.resolve();
+        }
+
+        // Default behavior for normal connection
         callback();
 
         return this.webex.internal.feature
@@ -269,32 +519,49 @@ const Mercury = WebexPlugin.extend({
           });
       })
       .catch((reason) => {
+        // For shutdown, simpler error handling - just callback for retry
+        if (isShutdownSwitchover) {
+          this.logger.info(
+            `${this.namespace}: [shutdown] switchover attempt failed for ${sessionId}`,
+            reason
+          );
+
+          return callback(reason);
+        }
+
+        // Normal connection error handling (existing complex logic)
         this.lastError = reason; // remember the last error
 
+        const backoffCall = this.backoffCalls.get(sessionId);
         // Suppress connection errors that appear to be network related. This
         // may end up suppressing metrics during outages, but we might not care
         // (especially since many of our outages happen in a way that client
         // metrics can't be trusted).
-        if (reason.code !== 1006 && this.backoffCall && this.backoffCall?.getNumRetries() > 0) {
-          this._emit('connection_failed', reason, {retries: this.backoffCall?.getNumRetries()});
+        if (reason.code !== 1006 && backoffCall && backoffCall?.getNumRetries() > 0) {
+          this._emit(sessionId, 'connection_failed', reason, {
+            sessionId,
+            retries: backoffCall?.getNumRetries(),
+          });
         }
         this.logger.info(
-          `${this.namespace}: connection attempt failed`,
+          `${this.namespace}: connection attempt failed for ${sessionId}`,
           reason,
-          this.backoffCall?.getNumRetries() === 0 ? reason.stack : ''
+          backoffCall?.getNumRetries() === 0 ? reason.stack : ''
         );
         // UnknownResponse is produced by IE for any 4XXX; treated it like a bad
         // web socket url and let WDM handle the token checking
         if (reason instanceof UnknownResponse) {
           this.logger.info(
-            `${this.namespace}: received unknown response code, refreshing device registration`
+            `${this.namespace}: received unknown response code for ${sessionId}, refreshing device registration`
           );
 
           return this.webex.internal.device.refresh().then(() => callback(reason));
         }
         // NotAuthorized implies expired token
         if (reason instanceof NotAuthorized) {
-          this.logger.info(`${this.namespace}: received authorization error, reauthorizing`);
+          this.logger.info(
+            `${this.namespace}: received authorization error for ${sessionId}, reauthorizing`
+          );
 
           return this.webex.credentials.refresh({force: true}).then(() => callback(reason));
         }
@@ -307,8 +574,10 @@ const Mercury = WebexPlugin.extend({
         // BadRequest implies current credentials are for a Service Account
         // Forbidden implies current user is not entitle for Webex
         if (reason instanceof BadRequest || reason instanceof Forbidden) {
-          this.logger.warn(`${this.namespace}: received unrecoverable response from mercury`);
-          this.backoffCall.abort();
+          this.logger.warn(
+            `${this.namespace}: received unrecoverable response from mercury for ${sessionId}`
+          );
+          backoffCall?.abort();
 
           return callback(reason);
         }
@@ -318,10 +587,10 @@ const Mercury = WebexPlugin.extend({
             .then((haMessagingEnabled) => {
               if (haMessagingEnabled) {
                 this.logger.info(
-                  `${this.namespace}: received a generic connection error, will try to connect to another datacenter. failed, action: 'failed', url: ${attemptWSUrl} error: ${reason.message}`
+                  `${this.namespace}: received a generic connection error for ${sessionId}, will try to connect to another datacenter. failed, action: 'failed', url: ${newWSUrl} error: ${reason.message}`
                 );
 
-                return this.webex.internal.services.markFailedUrl(attemptWSUrl);
+                return this.webex.internal.services.markFailedUrl(newWSUrl);
               }
 
               return null;
@@ -332,42 +601,103 @@ const Mercury = WebexPlugin.extend({
         return callback(reason);
       })
       .catch((reason) => {
-        this.logger.error(`${this.namespace}: failed to handle connection failure`, reason);
+        this.logger.error(
+          `${this.namespace}: failed to handle connection failure for ${sessionId}`,
+          reason
+        );
         callback(reason);
       });
   },
 
-  _connectWithBackoff(webSocketUrl) {
+  _prepareAndOpenSocket(socket, socketUrl, sessionId, isShutdownSwitchover = false) {
+    const logPrefix = isShutdownSwitchover ? '[shutdown] switchover' : 'connection';
+
+    return Promise.all([this._prepareUrl(socketUrl), this.webex.credentials.getUserToken()]).then(
+      ([webSocketUrl, token]) => {
+        let options = {
+          forceCloseDelay: this.config.forceCloseDelay,
+          pingInterval: this.config.pingInterval,
+          pongTimeout: this.config.pongTimeout,
+          token: token.toString(),
+          trackingId: `${this.webex.sessionId}_${Date.now()}`,
+          logger: this.logger,
+        };
+
+        if (this.webex.config.defaultMercuryOptions) {
+          const customOptionsMsg = isShutdownSwitchover
+            ? 'setting custom options for switchover'
+            : 'setting custom options';
+
+          this.logger.info(`${this.namespace}: ${customOptionsMsg}`);
+          options = {...options, ...this.webex.config.defaultMercuryOptions};
+        }
+
+        // Set the socket before opening it. This allows a disconnect() to close
+        // the socket if it is in the process of being opened.
+        this.sockets.set(sessionId, socket);
+        this.socket = this.sockets.get(this.defaultSessionId) || socket;
+
+        this.logger.info(`${this.namespace} ${logPrefix} url for ${sessionId}: ${webSocketUrl}`);
+
+        return socket.open(webSocketUrl, options).then(() => webSocketUrl);
+      }
+    );
+  },
+
+  _connectWithBackoff(webSocketUrl, sessionId, context = {}) {
+    const {isShutdownSwitchover = false, attemptOptions = {}} = context;
+
     return new Promise((resolve, reject) => {
-      // eslint gets confused about whether or not call is actually used
+      // eslint gets confused about whether call is actually used
       // eslint-disable-next-line prefer-const
       let call;
-      const onComplete = (err) => {
-        this.connecting = false;
-
-        this.backoffCall = undefined;
+      const onComplete = (err, sid = sessionId) => {
+        if (isShutdownSwitchover) {
+          this._shutdownSwitchoverBackoffCalls.delete(sid);
+        } else {
+          this.backoffCalls.delete(sid);
+        }
         if (err) {
+          const msg = isShutdownSwitchover
+            ? `[shutdown] switchover failed after ${call.getNumRetries()} retries`
+            : `failed to connect after ${call.getNumRetries()} retries`;
+
           this.logger.info(
-            `${
-              this.namespace
-            }: failed to connect after ${call.getNumRetries()} retries; log statement about next retry was inaccurate; ${err}`
+            `${this.namespace}: ${msg}; log statement about next retry was inaccurate; ${err}`
           );
 
           return reject(err);
         }
-        this.connected = true;
-        this.hasEverConnected = true;
-        this._emit('online');
-        this.webex.internal.newMetrics.callDiagnosticMetrics.setMercuryConnectedStatus(true);
+        // Update overall connected status
+        const sessionSocket = this.sockets.get(sid);
+        if (sessionSocket) {
+          sessionSocket.connecting = false;
+          sessionSocket.connected = true;
+        }
+        // Default success handling for normal connections
+        if (!isShutdownSwitchover) {
+          this.connecting = this.hasConnectingSockets();
+          this.connected = this.hasConnectedSockets();
+          this.hasEverConnected = true;
+          this._emit(sid, 'online', {sessionId: sid});
+          this.webex.internal.newMetrics.callDiagnosticMetrics.setMercuryConnectedStatus(true);
+        }
 
         return resolve();
       };
-
       // eslint-disable-next-line prefer-reflect
-      call = backoff.call((callback) => {
-        this.logger.info(`${this.namespace}: executing connection attempt ${call.getNumRetries()}`);
-        this._attemptConnection(webSocketUrl, callback);
-      }, onComplete);
+      call = backoff.call(
+        (callback) => {
+          const attemptNum = call.getNumRetries();
+          const logPrefix = isShutdownSwitchover ? '[shutdown] switchover' : 'connection';
+
+          this.logger.info(
+            `${this.namespace}: executing ${logPrefix} attempt ${attemptNum} for ${sessionId}`
+          );
+          this._attemptConnection(webSocketUrl, sessionId, callback, attemptOptions);
+        },
+        (err) => onComplete(err, sessionId)
+      );
 
       call.setStrategy(
         new backoff.ExponentialStrategy({
@@ -376,15 +706,29 @@ const Mercury = WebexPlugin.extend({
         })
       );
 
-      if (this.config.initialConnectionMaxRetries && !this.hasEverConnected) {
+      if (
+        this.config.initialConnectionMaxRetries &&
+        !this.hasEverConnected &&
+        !isShutdownSwitchover
+      ) {
         call.failAfter(this.config.initialConnectionMaxRetries);
       } else if (this.config.maxRetries) {
         call.failAfter(this.config.maxRetries);
       }
 
+      // Store the call BEFORE setting up event handlers to prevent race conditions
+      // Store backoff call reference BEFORE starting (so it's available in _attemptConnection)
+      if (isShutdownSwitchover) {
+        this._shutdownSwitchoverBackoffCalls.set(sessionId, call);
+      } else {
+        this.backoffCalls.set(sessionId, call);
+      }
+
       call.on('abort', () => {
-        this.logger.info(`${this.namespace}: connection aborted`);
-        reject(new Error('Mercury Connection Aborted'));
+        const msg = isShutdownSwitchover ? 'Shutdown Switchover' : 'Connection';
+
+        this.logger.info(`${this.namespace}: ${msg} aborted for ${sessionId}`);
+        reject(new Error(`Mercury ${msg} Aborted for ${sessionId}`));
       });
 
       call.on('callback', (err) => {
@@ -392,8 +736,12 @@ const Mercury = WebexPlugin.extend({
           const number = call.getNumRetries();
           const delay = Math.min(call.strategy_.nextBackoffDelay_, this.config.backoffTimeMax);
 
+          const logPrefix = isShutdownSwitchover ? '[shutdown] switchover' : '';
+
           this.logger.info(
-            `${this.namespace}: failed to connect; attempting retry ${number + 1} in ${delay} ms`
+            `${this.namespace}: ${logPrefix} failed to connect; attempting retry ${
+              number + 1
+            } in ${delay} ms for ${sessionId}`
           );
           /* istanbul ignore if */
           if (process.env.NODE_ENV === 'development') {
@@ -402,25 +750,52 @@ const Mercury = WebexPlugin.extend({
 
           return;
         }
-        this.logger.info(`${this.namespace}: connected`);
+        this.logger.info(`${this.namespace}: connected ${sessionId}`);
       });
 
       call.start();
-
-      this.backoffCall = call;
     });
   },
 
   _emit(...args) {
     try {
-      this.trigger(...args);
+      if (!args || args.length === 0) {
+        return;
+      }
+
+      // New signature: _emit(sessionId, eventName, ...rest)
+      // Backwards compatibility: if the first arg isn't a known sessionId (or defaultSessionId),
+      // treat the call as the old signature and forward directly to trigger(...)
+      const [first, second, ...rest] = args;
+
+      if (
+        typeof first === 'string' &&
+        (this.sockets.has(first) || first === this.defaultSessionId) &&
+        typeof second === 'string'
+      ) {
+        const sessionId = first;
+        const eventName = second;
+        const suffix = sessionId === this.defaultSessionId ? '' : `:${sessionId}`;
+
+        this.trigger(`${eventName}${suffix}`, ...rest);
+      } else {
+        // Old usage: _emit(eventName, ...args)
+        this.trigger(...args);
+      }
     } catch (error) {
-      this.logger.error(
-        `${this.namespace}: error occurred in event handler:`,
-        error,
-        ' with args: ',
-        args
-      );
+      // Safely handle errors without causing additional issues during cleanup
+      try {
+        this.logger.error(
+          `${this.namespace}: error occurred in event handler:`,
+          error,
+          ' with args: ',
+          args
+        );
+      } catch (logError) {
+        // If even logging fails, just ignore to prevent cascading errors during cleanup
+        // eslint-disable-next-line no-console
+        console.error('Mercury _emit error handling failed:', logError);
+      }
     }
   },
 
@@ -444,78 +819,152 @@ const Mercury = WebexPlugin.extend({
     return handlers;
   },
 
-  _onclose(event) {
+  _onclose(sessionId, event, sourceSocket) {
     // I don't see any way to avoid the complexity or statement count in here.
     /* eslint complexity: [0] */
 
     try {
       const reason = event.reason && event.reason.toLowerCase();
-      const socketUrl = this.socket.url;
+      let sessionSocket = this.sockets.get(sessionId);
+      let socketUrl;
+      event.sessionId = sessionId;
 
-      this.socket.removeAllListeners();
-      this.unset('socket');
-      this.connected = false;
-      this._emit('offline', event);
-      this.webex.internal.newMetrics.callDiagnosticMetrics.setMercuryConnectedStatus(false);
+      const isActiveSocket = sourceSocket === sessionSocket;
+      if (sourceSocket) {
+        socketUrl = sourceSocket.url;
+      }
+      this.sockets.delete(sessionId);
+
+      if (isActiveSocket) {
+        // Only tear down state if the currently active socket closed
+        if (sessionSocket) {
+          sessionSocket.removeAllListeners();
+          sessionSocket = null;
+          this._emit(sessionId, 'offline', event);
+        }
+        // Update overall connected status
+        this.connecting = this.hasConnectingSockets();
+        this.connected = this.hasConnectedSockets();
+
+        if (!this.connected) {
+          this.webex.internal.newMetrics.callDiagnosticMetrics.setMercuryConnectedStatus(false);
+        }
+      } else {
+        // Old socket closed; do not flip connection state
+        this.logger.info(
+          `${this.namespace}: [shutdown] non-active socket closed, code=${event.code} for ${sessionId}`
+        );
+        // Clean up listeners from old socket now that it's closed
+        if (sourceSocket) {
+          sourceSocket.removeAllListeners();
+        }
+      }
 
       switch (event.code) {
         case 1003:
           // metric: disconnect
           this.logger.info(
-            `${this.namespace}: Mercury service rejected last message; will not reconnect: ${event.reason}`
+            `${this.namespace}: Mercury service rejected last message for ${sessionId}; will not reconnect: ${event.reason}`
           );
-          this._emit('offline.permanent', event);
+          if (isActiveSocket) this._emit(sessionId, 'offline.permanent', event);
           break;
         case 4000:
           // metric: disconnect
-          this.logger.info(`${this.namespace}: socket replaced; will not reconnect`);
-          this._emit('offline.replaced', event);
+          this.logger.info(`${this.namespace}: socket ${sessionId} replaced; will not reconnect`);
+          if (isActiveSocket) this._emit(sessionId, 'offline.replaced', event);
+          // If not active, nothing to do
+          break;
+        case 4001:
+          // replaced during shutdown
+          if (isActiveSocket) {
+            // Server closed active socket with 4001, meaning it expected this connection
+            // to be replaced, but the switchover in _handleImminentShutdown failed.
+            // This is a permanent failure - do not reconnect.
+            this.logger.warn(
+              `${this.namespace}: active socket closed with 4001; shutdown switchover failed for ${sessionId}`
+            );
+            this._emit(sessionId, 'offline.permanent', event);
+          } else {
+            // Expected: old socket closed after successful switchover
+            this.logger.info(
+              `${this.namespace}: old socket closed with 4001 (replaced during shutdown); no reconnect needed for ${sessionId}`
+            );
+            this._emit(sessionId, 'offline.replaced', event);
+          }
           break;
         case 1001:
         case 1005:
         case 1006:
         case 1011:
-          this.logger.info(`${this.namespace}: socket disconnected; reconnecting`);
-          this._emit('offline.transient', event);
-          this._reconnect(socketUrl);
+          this.logger.info(`${this.namespace}: socket ${sessionId} disconnected; reconnecting`);
+          if (isActiveSocket) {
+            this._emit(sessionId, 'offline.transient', event);
+            this.logger.info(
+              `${this.namespace}: [shutdown] reconnecting active socket to recover for ${sessionId}`
+            );
+            this._reconnect(socketUrl, sessionId);
+          }
           // metric: disconnect
           // if (code == 1011 && reason !== ping error) metric: unexpected disconnect
           break;
         case 1000:
         case 3050: // 3050 indicates logout form of closure, default to old behavior, use config reason defined by consumer to proceed with the permanent block
           if (normalReconnectReasons.includes(reason)) {
-            this.logger.info(`${this.namespace}: socket disconnected; reconnecting`);
-            this._emit('offline.transient', event);
-            this._reconnect(socketUrl);
+            this.logger.info(`${this.namespace}: socket ${sessionId} disconnected; reconnecting`);
+            if (isActiveSocket) {
+              this._emit(sessionId, 'offline.transient', event);
+              this.logger.info(
+                `${this.namespace}: [shutdown] reconnecting due to normal close for ${sessionId}`
+              );
+              this._reconnect(socketUrl, sessionId);
+            }
             // metric: disconnect
             // if (reason === done forced) metric: force closure
           } else {
             this.logger.info(
-              `${this.namespace}: socket disconnected; will not reconnect: ${event.reason}`
+              `${this.namespace}: socket ${sessionId} disconnected; will not reconnect: ${event.reason}`
             );
-            this._emit('offline.permanent', event);
+            if (isActiveSocket) this._emit(sessionId, 'offline.permanent', event);
           }
           break;
         default:
           this.logger.info(
-            `${this.namespace}: socket disconnected unexpectedly; will not reconnect`
+            `${this.namespace}: socket ${sessionId} disconnected unexpectedly; will not reconnect`
           );
           // unexpected disconnect
-          this._emit('offline.permanent', event);
+          if (isActiveSocket) this._emit(sessionId, 'offline.permanent', event);
       }
     } catch (error) {
-      this.logger.error(`${this.namespace}: error occurred in close handler`, error);
+      this.logger.error(
+        `${this.namespace}: error occurred in close handler for ${sessionId}`,
+        error
+      );
     }
   },
 
-  _onmessage(event) {
-    this._setTimeOffset(event);
+  _onmessage(sessionId, event) {
+    this._setTimeOffset(sessionId, event);
     const envelope = event.data;
 
     if (process.env.ENABLE_MERCURY_LOGGING) {
-      this.logger.debug(`${this.namespace}: message envelope: `, envelope);
+      this.logger.debug(`${this.namespace}: message envelope from ${sessionId}: `, envelope);
+    }
+
+    envelope.sessionId = sessionId;
+
+    // Handle shutdown message shape: { type: 'shutdown' }
+    if (envelope && envelope.type === 'shutdown') {
+      this.logger.info(
+        `${this.namespace}: [shutdown] imminent shutdown message received for ${sessionId}`
+      );
+      this._emit(sessionId, 'event:mercury_shutdown_imminent', envelope);
+
+      this._handleImminentShutdown(sessionId);
+
+      return Promise.resolve();
     }
 
+    envelope.sessionId = sessionId;
     const {data} = envelope;
 
     this._applyOverrides(data);
@@ -530,7 +979,7 @@ const Mercury = WebexPlugin.extend({
               resolve((this.webex[namespace] || this.webex.internal[namespace])[name](data))
             ).catch((reason) =>
               this.logger.error(
-                `${this.namespace}: error occurred in autowired event handler for ${data.eventType}`,
+                `${this.namespace}: error occurred in autowired event handler for ${data.eventType} from ${sessionId}`,
                 reason
               )
             );
@@ -538,32 +987,35 @@ const Mercury = WebexPlugin.extend({
         Promise.resolve()
       )
       .then(() => {
-        this._emit('event', event.data);
+        this._emit(sessionId, 'event', envelope);
         const [namespace] = data.eventType.split('.');
 
         if (namespace === data.eventType) {
-          this._emit(`event:${namespace}`, envelope);
+          this._emit(sessionId, `event:${namespace}`, envelope);
         } else {
-          this._emit(`event:${namespace}`, envelope);
-          this._emit(`event:${data.eventType}`, envelope);
+          this._emit(sessionId, `event:${namespace}`, envelope);
+          this._emit(sessionId, `event:${data.eventType}`, envelope);
         }
       })
       .catch((reason) => {
-        this.logger.error(`${this.namespace}: error occurred processing socket message`, reason);
+        this.logger.error(
+          `${this.namespace}: error occurred processing socket message from ${sessionId}`,
+          reason
+        );
       });
   },
 
-  _setTimeOffset(event) {
+  _setTimeOffset(sessionId, event) {
     const {wsWriteTimestamp} = event.data;
     if (typeof wsWriteTimestamp === 'number' && wsWriteTimestamp > 0) {
       this.mercuryTimeOffset = Date.now() - wsWriteTimestamp;
     }
   },
 
-  _reconnect(webSocketUrl) {
-    this.logger.info(`${this.namespace}: reconnecting`);
+  _reconnect(webSocketUrl, sessionId = this.defaultSessionId) {
+    this.logger.info(`${this.namespace}: reconnecting ${sessionId}`);
 
-    return this.connect(webSocketUrl);
+    return this.connect(webSocketUrl, sessionId);
   },
 });