@fluidframework/container-loader 0.52.0 → 0.54.0-47413

package/lib/deltaManager.js

@@ -4,86 +4,24 @@
  */
 import { default as AbortController } from "abort-controller";
 import { v4 as uuid } from "uuid";
-import { assert, Deferred, performance, TypedEventEmitter } from "@fluidframework/common-utils";
-import { TelemetryLogger, safeRaiseEvent, logIfFalse, normalizeError, wrapError, } from "@fluidframework/telemetry-utils";
+import { assert, TypedEventEmitter } from "@fluidframework/common-utils";
+import { normalizeError, logIfFalse, safeRaiseEvent, } from "@fluidframework/telemetry-utils";
 import { DriverErrorType, } from "@fluidframework/driver-definitions";
 import { isSystemMessage } from "@fluidframework/protocol-base";
-import { MessageType, ScopeType, } from "@fluidframework/protocol-definitions";
-import { canRetryOnError, createWriteError, createGenericNetworkError, getRetryDelayFromError, logNetworkFailure, waitForConnectedState, NonRetryableError, DeltaStreamConnectionForbiddenError, GenericNetworkError, } from "@fluidframework/driver-utils";
-import { ThrottlingWarning, CreateProcessingError, DataCorruptionError, GenericError, } from "@fluidframework/container-utils";
+import { MessageType, } from "@fluidframework/protocol-definitions";
+import { NonRetryableError, } from "@fluidframework/driver-utils";
+import { ThrottlingWarning, CreateProcessingError, DataCorruptionError, } from "@fluidframework/container-utils";
 import { DeltaQueue } from "./deltaQueue";
-const MaxReconnectDelayInMs = 8000;
-const InitialReconnectDelayInMs = 1000;
-const DefaultChunkSize = 16 * 1024;
-function getNackReconnectInfo(nackContent) {
-    const message = `Nack (${nackContent.type}): ${nackContent.message}`;
-    const canRetry = nackContent.code !== 403;
-    const retryAfterMs = nackContent.retryAfter !== undefined ? nackContent.retryAfter * 1000 : undefined;
-    return createGenericNetworkError(`nack [${nackContent.code}]`, message, canRetry, retryAfterMs, { statusCode: nackContent.code });
-}
-const createReconnectError = (fluidErrorCode, err) => wrapError(err, (errorMessage) => new GenericNetworkError(fluidErrorCode, errorMessage, true /* canRetry */));
-export var ReconnectMode;
-(function (ReconnectMode) {
-    ReconnectMode["Never"] = "Never";
-    ReconnectMode["Disabled"] = "Disabled";
-    ReconnectMode["Enabled"] = "Enabled";
-})(ReconnectMode || (ReconnectMode = {}));
-/**
- * Implementation of IDocumentDeltaConnection that does not support submitting
- * or receiving ops. Used in storage-only mode.
- */
-class NoDeltaStream extends TypedEventEmitter {
-    constructor() {
-        super(...arguments);
-        this.clientId = "storage-only client";
-        this.claims = {
-            scopes: [ScopeType.DocRead],
-        };
-        this.mode = "read";
-        this.existing = true;
-        this.maxMessageSize = 0;
-        this.version = "";
-        this.initialMessages = [];
-        this.initialSignals = [];
-        this.initialClients = [];
-        this.serviceConfiguration = {
-            maxMessageSize: 0,
-            blockSize: 0,
-            summary: undefined,
-        };
-        this.checkpointSequenceNumber = undefined;
-        this._disposed = false;
-    }
-    submit(messages) {
-        this.emit("nack", this.clientId, messages.map((operation) => {
-            return {
-                operation,
-                content: { message: "Cannot submit with storage-only connection", code: 403 },
-            };
-        }));
-    }
-    submitSignal(message) {
-        this.emit("nack", this.clientId, {
-            operation: message,
-            content: { message: "Cannot submit signal with storage-only connection", code: 403 },
-        });
-    }
-    get disposed() { return this._disposed; }
-    dispose() { this._disposed = true; }
-}
 /**
  * Manages the flow of both inbound and outbound messages. This class ensures that shared objects receive delta
  * messages in order regardless of possible network conditions or timings causing out of order delivery.
  */
 export class DeltaManager extends TypedEventEmitter {
-    constructor(serviceProvider, client, logger, reconnectAllowed, _active) {
+    constructor(serviceProvider, logger, _active, createConnectionManager) {
         super();
         this.serviceProvider = serviceProvider;
-        this.client = client;
         this.logger = logger;
         this._active = _active;
-        // tracks host requiring read-only mode.
-        this._forceReadonly = false;
         this.pending = [];
         // The minimum sequence number and last sequence number received from the server
         this.minSequenceNumber = 0;
@@ -100,82 +38,30 @@ export class DeltaManager extends TypedEventEmitter {
         this.baseTerm = 0;
         // The sequence number we initially loaded from
         this.initSequenceNumber = 0;
-        this.clientSequenceNumber = 0;
-        this.clientSequenceNumberObserved = 0;
-        // Counts the number of noops sent by the client which may not be acked.
-        this.trailingNoopCount = 0;
         this.closed = false;
-        this.deltaStreamDelayId = uuid();
-        this.deltaStorageDelayId = uuid();
-        this.messageBuffer = [];
-        this.connectFirstConnection = true;
         this.throttlingIdSet = new Set();
         this.timeTillThrottling = 0;
-        this.connectionStateProps = {};
-        // True if current connection has checkpoint information
-        // I.e. we know how far behind the client was at the time of establishing connection
-        this._hasCheckpointSequenceNumber = false;
         this.closeAbortController = new AbortController();
-        // True if there is pending (async) reconnection from "read" to "write"
-        this.pendingReconnect = false;
-        // downgrade "write" connection to "read"
-        this.downgradedConnection = false;
-        this.opHandler = (documentId, messagesArg) => {
-            const messages = Array.isArray(messagesArg) ? messagesArg : [messagesArg];
-            this.enqueueMessages(messages, "opHandler");
-        };
-        this.signalHandler = (message) => {
-            this._inboundSignal.push(message);
-        };
-        // Always connect in write mode after getting nacked.
-        this.nackHandler = (documentId, messages) => {
-            const message = messages[0];
-            // TODO: we should remove this check when service updates?
-            // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
-            if (this._readonlyPermissions) {
-                this.close(createWriteError("writeOnReadOnlyDocument"));
-            }
-            // check message.content for Back-compat with old service.
-            const reconnectInfo = message.content !== undefined
-                ? getNackReconnectInfo(message.content) :
-                createGenericNetworkError("nackReasonUnknown", undefined, true);
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-            this.reconnectOnError("write", reconnectInfo);
-        };
-        // Connection mode is always read on disconnect/error unless the system mode was write.
-        this.disconnectHandler = (disconnectReason) => {
-            // Note: we might get multiple disconnect calls on same socket, as early disconnect notification
-            // ("server_disconnect", ODSP-specific) is mapped to "disconnect"
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-            this.reconnectOnError(this.defaultReconnectionMode, createReconnectError("dmDocumentDeltaConnectionDisconnected", disconnectReason));
-        };
-        this.errorHandler = (error) => {
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-            this.reconnectOnError(this.defaultReconnectionMode, createReconnectError("dmDocumentDeltaConnectionError", error));
-        };
-        this.pongHandler = (latency) => {
-            this.emit("pong", latency);
+        this.deltaStorageDelayId = uuid();
+        this.deltaStreamDelayId = uuid();
+        this.messageBuffer = [];
+        const props = {
+            incomingOpHandler: (messages, reason) => this.enqueueMessages(messages, reason),
+            signalHandler: (message) => this._inboundSignal.push(message),
+            reconnectionDelayHandler: (delayMs, error) => this.emitDelayInfo(this.deltaStreamDelayId, delayMs, error),
+            closeHandler: (error) => this.close(error),
+            disconnectHandler: (reason) => this.disconnectHandler(reason),
+            connectHandler: (connection) => this.connectHandler(connection),
+            pongHandler: (latency) => this.emit("pong", latency),
+            readonlyChangeHandler: (readonly) => safeRaiseEvent(this, this.logger, "readonly", readonly),
         };
-        this.clientDetails = this.client.details;
-        this.defaultReconnectionMode = this.client.mode;
-        this._reconnectMode = reconnectAllowed ? ReconnectMode.Enabled : ReconnectMode.Never;
+        this.connectionManager = createConnectionManager(props);
         this._inbound = new DeltaQueue((op) => {
             this.processInboundMessage(op);
         });
         this._inbound.on("error", (error) => {
             this.close(CreateProcessingError(error, "deltaManagerInboundErrorHandler", this.lastMessage));
         });
-        // Outbound message queue. The outbound queue is represented as a queue of an array of ops. Ops contained
-        // within an array *must* fit within the maxMessageSize and are guaranteed to be ordered sequentially.
-        this._outbound = new DeltaQueue((messages) => {
-            if (this.connection === undefined) {
-                throw new Error("Attempted to submit an outbound message without connection");
-            }
-            this.connection.submit(messages);
-        });
-        this._outbound.on("error", (error) => {
-            this.close(normalizeError(error));
-        });
         // Inbound signal queue
         this._inboundSignal = new DeltaQueue((message) => {
             if (this.handler === undefined) {
@@ -196,21 +82,9 @@ export class DeltaManager extends TypedEventEmitter {
     get active() { return this._active(); }
     get disposed() { return this.closed; }
     get IDeltaSender() { return this; }
-    /**
-     * Tells if  current connection has checkpoint information.
-     * I.e. we know how far behind the client was at the time of establishing connection
-     */
-    get hasCheckpointSequenceNumber() {
-        // Valid to be called only if we have active connection.
-        assert(this.connection !== undefined, 0x0df /* "Missing active connection" */);
-        return this._hasCheckpointSequenceNumber;
-    }
     get inbound() {
         return this._inbound;
     }
-    get outbound() {
-        return this._outbound;
-    }
     get inboundSignal() {
         return this._inboundSignal;
     }
@@ -232,39 +106,22 @@ export class DeltaManager extends TypedEventEmitter {
     get minimumSequenceNumber() {
         return this.minSequenceNumber;
     }
-    get maxMessageSize() {
-        var _a, _b, _c;
-        return (_c = (_b = (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration) === null || _b === void 0 ? void 0 : _b.maxMessageSize) !== null && _c !== void 0 ? _c : DefaultChunkSize;
-    }
-    get version() {
-        if (this.connection === undefined) {
-            throw new Error("Cannot check version without a connection");
-        }
-        return this.connection.version;
-    }
-    get serviceConfiguration() {
-        var _a;
-        return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration;
-    }
-    get scopes() {
-        var _a;
-        return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.claims.scopes;
-    }
-    get socketDocumentId() {
-        var _a;
-        return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.claims.documentId;
-    }
     /**
-     * The current connection mode, initially read.
+     * Tells if  current connection has checkpoint information.
+     * I.e. we know how far behind the client was at the time of establishing connection
      */
-    get connectionMode() {
-        var _a;
-        assert(!this.downgradedConnection || ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.mode) === "write", 0x277 /* "Did we forget to reset downgradedConnection on new connection?" */);
-        if (this.connection === undefined || this.downgradedConnection) {
-            return "read";
-        }
-        return this.connection.mode;
-    }
+    get hasCheckpointSequenceNumber() {
+        // Valid to be called only if we have active connection.
+        assert(this.connectionManager.connected, 0x0df /* "Missing active connection" */);
+        return this._checkpointSequenceNumber !== undefined;
+    }
+    // Forwarding connection manager properties / IDeltaManager implementation
+    get maxMessageSize() { return this.connectionManager.maxMessageSize; }
+    get version() { return this.connectionManager.version; }
+    get serviceConfiguration() { return this.connectionManager.serviceConfiguration; }
+    get outbound() { return this.connectionManager.outbound; }
+    get readOnlyInfo() { return this.connectionManager.readOnlyInfo; }
+    get clientDetails() { return this.connectionManager.clientDetails; }
     /**
      * Tells if container is in read-only mode.
      * Data stores should listen for "readonly" notifications and disallow user
@@ -276,118 +133,50 @@ export class DeltaManager extends TypedEventEmitter {
      * @deprecated - use readOnlyInfo
      */
     get readonly() {
-        if (this._forceReadonly) {
-            return true;
-        }
-        return this._readonlyPermissions;
-    }
-    /**
-     * Tells if user has no write permissions for file in storage
-     * It is undefined if we have not yet established websocket connection
-     * and do not know if user has write access to a file.
-     * @deprecated - use readOnlyInfo
-     */
-    get readonlyPermissions() {
-        return this._readonlyPermissions;
-    }
-    get readOnlyInfo() {
-        const storageOnly = this.connection !== undefined && this.connection instanceof NoDeltaStream;
-        if (storageOnly || this._forceReadonly || this._readonlyPermissions === true) {
-            return {
-                readonly: true,
-                forced: this._forceReadonly,
-                permissions: this._readonlyPermissions,
-                storageOnly,
-            };
-        }
-        return { readonly: this._readonlyPermissions };
-    }
-    /**
-     * Automatic reconnecting enabled or disabled.
-     * If set to Never, then reconnecting will never be allowed.
-     */
-    get reconnectMode() {
-        return this._reconnectMode;
+        return this.readOnlyInfo.readonly;
     }
-    shouldJoinWrite() {
-        // We don't have to wait for ack for topmost NoOps. So subtract those.
-        return this.clientSequenceNumberObserved < (this.clientSequenceNumber - this.trailingNoopCount);
-    }
-    /**
-     * Returns set of props that can be logged in telemetry that provide some insights / statistics
-     * about current or last connection (if there is no connection at the moment)
-    */
-    connectionProps() {
-        const common = {
-            sequenceNumber: this.lastSequenceNumber,
+    submit(type, contents, batch = false, metadata) {
+        // Start adding trace for the op.
+        const traces = [
+            {
+                action: "start",
+                service: "client",
+                timestamp: Date.now(),
+            }
+        ];
+        const messagePartial = {
+            contents: JSON.stringify(contents),
+            metadata,
+            referenceSequenceNumber: this.lastProcessedSequenceNumber,
+            traces,
+            type,
         };
-        if (this.connection !== undefined) {
-            return Object.assign(Object.assign({}, common), { connectionMode: this.connectionMode });
+        if (!batch) {
+            this.flush();
         }
-        else {
-            return Object.assign(Object.assign({}, common), { 
-                // Report how many ops this client sent in last disconnected session
-                sentOps: this.clientSequenceNumber });
+        const message = this.connectionManager.prepareMessageToSend(messagePartial);
+        if (message === undefined) {
+            return -1;
         }
-    }
-    /**
-     * Enables or disables automatic reconnecting.
-     * Will throw an error if reconnectMode set to Never.
-     */
-    setAutoReconnect(mode) {
-        assert(mode !== ReconnectMode.Never && this._reconnectMode !== ReconnectMode.Never, 0x278 /* "API is not supported for non-connecting or closed container" */);
-        this._reconnectMode = mode;
-        if (mode !== ReconnectMode.Enabled) {
-            // immediately disconnect - do not rely on service eventually dropping connection.
-            this.disconnectFromDeltaStream("setAutoReconnect");
+        this.messageBuffer.push(message);
+        if (!batch) {
+            this.flush();
         }
+        this.emit("submitOp", message);
+        return message.clientSequenceNumber;
     }
-    /**
-     * Sends signal to runtime (and data stores) to be read-only.
-     * Hosts may have read only views, indicating to data stores that no edits are allowed.
-     * This is independent from this._readonlyPermissions (permissions) and this.connectionMode
-     * (server can return "write" mode even when asked for "read")
-     * Leveraging same "readonly" event as runtime & data stores should behave the same in such case
-     * as in read-only permissions.
-     * But this.active can be used by some DDSes to figure out if ops can be sent
-     * (for example, read-only view still participates in code proposals / upgrades decisions)
-     *
-     * Forcing Readonly does not prevent DDS from generating ops. It is up to user code to honour
-     * the readonly flag. If ops are generated, they will accumulate locally and not be sent. If
-     * there are pending in the outbound queue, it will stop sending until force readonly is
-     * cleared.
-     *
-     * @param readonly - set or clear force readonly.
-     */
-    forceReadonly(readonly) {
-        if (readonly !== this._forceReadonly) {
-            this.logger.sendTelemetryEvent({
-                eventName: "ForceReadOnly",
-                value: readonly,
-            });
-        }
-        const oldValue = this.readonly;
-        this._forceReadonly = readonly;
-        if (oldValue !== this.readonly) {
-            assert(this._reconnectMode !== ReconnectMode.Never, 0x279 /* "API is not supported for non-connecting or closed container" */);
-            let reconnect = false;
-            if (this.readonly === true) {
-                // If we switch to readonly while connected, we should disconnect first
-                // See comment in the "readonly" event handler to deltaManager set up by
-                // the ContainerRuntime constructor
-                if (this.shouldJoinWrite()) {
-                    // If we have pending changes, then we will never send them - it smells like
-                    // host logic error.
-                    this.logger.sendErrorEvent({ eventName: "ForceReadonlyPendingChanged" });
-                }
-                reconnect = this.disconnectFromDeltaStream("Force readonly");
-            }
-            safeRaiseEvent(this, this.logger, "readonly", this.readonly);
-            if (reconnect) {
-                // reconnect if we disconnected from before.
-                this.triggerConnect({ reason: "forceReadonly", mode: "read", fetchOpsFromStorage: false });
-            }
+    submitSignal(content) { return this.connectionManager.submitSignal(content); }
+    flush() {
+        if (this.messageBuffer.length === 0) {
+            return;
         }
+        // The prepareFlush event allows listeners to append metadata to the batch prior to submission.
+        this.emit("prepareSend", this.messageBuffer);
+        this.connectionManager.sendMessages(this.messageBuffer);
+        this.messageBuffer = [];
+    }
+    get connectionProps() {
+        return Object.assign({ sequenceNumber: this.lastSequenceNumber }, this.connectionManager.connectionProps);
     }
     /**
      * Log error event with a bunch of internal to DeltaManager information about state of op processing
@@ -397,20 +186,46 @@ export class DeltaManager extends TypedEventEmitter {
      */
     logConnectionIssue(event) {
         var _a;
-        assert(this.connection !== undefined, 0x238 /* "called only in connected state" */);
+        assert(this.connectionManager.connected, 0x238 /* "called only in connected state" */);
         const pendingSorted = this.pending.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
         this.logger.sendErrorEvent(Object.assign(Object.assign(Object.assign(Object.assign({}, event), { 
             // This directly tells us if fetching ops is in flight, and thus likely the reason of
             // stalled op processing
             fetchReason: this.fetchReason, 
             // A bunch of useful sequence numbers to understand if we are holding some ops from processing
-            lastQueuedSequenceNumber: this.lastQueuedSequenceNumber, lastProcessedSequenceNumber: this.lastProcessedSequenceNumber, lastObserved: this.lastObservedSeqNumber }), this.connectionStateProps), { pendingOps: this.pending.length, pendingFirst: (_a = pendingSorted[0]) === null || _a === void 0 ? void 0 : _a.sequenceNumber, haveHandler: this.handler !== undefined, inboundLength: this.inbound.length, inboundPaused: this.inbound.paused }));
+            lastQueuedSequenceNumber: this.lastQueuedSequenceNumber, lastProcessedSequenceNumber: this.lastProcessedSequenceNumber, lastObserved: this.lastObservedSeqNumber }), this.connectionManager.connectionVerboseProps), { pendingOps: this.pending.length, pendingFirst: (_a = pendingSorted[0]) === null || _a === void 0 ? void 0 : _a.sequenceNumber, haveHandler: this.handler !== undefined, inboundLength: this.inbound.length, inboundPaused: this.inbound.paused }));
     }
-    set_readonlyPermissions(readonly) {
-        const oldValue = this.readonly;
-        this._readonlyPermissions = readonly;
-        if (oldValue !== this.readonly) {
-            safeRaiseEvent(this, this.logger, "readonly", this.readonly);
+    connectHandler(connection) {
+        this.refreshDelayInfo(this.deltaStreamDelayId);
+        const props = this.connectionManager.connectionVerboseProps;
+        props.connectionLastQueuedSequenceNumber = this.lastQueuedSequenceNumber;
+        props.connectionLastObservedSeqNumber = this.lastObservedSeqNumber;
+        const checkpointSequenceNumber = connection.checkpointSequenceNumber;
+        this._checkpointSequenceNumber = checkpointSequenceNumber;
+        if (checkpointSequenceNumber !== undefined) {
+            this.updateLatestKnownOpSeqNumber(checkpointSequenceNumber);
+        }
+        // We cancel all ops on lost of connectivity, and rely on DDSes to resubmit them.
+        // Semantics are not well defined for batches (and they are broken right now on disconnects anyway),
+        // but it's safe to assume (until better design is put into place) that batches should not exist
+        // across multiple connections. Right now we assume runtime will not submit any ops in disconnected
+        // state. As requirements change, so should these checks.
+        assert(this.messageBuffer.length === 0, 0x0e9 /* "messageBuffer is not empty on new connection" */);
+        this.emit("connect", connection, checkpointSequenceNumber !== undefined ?
+            this.lastObservedSeqNumber - this.lastSequenceNumber : undefined);
+        // If we got some initial ops, then we know the gap and call above fetched ops to fill it.
+        // Same is true for "write" mode even if we have no ops - we will get "join" own op very very soon.
+        // However if we are connecting as view-only, then there is no good signal to realize if client is behind.
+        // Thus we have to hit storage to see if any ops are there.
+        if (checkpointSequenceNumber !== undefined) {
+            // We know how far we are behind (roughly). If it's non-zero gap, fetch ops right away.
+            if (checkpointSequenceNumber > this.lastQueuedSequenceNumber) {
+                this.fetchMissingDeltas("AfterConnection");
+            }
+            // we do not know the gap, and we will not learn about it if socket is quite - have to ask.
+        }
+        else if (connection.mode === "read") {
+            this.fetchMissingDeltas("AfterReadConnection");
         }
     }
     dispose() {
@@ -451,66 +266,16 @@ export class DeltaManager extends TypedEventEmitter {
             // (which in most cases will happen when we are done processing cached ops)
             if (cacheOnly) {
                 // fire and forget
-                this.fetchMissingDeltas("DocumentOpen", this.lastQueuedSequenceNumber);
+                this.fetchMissingDeltas("DocumentOpen");
             }
         }
         // Ensure there is no need to call this.processPendingOps() at the end of boot sequence
         assert(this.fetchReason !== undefined || this.pending.length === 0, 0x269 /* "pending ops are not dropped" */);
     }
-    static detailsFromConnection(connection) {
-        return {
-            claims: connection.claims,
-            clientId: connection.clientId,
-            existing: connection.existing,
-            checkpointSequenceNumber: connection.checkpointSequenceNumber,
-            get initialClients() { return connection.initialClients; },
-            maxMessageSize: connection.serviceConfiguration.maxMessageSize,
-            mode: connection.mode,
-            serviceConfiguration: connection.serviceConfiguration,
-            version: connection.version,
-        };
-    }
-    async connect(args) {
-        const connection = await this.connectCore(args);
-        return DeltaManager.detailsFromConnection(connection);
-    }
-    /**
-     * Start the connection. Any error should result in container being close.
-     * And report the error if it excape for any reason.
-     * @param args - The connection arguments
-     */
-    triggerConnect(args) {
-        assert(this.connection === undefined, 0x239 /* "called only in disconnected state" */);
-        if (this.reconnectMode !== ReconnectMode.Enabled) {
-            return;
-        }
-        this.connectCore(args).catch((err) => {
-            // Errors are raised as "error" event and close container.
-            // Have a catch-all case in case we missed something
-            if (!this.closed) {
-                this.logger.sendErrorEvent({ eventName: "ConnectException" }, err);
-            }
-        });
-    }
-    async connectCore(args) {
-        var _a, _b, _c;
-        assert(!this.closed, 0x26a /* "not closed" */);
-        if (this.connection !== undefined) {
-            return this.connection;
-        }
-        if (this.connectionP !== undefined) {
-            return this.connectionP;
-        }
+    connect(args) {
+        var _a;
         const fetchOpsFromStorage = (_a = args.fetchOpsFromStorage) !== null && _a !== void 0 ? _a : true;
-        let requestedMode = (_b = args.mode) !== null && _b !== void 0 ? _b : this.defaultReconnectionMode;
-        // if we have any non-acked ops from last connection, reconnect as "write".
-        // without that we would connect in view-only mode, which will result in immediate
-        // firing of "connected" event from Container and switch of current clientId (as tracked
-        // by all DDSes). This will make it impossible to figure out if ops actually made it through,
-        // so DDSes will immediately resubmit all pending ops, and some of them will be duplicates, corrupting document
-        if (this.shouldJoinWrite()) {
-            requestedMode = "write";
-        }
+        logIfFalse(this.handler !== undefined || !fetchOpsFromStorage, this.logger, "CantFetchWithoutBaseline"); // can't fetch if no baseline
         // Note: There is race condition here.
         // We want to issue request to storage as soon as possible, to
         // reduce latency of becoming current, thus this code here.
@@ -520,214 +285,11 @@ export class DeltaManager extends TypedEventEmitter {
         // own "join" message and realize any gap client has in ops.
         // But for view-only connection, we have no such signal, and with no traffic
         // on the wire, we might be always behind.
-        // See comment at the end of setupNewSuccessfulConnection()
-        logIfFalse(this.handler !== undefined || !fetchOpsFromStorage, this.logger, "CantFetchWithoutBaseline"); // can't fetch if no baseline
+        // See comment at the end of "connect" handler
         if (fetchOpsFromStorage) {
-            this.fetchMissingDeltas(args.reason, this.lastQueuedSequenceNumber);
-        }
-        const docService = this.serviceProvider();
-        if (docService === undefined) {
-            throw new Error("Container is not attached");
-        }
-        if (((_c = docService.policies) === null || _c === void 0 ? void 0 : _c.storageOnly) === true) {
-            const connection = new NoDeltaStream();
-            this.connectionP = Promise.resolve(connection); // to keep setupNewSuccessfulConnection happy
-            this.setupNewSuccessfulConnection(connection, "read");
-            return connection;
-        }
-        // The promise returned from connectCore will settle with a resolved connection or reject with error
-        const connectCore = async () => {
-            let connection;
-            let delayMs = InitialReconnectDelayInMs;
-            let connectRepeatCount = 0;
-            const connectStartTime = performance.now();
-            let lastError;
-            // This loop will keep trying to connect until successful, with a delay between each iteration.
-            while (connection === undefined) {
-                if (this.closed) {
-                    throw new Error("Attempting to connect a closed DeltaManager");
-                }
-                connectRepeatCount++;
-                try {
-                    this.client.mode = requestedMode;
-                    connection = await docService.connectToDeltaStream(this.client);
-                    if (connection.disposed) {
-                        // Nobody observed this connection, so drop it on the floor and retry.
-                        this.logger.sendTelemetryEvent({ eventName: "ReceivedClosedConnection" });
-                        connection = undefined;
-                    }
-                }
-                catch (origError) {
-                    if (typeof origError === "object" && origError !== null &&
-                        (origError === null || origError === void 0 ? void 0 : origError.errorType) === DeltaStreamConnectionForbiddenError.errorType) {
-                        connection = new NoDeltaStream();
-                        requestedMode = "read";
-                        break;
-                    }
-                    // Socket.io error when we connect to wrong socket, or hit some multiplexing bug
-                    if (!canRetryOnError(origError)) {
-                        const error = normalizeError(origError);
-                        this.close(error);
-                        throw error;
-                    }
-                    // Log error once - we get too many errors in logs when we are offline,
-                    // and unfortunately there is no reliable way to detect that.
-                    if (connectRepeatCount === 1) {
-                        logNetworkFailure(this.logger, {
-                            delay: delayMs,
-                            eventName: "DeltaConnectionFailureToConnect",
-                            duration: TelemetryLogger.formatTick(performance.now() - connectStartTime),
-                        }, origError);
-                    }
-                    lastError = origError;
-                    const retryDelayFromError = getRetryDelayFromError(origError);
-                    delayMs = retryDelayFromError !== null && retryDelayFromError !== void 0 ? retryDelayFromError : Math.min(delayMs * 2, MaxReconnectDelayInMs);
-                    if (retryDelayFromError !== undefined) {
-                        this.emitDelayInfo(this.deltaStreamDelayId, retryDelayFromError, origError);
-                    }
-                    await waitForConnectedState(delayMs);
-                }
-            }
-            // If we retried more than once, log an event about how long it took
-            if (connectRepeatCount > 1) {
-                this.logger.sendTelemetryEvent({
-                    eventName: "MultipleDeltaConnectionFailures",
-                    attempts: connectRepeatCount,
-                    duration: TelemetryLogger.formatTick(performance.now() - connectStartTime),
-                }, lastError);
-            }
-            this.setupNewSuccessfulConnection(connection, requestedMode);
-            return connection;
-        };
-        // This promise settles as soon as we know the outcome of the connection attempt
-        // Set it upfront, such that if connection is established (NoDeltaConnection) or rejected (bug in
-        // connectToDeltaStream() implementation - throwing exception vs. returning rejected promise) in
-        // synchronous way, we have this.connectionP setup for all the code to assert correctness of the flow.
-        const deferred = new Deferred();
-        this.connectionP = deferred.promise;
-        // Regardless of how the connection attempt concludes, we'll clear the promise and remove the listener
-        // Reject the connection promise if the DeltaManager gets closed during connection
-        const cleanupAndReject = (error) => {
-            this.connectionP = undefined;
-            this.removeListener("closed", cleanupAndReject);
-            // This error came from some logic error in this file. Fail-fast to learn and fix the issue faster
-            this.close(error);
-            deferred.reject(error);
-        };
-        this.on("closed", cleanupAndReject);
-        // Attempt the connection
-        connectCore().then((connection) => {
-            this.removeListener("closed", cleanupAndReject);
-            deferred.resolve(connection);
-        }).catch(cleanupAndReject);
-        return this.connectionP;
-    }
-    flush() {
-        if (this.messageBuffer.length === 0) {
-            return;
-        }
-        // The prepareFlush event allows listeners to append metadata to the batch prior to submission.
-        this.emit("prepareSend", this.messageBuffer);
-        this._outbound.push(this.messageBuffer);
-        this.messageBuffer = [];
-    }
-    /**
-     * Submits the given delta returning the client sequence number for the message. Contents is the actual
-     * contents of the message. appData is optional metadata that can be attached to the op by the app.
-     *
-     * If batch is set to true then the submit will be batched - and as a result guaranteed to be ordered sequentially
-     * in the global sequencing space. The batch will be flushed either when flush is called or when a non-batched
-     * op is submitted.
-     */
-    submit(type, contents, batch = false, metadata) {
-        // TODO need to fail if gets too large
-        // const serializedContent = JSON.stringify(this.messageBuffer);
-        // const maxOpSize = this.context.deltaManager.maxMessageSize;
-        var _a, _b;
-        if (this.readonly === true) {
-            assert(this.readOnlyInfo.readonly === true, 0x1f0 /* "Unexpected mismatch in readonly" */);
-            const error = new GenericError("deltaManagerReadonlySubmit", undefined /* error */, {
-                readonly: this.readOnlyInfo.readonly,
-                forcedReadonly: this.readOnlyInfo.forced,
-                readonlyPermissions: this.readOnlyInfo.permissions,
-                storageOnly: this.readOnlyInfo.storageOnly,
-            });
-            this.close(error);
-            return -1;
-        }
-        // reset clientSequenceNumber if we are using new clientId.
-        // we keep info about old connection as long as possible to be able to account for all non-acked ops
-        // that we pick up on next connection.
-        assert(!!this.connection, 0x0e4 /* "Lost old connection!" */);
-        if (this.lastSubmittedClientId !== ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId)) {
-            this.lastSubmittedClientId = (_b = this.connection) === null || _b === void 0 ? void 0 : _b.clientId;
-            this.clientSequenceNumber = 0;
-            this.clientSequenceNumberObserved = 0;
-        }
-        // If connection is "read" or implicit "read" (got leave op for "write" connection),
-        // then op can't make it through - we will get a nack if op is sent.
-        // We can short-circuit this process.
-        // Note that we also want nacks to be rare and be treated as catastrophic failures.
-        // Be careful with reentrancy though - disconnected event should not be be raised in the
-        // middle of the current workflow, but rather on clean stack!
-        if (this.connectionMode === "read") {
-            if (!this.pendingReconnect) {
-                this.pendingReconnect = true;
-                Promise.resolve().then(async () => {
-                    if (this.pendingReconnect) { // still valid?
-                        return this.reconnectOnErrorCore("write", // connectionMode
-                        "Switch to write");
-                    }
-                })
-                    .catch(() => { });
-            }
-            // Can return -1 here, but no other path does it (other than error path in Container),
-            // so it's better not to introduce new states.
-            return ++this.clientSequenceNumber;
-        }
-        const service = this.clientDetails.type === undefined || this.clientDetails.type === ""
-            ? "unknown"
-            : this.clientDetails.type;
-        // Start adding trace for the op.
-        const traces = [
-            {
-                action: "start",
-                service,
-                timestamp: Date.now(),
-            }
-        ];
-        const message = {
-            clientSequenceNumber: ++this.clientSequenceNumber,
-            contents: JSON.stringify(contents),
-            metadata,
-            referenceSequenceNumber: this.lastProcessedSequenceNumber,
-            traces,
-            type,
-        };
-        if (type === MessageType.NoOp) {
-            this.trailingNoopCount++;
-        }
-        else {
-            this.trailingNoopCount = 0;
-        }
-        this.emit("submitOp", message);
-        if (!batch) {
-            this.flush();
-            this.messageBuffer.push(message);
-            this.flush();
-        }
-        else {
-            this.messageBuffer.push(message);
-        }
-        return message.clientSequenceNumber;
-    }
-    submitSignal(content) {
-        if (this.connection !== undefined) {
-            this.connection.submitSignal(content);
-        }
-        else {
-            this.logger.sendErrorEvent({ eventName: "submitSignalDisconnected" });
+            this.fetchMissingDeltas(args.reason);
         }
+        this.connectionManager.connect(args.mode);
     }
     async getDeltas(from, // inclusive
     to, // exclusive
@@ -739,9 +301,6 @@ export class DeltaManager extends TypedEventEmitter {
         if (this.deltaStorage === undefined) {
             this.deltaStorage = await docService.connectToDeltaStorage();
         }
-        assert(this.closeAbortController.signal.onabort === null, 0x1e8 /* "reentrancy" */);
-        const controller = new AbortController();
-        this.closeAbortController.signal.onabort = () => controller.abort();
         let cancelFetch;
         if (to !== undefined) {
             const lastExpectedOp = to - 1; // make it inclusive!
@@ -749,7 +308,7 @@ export class DeltaManager extends TypedEventEmitter {
             // received through delta stream. Validate that before moving forward.
             if (this.lastQueuedSequenceNumber >= lastExpectedOp) {
                 this.logger.sendPerformanceEvent(Object.assign({ reason: this.fetchReason, eventName: "ExtraStorageCall", early: true, from,
-                    to }, this.connectionStateProps));
+                    to }, this.connectionManager.connectionVerboseProps));
                 return;
             }
             // Be prepared for the case where webSocket would receive the ops that we are trying to fill through
@@ -766,6 +325,7 @@ export class DeltaManager extends TypedEventEmitter {
             // That said, if we have socket connection, make sure we got ops up to checkpointSequenceNumber!
             cancelFetch = (op) => op.sequenceNumber >= this.lastObservedSeqNumber;
         }
+        const controller = new AbortController();
         let opsFromFetch = false;
         const opListener = (op) => {
             assert(op.sequenceNumber === this.lastQueuedSequenceNumber, 0x23a /* "seq#'s" */);
@@ -777,11 +337,13 @@ export class DeltaManager extends TypedEventEmitter {
                 this._inbound.off("push", opListener);
             }
         };
-        this._inbound.on("push", opListener);
         try {
+            this._inbound.on("push", opListener);
+            assert(this.closeAbortController.signal.onabort === null, 0x1e8 /* "reentrancy" */);
+            this.closeAbortController.signal.onabort = () => controller.abort();
             const stream = this.deltaStorage.fetchMessages(from, // inclusive
             to, // exclusive
-            controller.signal, cacheOnly);
+            controller.signal, cacheOnly, this.fetchReason);
             // eslint-disable-next-line no-constant-condition
             while (true) {
                 const result = await stream.read();
@@ -798,9 +360,9 @@ export class DeltaManager extends TypedEventEmitter {
             }
         }
         finally {
-            assert(!opsFromFetch, 0x289 /* "logic error" */);
             this.closeAbortController.signal.onabort = null;
             this._inbound.off("push", opListener);
+            assert(!opsFromFetch, 0x289 /* "logic error" */);
         }
     }
     /**
@@ -811,13 +373,9 @@ export class DeltaManager extends TypedEventEmitter {
             return;
         }
         this.closed = true;
-        // Ensure that things like triggerConnect() will short circuit
-        this._reconnectMode = ReconnectMode.Never;
+        this.connectionManager.dispose(error);
         this.closeAbortController.abort();
-        // This raises "disconnect" event if we have active connection.
-        this.disconnectFromDeltaStream(error !== undefined ? `${error.message}` : "Container closed");
         this._inbound.clear();
-        this._outbound.clear();
         this._inboundSignal.clear();
         // eslint-disable-next-line @typescript-eslint/no-floating-promises
         this._inbound.pause();
@@ -825,10 +383,6 @@ export class DeltaManager extends TypedEventEmitter {
         this._inboundSignal.pause();
         // Drop pending messages - this will ensure catchUp() does not go into infinite loop
         this.pending = [];
-        // Notify everyone we are in read-only state.
-        // Useful for data stores in case we hit some critical error,
-        // to switch to a mode where user edits are not accepted
-        this.set_readonlyPermissions(true);
         // This needs to be the last thing we do (before removing listeners), as it causes
         // Container to dispose context and break ability of data stores / runtime to "hear"
         // from delta manager, including notification (above) about readonly state.
@@ -841,6 +395,20 @@ export class DeltaManager extends TypedEventEmitter {
             this.timeTillThrottling = 0;
         }
     }
+    disconnectHandler(reason) {
+        if (this.messageBuffer.length > 0) {
+            // Behavior is not well defined here RE batches across connections / disconnect.
+            // DeltaManager overall policy - drop all ops on disconnection and rely on
+            // container runtime to deal with resubmitting any ops that did not make it through.
+            // So drop them, but also raise error event to look into details.
+            this.logger.sendErrorEvent({
+                eventName: "OpenBatchOnDisconnect",
+                length: this.messageBuffer.length,
+            });
+            this.messageBuffer.length = 0;
+        }
+        this.emit("disconnect", reason);
+    }
     /**
      * Emit info about a delay in service communication on account of throttling.
      * @param id - Id of the connection that is delayed
@@ -856,177 +424,6 @@ export class DeltaManager extends TypedEventEmitter {
             this.emit("throttled", throttlingWarning);
         }
     }
-    /**
-     * Once we've successfully gotten a connection, we need to set up state, attach event listeners, and process
-     * initial messages.
-     * @param connection - The newly established connection
-     */
-    setupNewSuccessfulConnection(connection, requestedMode) {
-        // Old connection should have been cleaned up before establishing a new one
-        assert(this.connection === undefined, 0x0e6 /* "old connection exists on new connection setup" */);
-        assert(this.connectionP !== undefined || this.closed, 0x27f /* "reentrancy may result in incorrect behavior" */);
-        assert(!connection.disposed, 0x28a /* "can't be disposed - Callers need to ensure that!" */);
-        this.connectionP = undefined;
-        this.connection = connection;
-        // Does information in scopes & mode matches?
-        // If we asked for "write" and got "read", then file is read-only
-        // But if we ask read, server can still give us write.
-        const readonly = !connection.claims.scopes.includes(ScopeType.DocWrite);
-        // This connection mode validation logic is moving to the driver layer in 0.44.  These two asserts can be
-        // removed after those packages have released and become ubiquitous.
-        assert(requestedMode === "read" || readonly === (this.connectionMode === "read"), 0x0e7 /* "claims/connectionMode mismatch" */);
-        assert(!readonly || this.connectionMode === "read", 0x0e8 /* "readonly perf with write connection" */);
-        this.set_readonlyPermissions(readonly);
-        this.refreshDelayInfo(this.deltaStreamDelayId);
-        if (this.closed) {
-            // Raise proper events, Log telemetry event and close connection.
-            this.disconnectFromDeltaStream(`Disconnect on close`);
-            return;
-        }
-        // We cancel all ops on lost of connectivity, and rely on DDSes to resubmit them.
-        // Semantics are not well defined for batches (and they are broken right now on disconnects anyway),
-        // but it's safe to assume (until better design is put into place) that batches should not exist
-        // across multiple connections. Right now we assume runtime will not submit any ops in disconnected
-        // state. As requirements change, so should these checks.
-        assert(this.messageBuffer.length === 0, 0x0e9 /* "messageBuffer is not empty on new connection" */);
-        this._outbound.resume();
-        connection.on("op", this.opHandler);
-        connection.on("signal", this.signalHandler);
-        connection.on("nack", this.nackHandler);
-        connection.on("disconnect", this.disconnectHandler);
-        connection.on("error", this.errorHandler);
-        connection.on("pong", this.pongHandler);
-        // Initial messages are always sorted. However, due to early op handler installed by drivers and appending those
-        // ops to initialMessages, resulting set is no longer sorted, which would result in client hitting storage to
-        // fill in gap. We will recover by cancelling this request once we process remaining ops, but it's a waste that
-        // we could avoid
-        const initialMessages = connection.initialMessages.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
-        this.connectionStateProps = {
-            connectionLastQueuedSequenceNumber: this.lastQueuedSequenceNumber,
-            connectionLastObservedSeqNumber: this.lastObservedSeqNumber,
-            clientId: connection.clientId,
-            mode: connection.mode,
-        };
-        this._hasCheckpointSequenceNumber = false;
-        // Some storages may provide checkpointSequenceNumber to identify how far client is behind.
-        const checkpointSequenceNumber = connection.checkpointSequenceNumber;
-        if (checkpointSequenceNumber !== undefined) {
-            this._hasCheckpointSequenceNumber = true;
-            this.updateLatestKnownOpSeqNumber(checkpointSequenceNumber);
-        }
-        // Update knowledge of how far we are behind, before raising "connect" event
-        // This is duplication of what enqueueMessages() does, but we have to raise event before we get there,
-        // so duplicating update logic here as well.
-        const last = initialMessages.length > 0 ? initialMessages[initialMessages.length - 1].sequenceNumber : -1;
-        if (initialMessages.length > 0) {
-            this._hasCheckpointSequenceNumber = true;
-            this.updateLatestKnownOpSeqNumber(last);
-        }
-        // Notify of the connection
-        // WARNING: This has to happen before processInitialMessages() call below.
-        // If not, we may not update Container.pendingClientId in time before seeing our own join session op.
-        this.emit("connect", DeltaManager.detailsFromConnection(connection), this._hasCheckpointSequenceNumber ? this.lastObservedSeqNumber - this.lastSequenceNumber : undefined);
-        this.enqueueMessages(initialMessages, this.connectFirstConnection ? "InitialOps" : "ReconnectOps");
-        if (connection.initialSignals !== undefined) {
-            for (const signal of connection.initialSignals) {
-                this._inboundSignal.push(signal);
-            }
-        }
-        // If we got some initial ops, then we know the gap and call above fetched ops to fill it.
-        // Same is true for "write" mode even if we have no ops - we will get self "join" ops very very soon.
-        // However if we are connecting as view-only, then there is no good signal to realize if client is behind.
-        // Thus we have to hit storage to see if any ops are there.
-        if (initialMessages.length === 0) {
-            if (checkpointSequenceNumber !== undefined) {
-                // We know how far we are behind (roughly). If it's non-zero gap, fetch ops right away.
-                if (checkpointSequenceNumber > this.lastQueuedSequenceNumber) {
-                    this.fetchMissingDeltas("AfterConnection", this.lastQueuedSequenceNumber);
-                }
-                // we do not know the gap, and we will not learn about it if socket is quite - have to ask.
-            }
-            else if (connection.mode === "read") {
-                this.fetchMissingDeltas("AfterReadConnection", this.lastQueuedSequenceNumber);
-            }
-        }
-        else {
-            this.connectionStateProps.connectionInitialOpsFrom = initialMessages[0].sequenceNumber;
-            this.connectionStateProps.connectionInitialOpsTo = last + 1;
-        }
-        this.connectFirstConnection = false;
-    }
-    /**
-     * Disconnect the current connection.
-     * @param reason - Text description of disconnect reason to emit with disconnect event
-     */
-    disconnectFromDeltaStream(reason) {
-        this.pendingReconnect = false;
-        this.downgradedConnection = false;
-        if (this.connection === undefined) {
-            return false;
-        }
-        assert(this.connectionP === undefined, 0x27b /* "reentrancy may result in incorrect behavior" */);
-        const connection = this.connection;
-        // Avoid any re-entrancy - clear object reference
-        this.connection = undefined;
-        // Remove listeners first so we don't try to retrigger this flow accidentally through reconnectOnError
-        connection.off("op", this.opHandler);
-        connection.off("signal", this.signalHandler);
-        connection.off("nack", this.nackHandler);
-        connection.off("disconnect", this.disconnectHandler);
-        connection.off("error", this.errorHandler);
-        connection.off("pong", this.pongHandler);
-        // eslint-disable-next-line @typescript-eslint/no-floating-promises
-        this._outbound.pause();
-        this._outbound.clear();
-        this.emit("disconnect", reason);
-        connection.dispose();
-        this.connectionStateProps = {};
-        return true;
-    }
-    /**
-     * Disconnect the current connection and reconnect.
-     * @param connection - The connection that wants to reconnect - no-op if it's different from this.connection
-     * @param requestedMode - Read or write
-     * @param error - Error reconnect information including whether or not to reconnect
-     * @returns A promise that resolves when the connection is reestablished or we stop trying
-     */
-    async reconnectOnError(requestedMode, error) {
-        return this.reconnectOnErrorCore(requestedMode, error.message, error);
-    }
-    /**
-     * Disconnect the current connection and reconnect.
-     * @param connection - The connection that wants to reconnect - no-op if it's different from this.connection
-     * @param requestedMode - Read or write
-     * @param error - Error reconnect information including whether or not to reconnect
-     * @returns A promise that resolves when the connection is reestablished or we stop trying
-     */
-    async reconnectOnErrorCore(requestedMode, disconnectMessage, error) {
-        // We quite often get protocol errors before / after observing nack/disconnect
-        // we do not want to run through same sequence twice.
-        // If we're already disconnected/disconnecting it's not appropriate to call this again.
-        assert(this.connection !== undefined, 0x0eb /* "Missing connection for reconnect" */);
-        this.disconnectFromDeltaStream(disconnectMessage);
-        const canRetry = error !== undefined ? canRetryOnError(error) : true;
-        // If reconnection is not an option, close the DeltaManager
-        if (this.reconnectMode === ReconnectMode.Never || !canRetry) {
-            // Do not raise container error if we are closing just because we lost connection.
-            // Those errors (like IdleDisconnect) would show up in telemetry dashboards and
-            // are very misleading, as first initial reaction - some logic is broken.
-            this.close(canRetry ? undefined : error);
-        }
-        // If closed then we can't reconnect
-        if (this.closed) {
-            return;
-        }
-        if (this.reconnectMode === ReconnectMode.Enabled) {
-            const delayMs = error !== undefined ? getRetryDelayFromError(error) : undefined;
-            if (delayMs !== undefined) {
-                this.emitDelayInfo(this.deltaStreamDelayId, delayMs, error);
-                await waitForConnectedState(delayMs);
-            }
-            this.triggerConnect({ reason: "reconnect", mode: requestedMode, fetchOpsFromStorage: false });
-        }
-    }
     // returns parts of message (in string format) that should never change for a given message.
     // Used for message comparison. It attempts to avoid comparing fields that potentially may differ.
     // for example, it's not clear if serverMetadata or timestamp property is a property of message or server state.
@@ -1038,7 +435,7 @@ export class DeltaManager extends TypedEventEmitter {
         return `${m.clientId}-${m.type}-${m.minimumSequenceNumber}-${m.referenceSequenceNumber}-${m.timestamp}`;
     }
     enqueueMessages(messages, reason, allowGaps = false) {
-        var _a, _b, _c;
+        var _a, _b;
         if (this.handler === undefined) {
             // We did not setup handler yet.
             // This happens when we connect to web socket faster than we get attributes for container
@@ -1098,7 +495,7 @@ export class DeltaManager extends TypedEventEmitter {
             // correctly take into account pending ops.
             if (eventName !== undefined) {
                 this.logger.sendPerformanceEvent(Object.assign({ eventName,
-                    reason, previousReason: this.prevEnqueueMessagesReason, from, to: last + 1, length: messages.length, fetchReason: this.fetchReason, duplicate: duplicate > 0 ? duplicate : undefined, initialGap: initialGap !== 0 ? initialGap : undefined, gap: gap > 0 ? gap : undefined, firstMissing, dmInitialSeqNumber: this.initialSequenceNumber }, this.connectionStateProps));
+                    reason, previousReason: this.prevEnqueueMessagesReason, from, to: last + 1, length: messages.length, fetchReason: this.fetchReason, duplicate: duplicate > 0 ? duplicate : undefined, initialGap: initialGap !== 0 ? initialGap : undefined, gap: gap > 0 ? gap : undefined, firstMissing, dmInitialSeqNumber: this.initialSequenceNumber }, this.connectionManager.connectionVerboseProps));
             }
         }
         this.updateLatestKnownOpSeqNumber(messages[messages.length - 1].sequenceNumber);
@@ -1114,7 +511,7 @@ export class DeltaManager extends TypedEventEmitter {
                     const message2 = this.comparableMessagePayload(message);
                     if (message1 !== message2) {
                         const error = new NonRetryableError("twoMessagesWithSameSeqNumAndDifferentPayload", undefined, DriverErrorType.fileOverwrittenInStorage, {
-                            clientId: (_c = this.connection) === null || _c === void 0 ? void 0 : _c.clientId,
+                            clientId: this.connectionManager.clientId,
                             sequenceNumber: message.sequenceNumber,
                             message1,
                             message2,
@@ -1125,7 +522,7 @@ export class DeltaManager extends TypedEventEmitter {
             }
             else if (message.sequenceNumber !== this.lastQueuedSequenceNumber + 1) {
                 this.pending.push(message);
-                this.fetchMissingDeltas(reason, this.lastQueuedSequenceNumber, message.sequenceNumber);
+                this.fetchMissingDeltas(reason, message.sequenceNumber);
             }
             else {
                 this.lastQueuedSequenceNumber = message.sequenceNumber;
@@ -1139,57 +536,34 @@ export class DeltaManager extends TypedEventEmitter {
         this.prevEnqueueMessagesReason = this.pending.length > 0 ? "unknown" : reason;
     }
     processInboundMessage(message) {
-        var _a, _b, _c;
         const startTime = Date.now();
         this.lastProcessedMessage = message;
         // All non-system messages are coming from some client, and should have clientId
         // System messages may have no clientId (but some do, like propose, noop, summarize)
         assert(message.clientId !== undefined
             || isSystemMessage(message), 0x0ed /* "non-system message have to have clientId" */);
-        // if we have connection, and message is local, then we better treat is as local!
-        assert(this.connection === undefined
-            || this.connection.clientId !== message.clientId
-            || this.lastSubmittedClientId === message.clientId, 0x0ee /* "Not accounting local messages correctly" */);
-        if (this.lastSubmittedClientId !== undefined && this.lastSubmittedClientId === message.clientId) {
-            const clientSequenceNumber = message.clientSequenceNumber;
-            assert(this.clientSequenceNumberObserved < clientSequenceNumber, 0x0ef /* "client seq# not growing" */);
-            assert(clientSequenceNumber <= this.clientSequenceNumber, 0x0f0 /* "Incoming local client seq# > generated by this client" */);
-            this.clientSequenceNumberObserved = clientSequenceNumber;
-        }
         // TODO Remove after SPO picks up the latest build.
         if (typeof message.contents === "string"
             && message.contents !== ""
             && message.type !== MessageType.ClientLeave) {
             message.contents = JSON.parse(message.contents);
         }
-        if (message.type === MessageType.ClientLeave) {
-            const systemLeaveMessage = message;
-            const clientId = JSON.parse(systemLeaveMessage.data);
-            if (clientId === ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId)) {
-                // We have been kicked out from quorum
-                this.logger.sendPerformanceEvent({ eventName: "ReadConnectionTransition" });
-                this.downgradedConnection = true;
-                assert(this.connectionMode === "read", 0x27c /* "effective connectionMode should be 'read' after downgrade" */);
-            }
-        }
+        this.connectionManager.beforeProcessingIncomingOp(message);
         // Add final ack trace.
         if (message.traces !== undefined && message.traces.length > 0) {
-            const service = this.clientDetails.type === undefined || this.clientDetails.type === ""
-                ? "unknown"
-                : this.clientDetails.type;
             message.traces.push({
                 action: "end",
-                service,
+                service: "client",
                 timestamp: Date.now(),
             });
         }
         // Watch the minimum sequence number and be ready to update as needed
         if (this.minSequenceNumber > message.minimumSequenceNumber) {
-            throw new DataCorruptionError("msnMovesBackwards", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId: (_b = this.connection) === null || _b === void 0 ? void 0 : _b.clientId }));
+            throw new DataCorruptionError("msnMovesBackwards", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId: this.connectionManager.clientId }));
         }
         this.minSequenceNumber = message.minimumSequenceNumber;
         if (message.sequenceNumber !== this.lastProcessedSequenceNumber + 1) {
-            throw new DataCorruptionError("nonSequentialSequenceNumber", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId: (_c = this.connection) === null || _c === void 0 ? void 0 : _c.clientId }));
+            throw new DataCorruptionError("nonSequentialSequenceNumber", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId: this.connectionManager.clientId }));
         }
         this.lastProcessedSequenceNumber = message.sequenceNumber;
         // a bunch of code assumes that this is true
@@ -1211,15 +585,15 @@ export class DeltaManager extends TypedEventEmitter {
     /**
      * Retrieves the missing deltas between the given sequence numbers
      */
-    fetchMissingDeltas(reasonArg, lastKnowOp, to) {
-        this.fetchMissingDeltasCore(reasonArg, false /* cacheOnly */, lastKnowOp, to).catch((error) => {
+    fetchMissingDeltas(reasonArg, to) {
+        this.fetchMissingDeltasCore(reasonArg, false /* cacheOnly */, to).catch((error) => {
             this.logger.sendErrorEvent({ eventName: "fetchMissingDeltasException" }, error);
         });
     }
     /**
     * Retrieves the missing deltas between the given sequence numbers
     */
-    async fetchMissingDeltasCore(reason, cacheOnly, lastKnowOp, to) {
+    async fetchMissingDeltasCore(reason, cacheOnly, to) {
         var _a;
         // Exit out early if we're already fetching deltas
         if (this.fetchReason !== undefined) {
@@ -1231,12 +605,11 @@ export class DeltaManager extends TypedEventEmitter {
         }
         if (this.handler === undefined) {
             // We do not poses yet any information
-            assert(lastKnowOp === 0, 0x26b /* "initial state" */);
+            assert(this.lastQueuedSequenceNumber === 0, 0x26b /* "initial state" */);
             return;
         }
         try {
-            assert(lastKnowOp === this.lastQueuedSequenceNumber, 0x0f1 /* "from arg" */);
-            let from = lastKnowOp + 1;
+            let from = this.lastQueuedSequenceNumber + 1;
             const n = (_a = this.previouslyProcessedMessage) === null || _a === void 0 ? void 0 : _a.sequenceNumber;
             if (n !== undefined) {
                 // If we already processed at least one op, then we have this.previouslyProcessedMessage populated
@@ -1244,7 +617,7 @@ export class DeltaManager extends TypedEventEmitter {
                 // Knowing about this mechanism, we could ask for op we already observed to increase validation.
                 // This is especially useful when coming out of offline mode or loading from
                 // very old cached (by client / driver) snapshot.
-                assert(n === lastKnowOp, 0x0f2 /* "previouslyProcessedMessage" */);
+                assert(n === this.lastQueuedSequenceNumber, 0x0f2 /* "previouslyProcessedMessage" */);
                 assert(from > 1, 0x0f3 /* "not positive" */);
                 from--;
             }
@@ -1292,7 +665,7 @@ export class DeltaManager extends TypedEventEmitter {
             // (the other 50%), and thus these errors below should be looked at even if code below results in
             // recovery.
             if (this.lastQueuedSequenceNumber < this.lastObservedSeqNumber) {
-                this.fetchMissingDeltas("OpsBehind", this.lastQueuedSequenceNumber);
+                this.fetchMissingDeltas("OpsBehind");
             }
         }
     }