@fluidframework/container-loader 0.52.1 → 0.54.0

package/dist/connectionManager.js

@@ -0,0 +1,664 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConnectionManager = void 0;
+const common_utils_1 = require("@fluidframework/common-utils");
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
+const driver_utils_1 = require("@fluidframework/driver-utils");
+const container_utils_1 = require("@fluidframework/container-utils");
+const deltaQueue_1 = require("./deltaQueue");
+const contracts_1 = require("./contracts");
+const MaxReconnectDelayInMs = 8000;
+const InitialReconnectDelayInMs = 1000;
+const DefaultChunkSize = 16 * 1024;
+const fatalConnectErrorProp = { fatalConnectError: true };
+function getNackReconnectInfo(nackContent) {
+    const message = `Nack (${nackContent.type}): ${nackContent.message}`;
+    const canRetry = nackContent.code !== 403;
+    const retryAfterMs = nackContent.retryAfter !== undefined ? nackContent.retryAfter * 1000 : undefined;
+    return driver_utils_1.createGenericNetworkError(`nack [${nackContent.code}]`, message, canRetry, retryAfterMs, { statusCode: nackContent.code });
+}
+const createReconnectError = (fluidErrorCode, err) => telemetry_utils_1.wrapError(err, (errorMessage) => new driver_utils_1.GenericNetworkError(fluidErrorCode, errorMessage, true /* canRetry */));
+/**
+ * Implementation of IDocumentDeltaConnection that does not support submitting
+ * or receiving ops. Used in storage-only mode.
+ */
+class NoDeltaStream extends common_utils_1.TypedEventEmitter {
+    constructor() {
+        super(...arguments);
+        this.clientId = "storage-only client";
+        this.claims = {
+            scopes: [protocol_definitions_1.ScopeType.DocRead],
+        };
+        this.mode = "read";
+        this.existing = true;
+        this.maxMessageSize = 0;
+        this.version = "";
+        this.initialMessages = [];
+        this.initialSignals = [];
+        this.initialClients = [];
+        this.serviceConfiguration = {
+            maxMessageSize: 0,
+            blockSize: 0,
+            summary: undefined,
+        };
+        this.checkpointSequenceNumber = undefined;
+        this._disposed = false;
+    }
+    submit(messages) {
+        this.emit("nack", this.clientId, messages.map((operation) => {
+            return {
+                operation,
+                content: { message: "Cannot submit with storage-only connection", code: 403 },
+            };
+        }));
+    }
+    submitSignal(message) {
+        this.emit("nack", this.clientId, {
+            operation: message,
+            content: { message: "Cannot submit signal with storage-only connection", code: 403 },
+        });
+    }
+    get disposed() { return this._disposed; }
+    dispose() { this._disposed = true; }
+}
+/**
+ * Implementation of IConnectionManager, used by Container class
+ * Implements constant connectivity to relay service, by reconnecting in case of loast connection or error.
+ * Exposes various controls to influecen this process, including manual reconnects, forced read-only mode, etc.
+ */
+class ConnectionManager {
+    constructor(serviceProvider, client, reconnectAllowed, logger, props) {
+        this.serviceProvider = serviceProvider;
+        this.client = client;
+        this.logger = logger;
+        this.props = props;
+        this.pendingConnection = false;
+        /** tracks host requiring read-only mode. */
+        this._forceReadonly = false;
+        /** True if there is pending (async) reconnection from "read" to "write" */
+        this.pendingReconnect = false;
+        /** downgrade "write" connection to "read" */
+        this.downgradedConnection = false;
+        this.clientSequenceNumber = 0;
+        this.clientSequenceNumberObserved = 0;
+        /** Counts the number of noops sent by the client which may not be acked. */
+        this.trailingNoopCount = 0;
+        this.connectFirstConnection = true;
+        this._connectionVerboseProps = {};
+        this._connectionProps = {};
+        this.closed = false;
+        this.opHandler = (documentId, messagesArg) => {
+            const messages = Array.isArray(messagesArg) ? messagesArg : [messagesArg];
+            this.props.incomingOpHandler(messages, "opHandler");
+        };
+        // Always connect in write mode after getting nacked.
+        this.nackHandler = (documentId, messages) => {
+            const message = messages[0];
+            // TODO: we should remove this check when service updates?
+            // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+            if (this._readonlyPermissions) {
+                this.props.closeHandler(driver_utils_1.createWriteError("writeOnReadOnlyDocument"));
+            }
+            // check message.content for Back-compat with old service.
+            const reconnectInfo = message.content !== undefined
+                ? getNackReconnectInfo(message.content) :
+                driver_utils_1.createGenericNetworkError("nackReasonUnknown", undefined, true);
+            this.reconnectOnError("write", reconnectInfo);
+        };
+        // Connection mode is always read on disconnect/error unless the system mode was write.
+        this.disconnectHandlerInternal = (disconnectReason) => {
+            // Note: we might get multiple disconnect calls on same socket, as early disconnect notification
+            // ("server_disconnect", ODSP-specific) is mapped to "disconnect"
+            this.reconnectOnError(this.defaultReconnectionMode, createReconnectError("dmDocumentDeltaConnectionDisconnected", disconnectReason));
+        };
+        this.errorHandler = (error) => {
+            this.reconnectOnError(this.defaultReconnectionMode, createReconnectError("dmDocumentDeltaConnectionError", error));
+        };
+        this.clientDetails = this.client.details;
+        this.defaultReconnectionMode = this.client.mode;
+        this._reconnectMode = reconnectAllowed ? contracts_1.ReconnectMode.Enabled : contracts_1.ReconnectMode.Never;
+        // Outbound message queue. The outbound queue is represented as a queue of an array of ops. Ops contained
+        // within an array *must* fit within the maxMessageSize and are guaranteed to be ordered sequentially.
+        this._outbound = new deltaQueue_1.DeltaQueue((messages) => {
+            if (this.connection === undefined) {
+                throw new Error("Attempted to submit an outbound message without connection");
+            }
+            this.connection.submit(messages);
+        });
+        this._outbound.on("error", (error) => {
+            this.props.closeHandler(telemetry_utils_1.normalizeError(error));
+        });
+    }
+    get connectionVerboseProps() { return this._connectionVerboseProps; }
+    /**
+     * The current connection mode, initially read.
+     */
+    get connectionMode() {
+        var _a;
+        common_utils_1.assert(!this.downgradedConnection || ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.mode) === "write", 0x277 /* "Did we forget to reset downgradedConnection on new connection?" */);
+        if (this.connection === undefined) {
+            return "read";
+        }
+        return this.connection.mode;
+    }
+    get connected() { return this.connection !== undefined; }
+    get clientId() { var _a; return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId; }
+    /**
+     * Automatic reconnecting enabled or disabled.
+     * If set to Never, then reconnecting will never be allowed.
+     */
+    get reconnectMode() {
+        return this._reconnectMode;
+    }
+    get maxMessageSize() {
+        var _a, _b, _c;
+        return (_c = (_b = (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration) === null || _b === void 0 ? void 0 : _b.maxMessageSize) !== null && _c !== void 0 ? _c : DefaultChunkSize;
+    }
+    get version() {
+        if (this.connection === undefined) {
+            throw new Error("Cannot check version without a connection");
+        }
+        return this.connection.version;
+    }
+    get serviceConfiguration() {
+        var _a;
+        return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration;
+    }
+    get scopes() {
+        var _a;
+        return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.claims.scopes;
+    }
+    get outbound() {
+        return this._outbound;
+    }
+    /**
+     * Returns set of props that can be logged in telemetry that provide some insights / statistics
+     * about current or last connection (if there is no connection at the moment)
+    */
+    get connectionProps() {
+        if (this.connection !== undefined) {
+            return this._connectionProps;
+        }
+        else {
+            return Object.assign(Object.assign({}, this._connectionProps), { 
+                // Report how many ops this client sent in last disconnected session
+                sentOps: this.clientSequenceNumber });
+        }
+    }
+    shouldJoinWrite() {
+        // We don't have to wait for ack for topmost NoOps. So subtract those.
+        return this.clientSequenceNumberObserved < (this.clientSequenceNumber - this.trailingNoopCount);
+    }
+    /**
+     * Tells if container is in read-only mode.
+     * Data stores should listen for "readonly" notifications and disallow user
+     * making changes to data stores.
+     * Readonly state can be because of no storage write permission,
+     * or due to host forcing readonly mode for container.
+     * It is undefined if we have not yet established websocket connection
+     * and do not know if user has write access to a file.
+     */
+    get readonly() {
+        if (this._forceReadonly) {
+            return true;
+        }
+        return this._readonlyPermissions;
+    }
+    get readOnlyInfo() {
+        const storageOnly = this.connection !== undefined && this.connection instanceof NoDeltaStream;
+        if (storageOnly || this._forceReadonly || this._readonlyPermissions === true) {
+            return {
+                readonly: true,
+                forced: this._forceReadonly,
+                permissions: this._readonlyPermissions,
+                storageOnly,
+            };
+        }
+        return { readonly: this._readonlyPermissions };
+    }
+    static detailsFromConnection(connection) {
+        return {
+            claims: connection.claims,
+            clientId: connection.clientId,
+            existing: connection.existing,
+            checkpointSequenceNumber: connection.checkpointSequenceNumber,
+            get initialClients() { return connection.initialClients; },
+            mode: connection.mode,
+            serviceConfiguration: connection.serviceConfiguration,
+            version: connection.version,
+        };
+    }
+    dispose(error) {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        this.pendingConnection = false;
+        // Ensure that things like triggerConnect() will short circuit
+        this._reconnectMode = contracts_1.ReconnectMode.Never;
+        this._outbound.clear();
+        const disconnectReason = error !== undefined
+            ? `Closing DeltaManager (${error.message})`
+            : "Closing DeltaManager";
+        // This raises "disconnect" event if we have active connection.
+        this.disconnectFromDeltaStream(disconnectReason);
+        // Notify everyone we are in read-only state.
+        // Useful for data stores in case we hit some critical error,
+        // to switch to a mode where user edits are not accepted
+        this.set_readonlyPermissions(true);
+    }
+    /**
+     * Enables or disables automatic reconnecting.
+     * Will throw an error if reconnectMode set to Never.
+    */
+    setAutoReconnect(mode) {
+        common_utils_1.assert(mode !== contracts_1.ReconnectMode.Never && this._reconnectMode !== contracts_1.ReconnectMode.Never, 0x278 /* "API is not supported for non-connecting or closed container" */);
+        this._reconnectMode = mode;
+        if (mode !== contracts_1.ReconnectMode.Enabled) {
+            // immediately disconnect - do not rely on service eventually dropping connection.
+            this.disconnectFromDeltaStream("setAutoReconnect");
+        }
+    }
+    /**
+     * Sends signal to runtime (and data stores) to be read-only.
+     * Hosts may have read only views, indicating to data stores that no edits are allowed.
+     * This is independent from this._readonlyPermissions (permissions) and this.connectionMode
+     * (server can return "write" mode even when asked for "read")
+     * Leveraging same "readonly" event as runtime & data stores should behave the same in such case
+     * as in read-only permissions.
+     * But this.active can be used by some DDSes to figure out if ops can be sent
+     * (for example, read-only view still participates in code proposals / upgrades decisions)
+     *
+     * Forcing Readonly does not prevent DDS from generating ops. It is up to user code to honour
+     * the readonly flag. If ops are generated, they will accumulate locally and not be sent. If
+     * there are pending in the outbound queue, it will stop sending until force readonly is
+     * cleared.
+     *
+     * @param readonly - set or clear force readonly.
+     */
+    forceReadonly(readonly) {
+        if (readonly !== this._forceReadonly) {
+            this.logger.sendTelemetryEvent({
+                eventName: "ForceReadOnly",
+                value: readonly,
+            });
+        }
+        const oldValue = this.readonly;
+        this._forceReadonly = readonly;
+        if (oldValue !== this.readonly) {
+            common_utils_1.assert(this._reconnectMode !== contracts_1.ReconnectMode.Never, 0x279 /* "API is not supported for non-connecting or closed container" */);
+            let reconnect = false;
+            if (this.readonly === true) {
+                // If we switch to readonly while connected, we should disconnect first
+                // See comment in the "readonly" event handler to deltaManager set up by
+                // the ContainerRuntime constructor
+                if (this.shouldJoinWrite()) {
+                    // If we have pending changes, then we will never send them - it smells like
+                    // host logic error.
+                    this.logger.sendErrorEvent({ eventName: "ForceReadonlyPendingChanged" });
+                }
+                reconnect = this.disconnectFromDeltaStream("Force readonly");
+            }
+            this.props.readonlyChangeHandler(this.readonly);
+            if (reconnect) {
+                // reconnect if we disconnected from before.
+                this.triggerConnect("read");
+            }
+        }
+    }
+    set_readonlyPermissions(readonly) {
+        const oldValue = this.readonly;
+        this._readonlyPermissions = readonly;
+        if (oldValue !== this.readonly) {
+            this.props.readonlyChangeHandler(this.readonly);
+        }
+    }
+    connect(connectionMode) {
+        this.connectCore(connectionMode).catch((error) => {
+            const normalizedError = telemetry_utils_1.normalizeError(error, { props: fatalConnectErrorProp });
+            this.props.closeHandler(normalizedError);
+        });
+    }
+    async connectCore(connectionMode) {
+        var _a;
+        common_utils_1.assert(!this.closed, 0x26a /* "not closed" */);
+        if (this.connection !== undefined || this.pendingConnection) {
+            return;
+        }
+        let requestedMode = connectionMode !== null && connectionMode !== void 0 ? connectionMode : this.defaultReconnectionMode;
+        // if we have any non-acked ops from last connection, reconnect as "write".
+        // without that we would connect in view-only mode, which will result in immediate
+        // firing of "connected" event from Container and switch of current clientId (as tracked
+        // by all DDSes). This will make it impossible to figure out if ops actually made it through,
+        // so DDSes will immediately resubmit all pending ops, and some of them will be duplicates, corrupting document
+        if (this.shouldJoinWrite()) {
+            requestedMode = "write";
+        }
+        const docService = this.serviceProvider();
+        common_utils_1.assert(docService !== undefined, 0x2a7 /* "Container is not attached" */);
+        let connection;
+        if (((_a = docService.policies) === null || _a === void 0 ? void 0 : _a.storageOnly) === true) {
+            connection = new NoDeltaStream();
+            // to keep setupNewSuccessfulConnection happy
+            this.pendingConnection = true;
+            this.setupNewSuccessfulConnection(connection, "read");
+            common_utils_1.assert(!this.pendingConnection, 0x2b3 /* "logic error" */);
+            return;
+        }
+        // this.pendingConnection resets to false as soon as we know the outcome of the connection attempt
+        this.pendingConnection = true;
+        let delayMs = InitialReconnectDelayInMs;
+        let connectRepeatCount = 0;
+        const connectStartTime = common_utils_1.performance.now();
+        let lastError;
+        // This loop will keep trying to connect until successful, with a delay between each iteration.
+        while (connection === undefined) {
+            if (this.closed) {
+                throw new Error("Attempting to connect a closed DeltaManager");
+            }
+            connectRepeatCount++;
+            try {
+                this.client.mode = requestedMode;
+                connection = await docService.connectToDeltaStream(this.client);
+                if (connection.disposed) {
+                    // Nobody observed this connection, so drop it on the floor and retry.
+                    this.logger.sendTelemetryEvent({ eventName: "ReceivedClosedConnection" });
+                    connection = undefined;
+                }
+            }
+            catch (origError) {
+                if (typeof origError === "object" && origError !== null &&
+                    (origError === null || origError === void 0 ? void 0 : origError.errorType) === driver_utils_1.DeltaStreamConnectionForbiddenError.errorType) {
+                    connection = new NoDeltaStream();
+                    requestedMode = "read";
+                    break;
+                }
+                // Socket.io error when we connect to wrong socket, or hit some multiplexing bug
+                if (!driver_utils_1.canRetryOnError(origError)) {
+                    const error = telemetry_utils_1.normalizeError(origError, { props: fatalConnectErrorProp });
+                    this.props.closeHandler(error);
+                    throw error;
+                }
+                // Log error once - we get too many errors in logs when we are offline,
+                // and unfortunately there is no reliable way to detect that.
+                if (connectRepeatCount === 1) {
+                    driver_utils_1.logNetworkFailure(this.logger, {
+                        delay: delayMs,
+                        eventName: "DeltaConnectionFailureToConnect",
+                        duration: telemetry_utils_1.TelemetryLogger.formatTick(common_utils_1.performance.now() - connectStartTime),
+                    }, origError);
+                }
+                lastError = origError;
+                const retryDelayFromError = driver_utils_1.getRetryDelayFromError(origError);
+                delayMs = retryDelayFromError !== null && retryDelayFromError !== void 0 ? retryDelayFromError : Math.min(delayMs * 2, MaxReconnectDelayInMs);
+                if (retryDelayFromError !== undefined) {
+                    this.props.reconnectionDelayHandler(retryDelayFromError, origError);
+                }
+                await driver_utils_1.waitForConnectedState(delayMs);
+            }
+        }
+        // If we retried more than once, log an event about how long it took
+        if (connectRepeatCount > 1) {
+            this.logger.sendTelemetryEvent({
+                eventName: "MultipleDeltaConnectionFailures",
+                attempts: connectRepeatCount,
+                duration: telemetry_utils_1.TelemetryLogger.formatTick(common_utils_1.performance.now() - connectStartTime),
+            }, lastError);
+        }
+        this.setupNewSuccessfulConnection(connection, requestedMode);
+    }
+    /**
+     * Start the connection. Any error should result in container being close.
+     * And report the error if it excape for any reason.
+     * @param args - The connection arguments
+     */
+    triggerConnect(connectionMode) {
+        common_utils_1.assert(this.connection === undefined, 0x239 /* "called only in disconnected state" */);
+        if (this.reconnectMode !== contracts_1.ReconnectMode.Enabled) {
+            return;
+        }
+        this.connect(connectionMode);
+    }
+    /**
+     * Disconnect the current connection.
+     * @param reason - Text description of disconnect reason to emit with disconnect event
+     */
+    disconnectFromDeltaStream(reason) {
+        this.pendingReconnect = false;
+        this.downgradedConnection = false;
+        if (this.connection === undefined) {
+            return false;
+        }
+        common_utils_1.assert(!this.pendingConnection, 0x27b /* "reentrancy may result in incorrect behavior" */);
+        const connection = this.connection;
+        // Avoid any re-entrancy - clear object reference
+        this.connection = undefined;
+        // Remove listeners first so we don't try to retrigger this flow accidentally through reconnectOnError
+        connection.off("op", this.opHandler);
+        connection.off("signal", this.props.signalHandler);
+        connection.off("nack", this.nackHandler);
+        connection.off("disconnect", this.disconnectHandlerInternal);
+        connection.off("error", this.errorHandler);
+        connection.off("pong", this.props.pongHandler);
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        this._outbound.pause();
+        this._outbound.clear();
+        this.props.disconnectHandler(reason);
+        connection.dispose();
+        this._connectionVerboseProps = {};
+        return true;
+    }
+    /**
+     * Once we've successfully gotten a connection, we need to set up state, attach event listeners, and process
+     * initial messages.
+     * @param connection - The newly established connection
+     */
+    setupNewSuccessfulConnection(connection, requestedMode) {
+        // Old connection should have been cleaned up before establishing a new one
+        common_utils_1.assert(this.connection === undefined, 0x0e6 /* "old connection exists on new connection setup" */);
+        common_utils_1.assert(!connection.disposed, 0x28a /* "can't be disposed - Callers need to ensure that!" */);
+        if (this.pendingConnection) {
+            this.pendingConnection = false;
+        }
+        else {
+            common_utils_1.assert(this.closed, 0x27f /* "reentrancy may result in incorrect behavior" */);
+        }
+        this.connection = connection;
+        // Does information in scopes & mode matches?
+        // If we asked for "write" and got "read", then file is read-only
+        // But if we ask read, server can still give us write.
+        const readonly = !connection.claims.scopes.includes(protocol_definitions_1.ScopeType.DocWrite);
+        // This connection mode validation logic is moving to the driver layer in 0.44.  These two asserts can be
+        // removed after those packages have released and become ubiquitous.
+        common_utils_1.assert(requestedMode === "read" || readonly === (this.connectionMode === "read"), 0x0e7 /* "claims/connectionMode mismatch" */);
+        common_utils_1.assert(!readonly || this.connectionMode === "read", 0x0e8 /* "readonly perf with write connection" */);
+        this.set_readonlyPermissions(readonly);
+        if (this.closed) {
+            // Raise proper events, Log telemetry event and close connection.
+            this.disconnectFromDeltaStream("ConnectionManager already closed");
+            return;
+        }
+        this._outbound.resume();
+        connection.on("op", this.opHandler);
+        connection.on("signal", this.props.signalHandler);
+        connection.on("nack", this.nackHandler);
+        connection.on("disconnect", this.disconnectHandlerInternal);
+        connection.on("error", this.errorHandler);
+        connection.on("pong", this.props.pongHandler);
+        // Initial messages are always sorted. However, due to early op handler installed by drivers and appending those
+        // ops to initialMessages, resulting set is no longer sorted, which would result in client hitting storage to
+        // fill in gap. We will recover by cancelling this request once we process remaining ops, but it's a waste that
+        // we could avoid
+        const initialMessages = connection.initialMessages.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
+        // Some storages may provide checkpointSequenceNumber to identify how far client is behind.
+        let checkpointSequenceNumber = connection.checkpointSequenceNumber;
+        this._connectionVerboseProps = {
+            clientId: connection.clientId,
+            mode: connection.mode,
+        };
+        // reset connection props
+        this._connectionProps = {};
+        if (connection.relayServiceAgent !== undefined) {
+            this._connectionVerboseProps.relayServiceAgent = connection.relayServiceAgent;
+            this._connectionProps.relayServiceAgent = connection.relayServiceAgent;
+        }
+        this._connectionProps.socketDocumentId = connection.claims.documentId;
+        this._connectionProps.connectionMode = connection.mode;
+        let last = -1;
+        if (initialMessages.length !== 0) {
+            this._connectionVerboseProps.connectionInitialOpsFrom = initialMessages[0].sequenceNumber;
+            last = initialMessages[initialMessages.length - 1].sequenceNumber;
+            this._connectionVerboseProps.connectionInitialOpsTo = last + 1;
+            // Update knowledge of how far we are behind, before raising "connect" event
+            // This is duplication of what incomingOpHandler() does, but we have to raise event before we get there,
+            // so duplicating update logic here as well.
+            if (checkpointSequenceNumber === undefined || checkpointSequenceNumber < last) {
+                checkpointSequenceNumber = last;
+            }
+        }
+        this.props.incomingOpHandler(initialMessages, this.connectFirstConnection ? "InitialOps" : "ReconnectOps");
+        if (connection.initialSignals !== undefined) {
+            for (const signal of connection.initialSignals) {
+                this.props.signalHandler(signal);
+            }
+        }
+        const details = ConnectionManager.detailsFromConnection(connection);
+        details.checkpointSequenceNumber = checkpointSequenceNumber;
+        this.props.connectHandler(details);
+        this.connectFirstConnection = false;
+    }
+    /**
+     * Disconnect the current connection and reconnect.
+     * @param connection - The connection that wants to reconnect - no-op if it's different from this.connection
+     * @param requestedMode - Read or write
+     * @param error - Error reconnect information including whether or not to reconnect
+     * @returns A promise that resolves when the connection is reestablished or we stop trying
+     */
+    reconnectOnError(requestedMode, error) {
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        this.reconnectOnErrorCore(requestedMode, error.message, error);
+    }
+    /**
+     * Disconnect the current connection and reconnect.
+     * @param connection - The connection that wants to reconnect - no-op if it's different from this.connection
+     * @param requestedMode - Read or write
+     * @param error - Error reconnect information including whether or not to reconnect
+     * @returns A promise that resolves when the connection is reestablished or we stop trying
+     */
+    async reconnectOnErrorCore(requestedMode, disconnectMessage, error) {
+        // We quite often get protocol errors before / after observing nack/disconnect
+        // we do not want to run through same sequence twice.
+        // If we're already disconnected/disconnecting it's not appropriate to call this again.
+        common_utils_1.assert(this.connection !== undefined, 0x0eb /* "Missing connection for reconnect" */);
+        this.disconnectFromDeltaStream(disconnectMessage);
+        const canRetry = error !== undefined ? driver_utils_1.canRetryOnError(error) : true;
+        // If reconnection is not an option, close the DeltaManager
+        if (!canRetry) {
+            this.props.closeHandler(telemetry_utils_1.normalizeError(error, { props: fatalConnectErrorProp }));
+        }
+        else if (this.reconnectMode === contracts_1.ReconnectMode.Never) {
+            // Do not raise container error if we are closing just because we lost connection.
+            // Those errors (like IdleDisconnect) would show up in telemetry dashboards and
+            // are very misleading, as first initial reaction - some logic is broken.
+            this.props.closeHandler();
+        }
+        // If closed then we can't reconnect
+        if (this.closed || this.reconnectMode !== contracts_1.ReconnectMode.Enabled) {
+            return;
+        }
+        const delayMs = error !== undefined ? driver_utils_1.getRetryDelayFromError(error) : undefined;
+        if (delayMs !== undefined) {
+            this.props.reconnectionDelayHandler(delayMs, error);
+            await driver_utils_1.waitForConnectedState(delayMs);
+        }
+        this.triggerConnect(requestedMode);
+    }
+    prepareMessageToSend(message) {
+        var _a, _b;
+        if (this.readonly === true) {
+            common_utils_1.assert(this.readOnlyInfo.readonly === true, 0x1f0 /* "Unexpected mismatch in readonly" */);
+            const error = new container_utils_1.GenericError("deltaManagerReadonlySubmit", undefined /* error */, {
+                readonly: this.readOnlyInfo.readonly,
+                forcedReadonly: this.readOnlyInfo.forced,
+                readonlyPermissions: this.readOnlyInfo.permissions,
+                storageOnly: this.readOnlyInfo.storageOnly,
+            });
+            this.props.closeHandler(error);
+            return undefined;
+        }
+        // reset clientSequenceNumber if we are using new clientId.
+        // we keep info about old connection as long as possible to be able to account for all non-acked ops
+        // that we pick up on next connection.
+        common_utils_1.assert(!!this.connection, 0x0e4 /* "Lost old connection!" */);
+        if (this.lastSubmittedClientId !== ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId)) {
+            this.lastSubmittedClientId = (_b = this.connection) === null || _b === void 0 ? void 0 : _b.clientId;
+            this.clientSequenceNumber = 0;
+            this.clientSequenceNumberObserved = 0;
+        }
+        if (message.type === protocol_definitions_1.MessageType.NoOp) {
+            this.trailingNoopCount++;
+        }
+        else {
+            this.trailingNoopCount = 0;
+        }
+        return Object.assign(Object.assign({}, message), { clientSequenceNumber: ++this.clientSequenceNumber });
+    }
+    submitSignal(content) {
+        if (this.connection !== undefined) {
+            this.connection.submitSignal(content);
+        }
+        else {
+            this.logger.sendErrorEvent({ eventName: "submitSignalDisconnected" });
+        }
+    }
+    sendMessages(messages) {
+        common_utils_1.assert(this.connected, 0x2b4 /* "not connected on sending ops!" */);
+        // If connection is "read" or implicit "read" (got leave op for "write" connection),
+        // then op can't make it through - we will get a nack if op is sent.
+        // We can short-circuit this process.
+        // Note that we also want nacks to be rare and be treated as catastrophic failures.
+        // Be careful with reentrancy though - disconnected event should not be be raised in the
+        // middle of the current workflow, but rather on clean stack!
+        if (this.connectionMode === "read" || this.downgradedConnection) {
+            if (!this.pendingReconnect) {
+                this.pendingReconnect = true;
+                Promise.resolve().then(async () => {
+                    if (this.pendingReconnect) { // still valid?
+                        await this.reconnectOnErrorCore("write", // connectionMode
+                        "Switch to write");
+                    }
+                })
+                    .catch(() => { });
+            }
+            return;
+        }
+        common_utils_1.assert(!this.pendingReconnect, 0x2b5 /* "logic error" */);
+        this._outbound.push(messages);
+    }
+    beforeProcessingIncomingOp(message) {
+        // if we have connection, and message is local, then we better treat is as local!
+        common_utils_1.assert(this.clientId !== message.clientId || this.lastSubmittedClientId === message.clientId, 0x0ee /* "Not accounting local messages correctly" */);
+        if (this.lastSubmittedClientId !== undefined && this.lastSubmittedClientId === message.clientId) {
+            const clientSequenceNumber = message.clientSequenceNumber;
+            common_utils_1.assert(this.clientSequenceNumberObserved < clientSequenceNumber, 0x0ef /* "client seq# not growing" */);
+            common_utils_1.assert(clientSequenceNumber <= this.clientSequenceNumber, 0x0f0 /* "Incoming local client seq# > generated by this client" */);
+            this.clientSequenceNumberObserved = clientSequenceNumber;
+        }
+        if (message.type === protocol_definitions_1.MessageType.ClientLeave) {
+            const systemLeaveMessage = message;
+            const clientId = JSON.parse(systemLeaveMessage.data);
+            if (clientId === this.clientId) {
+                // We have been kicked out from quorum
+                this.logger.sendPerformanceEvent({ eventName: "ReadConnectionTransition" });
+                this.downgradedConnection = true;
+            }
+        }
+    }
+}
+exports.ConnectionManager = ConnectionManager;
+//# sourceMappingURL=connectionManager.js.map