@fluidframework/container-loader 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.224419

package/lib/{connectionManager.js → connectionManager.mjs}

@@ -2,18 +2,21 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { default as AbortController } from "abort-controller";
-import { assert, performance, TypedEventEmitter } from "@fluidframework/common-utils";
-import { GenericError, UsageError } from "@fluidframework/container-utils";
-import { canRetryOnError, createWriteError, createGenericNetworkError, getRetryDelayFromError, waitForConnectedState, DeltaStreamConnectionForbiddenError, logNetworkFailure,
-// isRuntimeMessage,
- } from "@fluidframework/driver-utils";
+import { LogLevel } from "@fluidframework/core-interfaces";
+import { assert } from "@fluidframework/core-utils";
+import { performance, TypedEventEmitter } from "@fluid-internal/client-utils";
+import { 
+// eslint-disable-next-line import/no-deprecated
+DriverErrorType, } from "@fluidframework/driver-definitions";
+import { canRetryOnError, createWriteError, createGenericNetworkError, getRetryDelayFromError, logNetworkFailure, isRuntimeMessage, calculateMaxWaitTime, } from "@fluidframework/driver-utils";
 import { MessageType, ScopeType, } from "@fluidframework/protocol-definitions";
-import { TelemetryLogger, normalizeError, } from "@fluidframework/telemetry-utils";
-import { ReconnectMode, } from "./contracts";
-import { DeltaQueue } from "./deltaQueue";
-const MaxReconnectDelayInMs = 8000;
-const InitialReconnectDelayInMs = 1000;
+import { formatTick, GenericError, isFluidError, normalizeError, UsageError, } from "@fluidframework/telemetry-utils";
+import { ReconnectMode, } from "./contracts.mjs";
+import { DeltaQueue } from "./deltaQueue.mjs";
+import { SignalType } from "./protocol.mjs";
+import { isDeltaStreamConnectionForbiddenError } from "./utils.mjs";
+// We double this value in first try in when we calculate time to wait for in "calculateMaxWaitTime" function.
+const InitialReconnectDelayInMs = 500;
 const DefaultChunkSize = 16 * 1024;
 const fatalConnectErrorProp = { fatalConnectError: true };
 function getNackReconnectInfo(nackContent) {
@@ -26,10 +29,25 @@ function getNackReconnectInfo(nackContent) {
  * Implementation of IDocumentDeltaConnection that does not support submitting
  * or receiving ops. Used in storage-only mode.
  */
+const clientNoDeltaStream = {
+    mode: "read",
+    details: { capabilities: { interactive: true } },
+    permission: [],
+    user: { id: "storage-only client" },
+    scopes: [],
+};
+const clientIdNoDeltaStream = "storage-only client";
 class NoDeltaStream extends TypedEventEmitter {
-    constructor() {
-        super(...arguments);
-        this.clientId = "storage-only client";
+    /**
+     * Connection which is not connected to socket.
+     * @param storageOnlyReason - Reason on why the connection to delta stream is not allowed.
+     * @param readonlyConnectionReason - reason/error if any which lead to using NoDeltaStream.
+     */
+    constructor(storageOnlyReason, readonlyConnectionReason) {
+        super();
+        this.storageOnlyReason = storageOnlyReason;
+        this.readonlyConnectionReason = readonlyConnectionReason;
+        this.clientId = clientIdNoDeltaStream;
         this.claims = {
             scopes: [ScopeType.DocRead],
         };
@@ -39,11 +57,12 @@ class NoDeltaStream extends TypedEventEmitter {
         this.version = "";
         this.initialMessages = [];
         this.initialSignals = [];
-        this.initialClients = [];
+        this.initialClients = [
+            { client: clientNoDeltaStream, clientId: clientIdNoDeltaStream },
+        ];
         this.serviceConfiguration = {
             maxMessageSize: 0,
             blockSize: 0,
-            summary: undefined,
         };
         this.checkpointSequenceNumber = undefined;
         this._disposed = false;
@@ -62,85 +81,49 @@ class NoDeltaStream extends TypedEventEmitter {
             content: { message: "Cannot submit signal with storage-only connection", code: 403 },
         });
     }
-    get disposed() { return this._disposed; }
-    dispose() { this._disposed = true; }
+    get disposed() {
+        return this._disposed;
+    }
+    dispose() {
+        this._disposed = true;
+    }
+}
+function isNoDeltaStreamConnection(connection) {
+    return connection instanceof NoDeltaStream;
 }
+const waitForOnline = async () => {
+    // Only wait if we have a strong signal that we're offline - otherwise assume we're online.
+    if (globalThis.navigator?.onLine === false && globalThis.addEventListener !== undefined) {
+        return new Promise((resolve) => {
+            const resolveAndRemoveListener = () => {
+                resolve();
+                globalThis.removeEventListener("online", resolveAndRemoveListener);
+            };
+            globalThis.addEventListener("online", resolveAndRemoveListener);
+        });
+    }
+};
 /**
  * Implementation of IConnectionManager, used by Container class
- * Implements constant connectivity to relay service, by reconnecting in case of loast connection or error.
- * Exposes various controls to influecen this process, including manual reconnects, forced read-only mode, etc.
+ * Implements constant connectivity to relay service, by reconnecting in case of lost connection or error.
+ * Exposes various controls to influence this process, including manual reconnects, forced read-only mode, etc.
  */
 export class ConnectionManager {
-    constructor(serviceProvider, client, reconnectAllowed, logger, props) {
-        this.serviceProvider = serviceProvider;
-        this.client = client;
-        this.logger = logger;
-        this.props = props;
-        /** tracks host requiring read-only mode. */
-        this._forceReadonly = false;
-        /** True if there is pending (async) reconnection from "read" to "write" */
-        this.pendingReconnect = false;
-        this.clientSequenceNumber = 0;
-        this.clientSequenceNumberObserved = 0;
-        /** Counts the number of noops sent by the client which may not be acked. */
-        this.trailingNoopCount = 0;
-        this.connectFirstConnection = true;
-        this._connectionVerboseProps = {};
-        this._connectionProps = {};
-        this.closed = false;
-        this.opHandler = (documentId, messagesArg) => {
-            const messages = Array.isArray(messagesArg) ? messagesArg : [messagesArg];
-            this.props.incomingOpHandler(messages, "opHandler");
-        };
-        // Always connect in write mode after getting nacked.
-        this.nackHandler = (documentId, messages) => {
-            const message = messages[0];
-            if (this._readonlyPermissions === true) {
-                this.props.closeHandler(createWriteError("writeOnReadOnlyDocument", { driverVersion: undefined }));
-                return;
-            }
-            const reconnectInfo = getNackReconnectInfo(message.content);
-            // If the nack indicates we cannot retry, then close the container outright
-            if (!reconnectInfo.canRetry) {
-                this.props.closeHandler(reconnectInfo);
-                return;
-            }
-            this.reconnectOnError("write", reconnectInfo);
-        };
-        // Connection mode is always read on disconnect/error unless the system mode was write.
-        this.disconnectHandlerInternal = (disconnectReason) => {
-            // Note: we might get multiple disconnect calls on same socket, as early disconnect notification
-            // ("server_disconnect", ODSP-specific) is mapped to "disconnect"
-            this.reconnectOnError(this.defaultReconnectionMode, disconnectReason);
-        };
-        this.errorHandler = (error) => {
-            this.reconnectOnError(this.defaultReconnectionMode, error);
-        };
-        this.clientDetails = this.client.details;
-        this.defaultReconnectionMode = this.client.mode;
-        this._reconnectMode = reconnectAllowed ? ReconnectMode.Enabled : ReconnectMode.Never;
-        // Outbound message queue. The outbound queue is represented as a queue of an array of ops. Ops contained
-        // within an array *must* fit within the maxMessageSize and are guaranteed to be ordered sequentially.
-        this._outbound = new DeltaQueue((messages) => {
-            if (this.connection === undefined) {
-                throw new Error("Attempted to submit an outbound message without connection");
-            }
-            this.connection.submit(messages);
-        });
-        this._outbound.on("error", (error) => {
-            this.props.closeHandler(normalizeError(error));
-        });
+    get connectionVerboseProps() {
+        return this._connectionVerboseProps;
     }
-    get connectionVerboseProps() { return this._connectionVerboseProps; }
     /**
      * The current connection mode, initially read.
      */
     get connectionMode() {
-        var _a, _b;
-        return (_b = (_a = this.connection) === null || _a === void 0 ? void 0 : _a.mode) !== null && _b !== void 0 ? _b : "read";
+        return this.connection?.mode ?? "read";
+    }
+    get connected() {
+        return this.connection !== undefined;
+    }
+    get clientId() {
+        return this.connection?.clientId;
     }
-    get connected() { return this.connection !== undefined; }
-    get clientId() { var _a; return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId; }
     /**
      * Automatic reconnecting enabled or disabled.
      * If set to Never, then reconnecting will never be allowed.
@@ -149,8 +132,7 @@ export class ConnectionManager {
         return this._reconnectMode;
     }
     get maxMessageSize() {
-        var _a, _b, _c;
-        return (_c = (_b = (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration) === null || _b === void 0 ? void 0 : _b.maxMessageSize) !== null && _c !== void 0 ? _c : DefaultChunkSize;
+        return this.connection?.serviceConfiguration?.maxMessageSize ?? DefaultChunkSize;
     }
     get version() {
         if (this.connection === undefined) {
@@ -159,12 +141,10 @@ export class ConnectionManager {
         return this.connection.version;
     }
     get serviceConfiguration() {
-        var _a;
-        return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration;
+        return this.connection?.serviceConfiguration;
     }
     get scopes() {
-        var _a;
-        return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.claims.scopes;
+        return this.connection?.claims.scopes;
     }
     get outbound() {
         return this._outbound;
@@ -172,20 +152,31 @@ export class ConnectionManager {
     /**
      * Returns set of props that can be logged in telemetry that provide some insights / statistics
      * about current or last connection (if there is no connection at the moment)
-    */
+     */
     get connectionProps() {
-        if (this.connection !== undefined) {
-            return this._connectionProps;
-        }
-        else {
-            return Object.assign(Object.assign({}, this._connectionProps), { 
+        return this.connection !== undefined
+            ? this._connectionProps
+            : {
+                ...this._connectionProps,
                 // Report how many ops this client sent in last disconnected session
-                sentOps: this.clientSequenceNumber });
-        }
+                sentOps: this.clientSequenceNumber,
+            };
     }
     shouldJoinWrite() {
         // We don't have to wait for ack for topmost NoOps. So subtract those.
-        return this.clientSequenceNumberObserved < (this.clientSequenceNumber - this.trailingNoopCount);
+        const outstandingOps = this.clientSequenceNumberObserved < this.clientSequenceNumber - this.localOpsToIgnore;
+        // Previous behavior was to force write mode here only when there are outstanding ops (besides
+        // no-ops). The dirty signal from runtime should provide the same behavior, but also support
+        // stashed ops that weren't submitted to container layer yet. For safety, we want to retain the
+        // same behavior whenever dirty is false.
+        const isDirty = this.containerDirty();
+        if (outstandingOps !== isDirty) {
+            this.logger.sendTelemetryEvent({
+                eventName: "DesiredConnectionModeMismatch",
+                details: JSON.stringify({ outstandingOps, isDirty }),
+            });
+        }
+        return outstandingOps || isDirty;
     }
     /**
      * Tells if container is in read-only mode.
@@ -197,82 +188,141 @@ export class ConnectionManager {
      * and do not know if user has write access to a file.
      */
     get readonly() {
-        if (this._forceReadonly) {
-            return true;
-        }
-        return this._readonlyPermissions;
+        return this.readOnlyInfo.readonly;
     }
     get readOnlyInfo() {
-        const storageOnly = this.connection !== undefined && this.connection instanceof NoDeltaStream;
+        let storageOnly = false;
+        let storageOnlyReason;
+        if (isNoDeltaStreamConnection(this.connection)) {
+            storageOnly = true;
+            storageOnlyReason = this.connection.storageOnlyReason;
+        }
         if (storageOnly || this._forceReadonly || this._readonlyPermissions === true) {
             return {
                 readonly: true,
                 forced: this._forceReadonly,
                 permissions: this._readonlyPermissions,
                 storageOnly,
+                storageOnlyReason,
             };
         }
         return { readonly: this._readonlyPermissions };
     }
-    static detailsFromConnection(connection) {
+    static detailsFromConnection(connection, reason) {
         return {
             claims: connection.claims,
             clientId: connection.clientId,
-            existing: connection.existing,
             checkpointSequenceNumber: connection.checkpointSequenceNumber,
-            get initialClients() { return connection.initialClients; },
+            get initialClients() {
+                return connection.initialClients;
+            },
             mode: connection.mode,
             serviceConfiguration: connection.serviceConfiguration,
             version: connection.version,
+            reason,
         };
     }
-    dispose(error) {
-        if (this.closed) {
+    constructor(serviceProvider, containerDirty, client, reconnectAllowed, logger, props) {
+        this.serviceProvider = serviceProvider;
+        this.containerDirty = containerDirty;
+        this.client = client;
+        this.logger = logger;
+        this.props = props;
+        /** tracks host requiring read-only mode. */
+        this._forceReadonly = false;
+        /** True if there is pending (async) reconnection from "read" to "write" */
+        this.pendingReconnect = false;
+        this.clientSequenceNumber = 0;
+        this.clientSequenceNumberObserved = 0;
+        /** Counts the number of non-runtime ops sent by the client which may not be acked. */
+        this.localOpsToIgnore = 0;
+        this.connectFirstConnection = true;
+        this._connectionVerboseProps = {};
+        this._connectionProps = {};
+        this._disposed = false;
+        this.opHandler = (documentId, messagesArg) => {
+            const messages = Array.isArray(messagesArg) ? messagesArg : [messagesArg];
+            this.props.incomingOpHandler(messages, "opHandler");
+        };
+        this.signalHandler = (signalsArg) => {
+            const signals = Array.isArray(signalsArg) ? signalsArg : [signalsArg];
+            this.props.signalHandler(signals);
+        };
+        // Always connect in write mode after getting nacked.
+        this.nackHandler = (documentId, messages) => {
+            const message = messages[0];
+            if (this._readonlyPermissions === true) {
+                this.props.closeHandler(createWriteError("writeOnReadOnlyDocument", { driverVersion: undefined }));
+                return;
+            }
+            const reconnectInfo = getNackReconnectInfo(message.content);
+            // If the nack indicates we cannot retry, then close the container outright
+            if (!reconnectInfo.canRetry) {
+                this.props.closeHandler(reconnectInfo);
+                return;
+            }
+            this.reconnectOnError("write", reconnectInfo);
+        };
+        // Connection mode is always read on disconnect/error unless the system mode was write.
+        this.disconnectHandlerInternal = (disconnectReason) => {
+            // Note: we might get multiple disconnect calls on same socket, as early disconnect notification
+            // ("server_disconnect", ODSP-specific) is mapped to "disconnect"
+            this.reconnectOnError(this.defaultReconnectionMode, disconnectReason);
+        };
+        this.errorHandler = (error) => {
+            this.reconnectOnError(this.defaultReconnectionMode, error);
+        };
+        this.clientDetails = this.client.details;
+        this.defaultReconnectionMode = this.client.mode;
+        this._reconnectMode = reconnectAllowed ? ReconnectMode.Enabled : ReconnectMode.Never;
+        // Outbound message queue. The outbound queue is represented as a queue of an array of ops. Ops contained
+        // within an array *must* fit within the maxMessageSize and are guaranteed to be ordered sequentially.
+        this._outbound = new DeltaQueue((messages) => {
+            if (this.connection === undefined) {
+                throw new Error("Attempted to submit an outbound message without connection");
+            }
+            this.connection.submit(messages);
+        });
+        this._outbound.on("error", (error) => {
+            this.props.closeHandler(normalizeError(error));
+        });
+    }
+    dispose(error, switchToReadonly = true) {
+        if (this._disposed) {
             return;
         }
-        this.closed = true;
-        this.pendingConnection = undefined;
+        this._disposed = true;
         // Ensure that things like triggerConnect() will short circuit
         this._reconnectMode = ReconnectMode.Never;
         this._outbound.clear();
-        const disconnectReason = error !== undefined
-            ? `Closing DeltaManager (${error.message})`
-            : "Closing DeltaManager";
+        const disconnectReason = {
+            text: "Closing DeltaManager",
+            error,
+        };
+        const oldReadonlyValue = this.readonly;
         // This raises "disconnect" event if we have active connection.
         this.disconnectFromDeltaStream(disconnectReason);
-        // Notify everyone we are in read-only state.
-        // Useful for data stores in case we hit some critical error,
-        // to switch to a mode where user edits are not accepted
-        this.set_readonlyPermissions(true);
+        if (switchToReadonly) {
+            // Notify everyone we are in read-only state.
+            // Useful for data stores in case we hit some critical error,
+            // to switch to a mode where user edits are not accepted
+            this.set_readonlyPermissions(true, oldReadonlyValue, disconnectReason);
+        }
     }
     /**
      * Enables or disables automatic reconnecting.
      * Will throw an error if reconnectMode set to Never.
-    */
-    setAutoReconnect(mode) {
+     */
+    setAutoReconnect(mode, reason) {
         assert(mode !== ReconnectMode.Never && this._reconnectMode !== ReconnectMode.Never, 0x278 /* "API is not supported for non-connecting or closed container" */);
         this._reconnectMode = mode;
         if (mode !== ReconnectMode.Enabled) {
             // immediately disconnect - do not rely on service eventually dropping connection.
-            this.disconnectFromDeltaStream("setAutoReconnect");
+            this.disconnectFromDeltaStream(reason);
         }
     }
     /**
-     * Sends signal to runtime (and data stores) to be read-only.
-     * Hosts may have read only views, indicating to data stores that no edits are allowed.
-     * This is independent from this._readonlyPermissions (permissions) and this.connectionMode
-     * (server can return "write" mode even when asked for "read")
-     * Leveraging same "readonly" event as runtime & data stores should behave the same in such case
-     * as in read-only permissions.
-     * But this.active can be used by some DDSes to figure out if ops can be sent
-     * (for example, read-only view still participates in code proposals / upgrades decisions)
-     *
-     * Forcing Readonly does not prevent DDS from generating ops. It is up to user code to honour
-     * the readonly flag. If ops are generated, they will accumulate locally and not be sent. If
-     * there are pending in the outbound queue, it will stop sending until force readonly is
-     * cleared.
-     *
-     * @param readonly - set or clear force readonly.
+     * {@inheritDoc Container.forceReadonly}
      */
     forceReadonly(readonly) {
         if (readonly !== this._forceReadonly) {
@@ -297,42 +347,40 @@ export class ConnectionManager {
                     // host logic error.
                     this.logger.sendErrorEvent({ eventName: "ForceReadonlyPendingChanged" });
                 }
-                reconnect = this.disconnectFromDeltaStream("Force readonly");
+                reconnect = this.disconnectFromDeltaStream({ text: "Force readonly" });
             }
             this.props.readonlyChangeHandler(this.readonly);
             if (reconnect) {
                 // reconnect if we disconnected from before.
-                this.triggerConnect("read");
+                this.triggerConnect({ text: "Force Readonly" }, "read");
             }
         }
     }
-    set_readonlyPermissions(readonly) {
-        const oldValue = this.readonly;
-        this._readonlyPermissions = readonly;
-        if (oldValue !== this.readonly) {
-            this.props.readonlyChangeHandler(this.readonly);
+    set_readonlyPermissions(newReadonlyValue, oldReadonlyValue, readonlyConnectionReason) {
+        this._readonlyPermissions = newReadonlyValue;
+        if (oldReadonlyValue !== this.readonly) {
+            this.props.readonlyChangeHandler(this.readonly, readonlyConnectionReason);
         }
     }
-    connect(connectionMode) {
-        this.connectCore(connectionMode).catch((error) => {
-            const normalizedError = normalizeError(error, { props: fatalConnectErrorProp });
+    connect(reason, connectionMode) {
+        this.connectCore(reason, connectionMode).catch((e) => {
+            const normalizedError = normalizeError(e, { props: fatalConnectErrorProp });
             this.props.closeHandler(normalizedError);
         });
     }
-    async connectCore(connectionMode) {
-        var _a, _b;
-        assert(!this.closed, 0x26a /* "not closed" */);
+    async connectCore(reason, connectionMode) {
+        assert(!this._disposed, 0x26a /* "not closed" */);
         if (this.connection !== undefined) {
             return; // Connection attempt already completed successfully
         }
         let pendingConnectionMode;
         if (this.pendingConnection !== undefined) {
             pendingConnectionMode = this.pendingConnection.connectionMode;
-            this.cancelConnection(); // Throw out in-progress connection attempt in favor of new attempt
+            this.cancelConnection(reason); // Throw out in-progress connection attempt in favor of new attempt
             assert(this.pendingConnection === undefined, 0x344 /* this.pendingConnection should be undefined */);
         }
         // If there is no specified ConnectionMode, try the previous mode, if there is no previous mode use default
-        let requestedMode = (_a = connectionMode !== null && connectionMode !== void 0 ? connectionMode : pendingConnectionMode) !== null && _a !== void 0 ? _a : this.defaultReconnectionMode;
+        let requestedMode = connectionMode ?? pendingConnectionMode ?? this.defaultReconnectionMode;
         // if we have any non-acked ops from last connection, reconnect as "write".
         // without that we would connect in view-only mode, which will result in immediate
         // firing of "connected" event from Container and switch of current clientId (as tracked
@@ -344,9 +392,9 @@ export class ConnectionManager {
         const docService = this.serviceProvider();
         assert(docService !== undefined, 0x2a7 /* "Container is not attached" */);
         let connection;
-        if (((_b = docService.policies) === null || _b === void 0 ? void 0 : _b.storageOnly) === true) {
+        if (docService.policies?.storageOnly === true) {
             connection = new NoDeltaStream();
-            this.setupNewSuccessfulConnection(connection, "read");
+            this.setupNewSuccessfulConnection(connection, "read", reason);
             assert(this.pendingConnection === undefined, 0x2b3 /* "logic error" */);
             return;
         }
@@ -356,17 +404,23 @@ export class ConnectionManager {
         let lastError;
         const abortController = new AbortController();
         const abortSignal = abortController.signal;
-        this.pendingConnection = { abort: () => { abortController.abort(); }, connectionMode: requestedMode };
+        this.pendingConnection = {
+            abort: () => {
+                abortController.abort();
+            },
+            connectionMode: requestedMode,
+        };
+        this.props.establishConnectionHandler(reason);
         // This loop will keep trying to connect until successful, with a delay between each iteration.
         while (connection === undefined) {
-            if (this.closed) {
+            if (this._disposed) {
                 throw new Error("Attempting to connect a closed DeltaManager");
             }
             if (abortSignal.aborted === true) {
                 this.logger.sendTelemetryEvent({
                     eventName: "ConnectionAttemptCancelled",
                     attempts: connectRepeatCount,
-                    duration: TelemetryLogger.formatTick(performance.now() - connectStartTime),
+                    duration: formatTick(performance.now() - connectStartTime),
                     connectionEstablished: false,
                 });
                 return;
@@ -374,17 +428,42 @@ export class ConnectionManager {
             connectRepeatCount++;
             try {
                 this.client.mode = requestedMode;
-                connection = await docService.connectToDeltaStream(Object.assign(Object.assign({}, this.client), { mode: requestedMode }));
+                connection = await docService.connectToDeltaStream({
+                    ...this.client,
+                    mode: requestedMode,
+                });
                 if (connection.disposed) {
                     // Nobody observed this connection, so drop it on the floor and retry.
                     this.logger.sendTelemetryEvent({ eventName: "ReceivedClosedConnection" });
                     connection = undefined;
                 }
+                this.logger.sendTelemetryEvent({
+                    eventName: "ConnectionReceived",
+                    connected: connection !== undefined && connection.disposed === false,
+                }, undefined, LogLevel.verbose);
             }
             catch (origError) {
-                if (typeof origError === "object" && origError !== null &&
-                    (origError === null || origError === void 0 ? void 0 : origError.errorType) === DeltaStreamConnectionForbiddenError.errorType) {
-                    connection = new NoDeltaStream();
+                this.logger.sendTelemetryEvent({
+                    eventName: "ConnectToDeltaStreamException",
+                    connected: connection !== undefined && connection.disposed === false,
+                }, undefined, LogLevel.verbose);
+                if (isDeltaStreamConnectionForbiddenError(origError)) {
+                    connection = new NoDeltaStream(origError.storageOnlyReason, {
+                        text: origError.message,
+                        error: origError,
+                    });
+                    requestedMode = "read";
+                    break;
+                }
+                else if (isFluidError(origError) &&
+                    // eslint-disable-next-line import/no-deprecated
+                    origError.errorType === DriverErrorType.outOfStorageError) {
+                    // If we get out of storage error from calling joinsession, then use the NoDeltaStream object so
+                    // that user can at least load the container.
+                    connection = new NoDeltaStream(undefined, {
+                        text: origError.message,
+                        error: origError,
+                    });
                     requestedMode = "read";
                     break;
                 }
@@ -399,15 +478,36 @@ export class ConnectionManager {
                     attempts: connectRepeatCount,
                     delay: delayMs,
                     eventName: "DeltaConnectionFailureToConnect",
-                    duration: TelemetryLogger.formatTick(performance.now() - connectStartTime),
+                    duration: formatTick(performance.now() - connectStartTime),
                 }, origError);
                 lastError = origError;
+                const waitStartTime = performance.now();
                 const retryDelayFromError = getRetryDelayFromError(origError);
-                delayMs = retryDelayFromError !== null && retryDelayFromError !== void 0 ? retryDelayFromError : Math.min(delayMs * 2, MaxReconnectDelayInMs);
-                if (retryDelayFromError !== undefined) {
-                    this.props.reconnectionDelayHandler(retryDelayFromError, origError);
+                // If the error told us to wait or browser signals us that we are offline, then calculate the time we
+                // want to wait for before retrying. then we wait for that time. If the error didn't tell us to wait,
+                // let's still wait a little bit before retrying. We can skip this delay if we're confident we're offline,
+                // because we probably just need to wait to come back online. But we never have strong signal of being
+                // offline, so we at least wait for sometime.
+                if (retryDelayFromError !== undefined || globalThis.navigator?.onLine !== false) {
+                    delayMs = calculateMaxWaitTime(delayMs, origError);
                 }
-                await waitForConnectedState(delayMs);
+                // Raise event in case the delay was there.
+                this.props.reconnectionDelayHandler(delayMs, origError);
+                await new Promise((resolve) => {
+                    setTimeout(resolve, delayMs);
+                });
+                // If we believe we're offline, we assume there's no point in trying until we at least think we're online.
+                // NOTE: This isn't strictly true for drivers that don't require network (e.g. local driver).  Really this logic
+                // should probably live in the driver.
+                await waitForOnline();
+                this.logger.sendPerformanceEvent({
+                    eventName: "WaitBetweenConnectionAttempts",
+                    duration: performance.now() - waitStartTime,
+                    details: JSON.stringify({
+                        retryDelayFromError,
+                        delayMs,
+                    }),
+                });
             }
         }
         // If we retried more than once, log an event about how long it took (this will not log to error table)
@@ -415,44 +515,50 @@ export class ConnectionManager {
             logNetworkFailure(this.logger, {
                 eventName: "MultipleDeltaConnectionFailures",
                 attempts: connectRepeatCount,
-                duration: TelemetryLogger.formatTick(performance.now() - connectStartTime),
+                duration: formatTick(performance.now() - connectStartTime),
             }, lastError);
         }
-        // Check for abort signal after while loop as well
-        if (abortSignal.aborted === true) {
+        // Check for abort signal after while loop as well or we've been disposed
+        if (abortSignal.aborted === true || this._disposed) {
             connection.dispose();
             this.logger.sendTelemetryEvent({
                 eventName: "ConnectionAttemptCancelled",
                 attempts: connectRepeatCount,
-                duration: TelemetryLogger.formatTick(performance.now() - connectStartTime),
+                duration: formatTick(performance.now() - connectStartTime),
                 connectionEstablished: true,
             });
             return;
         }
-        this.setupNewSuccessfulConnection(connection, requestedMode);
+        this.setupNewSuccessfulConnection(connection, requestedMode, reason);
     }
     /**
-     * Start the connection. Any error should result in container being close.
-     * And report the error if it excape for any reason.
+     * Start the connection. Any error should result in container being closed.
+     * And report the error if it escapes for any reason.
      * @param args - The connection arguments
      */
-    triggerConnect(connectionMode) {
-        assert(this.connection === undefined, 0x239 /* "called only in disconnected state" */);
+    triggerConnect(reason, connectionMode) {
+        // reconnect() includes async awaits, and that causes potential race conditions
+        // where we might already have a connection. If it were to happen, it's possible that we will connect
+        // with different mode to `connectionMode`. Glancing through the caller chains, it looks like code should be
+        // fine (if needed, reconnect flow will get triggered again). Places where new mode matters should encode it
+        // directly in connectCore - see this.shouldJoinWrite() test as an example.
+        // assert(this.connection === undefined, 0x239 /* "called only in disconnected state" */);
         if (this.reconnectMode !== ReconnectMode.Enabled) {
             return;
         }
-        this.connect(connectionMode);
+        this.connect(reason, connectionMode);
     }
     /**
      * Disconnect the current connection.
      * @param reason - Text description of disconnect reason to emit with disconnect event
+     * @param error - Error causing the disconnect if any.
      * @returns A boolean that indicates if there was an existing connection (or pending connection) to disconnect
      */
     disconnectFromDeltaStream(reason) {
         this.pendingReconnect = false;
         if (this.connection === undefined) {
             if (this.pendingConnection !== undefined) {
-                this.cancelConnection();
+                this.cancelConnection(reason);
                 return true;
             }
             return false;
@@ -463,7 +569,7 @@ export class ConnectionManager {
         this.connection = undefined;
         // Remove listeners first so we don't try to retrigger this flow accidentally through reconnectOnError
         connection.off("op", this.opHandler);
-        connection.off("signal", this.props.signalHandler);
+        connection.off("signal", this.signalHandler);
         connection.off("nack", this.nackHandler);
         connection.off("disconnect", this.disconnectHandlerInternal);
         connection.off("error", this.errorHandler);
@@ -471,48 +577,60 @@ export class ConnectionManager {
         // eslint-disable-next-line @typescript-eslint/no-floating-promises
         this._outbound.pause();
         this._outbound.clear();
-        this.props.disconnectHandler(reason);
         connection.dispose();
+        this.props.disconnectHandler(reason);
         this._connectionVerboseProps = {};
         return true;
     }
     /**
      * Cancel in-progress connection attempt.
      */
-    cancelConnection() {
+    cancelConnection(reason) {
         assert(this.pendingConnection !== undefined, 0x345 /* this.pendingConnection is undefined when trying to cancel */);
         this.pendingConnection.abort();
         this.pendingConnection = undefined;
         this.logger.sendTelemetryEvent({ eventName: "ConnectionCancelReceived" });
+        this.props.cancelConnectionHandler({
+            text: `Cancel Pending Connection due to ${reason.text}`,
+            error: reason.error,
+        });
     }
     /**
      * Once we've successfully gotten a connection, we need to set up state, attach event listeners, and process
      * initial messages.
      * @param connection - The newly established connection
      */
-    setupNewSuccessfulConnection(connection, requestedMode) {
+    setupNewSuccessfulConnection(connection, requestedMode, reason) {
         // Old connection should have been cleaned up before establishing a new one
         assert(this.connection === undefined, 0x0e6 /* "old connection exists on new connection setup" */);
         assert(!connection.disposed, 0x28a /* "can't be disposed - Callers need to ensure that!" */);
         this.pendingConnection = undefined;
+        const oldReadonlyValue = this.readonly;
         this.connection = connection;
         // Does information in scopes & mode matches?
         // If we asked for "write" and got "read", then file is read-only
         // But if we ask read, server can still give us write.
         const readonly = !connection.claims.scopes.includes(ScopeType.DocWrite);
+        if (connection.mode !== requestedMode) {
+            this.logger.sendTelemetryEvent({
+                eventName: "ConnectionModeMismatch",
+                requestedMode,
+                mode: connection.mode,
+            });
+        }
         // This connection mode validation logic is moving to the driver layer in 0.44.  These two asserts can be
         // removed after those packages have released and become ubiquitous.
         assert(requestedMode === "read" || readonly === (this.connectionMode === "read"), 0x0e7 /* "claims/connectionMode mismatch" */);
         assert(!readonly || this.connectionMode === "read", 0x0e8 /* "readonly perf with write connection" */);
-        this.set_readonlyPermissions(readonly);
-        if (this.closed) {
+        this.set_readonlyPermissions(readonly, oldReadonlyValue, isNoDeltaStreamConnection(connection) ? connection.readonlyConnectionReason : undefined);
+        if (this._disposed) {
             // Raise proper events, Log telemetry event and close connection.
-            this.disconnectFromDeltaStream("ConnectionManager already closed");
+            this.disconnectFromDeltaStream({ text: "ConnectionManager already closed" });
             return;
         }
         this._outbound.resume();
         connection.on("op", this.opHandler);
-        connection.on("signal", this.props.signalHandler);
+        connection.on("signal", this.signalHandler);
         connection.on("nack", this.nackHandler);
         connection.on("disconnect", this.disconnectHandlerInternal);
         connection.on("error", this.errorHandler);
@@ -538,7 +656,8 @@ export class ConnectionManager {
         this._connectionProps.connectionMode = connection.mode;
         let last = -1;
         if (initialMessages.length !== 0) {
-            this._connectionVerboseProps.connectionInitialOpsFrom = initialMessages[0].sequenceNumber;
+            this._connectionVerboseProps.connectionInitialOpsFrom =
+                initialMessages[0].sequenceNumber;
             last = initialMessages[initialMessages.length - 1].sequenceNumber;
             this._connectionVerboseProps.connectionInitialOpsTo = last + 1;
             // Update knowledge of how far we are behind, before raising "connect" event
@@ -549,15 +668,39 @@ export class ConnectionManager {
             }
         }
         this.props.incomingOpHandler(initialMessages, this.connectFirstConnection ? "InitialOps" : "ReconnectOps");
-        if (connection.initialSignals !== undefined) {
-            for (const signal of connection.initialSignals) {
-                this.props.signalHandler(signal);
-            }
-        }
-        const details = ConnectionManager.detailsFromConnection(connection);
+        const details = ConnectionManager.detailsFromConnection(connection, reason);
         details.checkpointSequenceNumber = checkpointSequenceNumber;
         this.props.connectHandler(details);
         this.connectFirstConnection = false;
+        // Synthesize clear & join signals out of initialClients state.
+        // This allows us to have single way to process signals, and makes it simpler to initialize
+        // protocol in Container.
+        const clearSignal = {
+            clientId: null,
+            content: JSON.stringify({
+                type: SignalType.Clear,
+            }),
+        };
+        // list of signals to process due to this new connection
+        let signalsToProcess = [clearSignal];
+        const clientJoinSignals = (connection.initialClients ?? []).map((priorClient) => ({
+            clientId: null,
+            content: JSON.stringify({
+                type: SignalType.ClientJoin,
+                content: priorClient, // ISignalClient
+            }),
+        }));
+        if (clientJoinSignals.length > 0) {
+            signalsToProcess = signalsToProcess.concat(clientJoinSignals);
+        }
+        // Unfortunately, there is no defined order between initialSignals (including join & leave signals)
+        // and connection.initialClients. In practice, connection.initialSignals quite often contains join signal
+        // for "self" and connection.initialClients does not contain "self", so we have to process them after
+        // "clear" signal above.
+        if (connection.initialSignals !== undefined && connection.initialSignals.length > 0) {
+            signalsToProcess = signalsToProcess.concat(connection.initialSignals);
+        }
+        this.props.signalHandler(signalsToProcess);
     }
     /**
      * Disconnect the current connection and reconnect. Closes the container if it fails.
@@ -567,8 +710,7 @@ export class ConnectionManager {
      * @returns A promise that resolves when the connection is reestablished or we stop trying
      */
     reconnectOnError(requestedMode, error) {
-        this.reconnect(requestedMode, error.message, error)
-            .catch(this.props.closeHandler);
+        this.reconnect(requestedMode, { text: error.message, error }).catch(this.props.closeHandler);
     }
     /**
      * Disconnect the current connection and reconnect.
@@ -577,20 +719,20 @@ export class ConnectionManager {
      * @param error - Error reconnect information including whether or not to reconnect
      * @returns A promise that resolves when the connection is reestablished or we stop trying
      */
-    async reconnect(requestedMode, disconnectMessage, error) {
+    async reconnect(requestedMode, reason) {
         // We quite often get protocol errors before / after observing nack/disconnect
         // we do not want to run through same sequence twice.
         // If we're already disconnected/disconnecting it's not appropriate to call this again.
         assert(this.connection !== undefined, 0x0eb /* "Missing connection for reconnect" */);
-        this.disconnectFromDeltaStream(disconnectMessage);
+        this.disconnectFromDeltaStream(reason);
         // We will always trigger reconnect, even if canRetry is false.
         // Any truly fatal error state will result in container close upon attempted reconnect,
         // which is a preferable to closing abruptly when a live connection fails.
-        if (error !== undefined && !error.canRetry) {
+        if (reason.error?.canRetry === false) {
             this.logger.sendTelemetryEvent({
                 eventName: "reconnectingDespiteFatalError",
                 reconnectMode: this.reconnectMode,
-            }, error);
+            }, reason.error);
         }
         if (this.reconnectMode === ReconnectMode.Never) {
             // Do not raise container error if we are closing just because we lost connection.
@@ -599,18 +741,29 @@ export class ConnectionManager {
             this.props.closeHandler();
         }
         // If closed then we can't reconnect
-        if (this.closed || this.reconnectMode !== ReconnectMode.Enabled) {
+        if (this._disposed || this.reconnectMode !== ReconnectMode.Enabled) {
             return;
         }
-        const delayMs = getRetryDelayFromError(error);
-        if (error !== undefined && delayMs !== undefined) {
-            this.props.reconnectionDelayHandler(delayMs, error);
-            await waitForConnectedState(delayMs);
+        // If the error tells us to wait before retrying, then do so.
+        const delayMs = getRetryDelayFromError(reason.error);
+        if (reason.error !== undefined && delayMs !== undefined) {
+            this.props.reconnectionDelayHandler(delayMs, reason.error);
+            await new Promise((resolve) => {
+                setTimeout(resolve, delayMs);
+            });
         }
-        this.triggerConnect(requestedMode);
+        // If we believe we're offline, we assume there's no point in trying again until we at least think we're online.
+        // NOTE: This isn't strictly true for drivers that don't require network (e.g. local driver).  Really this logic
+        // should probably live in the driver.
+        await waitForOnline();
+        this.triggerConnect({
+            text: reason.error !== undefined
+                ? "Reconnecting due to Error"
+                : `Reconnecting due to: ${reason.text}`,
+            error: reason.error,
+        }, requestedMode);
     }
     prepareMessageToSend(message) {
-        var _a, _b;
         if (this.readonly === true) {
             assert(this.readOnlyInfo.readonly === true, 0x1f0 /* "Unexpected mismatch in readonly" */);
             const error = new GenericError("deltaManagerReadonlySubmit", undefined /* error */, {
@@ -618,6 +771,7 @@ export class ConnectionManager {
                 forcedReadonly: this.readOnlyInfo.forced,
                 readonlyPermissions: this.readOnlyInfo.permissions,
                 storageOnly: this.readOnlyInfo.storageOnly,
+                storageOnlyReason: this.readOnlyInfo.storageOnlyReason,
             });
             this.props.closeHandler(error);
             return undefined;
@@ -626,22 +780,25 @@ export class ConnectionManager {
         // we keep info about old connection as long as possible to be able to account for all non-acked ops
         // that we pick up on next connection.
         assert(!!this.connection, 0x0e4 /* "Lost old connection!" */);
-        if (this.lastSubmittedClientId !== ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId)) {
-            this.lastSubmittedClientId = (_b = this.connection) === null || _b === void 0 ? void 0 : _b.clientId;
+        if (this.lastSubmittedClientId !== this.connection?.clientId) {
+            this.lastSubmittedClientId = this.connection?.clientId;
             this.clientSequenceNumber = 0;
             this.clientSequenceNumberObserved = 0;
         }
-        if (message.type === MessageType.NoOp) {
-            this.trailingNoopCount++;
+        if (!isRuntimeMessage(message)) {
+            this.localOpsToIgnore++;
         }
         else {
-            this.trailingNoopCount = 0;
+            this.localOpsToIgnore = 0;
         }
-        return Object.assign(Object.assign({}, message), { clientSequenceNumber: ++this.clientSequenceNumber });
+        return {
+            ...message,
+            clientSequenceNumber: ++this.clientSequenceNumber,
+        };
     }
-    submitSignal(content) {
+    submitSignal(content, targetClientId) {
         if (this.connection !== undefined) {
-            this.connection.submitSignal(content);
+            this.connection.submitSignal(content, targetClientId);
         }
         else {
             this.logger.sendErrorEvent({ eventName: "submitSignalDisconnected" });
@@ -658,10 +815,12 @@ export class ConnectionManager {
         if (this.connectionMode === "read") {
             if (!this.pendingReconnect) {
                 this.pendingReconnect = true;
-                Promise.resolve().then(async () => {
-                    if (this.pendingReconnect) { // still valid?
+                Promise.resolve()
+                    .then(async () => {
+                    if (this.pendingReconnect) {
+                        // still valid?
                         await this.reconnect("write", // connectionMode
-                        "Switch to write");
+                        { text: "Switch to write" });
                     }
                 })
                     .catch(() => { });
@@ -674,7 +833,8 @@ export class ConnectionManager {
     beforeProcessingIncomingOp(message) {
         // if we have connection, and message is local, then we better treat is as local!
         assert(this.clientId !== message.clientId || this.lastSubmittedClientId === message.clientId, 0x0ee /* "Not accounting local messages correctly" */);
-        if (this.lastSubmittedClientId !== undefined && this.lastSubmittedClientId === message.clientId) {
+        if (this.lastSubmittedClientId !== undefined &&
+            this.lastSubmittedClientId === message.clientId) {
             const clientSequenceNumber = message.clientSequenceNumber;
             assert(this.clientSequenceNumberObserved < clientSequenceNumber, 0x0ef /* "client seq# not growing" */);
             assert(clientSequenceNumber <= this.clientSequenceNumber, 0x0f0 /* "Incoming local client seq# > generated by this client" */);
@@ -693,11 +853,11 @@ export class ConnectionManager {
                 // Clients need to be able to transition to "read" state after some time of inactivity!
                 // Note - this may close container!
                 this.reconnect("read", // connectionMode
-                "Switch to read").catch((error) => {
+                { text: "Switch to read" }).catch((error) => {
                     this.logger.sendErrorEvent({ eventName: "SwitchToReadConnection" }, error);
                 });
             }
         }
     }
 }
-//# sourceMappingURL=connectionManager.js.map
+//# sourceMappingURL=connectionManager.mjs.map