@fluidframework/container-loader 2.0.0-dev.1.4.6.106135 → 2.0.0-dev.2.3.0.115467

package/src/container.ts

@@ -64,7 +64,6 @@ import {
     ISequencedClient,
     ISequencedDocumentMessage,
     ISequencedProposal,
-    ISignalClient,
     ISignalMessage,
     ISnapshotTree,
     ISummaryContent,
@@ -242,14 +241,14 @@ const getCodeProposal =
  * @param eventName - event name
  * @param action - functor to call and measure
  */
-async function ReportIfTooLong(
+export async function ReportIfTooLong(
     logger: ITelemetryLogger,
     eventName: string,
     action: () => Promise<ITelemetryProperties>,
 ) {
     const event = PerformanceEvent.start(logger, { eventName });
     const props = await action();
-    if (event.duration > 1000) {
+    if (event.duration > 200) {
         event.end(props);
     }
 }
@@ -412,7 +411,6 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
     private readonly clientDetailsOverride: IClientDetails | undefined;
     private readonly _deltaManager: DeltaManager<ConnectionManager>;
     private service: IDocumentService | undefined;
-    private _initialClients: ISignalClient[] | undefined;
 
     private _context: ContainerContext | undefined;
     private get context() {
@@ -619,7 +617,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.subLogger, "Container"));
 
         const summarizeProtocolTree =
-            this.mc.config.getBoolean("Fluid.Container.summarizeProtocolTree")
+            this.mc.config.getBoolean("Fluid.Container.summarizeProtocolTree2")
             ?? this.loader.services.options.summarizeProtocolTree;
 
         this.options = {
@@ -639,19 +637,37 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
                     }
                     this.logConnectionStateChangeTelemetry(value, oldState, reason);
                     if (this._lifecycleState === "loaded") {
-                        this.propagateConnectionState(false /* initial transition */);
+                        this.propagateConnectionState(
+                            false /* initial transition */,
+                            value === ConnectionState.Disconnected ? reason : undefined /* disconnectedReason */,
+                        );
                     }
                 },
                 shouldClientJoinWrite: () => this._deltaManager.connectionManager.shouldJoinWrite(),
                 maxClientLeaveWaitTime: this.loader.services.options.maxClientLeaveWaitTime,
                 logConnectionIssue: (eventName: string, details?: ITelemetryProperties) => {
+                    const mode = this.connectionMode;
                     // We get here when socket does not receive any ops on "write" connection, including
                     // its own join op. Attempt recovery option.
                     this._deltaManager.logConnectionIssue({
                         eventName,
+                        mode,
                         duration: performance.now() - this.connectionTransitionTimes[ConnectionState.CatchingUp],
                         ...(details === undefined ? {} : { details: JSON.stringify(details) }),
                     });
+
+                    // If this is "write" connection, it took too long to receive join op. But in most cases that's due
+                    // to very slow op fetches and we will eventually get there.
+                    // For "read" connections, we get here due to self join signal not arriving on time. We will need to
+                    // better understand when and why it may happen.
+                    // For now, attempt to recover by reconnecting. In future, maybe we can query relay service for
+                    // current state of audience.
+                    // Other possible recovery path - move to connected state (i.e. ConnectionStateHandler.joinOpTimer
+                    // to call this.applyForConnectedState("addMemberEvent") for "read" connections)
+                    if (mode === "read") {
+                        this.disconnect();
+                        this.connect();
+                    }
                 },
             },
             this.deltaManager,
@@ -1383,11 +1399,8 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
             attributes,
             quorumSnapshot,
             (key, value) => this.submitMessage(MessageType.Propose, JSON.stringify({ key, value })),
-            this._initialClients ?? [],
         );
 
-        this._initialClients = undefined;
-
         const protocolLogger = ChildLogger.create(this.subLogger, "ProtocolHandler");
 
         protocol.quorum.on("error", (error) => {
@@ -1512,22 +1525,8 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         deltaManager.inboundSignal.pause();
 
         deltaManager.on("connect", (details: IConnectionDetails, _opsBehind?: number) => {
-            if (this._protocolHandler === undefined) {
-                // Store the initial clients so that they can be submitted to the
-                // protocol handler when it is created.
-                this._initialClients = details.initialClients;
-            } else {
-                // When reconnecting, the protocol handler is already created,
-                // so we can update the audience right now.
-                this._protocolHandler.audience.clear();
-
-                for (const priorClient of details.initialClients ?? []) {
-                    this._protocolHandler.audience.addMember(priorClient.clientId, priorClient.client);
-                }
-            }
-
+            assert(this.connectionMode === details.mode, 0x4b7 /* mismatch */);
             this.connectionStateHandler.receivedConnectEvent(
-                this.connectionMode,
                 details,
             );
         });
@@ -1542,7 +1541,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
             // Some "warning" events come from outside the container and are logged
             // elsewhere (e.g. summarizing container). We shouldn't log these here.
             if (warn.logged !== true) {
-                this.logContainerError(warn);
+                this.mc.logger.sendTelemetryEvent({ eventName: "ContainerWarning" }, warn);
             }
             this.emit("warning", warn);
         });
@@ -1629,7 +1628,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         }
     }
 
-    private propagateConnectionState(initialTransition: boolean) {
+    private propagateConnectionState(initialTransition: boolean, disconnectedReason?: string) {
         // When container loaded, we want to propagate initial connection state.
         // After that, we communicate only transitions to Connected & Disconnected states, skipping all other states.
         // This can be changed in the future, for example we likely should add "CatchingUp" event on Container.
@@ -1652,7 +1651,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
 
         this.setContextConnectedState(state, this._deltaManager.connectionManager.readOnlyInfo.readonly ?? false);
         this.protocolHandler.setConnectionState(state, this.clientId);
-        raiseConnectedEvent(this.mc.logger, this, state, this.clientId);
+        raiseConnectedEvent(this.mc.logger, this, state, this.clientId, disconnectedReason);
 
         if (logOpsOnReconnect) {
             this.mc.logger.sendTelemetryEvent(
@@ -1687,7 +1686,8 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
                 MessageType.Operation,
                 message.contents,
                 true, // batch
-                message.metadata);
+                message.metadata,
+                message.compression);
         }
         this._deltaManager.flush();
         return clientSequenceNumber;
@@ -1706,7 +1706,11 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         return this.submitMessage(MessageType.Summarize, JSON.stringify(summary), false /* batch */);
     }
 
-    private submitMessage(type: MessageType, contents?: string, batch?: boolean, metadata?: any): number {
+    private submitMessage(type: MessageType,
+                          contents?: string,
+                          batch?: boolean,
+                          metadata?: any,
+                          compression?: string): number {
         if (this.connectionState !== ConnectionState.Connected) {
             this.mc.logger.sendErrorEvent({ eventName: "SubmitMessageWithNoConnection", type });
             return -1;
@@ -1714,7 +1718,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
 
         this.messageCountAfterDisconnection += 1;
         this.collabWindowTracker?.stopSequenceNumberUpdate();
-        return this._deltaManager.submit(type, contents, batch, metadata);
+        return this._deltaManager.submit(type, contents, batch, metadata, compression);
     }
 
     private processRemoteMessage(message: ISequencedDocumentMessage) {
@@ -1724,7 +1728,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         const result = this.protocolHandler.processMessage(message, local);
 
         // Forward messages to the loaded runtime for processing
-        this.context.process(message, local, undefined);
+        this.context.process(message, local);
 
         // Inactive (not in quorum or not writers) clients don't take part in the minimum sequence number calculation.
         if (this.activeConnection()) {
@@ -1740,9 +1744,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
                     (type) => {
                         assert(this.activeConnection(),
                             0x241 /* "disconnect should result in stopSequenceNumberUpdate() call" */);
-                        // back-compat: There is a bug in 1.2 runtime where clients cannot handle
-                        // ops whose contents are undefined
-                        this.submitMessage(type, null as any);
+                        this.submitMessage(type);
                     },
                     this.serviceConfiguration.noopTimeFrequency,
                     this.serviceConfiguration.noopCountFrequency,
@@ -1848,10 +1850,6 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         this.emit(dirty ? dirtyContainerEvent : savedContainerEvent);
     }
 
-    private logContainerError(warning: ContainerWarning) {
-        this.mc.logger.sendErrorEvent({ eventName: "ContainerWarning" }, warning);
-    }
-
     /**
      * Set the connected state of the ContainerContext
      * This controls the "connected" state of the ContainerRuntime as well

package/src/containerContext.ts

@@ -46,7 +46,7 @@ import {
     ISummaryContent,
 } from "@fluidframework/protocol-definitions";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
-import { Container } from "./container";
+import { Container, ReportIfTooLong } from "./container";
 
 const PackageNotFactoryError = "Code package does not implement IRuntimeFactory";
 
@@ -243,8 +243,8 @@ export class ContainerContext implements IContainerContext {
         runtime.setConnectionState(connected, clientId);
     }
 
-    public process(message: ISequencedDocumentMessage, local: boolean, context: any) {
-        this.runtime.process(message, local, context);
+    public process(message: ISequencedDocumentMessage, local: boolean) {
+        this.runtime.process(message, local);
     }
 
     public processSignal(message: ISignalMessage, local: boolean) {
@@ -324,7 +324,13 @@ export class ContainerContext implements IContainerContext {
 
     private async instantiateRuntime(existing: boolean) {
         const runtimeFactory = await this.getRuntimeFactory();
-        this._runtime = await runtimeFactory.instantiateRuntime(this, existing);
+        await ReportIfTooLong(
+            this.taggedLogger,
+            "instantiateRuntime",
+            async () => {
+                this._runtime = await runtimeFactory.instantiateRuntime(this, existing);
+                return {};
+            });
     }
 
     private attachListener() {

package/src/containerStorageAdapter.ts

@@ -176,7 +176,7 @@ class BlobOnlyStorage implements IDocumentStorageService {
             // some browsers may not populate stack unless exception is thrown
             throw new Error("BlobOnlyStorage not implemented method used");
         } catch (err) {
-            this.logger.sendErrorEvent({ eventName: "BlobOnlyStorageWrongCall" }, err);
+            this.logger.sendTelemetryEvent({ eventName: "BlobOnlyStorageWrongCall" }, err);
             throw err;
         }
     }

package/src/contracts.ts

@@ -148,15 +148,23 @@ export interface IConnectionManagerFactoryArgs {
 
     /**
      * Called whenever ping/pong messages are roundtripped on connection.
+     *
+     * @deprecated No replacement API intended.
      */
     readonly pongHandler: (latency: number) => void;
 
     /**
      * Called whenever connection type changes from writable to read-only or vice versa.
+     *
+     * @remarks
+     *
      * Connection can be read-only if user has no edit permissions, or if container forced
      * connection to be read-only.
      * This should not be confused with "read" / "write"connection mode which is internal
      * optimization.
+     *
+     * @param readonly - Whether or not the container is now read-only.
+     * `undefined` indicates that user permissions are not yet known.
      */
     readonly readonlyChangeHandler: (readonly?: boolean) => void;
 }

package/src/deltaManager.ts

@@ -25,8 +25,6 @@ import {
     normalizeError,
     logIfFalse,
     safeRaiseEvent,
-    MonitoringContext,
-    loggerToMonitoringContext,
 } from "@fluidframework/telemetry-utils";
 import {
     IDocumentDeltaStorageService,
@@ -42,20 +40,20 @@ import {
 } from "@fluidframework/protocol-definitions";
 import {
     NonRetryableError,
-    isClientMessage,
+    isRuntimeMessage,
+    MessageType2,
 } from "@fluidframework/driver-utils";
 import {
     ThrottlingWarning,
     DataCorruptionError,
     extractSafePropertiesFromMessage,
     DataProcessingError,
-    UsageError,
 } from "@fluidframework/container-utils";
 import { DeltaQueue } from "./deltaQueue";
 import {
     IConnectionManagerFactoryArgs,
     IConnectionManager,
- } from "./contracts";
+} from "./contracts";
 
 export interface IConnectionArgs {
     mode?: ConnectionMode;
@@ -72,6 +70,25 @@ export interface IDeltaManagerInternalEvents extends IDeltaManagerEvents {
     (event: "closed", listener: (error?: ICriticalContainerError) => void);
 }
 
+/**
+ * Determines if message was sent by client, not service
+ */
+function isClientMessage(message: ISequencedDocumentMessage | IDocumentMessage): boolean {
+    if (isRuntimeMessage(message)) {
+        return true;
+    }
+    switch (message.type) {
+        case MessageType.Propose:
+        case MessageType.Reject:
+        case MessageType.NoOp:
+        case MessageType2.Accept:
+        case MessageType.Summarize:
+            return true;
+        default:
+            return false;
+    }
+}
+
 /**
  * Manages the flow of both inbound and outbound messages. This class ensures that shared objects receive delta
  * messages in order regardless of possible network conditions or timings causing out of order delivery.
@@ -92,23 +109,17 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
     private pending: ISequencedDocumentMessage[] = [];
     private fetchReason: string | undefined;
 
-    private readonly mc: MonitoringContext;
-
     // A boolean used to assert that ops are not being sent while processing another op.
     private currentlyProcessingOps: boolean = false;
 
-    // Feature gate that closes a container when sending an op if the container is
-    // concurrently processing another op
-    private readonly preventConcurrentOpSend: boolean = true;
-
     // The minimum sequence number and last sequence number received from the server
     private minSequenceNumber: number = 0;
 
     // There are three numbers we track
     // * lastQueuedSequenceNumber is the last queued sequence number. If there are gaps in seq numbers, then this number
     //   is not updated until we cover that gap, so it increases each time by 1.
-    // * lastObservedSeqNumber is  an estimation of last known sequence number for container in storage. It's initially
-    //   populated at web socket connection time (if storage provides that info) and is  updated once ops shows up.
+    // * lastObservedSeqNumber is an estimation of last known sequence number for container in storage. It's initially
+    //   populated at web socket connection time (if storage provides that info) and is updated once ops shows up.
     //   It's never less than lastQueuedSequenceNumber
     // * lastProcessedSequenceNumber - last processed sequence number
     private lastQueuedSequenceNumber: number = 0;
@@ -185,7 +196,7 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
      * Tells if  current connection has checkpoint information.
      * I.e. we know how far behind the client was at the time of establishing connection
      */
-     public get hasCheckpointSequenceNumber() {
+    public get hasCheckpointSequenceNumber() {
         // Valid to be called only if we have active connection.
         assert(this.connectionManager.connected, 0x0df /* "Missing active connection" */);
         return this._checkpointSequenceNumber !== undefined;
@@ -199,15 +210,13 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
     public get readOnlyInfo() { return this.connectionManager.readOnlyInfo; }
     public get clientDetails() { return this.connectionManager.clientDetails; }
 
-    public submit(type: MessageType, contents?: string, batch = false, metadata?: any) {
-        if (this.currentlyProcessingOps && this.preventConcurrentOpSend) {
-            this.close(new UsageError("Making changes to data model is disallowed while processing ops."));
-        }
+    public submit(type: MessageType, contents?: string, batch = false, metadata?: any, compression?: string) {
         const messagePartial: Omit<IDocumentMessage, "clientSequenceNumber"> = {
             contents,
             metadata,
             referenceSequenceNumber: this.lastProcessedSequenceNumber,
             type,
+            compression,
         };
 
         if (!batch) {
@@ -218,7 +227,9 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
             return -1;
         }
 
-        if (contents !== undefined && contents !== null) {
+        assert(isClientMessage(message), 0x419 /* client sends non-client message */);
+
+        if (contents !== undefined) {
             this.opsSize += contents.length;
         }
 
@@ -321,8 +332,6 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
         };
 
         this.connectionManager = createConnectionManager(props);
-        this.mc = loggerToMonitoringContext(logger);
-        this.preventConcurrentOpSend = this.mc.config.getBoolean("Fluid.Container.ConcurrentOpSend") === true;
         this._inbound = new DeltaQueue<ISequencedDocumentMessage>(
             (op) => {
                 this.processInboundMessage(op);
@@ -390,7 +399,7 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
             if (checkpointSequenceNumber > this.lastQueuedSequenceNumber) {
                 this.fetchMissingDeltas("AfterConnection");
             }
-        // we do not know the gap, and we will not learn about it if socket is quite - have to ask.
+            // we do not know the gap, and we will not learn about it if socket is quite - have to ask.
         } else if (connection.mode === "read") {
             this.fetchMissingDeltas("AfterReadConnection");
         }
@@ -708,9 +717,9 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
             // Report if we found some issues
             if (duplicate !== 0 || gap !== 0 && !allowGaps || initialGap > 0 && this.fetchReason === undefined) {
                 eventName = "enqueueMessages";
-            // Also report if we are fetching ops, and same range comes in, thus making this fetch obsolete.
+                // Also report if we are fetching ops, and same range comes in, thus making this fetch obsolete.
             } else if (this.fetchReason !== undefined && this.fetchReason !== reason &&
-                    (from <= this.lastQueuedSequenceNumber + 1 && last > this.lastQueuedSequenceNumber)) {
+                (from <= this.lastQueuedSequenceNumber + 1 && last > this.lastQueuedSequenceNumber)) {
                 eventName = "enqueueMessagesExtraFetch";
             }
 
@@ -795,13 +804,16 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
         this.currentlyProcessingOps = true;
         this.lastProcessedMessage = message;
 
-        // All non-system messages are coming from some client, and should have clientId
-        // System messages may have no clientId (but some do, like propose, noop, summarize)
-        assert(
-            message.clientId !== undefined
-            || !(isClientMessage(message)),
-            0x0ed /* "non-system message have to have clientId" */,
-        );
+        const isString = typeof message.clientId === "string";
+        assert(message.clientId === null || isString, 0x41a /* undefined or string */);
+        // All client messages are coming from some client, and should have clientId,
+        // and non-client message should not have clientId. But, there are two exceptions:
+        // 1. (Legacy) We can see message.type === "attach" or "chunkedOp" for legacy files before RTM
+        // 2. Non-immediate noops (contents: null) can be sent by service without clientId
+        if (!isString && isClientMessage(message) && message.type !== MessageType.NoOp) {
+            throw new DataCorruptionError("Mismatch in clientId",
+                { ...extractSafePropertiesFromMessage(message), messageType: message.type });
+        }
 
         // TODO Remove after SPO picks up the latest build.
         if (
@@ -822,6 +834,16 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
                 clientId: this.connectionManager.clientId,
             });
         }
+
+        // Client ops: MSN has to be lower than sequence #, as client can continue to send ops with same
+        // reference sequence number as this op.
+        // System ops (when no clients are connected) are the only ops where equation is possible.
+        const diff = message.sequenceNumber - message.minimumSequenceNumber;
+        if (diff < 0 || diff === 0 && message.clientId !== null) {
+            throw new DataCorruptionError("MSN has to be lower than sequence #",
+                extractSafePropertiesFromMessage(message));
+        }
+
         this.minSequenceNumber = message.minimumSequenceNumber;
 
         if (message.sequenceNumber !== this.lastProcessedSequenceNumber + 1) {
@@ -858,15 +880,15 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
     /**
      * Retrieves the missing deltas between the given sequence numbers
      */
-     private fetchMissingDeltas(reasonArg: string, to?: number) {
-         this.fetchMissingDeltasCore(reasonArg, false /* cacheOnly */, to).catch((error) => {
-             this.logger.sendErrorEvent({ eventName: "fetchMissingDeltasException" }, error);
-         });
-     }
+    private fetchMissingDeltas(reasonArg: string, to?: number) {
+        this.fetchMissingDeltasCore(reasonArg, false /* cacheOnly */, to).catch((error) => {
+            this.logger.sendErrorEvent({ eventName: "fetchMissingDeltasException" }, error);
+        });
+    }
 
-     /**
-     * Retrieves the missing deltas between the given sequence numbers
-     */
+    /**
+    * Retrieves the missing deltas between the given sequence numbers
+    */
     private async fetchMissingDeltasCore(
         reason: string,
         cacheOnly: boolean,

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-loader";
-export const pkgVersion = "2.0.0-dev.1.4.6.106135";
+export const pkgVersion = "2.0.0-dev.2.3.0.115467";

package/src/protocol.ts

@@ -20,6 +20,13 @@ import {
 } from "@fluidframework/protocol-definitions";
 import { canBeCoalescedByService } from "@fluidframework/driver-utils";
 
+// ADO: #1986: Start using enum from protocol-base.
+export enum SignalType {
+    ClientJoin = "join", // same value as MessageType.ClientJoin,
+    ClientLeave = "leave", // same value as MessageType.ClientLeave,
+    Clear = "clear", // used only by client for synthetic signals
+}
+
 /**
  * Function to be used for creating a protocol handler.
  */
@@ -27,7 +34,6 @@ export type ProtocolHandlerBuilder = (
     attributes: IDocumentAttributes,
     snapshot: IQuorumSnapshot,
     sendProposal: (key: string, value: any) => number,
-    initialClients: ISignalClient[],
 ) => IProtocolHandler;
 
 export interface IProtocolHandler extends IBaseProtocolHandler {
@@ -40,7 +46,6 @@ export class ProtocolHandler extends ProtocolOpHandler implements IProtocolHandl
         attributes: IDocumentAttributes,
         quorumSnapshot: IQuorumSnapshot,
         sendProposal: (key: string, value: any) => number,
-        initialClients: ISignalClient[],
         readonly audience: IAudienceOwner,
     ) {
         super(
@@ -53,8 +58,11 @@ export class ProtocolHandler extends ProtocolOpHandler implements IProtocolHandl
             sendProposal,
         );
 
-        for (const initialClient of initialClients) {
-            this.audience.addMember(initialClient.clientId, initialClient.client);
+        // Join / leave signals are ignored for "write" clients in favor of join / leave ops
+        this.quorum.on("addMember", (clientId, details) => audience.addMember(clientId, details.client));
+        this.quorum.on("removeMember", (clientId) => audience.removeMember(clientId));
+        for (const [clientId, details] of this.quorum.getMembers()) {
+            this.audience.addMember(clientId, details.client);
         }
     }
 
@@ -81,14 +89,29 @@ export class ProtocolHandler extends ProtocolOpHandler implements IProtocolHandl
     public processSignal(message: ISignalMessage) {
         const innerContent = message.content as { content: any; type: string; };
         switch (innerContent.type) {
-            case MessageType.ClientJoin: {
+            case SignalType.Clear: {
+                const members = this.audience.getMembers();
+                for (const [clientId, client] of members) {
+                    if (client.mode === "read") {
+                        this.audience.removeMember(clientId);
+                    }
+                }
+                break;
+            }
+            case SignalType.ClientJoin: {
                 const newClient = innerContent.content as ISignalClient;
-                this.audience.addMember(newClient.clientId, newClient.client);
+                // Ignore write clients - quorum will control such clients.
+                if (newClient.client.mode === "read") {
+                    this.audience.addMember(newClient.clientId, newClient.client);
+                }
                 break;
             }
-            case MessageType.ClientLeave: {
+            case SignalType.ClientLeave: {
                 const leftClientId = innerContent.content as string;
-                this.audience.removeMember(leftClientId);
+                // Ignore write clients - quorum will control such clients.
+                if (this.audience.getMember(leftClientId)?.mode === "read") {
+                    this.audience.removeMember(leftClientId);
+                }
                 break;
             }
             default: break;

package/src/retriableDocumentStorageService.ts

@@ -104,12 +104,16 @@ export class RetriableDocumentStorageService implements IDocumentStorageService,
         );
     }
 
-    private checkStorageDisposed() {
+    private checkStorageDisposed(callName: string, error: unknown) {
         if (this._disposed) {
+            this.logger.sendTelemetryEvent({
+                eventName: `${callName}_abortedStorageDisposed`,
+                fetchCallName: callName, // fetchCallName matches logs in runWithRetry.ts
+            }, error);
             // pre-0.58 error message: storageServiceDisposedCannotRetry
             throw new GenericError("Storage Service is disposed. Cannot retry", { canRetry: false });
         }
-        return undefined;
+        return;
     }
 
     private async runWithRetry<T>(api: () => Promise<T>, callName: string): Promise<T> {
@@ -118,7 +122,7 @@ export class RetriableDocumentStorageService implements IDocumentStorageService,
             callName,
             this.logger,
             {
-                onRetry: () => this.checkStorageDisposed(),
+                onRetry: (_delayInMs: number, error: unknown) => this.checkStorageDisposed(callName, error),
             },
         );
     }