@fluidframework/container-loader 2.0.0-dev.1.4.6.106135 → 2.0.0-dev.2.3.0.115467

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-loader",
-  "version": "2.0.0-dev.1.4.6.106135",
+  "version": "2.0.0-dev.2.3.0.115467",
   "description": "Fluid container loader",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -28,17 +28,19 @@
     "clean": "rimraf dist lib *.tsbuildinfo *.build.log",
     "eslint": "eslint --format stylish src",
     "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
+    "format": "npm run prettier:fix",
     "lint": "npm run eslint",
     "lint:fix": "npm run eslint:fix",
+    "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
+    "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
     "test": "npm run test:mocha",
     "test:coverage": "nyc npm test -- --reporter xunit --reporter-option output=nyc/junit-report.xml",
     "test:mocha": "mocha --ignore 'dist/test/types/*' --recursive dist/test -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
     "tsc": "tsc",
     "tsc:watch": "tsc --watch",
-    "tsfmt": "tsfmt --verify",
-    "tsfmt:fix": "tsfmt --replace",
-    "typetests:gen": "fluid-type-validator -g -d ."
+    "typetests:gen": "flub generate typetests --generate --dir .",
+    "typetests:prepare": "flub generate typetests --prepare --dir . --pin"
   },
   "nyc": {
     "all": true,
@@ -63,14 +65,14 @@
   "dependencies": {
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^1.0.0",
-    "@fluidframework/container-definitions": "2.0.0-dev.1.4.6.106135",
-    "@fluidframework/container-utils": "2.0.0-dev.1.4.6.106135",
-    "@fluidframework/core-interfaces": "2.0.0-dev.1.4.6.106135",
-    "@fluidframework/driver-definitions": "2.0.0-dev.1.4.6.106135",
-    "@fluidframework/driver-utils": "2.0.0-dev.1.4.6.106135",
-    "@fluidframework/protocol-base": "^0.1037.2001",
-    "@fluidframework/protocol-definitions": "^1.0.0",
-    "@fluidframework/telemetry-utils": "2.0.0-dev.1.4.6.106135",
+    "@fluidframework/container-definitions": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
+    "@fluidframework/container-utils": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
+    "@fluidframework/core-interfaces": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
+    "@fluidframework/driver-definitions": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
+    "@fluidframework/driver-utils": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
+    "@fluidframework/protocol-base": "^0.1038.2000",
+    "@fluidframework/protocol-definitions": "^1.1.0",
+    "@fluidframework/telemetry-utils": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
     "abort-controller": "^3.0.0",
     "double-ended-queue": "^2.1.0-0",
     "lodash": "^4.17.21",
@@ -78,12 +80,13 @@
     "uuid": "^8.3.1"
   },
   "devDependencies": {
-    "@fluidframework/build-common": "^1.0.0",
-    "@fluidframework/build-tools": "^0.4.4000",
-    "@fluidframework/container-loader-previous": "npm:@fluidframework/container-loader@^1.0.0",
-    "@fluidframework/eslint-config-fluid": "^1.0.0",
-    "@fluidframework/mocha-test-setup": "2.0.0-dev.1.4.6.106135",
-    "@fluidframework/test-loader-utils": "2.0.0-dev.1.4.6.106135",
+    "@fluid-tools/build-cli": "^0.7.0",
+    "@fluidframework/build-common": "^1.1.0",
+    "@fluidframework/build-tools": "^0.7.0",
+    "@fluidframework/container-loader-previous": "npm:@fluidframework/container-loader@2.0.0-internal.2.2.0",
+    "@fluidframework/eslint-config-fluid": "^1.2.0",
+    "@fluidframework/mocha-test-setup": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
+    "@fluidframework/test-loader-utils": ">=2.0.0-dev.2.3.0.115467 <2.0.0-dev.3.0.0",
     "@microsoft/api-extractor": "^7.22.2",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/double-ended-queue": "^2.1.0",
@@ -97,20 +100,15 @@
     "eslint": "~8.6.0",
     "mocha": "^10.0.0",
     "nyc": "^15.0.0",
+    "prettier": "~2.6.2",
     "rimraf": "^2.6.2",
     "sinon": "^7.4.2",
-    "typescript": "~4.5.5",
-    "typescript-formatter": "7.1.0"
+    "typescript": "~4.5.5"
   },
   "typeValidation": {
-    "version": "2.0.0",
-    "broken": {
-      "EnumDeclaration_ConnectionState": {
-        "forwardCompat": false
-      },
-      "ClassDeclaration_Container": {
-        "forwardCompat": false
-      }
-    }
+    "version": "2.0.0-internal.2.3.0",
+    "baselineRange": ">=2.0.0-internal.2.2.0 <2.0.0-internal.2.3.0",
+    "baselineVersion": "2.0.0-internal.2.2.0",
+    "broken": {}
   }
 }

package/prettier.config.cjs

@@ -0,0 +1,8 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+module.exports = {
+    ...require("@fluidframework/build-common/prettier.config.cjs"),
+};

package/src/audience.ts

@@ -3,6 +3,7 @@
  * Licensed under the MIT License.
  */
 import { EventEmitter } from "events";
+import { assert } from "@fluidframework/common-utils";
 import { IAudienceOwner } from "@fluidframework/container-definitions";
 import { IClient } from "@fluidframework/protocol-definitions";
 
@@ -21,8 +22,16 @@ export class Audience extends EventEmitter implements IAudienceOwner {
      * Adds a new client to the audience
      */
     public addMember(clientId: string, details: IClient) {
-        this.members.set(clientId, details);
-        this.emit("addMember", clientId, details);
+        // Given that signal delivery is unreliable process, we might observe same client being added twice
+        // In such case we should see exactly same payload (IClient), and should not raise event twice!
+        if (this.members.has(clientId)) {
+            const client = this.members.get(clientId);
+            assert(JSON.stringify(client) === JSON.stringify(details), 0x4b2 /* new client has different payload from existing one */);
+        }
+        else {
+            this.members.set(clientId, details);
+            this.emit("addMember", clientId, details);
+        }
     }
 
     /**
@@ -53,14 +62,4 @@ export class Audience extends EventEmitter implements IAudienceOwner {
     public getMember(clientId: string): IClient | undefined {
         return this.members.get(clientId);
     }
-
-    /**
-     * Clears the audience
-     */
-    public clear(): void {
-        const clientIds = this.members.keys();
-        for (const clientId of clientIds) {
-            this.removeMember(clientId);
-        }
-    }
 }

package/src/collabWindowTracker.ts

@@ -73,11 +73,11 @@ export class CollabWindowTracker {
             // Ensure we only send noop after a batch of many ops is processed
             // eslint-disable-next-line @typescript-eslint/no-floating-promises
             Promise.resolve().then(() => {
-                assert(this.opsCountSinceNoop >= this.NoopCountFrequency,
-                    0x3ae /* not enough ops were sent to reach the noop frequency */);
-                this.submitNoop(false /* immediate */);
-                // reset count now that all ops are processed
-                this.opsCountSinceNoop = 0;
+                if (this.opsCountSinceNoop >= this.NoopCountFrequency) {
+                    this.submitNoop(false /* immediate */);
+                    // reset count now that all ops are processed
+                    this.opsCountSinceNoop = 0;
+                }
                 return;
             });
         }

package/src/connectionManager.ts

@@ -18,6 +18,7 @@ import {
 } from "@fluidframework/container-definitions";
 import { GenericError, UsageError } from "@fluidframework/container-utils";
 import {
+    IAnyDriverError,
     IDocumentService,
     IDocumentDeltaConnection,
     IDocumentDeltaConnectionEvents,
@@ -27,7 +28,6 @@ import {
     createWriteError,
     createGenericNetworkError,
     getRetryDelayFromError,
-    IAnyDriverError,
     waitForConnectedState,
     DeltaStreamConnectionForbiddenError,
     logNetworkFailure,
@@ -59,6 +59,7 @@ import {
     IConnectionManagerFactoryArgs,
 } from "./contracts";
 import { DeltaQueue } from "./deltaQueue";
+import { SignalType } from "./protocol";
 
 const MaxReconnectDelayInMs = 8000;
 const InitialReconnectDelayInMs = 1000;
@@ -80,10 +81,19 @@ function getNackReconnectInfo(nackContent: INackContent) {
  * Implementation of IDocumentDeltaConnection that does not support submitting
  * or receiving ops. Used in storage-only mode.
  */
+const clientNoDeltaStream: IClient = {
+    mode: "read",
+    details: { capabilities: { interactive: true } },
+    permission: [],
+    user: { id: "storage-only client" }, // we need some "fake" ID here.
+    scopes: [],
+};
+const clientIdNoDeltaStream: string = "storage-only client";
+
 class NoDeltaStream
     extends TypedEventEmitter<IDocumentDeltaConnectionEvents>
     implements IDocumentDeltaConnection, IDisposable {
-    clientId: string = "storage-only client";
+    clientId = clientIdNoDeltaStream;
     claims: ITokenClaims = {
         scopes: [ScopeType.DocRead],
     } as any;
@@ -93,7 +103,7 @@ class NoDeltaStream
     version: string = "";
     initialMessages: ISequencedDocumentMessage[] = [];
     initialSignals: ISignalMessage[] = [];
-    initialClients: ISignalClient[] = [];
+    initialClients: ISignalClient[] = [{ client: clientNoDeltaStream, clientId: clientIdNoDeltaStream }];
     serviceConfiguration: IClientConfiguration = {
         maxMessageSize: 0,
         blockSize: 0,
@@ -257,7 +267,7 @@ export class ConnectionManager implements IConnectionManager {
      * It is undefined if we have not yet established websocket connection
      * and do not know if user has write access to a file.
      */
-    private get readonly() {
+    private get readonly(): boolean | undefined {
         if (this._forceReadonly) {
             return true;
         }
@@ -570,7 +580,13 @@ export class ConnectionManager implements IConnectionManager {
      * @param args - The connection arguments
      */
     private triggerConnect(connectionMode: ConnectionMode) {
-        assert(this.connection === undefined, 0x239 /* "called only in disconnected state" */);
+        // reconnect() has async await of waitForConnectedState(), and that causes potential race conditions
+        // where we might already have a connection. If it were to happen, it's possible that we will connect
+        // with different mode to `connectionMode`. Glancing through the caller chains, it looks like code should be
+        // fine (if needed, reconnect flow will get triggered again). Places where new mode matters should encode it
+        // directly in connectCore - see this.shouldJoinWrite() test as an example.
+        // assert(this.connection === undefined, 0x239 /* "called only in disconnected state" */);
+
         if (this.reconnectMode !== ReconnectMode.Enabled) {
             return;
         }
@@ -649,6 +665,9 @@ export class ConnectionManager implements IConnectionManager {
         // But if we ask read, server can still give us write.
         const readonly = !connection.claims.scopes.includes(ScopeType.DocWrite);
 
+        if (connection.mode !== requestedMode) {
+            this.logger.sendTelemetryEvent({ eventName: "ConnectionModeMismatch", requestedMode, mode: connection.mode });
+        }
         // This connection mode validation logic is moving to the driver layer in 0.44.  These two asserts can be
         // removed after those packages have released and become ubiquitous.
         assert(requestedMode === "read" || readonly === (this.connectionMode === "read"),
@@ -713,17 +732,43 @@ export class ConnectionManager implements IConnectionManager {
             initialMessages,
             this.connectFirstConnection ? "InitialOps" : "ReconnectOps");
 
-        if (connection.initialSignals !== undefined) {
-            for (const signal of connection.initialSignals) {
-                this.props.signalHandler(signal);
-            }
-        }
-
         const details = ConnectionManager.detailsFromConnection(connection);
         details.checkpointSequenceNumber = checkpointSequenceNumber;
         this.props.connectHandler(details);
 
         this.connectFirstConnection = false;
+
+        // Synthesize clear & join signals out of initialClients state.
+        // This allows us to have single way to process signals, and makes it simpler to initialize
+        // protocol in Container.
+        const clearSignal: ISignalMessage = {
+            clientId: null, // system message
+            content: JSON.stringify({
+                type: SignalType.Clear,
+            }),
+        };
+        this.props.signalHandler(clearSignal);
+
+        for (const priorClient of connection.initialClients ?? []) {
+            const joinSignal: ISignalMessage = {
+                clientId: null, // system signal
+                content: JSON.stringify({
+                    type: SignalType.ClientJoin,
+                    content: priorClient, // ISignalClient
+                }),
+            };
+            this.props.signalHandler(joinSignal);
+        }
+
+        // Unfortunately, there is no defined order between initialSignals (including join & leave signals)
+        // and connection.initialClients. In practice, connection.initialSignals quite often contains join signal
+        // for "self" and connection.initialClients does not contain "self", so we have to process them after
+        // "clear" signal above.
+        if (connection.initialSignals !== undefined) {
+            for (const signal of connection.initialSignals) {
+                this.props.signalHandler(signal);
+            }
+        }
     }
 
     /**

package/src/connectionStateHandler.ts

@@ -7,14 +7,20 @@ import { ITelemetryLogger, ITelemetryProperties } from "@fluidframework/common-d
 import { assert, Timer } from "@fluidframework/common-utils";
 import { IConnectionDetails, IDeltaManager } from "@fluidframework/container-definitions";
 import { ILocalSequencedClient } from "@fluidframework/protocol-base";
-import { ConnectionMode } from "@fluidframework/protocol-definitions";
+import { ISequencedClient, IClient } from "@fluidframework/protocol-definitions";
 import { PerformanceEvent, loggerToMonitoringContext } from "@fluidframework/telemetry-utils";
 import { ConnectionState } from "./connectionState";
 import { CatchUpMonitor, ICatchUpMonitor } from "./catchUpMonitor";
 import { IProtocolHandler } from "./protocol";
 
+// Based on recent data, it looks like majority of cases where we get stuck are due to really slow or
+// timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so
+// if retrying fixes the problem, we should not see these events.
 const JoinOpTimeoutMs = 45000;
 
+// Timeout waiting for "self" join signal, before giving up
+const JoinSignalTimeoutMs = 5000;
+
 /** Constructor parameter type for passing in dependencies needed by the ConnectionStateHandler */
 export interface IConnectionStateHandlerInputs {
     logger: ITelemetryLogger;
@@ -39,7 +45,7 @@ export interface IConnectionStateHandler {
     containerSaved(): void;
     dispose(): void;
     initProtocol(protocol: IProtocolHandler): void;
-    receivedConnectEvent(connectionMode: ConnectionMode, details: IConnectionDetails): void;
+    receivedConnectEvent(details: IConnectionDetails): void;
     receivedDisconnectEvent(reason: string): void;
 }
 
@@ -50,7 +56,8 @@ export function createConnectionStateHandler(
 ) {
     const mc = loggerToMonitoringContext(inputs.logger);
     return createConnectionStateHandlerCore(
-        mc.config.getBoolean("Fluid.Container.CatchUpBeforeDeclaringConnected") === true,
+        mc.config.getBoolean("Fluid.Container.CatchUpBeforeDeclaringConnected") === true, // connectedRaisedWhenCaughtUp
+        mc.config.getBoolean("Fluid.Container.DisableJoinSignalWait") !== true, // readClientsWaitForJoinSignal
         inputs,
         deltaManager,
         clientId,
@@ -58,20 +65,34 @@ export function createConnectionStateHandler(
 }
 
 export function createConnectionStateHandlerCore(
-    wait: boolean,
+    connectedRaisedWhenCaughtUp: boolean,
+    readClientsWaitForJoinSignal: boolean,
     inputs: IConnectionStateHandlerInputs,
     deltaManager: IDeltaManager<any, any>,
     clientId?: string,
 ) {
-    if (!wait) {
-        return new ConnectionStateHandler(inputs, clientId);
+    if (!connectedRaisedWhenCaughtUp) {
+        return new ConnectionStateHandler(inputs, readClientsWaitForJoinSignal, clientId);
     }
     return new ConnectionStateCatchup(
         inputs,
-        (handler: IConnectionStateHandlerInputs) => new ConnectionStateHandler(handler, clientId),
+        (handler: IConnectionStateHandlerInputs) => new ConnectionStateHandler(
+            handler,
+            readClientsWaitForJoinSignal,
+            clientId),
         deltaManager);
 }
 
+/**
+ * Helper internal interface to abstract away Audience & Quorum
+ */
+interface IMembership {
+    on(
+        eventName: "addMember" | "removeMember",
+        listener: (clientId: string, details: IClient | ISequencedClient) => void);
+    getMember(clientId: string): undefined | unknown;
+}
+
 /**
  * Class that can be used as a base class for building IConnectionStateHandler adapters / pipeline.
  * It implements both ends of communication interfaces and passes data back and forward
@@ -97,8 +118,8 @@ class ConnectionStateHandlerPassThrough implements IConnectionStateHandler, ICon
     public initProtocol(protocol: IProtocolHandler) { return this.pimpl.initProtocol(protocol); }
     public receivedDisconnectEvent(reason: string) { return this.pimpl.receivedDisconnectEvent(reason); }
 
-    public receivedConnectEvent(connectionMode: ConnectionMode, details: IConnectionDetails) {
-        return this.pimpl.receivedConnectEvent(connectionMode, details);
+    public receivedConnectEvent(details: IConnectionDetails) {
+        return this.pimpl.receivedConnectEvent(details);
     }
 
     /**
@@ -201,8 +222,8 @@ class ConnectionStateCatchup extends ConnectionStateHandlerPassThrough {
  *
  * For (a) we give up waiting after some time (same timeout as server uses), and go ahead and transition to Connected.
  *
- * For (b) we log telemetry if it takes too long, but still only transition to Connected when the Join op is processed
- * and we are added to the Quorum.
+ * For (b) we log telemetry if it takes too long, but still only transition to Connected when the Join op/signal is
+ * processed.
  *
  * For (c) this is optional behavior, controlled by the parameters of receivedConnectEvent
  */
@@ -212,6 +233,8 @@ class ConnectionStateHandler implements IConnectionStateHandler {
     private readonly prevClientLeftTimer: Timer;
     private readonly joinOpTimer: Timer;
     private protocol?: IProtocolHandler;
+    private connection?: IConnectionDetails;
+    private _clientId?: string;
 
     private waitEvent: PerformanceEvent | undefined;
 
@@ -229,8 +252,10 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 
     constructor(
         private readonly handler: IConnectionStateHandlerInputs,
-        private _clientId?: string,
+        private readonly readClientsWaitForJoinSignal: boolean,
+        clientIdFromPausedSession?: string,
     ) {
+        this._clientId = clientIdFromPausedSession;
         this.prevClientLeftTimer = new Timer(
             // Default is 5 min for which we are going to wait for its own "leave" message. This is same as
             // the max time on server after which leave op is sent.
@@ -242,11 +267,8 @@ class ConnectionStateHandler implements IConnectionStateHandler {
             },
         );
 
-        // Based on recent data, it looks like majority of cases where we get stuck are due to really slow or
-        // timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so
-        // if retrying fixes the problem, we should not see these events.
         this.joinOpTimer = new Timer(
-            JoinOpTimeoutMs,
+            0, // default value is not used - startJoinOpTimer() explicitly provides timeout
             () => {
                 // I've observed timer firing within couple ms from disconnect event, looks like
                 // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.
@@ -266,7 +288,10 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 
     private startJoinOpTimer() {
         assert(!this.joinOpTimer.hasTimer, 0x234 /* "has joinOpTimer" */);
-        this.joinOpTimer.start();
+        assert(this.connection !== undefined, 0x4b3 /* have connection */);
+        this.joinOpTimer.start(
+            this.connection.mode === "write" ? JoinOpTimeoutMs : JoinSignalTimeoutMs,
+        );
     }
 
     private stopJoinOpTimer() {
@@ -297,8 +322,8 @@ class ConnectionStateHandler implements IConnectionStateHandler {
         if (clientId === this.pendingClientId) {
             if (this.joinOpTimer.hasTimer) {
                 this.stopJoinOpTimer();
-            } else {
-                // timer has already fired, meaning it took too long to get join on.
+            } else if (this.shouldWaitForJoinSignal()) {
+                // timer has already fired, meaning it took too long to get join op/signal.
                 // Record how long it actually took to recover.
                 this.handler.logConnectionIssue("ReceivedJoinOp");
             }
@@ -333,9 +358,11 @@ class ConnectionStateHandler implements IConnectionStateHandler {
             this.setConnectionState(ConnectionState.Connected);
         } else {
             // Adding this event temporarily so that we can get help debugging if something goes wrong.
+            // We may not see any ops due to being disconnected all that time - that's not an error!
+            const error = source === "timeout" && this.connectionState !== ConnectionState.Disconnected;
             this.handler.logger.sendTelemetryEvent({
                 eventName: "connectedStateRejected",
-                category: source === "timeout" ? "error" : "generic",
+                category: error ? "error" : "generic",
                 details: JSON.stringify({
                     source,
                     pendingClientId: this.pendingClientId,
@@ -356,29 +383,35 @@ class ConnectionStateHandler implements IConnectionStateHandler {
     }
 
     public receivedDisconnectEvent(reason: string) {
+        this.connection = undefined;
         this.setConnectionState(ConnectionState.Disconnected, reason);
     }
 
+    private shouldWaitForJoinSignal() {
+        assert(this.connection !== undefined, 0x4b4 /* all callers call here with active connection */);
+        return this.connection.mode === "write" || this.readClientsWaitForJoinSignal;
+    }
+
     /**
      * The "connect" event indicates the connection to the Relay Service is live.
      * However, some additional conditions must be met before we can fully transition to
      * "Connected" state. This function handles that interim period, known as "Connecting" state.
-     * @param connectionMode - Read or Write connection
      * @param details - Connection details returned from the Relay Service
      * @param deltaManager - DeltaManager to be used for delaying Connected transition until caught up.
      * If it's undefined, then don't delay and transition to Connected as soon as Leave/Join op are accounted for
      */
     public receivedConnectEvent(
-        connectionMode: ConnectionMode,
         details: IConnectionDetails,
     ) {
+        this.connection = details;
+
         const oldState = this._connectionState;
         this._connectionState = ConnectionState.CatchingUp;
 
-        const writeConnection = connectionMode === "write";
-
         // The following checks are wrong. They are only valid if user has write access to a file.
         // If user lost such access mid-session, user will not be able to get "write" connection.
+        //
+        // const writeConnection = details.mode === "write";
         // assert(!this.handler.shouldClientJoinWrite() || writeConnection,
         //    0x30a /* shouldClientJoinWrite should imply this is a writeConnection */);
         // assert(!this.waitingForLeaveOp || writeConnection,
@@ -395,16 +428,14 @@ class ConnectionStateHandler implements IConnectionStateHandler {
         // IMPORTANT: Report telemetry after we set _pendingClientId, but before transitioning to Connected state
         this.handler.connectionStateChanged(ConnectionState.CatchingUp, oldState);
 
-        // For write connections, this pending clientId could be in the quorum already (i.e. join op already processed).
-        // We are fetching ops from storage in parallel to connecting to Relay Service,
-        // and given async processes, it's possible that we have already processed our own join message before
-        // connection was fully established.
-        // If protocol is not initialized yet, we expect it will process the join op after it's initialized.
-        const waitingForJoinOp = writeConnection && !this.hasMember(this._pendingClientId);
-
-        if (waitingForJoinOp) {
-            // Previous client left, and we are waiting for our own join op. When it is processed we'll join the quorum
-            // and attempt to transition to Connected state via receivedAddMemberEvent.
+        // Check if we need to wait for join op/signal, and if we need to wait for leave op from previous connection.
+        // Pending clientId could have joined already (i.e. join op/signal already processed):
+        //    We are fetching ops from storage in parallel to connecting to Relay Service,
+        //    and given async processes, it's possible that we have already processed our own join message before
+        //    connection was fully established.
+        if (!this.hasMember(this._pendingClientId) && this.shouldWaitForJoinSignal()) {
+            // We are waiting for our own join op / signal. When it is processed
+            // we'll attempt to transition to Connected state via receivedAddMemberEvent() flow.
             this.startJoinOpTimer();
         } else if (!this.waitingForLeaveOp) {
             // We're not waiting for Join or Leave op (if read-only connection those don't even apply),
@@ -426,6 +457,10 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 
         const oldState = this._connectionState;
         this._connectionState = value;
+
+        // This is the only place in code that deals with quorum. The rest works with audience
+        // The code below ensures that we do not send ops until we know that old "write" client's disconnect
+        // produced (and sequenced) leave op
         let client: ILocalSequencedClient | undefined;
         if (this._clientId !== undefined) {
             client = this.protocol?.quorum?.getMember(this._clientId);
@@ -476,23 +511,36 @@ class ConnectionStateHandler implements IConnectionStateHandler {
     // Helper method to switch between quorum and audience.
     // Old design was checking only quorum for "write" clients.
     // Latest change checks audience for all types of connections.
-    protected get membership() {
-        return this.protocol?.quorum;
+    protected get membership(): IMembership | undefined {
+        // We could always use audience here, and in practice it will probably be correct.
+        // (including case when this.readClientsWaitForJoinSignal === false).
+        // But only if it's superset of quorum, i.e. when filtered to "write" clients, they are always identical!
+        // It's safer to assume that we have bugs and engaging kill-bit switch should bring us back to well-known
+        // and tested state!
+        return this.readClientsWaitForJoinSignal ? this.protocol?.audience : this.protocol?.quorum;
     }
 
     public initProtocol(protocol: IProtocolHandler) {
         this.protocol = protocol;
 
-        this.membership?.on("addMember", (clientId) => {
+        this.membership?.on("addMember", (clientId, details) => {
+            assert((details as IClient).mode === "read" || protocol.quorum.getMember(clientId) !== undefined,
+                0x4b5 /* Audience is subset of quorum */);
             this.receivedAddMemberEvent(clientId);
         });
 
         this.membership?.on("removeMember", (clientId) => {
+            assert(protocol.quorum.getMember(clientId) === undefined, 0x4b6 /* Audience is subset of quorum */);
             this.receivedRemoveMemberEvent(clientId);
         });
 
-        // Very unlikely race condition, but theoretically can happen - our new connection is already
-        // summarized and we are loading from such summary.
+        /* There is a tiny tiny race possible, where these events happen in this order:
+          1. A connection is established (no "cached" mode is used, so it happens in parallel / faster than other steps)
+          2. Some other client produces a summary
+          3. We get "lucky" and load from that summary as our initial snapshot
+          4. ConnectionStateHandler.initProtocol is called, "self" is already in the quorum.
+        We could avoid this sequence (and delete test case for it) if we move connection lower in Container.load()
+        */
         if (this.hasMember(this.pendingClientId)) {
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             this.receivedAddMemberEvent(this.pendingClientId!);