@fluidframework/container-loader 0.59.4001 → 1.1.0-75972

package/src/connectionStateHandler.ts

@@ -3,36 +3,58 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLogger, ITelemetryProperties } from "@fluidframework/common-definitions";
+import { assert, Timer } from "@fluidframework/common-utils";
 import { IConnectionDetails } from "@fluidframework/container-definitions";
-import { ConnectionMode, IQuorumClients, ISequencedClient } from "@fluidframework/protocol-definitions";
+import { ILocalSequencedClient, IProtocolHandler } from "@fluidframework/protocol-base";
+import { ConnectionMode, IQuorumClients } from "@fluidframework/protocol-definitions";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
-import { assert, Timer } from "@fluidframework/common-utils";
 import { ConnectionState } from "./connectionState";
 
-export interface IConnectionStateHandler {
+/** Constructor parameter type for passing in dependencies needed by the ConnectionStateHandler */
+export interface IConnectionStateHandlerInputs {
+    /** Provides access to the clients currently in the quorum */
     quorumClients: () => IQuorumClients | undefined;
-    logConnectionStateChangeTelemetry: (
-        value: ConnectionState,
-        oldState: ConnectionState,
-        reason?: string | undefined
-    ) => void;
+    /** Log to telemetry any change in state, included to Connecting */
+    logConnectionStateChangeTelemetry:
+        (value: ConnectionState, oldState: ConnectionState, reason?: string | undefined) => void;
+    /** Whether to expect the client to join in write mode on next connection */
     shouldClientJoinWrite: () => boolean;
+    /** (Optional) How long should we wait on our previous client's Leave op before transitioning to Connected again */
     maxClientLeaveWaitTime: number | undefined;
-    logConnectionIssue: (eventName: string) => void;
+    /** Log an issue encountered while in the Connecting state. details will be logged as a JSON string */
+    logConnectionIssue: (eventName: string, details?: ITelemetryProperties) => void;
+    /** Callback whenever the ConnectionState changes between Disconnected and Connected */
     connectionStateChanged: () => void;
 }
 
-export interface ILocalSequencedClient extends ISequencedClient {
-    shouldHaveLeft?: boolean;
-}
-
-const JoinOpTimer = 45000;
+const JoinOpTimeoutMs = 45000;
 
+/**
+ * In the lifetime of a container, the connection will likely disconnect and reconnect periodically.
+ * This class ensures that any ops sent by this container instance on previous connection are either
+ * sequenced or blocked by the server before emitting the new "connected" event and allowing runtime to resubmit ops.
+ *
+ * Each connection is assigned a clientId by the service, and the connection is book-ended by a Join and a Leave op
+ * generated by the service. Due to the distributed nature of the ordering service, in the case of reconnect we cannot
+ * make any assumptions about ordering of operations between the old and new connections - i.e. new Join op could
+ * be sequenced before old Leave op (and some acks from pending ops that were in flight when we disconnected).
+ *
+ * The job of this class is to encapsulate the transition period during reconnect, which is identified by
+ * ConnectionState.CatchingUp. Specifically, before moving to Connected state with the new clientId, it ensures that:
+ * (A) We process the Leave op for the previous clientId. This allows us to properly handle any acks from in-flight ops
+ *     that got sequenced with the old clientId (we'll recognize them as local ops). After the Leave op, any other
+ *     pending ops can safely be submitted with the new clientId without fear of duplication in the sequenced op stream.
+ * (B) We process the Join op for the new clientId (identified when the underlying connection was first established),
+ *     indicating the service is ready to sequence ops sent with the new clientId.
+ *
+ * For (A) we give up waiting after some time (same timeout as server uses), and go ahead and transition to Connected.
+ * For (B) we log telemetry if it takes too long, but still only transition to Connected when the Join op is processed
+ * and we are added to the Quorum.
+ */
 export class ConnectionStateHandler {
     private _connectionState = ConnectionState.Disconnected;
     private _pendingClientId: string | undefined;
-    private _clientId: string | undefined;
     private readonly prevClientLeftTimer: Timer;
     private readonly joinOpTimer: Timer;
 
@@ -55,8 +77,9 @@ export class ConnectionStateHandler {
     }
 
     constructor(
-        private readonly handler: IConnectionStateHandler,
+        private readonly handler: IConnectionStateHandlerInputs,
         private readonly logger: ITelemetryLogger,
+        private _clientId?: string,
     ) {
         this.prevClientLeftTimer = new Timer(
             // Default is 5 min for which we are going to wait for its own "leave" message. This is same as
@@ -73,13 +96,21 @@ export class ConnectionStateHandler {
         // timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so
         // if retrying fixes the problem, we should not see these events.
         this.joinOpTimer = new Timer(
-            JoinOpTimer,
+            JoinOpTimeoutMs,
             () => {
                 // I've observed timer firing within couple ms from disconnect event, looks like
                 // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.
-                if (this.connectionState === ConnectionState.Connecting) {
-                    this.handler.logConnectionIssue("NoJoinOp");
+                if (this.connectionState !== ConnectionState.CatchingUp) {
+                    return;
                 }
+                const quorumClients = this.handler.quorumClients();
+                const details = {
+                    quorumInitialized: quorumClients !== undefined,
+                    hasPendingClientId: this.pendingClientId !== undefined,
+                    inQuorum: quorumClients?.getMember(this.pendingClientId ?? "") !== undefined,
+                    waitingForLeaveOp: this.waitingForLeaveOp,
+                };
+                this.handler.logConnectionIssue("NoJoinOp", details);
             },
         );
     }
@@ -94,6 +125,10 @@ export class ConnectionStateHandler {
         this.joinOpTimer.clear();
     }
 
+    private get waitingForLeaveOp() {
+        return this.prevClientLeftTimer.hasTimer;
+    }
+
     public dispose() {
         assert(!this.joinOpTimer.hasTimer, 0x2a5 /* "join timer" */);
         this.prevClientLeftTimer.clear();
@@ -102,13 +137,13 @@ export class ConnectionStateHandler {
     public containerSaved() {
         // If we were waiting for moving to Connected state, then only apply for state change. Since the container
         // is now saved and we don't have any ops to roundtrip, we can clear the timer and apply for connected state.
-        if (this.prevClientLeftTimer.hasTimer) {
+        if (this.waitingForLeaveOp) {
             this.prevClientLeftTimer.clear();
             this.applyForConnectedState("containerSaved");
         }
     }
 
-    public receivedAddMemberEvent(clientId: string) {
+    private receivedAddMemberEvent(clientId: string) {
         // This is the only one that requires the pending client ID
         if (clientId === this.pendingClientId) {
             if (this.joinOpTimer.hasTimer) {
@@ -119,11 +154,13 @@ export class ConnectionStateHandler {
                 this.handler.logConnectionIssue("ReceivedJoinOp");
             }
             // Start the event in case we are waiting for leave or timeout.
-            if (this.prevClientLeftTimer.hasTimer) {
+            if (this.waitingForLeaveOp) {
                 this.waitEvent = PerformanceEvent.start(this.logger, {
                     eventName: "WaitBeforeClientLeave",
-                    waitOnClientId: this._clientId,
-                    hadOutstandingOps: this.handler.shouldClientJoinWrite(),
+                    details: JSON.stringify({
+                        waitOnClientId: this._clientId,
+                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),
+                    }),
                 });
             }
             this.applyForConnectedState("addMemberEvent");
@@ -133,13 +170,18 @@ export class ConnectionStateHandler {
     private applyForConnectedState(source: "removeMemberEvent" | "addMemberEvent" | "timeout" | "containerSaved") {
         const quorumClients = this.handler.quorumClients();
         assert(quorumClients !== undefined, 0x236 /* "In all cases it should be already installed" */);
+
+        assert(this.waitingForLeaveOp === false ||
+            (this.clientId !== undefined && quorumClients.getMember(this.clientId) !== undefined),
+            0x2e2 /* "Must only wait for leave message when clientId in quorum" */);
+
         // Move to connected state only if we are in Connecting state, we have seen our join op
         // and there is no timer running which means we are not waiting for previous client to leave
-        // or timeout has occured while doing so.
+        // or timeout has occurred while doing so.
         if (this.pendingClientId !== this.clientId
             && this.pendingClientId !== undefined
             && quorumClients.getMember(this.pendingClientId) !== undefined
-            && !this.prevClientLeftTimer.hasTimer
+            && !this.waitingForLeaveOp
         ) {
             this.waitEvent?.end({ source });
             this.setConnectionState(ConnectionState.Connected);
@@ -148,17 +190,18 @@ export class ConnectionStateHandler {
             this.logger.sendTelemetryEvent({
                 eventName: "connectedStateRejected",
                 category: source === "timeout" ? "error" : "generic",
-                source,
-                pendingClientId: this.pendingClientId,
-                clientId: this.clientId,
-                hasTimer: this.prevClientLeftTimer.hasTimer,
-                inQuorum: quorumClients !== undefined && this.pendingClientId !== undefined
-                    && quorumClients.getMember(this.pendingClientId) !== undefined,
+                details: JSON.stringify({
+                    source,
+                    pendingClientId: this.pendingClientId,
+                    clientId: this.clientId,
+                    waitingForLeaveOp: this.waitingForLeaveOp,
+                    inQuorum: quorumClients?.getMember(this.pendingClientId ?? "") !== undefined,
+                }),
             });
         }
     }
 
-    public receivedRemoveMemberEvent(clientId: string) {
+    private receivedRemoveMemberEvent(clientId: string) {
         // If the client which has left was us, then finish the timer.
         if (this.clientId === clientId) {
             this.prevClientLeftTimer.clear();
@@ -173,43 +216,63 @@ export class ConnectionStateHandler {
         this.setConnectionState(ConnectionState.Disconnected, reason);
     }
 
+    /**
+     * The "connect" event indicates the connection to the Relay Service is live.
+     * However, some additional conditions must be met before we can fully transition to
+     * "Connected" state. This function handles that interim period, known as "Connecting" state.
+     * @param connectionMode - Read or Write connection
+     * @param details - Connection details returned from the ordering service
+     */
     public receivedConnectEvent(
         connectionMode: ConnectionMode,
         details: IConnectionDetails,
     ) {
         const oldState = this._connectionState;
-        this._connectionState = ConnectionState.Connecting;
+        this._connectionState = ConnectionState.CatchingUp;
+
+        const writeConnection = connectionMode === "write";
+        assert(writeConnection || !this.handler.shouldClientJoinWrite(),
+            0x30a /* shouldClientJoinWrite should imply this is a writeConnection */);
+        assert(writeConnection || !this.waitingForLeaveOp,
+            0x2a6 /* "waitingForLeaveOp should imply writeConnection (we need to be ready to flush pending ops)" */);
+
+        // Note that this may be undefined since the connection is established proactively on load
+        // and the quorum may still be under initialization.
+        const quorumClients: IQuorumClients | undefined = this.handler.quorumClients();
 
         // Stash the clientID to detect when transitioning from connecting (socket.io channel open) to connected
         // (have received the join message for the client ID)
         // This is especially important in the reconnect case. It's possible there could be outstanding
         // ops sent by this client, so we should keep the old client id until we see our own client's
-        // join message. after we see the join message for out new connection with our new client id,
+        // join message. after we see the join message for our new connection with our new client id,
         // we know there can no longer be outstanding ops that we sent with the previous client id.
         this._pendingClientId = details.clientId;
 
-        // Report telemetry after we set client id, but before transitioning to Connected state below!
-        this.handler.logConnectionStateChangeTelemetry(ConnectionState.Connecting, oldState);
+        // IMPORTANT: Report telemetry after we set _pendingClientId, but before transitioning to Connected state
+        this.handler.logConnectionStateChangeTelemetry(ConnectionState.CatchingUp, oldState);
 
-        const quorumClients = this.handler.quorumClients();
-        // Check if we already processed our own join op through delta storage!
-        // we are fetching ops from storage in parallel to connecting to ordering service
-        // Given async processes, it's possible that we have already processed our own join message before
+        // For write connections, this pending clientId could be in the quorum already (i.e. join op already processed).
+        // We are fetching ops from storage in parallel to connecting to Relay Service,
+        // and given async processes, it's possible that we have already processed our own join message before
         // connection was fully established.
-        // Note that we might be still initializing quorum - connection is established proactively on load!
-        if (quorumClients?.getMember(details.clientId) !== undefined
-            || connectionMode === "read"
-        ) {
-            assert(!this.prevClientLeftTimer.hasTimer, 0x2a6 /* "there should be no timer for 'read' connections" */);
-            this.setConnectionState(ConnectionState.Connected);
-        } else if (connectionMode === "write") {
+        // If quorumClients itself is undefined, we expect it will process the join op after it's initialized.
+        const waitingForJoinOp = writeConnection && quorumClients?.getMember(this._pendingClientId) === undefined;
+
+        if (waitingForJoinOp) {
+            // Previous client left, and we are waiting for our own join op. When it is processed we'll join the quorum
+            // and attempt to transition to Connected state via receivedAddMemberEvent.
             this.startJoinOpTimer();
+        } else if (!this.waitingForLeaveOp) {
+            // We're not waiting for Join or Leave op (if read-only connection those don't even apply),
+            // go ahead and declare the state to be Connected!
+            // If we are waiting for Leave op still, do nothing for now, we will transition to Connected later.
+            this.setConnectionState(ConnectionState.Connected);
         }
     }
 
-    private setConnectionState(value: ConnectionState.Disconnected, reason: string);
-    private setConnectionState(value: ConnectionState.Connected);
-    private setConnectionState(value: ConnectionState, reason?: string) {
+    private setConnectionState(value: ConnectionState.Disconnected, reason: string): void;
+    private setConnectionState(value: ConnectionState.Connected): void;
+    private setConnectionState(value: ConnectionState, reason?: string): void {
         if (this.connectionState === value) {
             // Already in the desired state - exit early
             this.logger.sendErrorEvent({ eventName: "setConnectionStateSame", value });
@@ -224,7 +287,7 @@ export class ConnectionStateHandler {
             client = quorumClients?.getMember(this._clientId);
         }
         if (value === ConnectionState.Connected) {
-            assert(oldState === ConnectionState.Connecting,
+            assert(oldState === ConnectionState.CatchingUp,
                 0x1d8 /* "Should only transition from Connecting state" */);
             // Mark our old client should have left in the quorum if it's still there
             if (client !== undefined) {
@@ -249,9 +312,11 @@ export class ConnectionStateHandler {
                 // Adding this event temporarily so that we can get help debugging if something goes wrong.
                 this.logger.sendTelemetryEvent({
                     eventName: "noWaitOnDisconnected",
-                    inQuorum: client !== undefined,
-                    hasTimer: this.prevClientLeftTimer.hasTimer,
-                    shouldClientJoinWrite: this.handler.shouldClientJoinWrite(),
+                    details: JSON.stringify({
+                        inQuorum: client !== undefined,
+                        waitingForLeaveOp: this.waitingForLeaveOp,
+                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),
+                    }),
                 });
             }
         }
@@ -262,4 +327,19 @@ export class ConnectionStateHandler {
         // Propagate event across layers
         this.handler.connectionStateChanged();
     }
+
+    public initProtocol(protocol: IProtocolHandler) {
+        protocol.quorum.on("addMember", (clientId, _details) => {
+            this.receivedAddMemberEvent(clientId);
+        });
+
+        protocol.quorum.on("removeMember", (clientId) => {
+            this.receivedRemoveMemberEvent(clientId);
+        });
+
+        // if we have a clientId from a previous container we need to wait for its leave message
+        if (this.clientId !== undefined && protocol.quorum.getMember(this.clientId) !== undefined) {
+            this.prevClientLeftTimer.restart();
+        }
+    }
 }