@fluidframework/container-loader 2.0.0-internal.2.1.2 → 2.0.0-internal.2.2.0

package/lib/connectionStateHandler.js

@@ -6,16 +6,23 @@ import { assert, Timer } from "@fluidframework/common-utils";
 import { PerformanceEvent, loggerToMonitoringContext } from "@fluidframework/telemetry-utils";
 import { ConnectionState } from "./connectionState";
 import { CatchUpMonitor } from "./catchUpMonitor";
+// Based on recent data, it looks like majority of cases where we get stuck are due to really slow or
+// timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so
+// if retrying fixes the problem, we should not see these events.
 const JoinOpTimeoutMs = 45000;
+// Timeout waiting for "self" join signal, before giving up
+const JoinSignalTimeoutMs = 5000;
 export function createConnectionStateHandler(inputs, deltaManager, clientId) {
     const mc = loggerToMonitoringContext(inputs.logger);
-    return createConnectionStateHandlerCore(mc.config.getBoolean("Fluid.Container.CatchUpBeforeDeclaringConnected") === true, inputs, deltaManager, clientId);
+    return createConnectionStateHandlerCore(mc.config.getBoolean("Fluid.Container.CatchUpBeforeDeclaringConnected") === true, // connectedRaisedWhenCaughtUp
+    mc.config.getBoolean("Fluid.Container.DisableJoinSignalWait") !== true, // readClientsWaitForJoinSignal
+    inputs, deltaManager, clientId);
 }
-export function createConnectionStateHandlerCore(wait, inputs, deltaManager, clientId) {
-    if (!wait) {
-        return new ConnectionStateHandler(inputs, clientId);
+export function createConnectionStateHandlerCore(connectedRaisedWhenCaughtUp, readClientsWaitForJoinSignal, inputs, deltaManager, clientId) {
+    if (!connectedRaisedWhenCaughtUp) {
+        return new ConnectionStateHandler(inputs, readClientsWaitForJoinSignal, clientId);
     }
-    return new ConnectionStateCatchup(inputs, (handler) => new ConnectionStateHandler(handler, clientId), deltaManager);
+    return new ConnectionStateCatchup(inputs, (handler) => new ConnectionStateHandler(handler, readClientsWaitForJoinSignal, clientId), deltaManager);
 }
 /**
  * Class that can be used as a base class for building IConnectionStateHandler adapters / pipeline.
@@ -35,8 +42,8 @@ class ConnectionStateHandlerPassThrough {
     dispose() { return this.pimpl.dispose(); }
     initProtocol(protocol) { return this.pimpl.initProtocol(protocol); }
     receivedDisconnectEvent(reason) { return this.pimpl.receivedDisconnectEvent(reason); }
-    receivedConnectEvent(connectionMode, details) {
-        return this.pimpl.receivedConnectEvent(connectionMode, details);
+    receivedConnectEvent(details) {
+        return this.pimpl.receivedConnectEvent(details);
     }
     /**
      * IConnectionStateHandlerInputs
@@ -123,17 +130,18 @@ class ConnectionStateCatchup extends ConnectionStateHandlerPassThrough {
  *
  * For (a) we give up waiting after some time (same timeout as server uses), and go ahead and transition to Connected.
  *
- * For (b) we log telemetry if it takes too long, but still only transition to Connected when the Join op is processed
- * and we are added to the Quorum.
+ * For (b) we log telemetry if it takes too long, but still only transition to Connected when the Join op/signal is
+ * processed.
  *
  * For (c) this is optional behavior, controlled by the parameters of receivedConnectEvent
  */
 class ConnectionStateHandler {
-    constructor(handler, _clientId) {
+    constructor(handler, readClientsWaitForJoinSignal, clientIdFromPausedSession) {
         var _a;
         this.handler = handler;
-        this._clientId = _clientId;
+        this.readClientsWaitForJoinSignal = readClientsWaitForJoinSignal;
         this._connectionState = ConnectionState.Disconnected;
+        this._clientId = clientIdFromPausedSession;
         this.prevClientLeftTimer = new Timer(
         // Default is 5 min for which we are going to wait for its own "leave" message. This is same as
         // the max time on server after which leave op is sent.
@@ -141,10 +149,8 @@ class ConnectionStateHandler {
             assert(this.connectionState !== ConnectionState.Connected, 0x2ac /* "Connected when timeout waiting for leave from previous session fired!" */);
             this.applyForConnectedState("timeout");
         });
-        // Based on recent data, it looks like majority of cases where we get stuck are due to really slow or
-        // timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so
-        // if retrying fixes the problem, we should not see these events.
-        this.joinOpTimer = new Timer(JoinOpTimeoutMs, () => {
+        this.joinOpTimer = new Timer(0, // default value is not used - startJoinOpTimer() explicitly provides timeout
+        () => {
             // I've observed timer firing within couple ms from disconnect event, looks like
             // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.
             if (this.connectionState !== ConnectionState.CatchingUp) {
@@ -170,7 +176,8 @@ class ConnectionStateHandler {
     }
     startJoinOpTimer() {
         assert(!this.joinOpTimer.hasTimer, 0x234 /* "has joinOpTimer" */);
-        this.joinOpTimer.start();
+        assert(this.connection !== undefined, 0x4b3 /* have connection */);
+        this.joinOpTimer.start(this.connection.mode === "write" ? JoinOpTimeoutMs : JoinSignalTimeoutMs);
     }
     stopJoinOpTimer() {
         assert(this.joinOpTimer.hasTimer, 0x235 /* "no joinOpTimer" */);
@@ -197,8 +204,8 @@ class ConnectionStateHandler {
             if (this.joinOpTimer.hasTimer) {
                 this.stopJoinOpTimer();
             }
-            else {
-                // timer has already fired, meaning it took too long to get join on.
+            else if (this.shouldWaitForJoinSignal()) {
+                // timer has already fired, meaning it took too long to get join op/signal.
                 // Record how long it actually took to recover.
                 this.handler.logConnectionIssue("ReceivedJoinOp");
             }
@@ -230,9 +237,11 @@ class ConnectionStateHandler {
         }
         else {
             // Adding this event temporarily so that we can get help debugging if something goes wrong.
+            // We may not see any ops due to being disconnected all that time - that's not an error!
+            const error = source === "timeout" && this.connectionState !== ConnectionState.Disconnected;
             this.handler.logger.sendTelemetryEvent({
                 eventName: "connectedStateRejected",
-                category: source === "timeout" ? "error" : "generic",
+                category: error ? "error" : "generic",
                 details: JSON.stringify({
                     source,
                     pendingClientId: this.pendingClientId,
@@ -251,23 +260,29 @@ class ConnectionStateHandler {
         }
     }
     receivedDisconnectEvent(reason) {
+        this.connection = undefined;
         this.setConnectionState(ConnectionState.Disconnected, reason);
     }
+    shouldWaitForJoinSignal() {
+        assert(this.connection !== undefined, 0x4b4 /* all callers call here with active connection */);
+        return this.connection.mode === "write" || this.readClientsWaitForJoinSignal;
+    }
     /**
      * The "connect" event indicates the connection to the Relay Service is live.
      * However, some additional conditions must be met before we can fully transition to
      * "Connected" state. This function handles that interim period, known as "Connecting" state.
-     * @param connectionMode - Read or Write connection
      * @param details - Connection details returned from the Relay Service
      * @param deltaManager - DeltaManager to be used for delaying Connected transition until caught up.
      * If it's undefined, then don't delay and transition to Connected as soon as Leave/Join op are accounted for
      */
-    receivedConnectEvent(connectionMode, details) {
+    receivedConnectEvent(details) {
+        this.connection = details;
         const oldState = this._connectionState;
         this._connectionState = ConnectionState.CatchingUp;
-        const writeConnection = connectionMode === "write";
         // The following checks are wrong. They are only valid if user has write access to a file.
         // If user lost such access mid-session, user will not be able to get "write" connection.
+        //
+        // const writeConnection = details.mode === "write";
         // assert(!this.handler.shouldClientJoinWrite() || writeConnection,
         //    0x30a /* shouldClientJoinWrite should imply this is a writeConnection */);
         // assert(!this.waitingForLeaveOp || writeConnection,
@@ -281,15 +296,14 @@ class ConnectionStateHandler {
         this._pendingClientId = details.clientId;
         // IMPORTANT: Report telemetry after we set _pendingClientId, but before transitioning to Connected state
         this.handler.connectionStateChanged(ConnectionState.CatchingUp, oldState);
-        // For write connections, this pending clientId could be in the quorum already (i.e. join op already processed).
-        // We are fetching ops from storage in parallel to connecting to Relay Service,
-        // and given async processes, it's possible that we have already processed our own join message before
-        // connection was fully established.
-        // If protocol is not initialized yet, we expect it will process the join op after it's initialized.
-        const waitingForJoinOp = writeConnection && !this.hasMember(this._pendingClientId);
-        if (waitingForJoinOp) {
-            // Previous client left, and we are waiting for our own join op. When it is processed we'll join the quorum
-            // and attempt to transition to Connected state via receivedAddMemberEvent.
+        // Check if we need to wait for join op/signal, and if we need to wait for leave op from previous connection.
+        // Pending clientId could have joined already (i.e. join op/signal already processed):
+        //    We are fetching ops from storage in parallel to connecting to Relay Service,
+        //    and given async processes, it's possible that we have already processed our own join message before
+        //    connection was fully established.
+        if (!this.hasMember(this._pendingClientId) && this.shouldWaitForJoinSignal()) {
+            // We are waiting for our own join op / signal. When it is processed
+            // we'll attempt to transition to Connected state via receivedAddMemberEvent() flow.
             this.startJoinOpTimer();
         }
         else if (!this.waitingForLeaveOp) {
@@ -309,6 +323,9 @@ class ConnectionStateHandler {
         }
         const oldState = this._connectionState;
         this._connectionState = value;
+        // This is the only place in code that deals with quorum. The rest works with audience
+        // The code below ensures that we do not send ops until we know that old "write" client's disconnect
+        // produced (and sequenced) leave op
         let client;
         if (this._clientId !== undefined) {
             client = (_b = (_a = this.protocol) === null || _a === void 0 ? void 0 : _a.quorum) === null || _b === void 0 ? void 0 : _b.getMember(this._clientId);
@@ -357,20 +374,32 @@ class ConnectionStateHandler {
     // Old design was checking only quorum for "write" clients.
     // Latest change checks audience for all types of connections.
     get membership() {
-        var _a;
-        return (_a = this.protocol) === null || _a === void 0 ? void 0 : _a.quorum;
+        var _a, _b;
+        // We could always use audience here, and in practice it will probably be correct.
+        // (including case when this.readClientsWaitForJoinSignal === false).
+        // But only if it's superset of quorum, i.e. when filtered to "write" clients, they are always identical!
+        // It's safer to assume that we have bugs and engaging kill-bit switch should bring us back to well-known
+        // and tested state!
+        return this.readClientsWaitForJoinSignal ? (_a = this.protocol) === null || _a === void 0 ? void 0 : _a.audience : (_b = this.protocol) === null || _b === void 0 ? void 0 : _b.quorum;
     }
     initProtocol(protocol) {
         var _a, _b;
         this.protocol = protocol;
-        (_a = this.membership) === null || _a === void 0 ? void 0 : _a.on("addMember", (clientId) => {
+        (_a = this.membership) === null || _a === void 0 ? void 0 : _a.on("addMember", (clientId, details) => {
+            assert(details.mode === "read" || protocol.quorum.getMember(clientId) !== undefined, 0x4b5 /* Audience is subset of quorum */);
             this.receivedAddMemberEvent(clientId);
         });
         (_b = this.membership) === null || _b === void 0 ? void 0 : _b.on("removeMember", (clientId) => {
+            assert(protocol.quorum.getMember(clientId) === undefined, 0x4b6 /* Audience is subset of quorum */);
             this.receivedRemoveMemberEvent(clientId);
         });
-        // Very unlikely race condition, but theoretically can happen - our new connection is already
-        // summarized and we are loading from such summary.
+        /* There is a tiny tiny race possible, where these events happen in this order:
+          1. A connection is established (no "cached" mode is used, so it happens in parallel / faster than other steps)
+          2. Some other client produces a summary
+          3. We get "lucky" and load from that summary as our initial snapshot
+          4. ConnectionStateHandler.initProtocol is called, "self" is already in the quorum.
+        We could avoid this sequence (and delete test case for it) if we move connection lower in Container.load()
+        */
         if (this.hasMember(this.pendingClientId)) {
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             this.receivedAddMemberEvent(this.pendingClientId);

package/lib/connectionStateHandler.js.map

@@ -1 +1 @@
-{"version":3,"file":"connectionStateHandler.js","sourceRoot":"","sources":["../src/connectionStateHandler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAI7D,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAC9F,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAmB,MAAM,kBAAkB,CAAC;AAGnE,MAAM,eAAe,GAAG,KAAK,CAAC;AA8B9B,MAAM,UAAU,4BAA4B,CACxC,MAAqC,EACrC,YAAqC,EACrC,QAAiB;IAEjB,MAAM,EAAE,GAAG,yBAAyB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpD,OAAO,gCAAgC,CACnC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iDAAiD,CAAC,KAAK,IAAI,EAChF,MAAM,EACN,YAAY,EACZ,QAAQ,CACX,CAAC;AACN,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC5C,IAAa,EACb,MAAqC,EACrC,YAAqC,EACrC,QAAiB;IAEjB,IAAI,CAAC,IAAI,EAAE;QACP,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;KACvD;IACD,OAAO,IAAI,sBAAsB,CAC7B,MAAM,EACN,CAAC,OAAsC,EAAE,EAAE,CAAC,IAAI,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,EACzF,YAAY,CAAC,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,iCAAiC;IAGnC,YACuB,MAAqC,EACxD,YAAiF;QAD9D,WAAM,GAAN,MAAM,CAA+B;QAGxD,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,IAAW,eAAe,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;IACnE,IAAW,eAAe,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;IAE5D,cAAc,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACxD,OAAO,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC1C,YAAY,CAAC,QAA0B,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACtF,uBAAuB,CAAC,MAAc,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE9F,oBAAoB,CAAC,cAA8B,EAAE,OAA2B;QACnF,OAAO,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IAEH,IAAW,MAAM,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3C,sBAAsB,CACzB,KAAsB,EACtB,QAAyB,EACzB,MAA2B;QAE3B,OAAO,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvE,CAAC;IACM,qBAAqB,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAW,sBAAsB,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAC3E,kBAAkB,CAAC,SAAiB,EAAE,OAA8B;QACvE,OAAO,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;CACJ;AAED;;;GAGG;AACH,MAAM,sBAAuB,SAAQ,iCAAiC;IAGlE,YACI,MAAqC,EACrC,YAAiF,EAChE,YAAqC;QAEtD,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAFX,iBAAY,GAAZ,YAAY,CAAyB;QAqCzC,+BAA0B,GAAG,GAAG,EAAE;YAC/C,kFAAkF;YAClF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;YACzC,MAAM,CAAC,KAAK,KAAK,eAAe,CAAC,SAAS,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC3F,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC,SAAS,CAAC;YAClD,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,eAAe,CAAC,SAAS,EAAE,eAAe,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC3G,CAAC,CAAC;QAzCE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;IACvD,CAAC;IAGD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEM,sBAAsB,CAAC,KAAsB,EAAE,QAAyB,EAAE,MAA2B;;QACxG,QAAQ,KAAK,EAAE;YACX,KAAK,eAAe,CAAC,SAAS;gBAC1B,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;gBACnG,mGAAmG;gBACnG,qGAAqG;gBACrG,oGAAoG;gBACpG,qGAAqG;gBACrG,qGAAqG;gBACrG,2CAA2C;gBAC3C,MAAM,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBACrF,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBAC7F,OAAO;YACX,KAAK,eAAe,CAAC,YAAY;gBAC7B,MAAA,IAAI,CAAC,cAAc,0CAAE,OAAO,EAAE,CAAC;gBAC/B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;gBAChC,MAAM;YACV,KAAK,eAAe,CAAC,UAAU;gBAC3B,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,eAAe,CAAC,YAAY,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;gBACrG,MAAM;YACV,QAAQ;SACX;QACD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAChE,CAAC;CAUJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,sBAAsB;IAqBxB,YACqB,OAAsC,EAC/C,SAAkB;;QADT,YAAO,GAAP,OAAO,CAA+B;QAC/C,cAAS,GAAT,SAAS,CAAS;QAtBtB,qBAAgB,GAAG,eAAe,CAAC,YAAY,CAAC;QAwBpD,IAAI,CAAC,mBAAmB,GAAG,IAAI,KAAK;QAChC,+FAA+F;QAC/F,uDAAuD;QACvD,MAAA,IAAI,CAAC,OAAO,CAAC,sBAAsB,mCAAI,MAAM,EAC7C,GAAG,EAAE;YACD,MAAM,CAAC,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,SAAS,EACrD,KAAK,CAAC,6EAA6E,CAAC,CAAC;YACzF,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC,CACJ,CAAC;QAEF,qGAAqG;QACrG,kGAAkG;QAClG,iEAAiE;QACjE,IAAI,CAAC,WAAW,GAAG,IAAI,KAAK,CACxB,eAAe,EACf,GAAG,EAAE;YACD,gFAAgF;YAChF,iGAAiG;YACjG,IAAI,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;gBACrD,OAAO;aACV;YACD,MAAM,OAAO,GAAG;gBACZ,mBAAmB,EAAE,IAAI,CAAC,QAAQ,KAAK,SAAS;gBAChD,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC;gBAClD,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;aAC5C,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC,CACJ,CAAC;IACN,CAAC;IA/CD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,IAAY,QAAQ;QAChB,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAuCO,gBAAgB;QACpB,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAEO,eAAe;QACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAEM,OAAO;QACV,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC7D,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAEM,cAAc;QACjB,0GAA0G;QAC1G,6GAA6G;QAC7G,IAAI,IAAI,CAAC,iBAAiB,EAAE;YACxB,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,QAAgB;QAC3C,2DAA2D;QAC3D,IAAI,QAAQ,KAAK,IAAI,CAAC,eAAe,EAAE;YACnC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;aAC1B;iBAAM;gBACH,oEAAoE;gBACpE,+CAA+C;gBAC/C,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;aACrD;YACD,+DAA+D;YAC/D,IAAI,IAAI,CAAC,iBAAiB,EAAE;gBACxB,IAAI,CAAC,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;oBACzD,SAAS,EAAE,uBAAuB;oBAClC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,cAAc,EAAE,IAAI,CAAC,SAAS;wBAC9B,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;qBAC1D,CAAC;iBACL,CAAC,CAAC;aACN;YACD,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,MAA6E;;QACxG,MAAM,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAE/F,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAC3D,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAE5E,uFAAuF;QACvF,4FAA4F;QAC5F,0CAA0C;QAC1C,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC,QAAQ;eACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC;eACpC,CAAC,IAAI,CAAC,iBAAiB,EAC5B;YACE,MAAA,IAAI,CAAC,SAAS,0CAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;aAAM;YACH,2FAA2F;YAC3F,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBACnC,SAAS,EAAE,wBAAwB;gBACnC,QAAQ,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;gBACpD,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,MAAM;oBACN,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;oBACzC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC;iBACrD,CAAC;aACL,CAAC,CAAC;SACN;IACL,CAAC;IAEO,yBAAyB,CAAC,QAAgB;QAC9C,8DAA8D;QAC9D,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAC5B,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,mBAAmB,CAAC,CAAC;SACpD;IACL,CAAC;IAEM,uBAAuB,CAAC,MAAc;QACzC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;;;OAQG;IACI,oBAAoB,CACvB,cAA8B,EAC9B,OAA2B;QAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC,UAAU,CAAC;QAEnD,MAAM,eAAe,GAAG,cAAc,KAAK,OAAO,CAAC;QAEnD,0FAA0F;QAC1F,yFAAyF;QACzF,mEAAmE;QACnE,gFAAgF;QAChF,qDAAqD;QACrD,+GAA+G;QAE/G,wGAAwG;QACxG,qDAAqD;QACrD,+FAA+F;QAC/F,6FAA6F;QAC7F,6FAA6F;QAC7F,2FAA2F;QAC3F,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEzC,yGAAyG;QACzG,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAE1E,gHAAgH;QAChH,+EAA+E;QAC/E,sGAAsG;QACtG,oCAAoC;QACpC,oGAAoG;QACpG,MAAM,gBAAgB,GAAG,eAAe,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAEnF,IAAI,gBAAgB,EAAE;YAClB,2GAA2G;YAC3G,2EAA2E;YAC3E,IAAI,CAAC,gBAAgB,EAAE,CAAC;SAC3B;aAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAChC,2FAA2F;YAC3F,kDAAkD;YAClD,mGAAmG;YACnG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;QACD,sGAAsG;IAC1G,CAAC;IAIO,kBAAkB,CAAC,KAA+D,EAAE,MAAe;;QACvG,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE;YAChC,4CAA4C;YAC5C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,wBAAwB,EAAE,KAAK,EAAE,CAAC,CAAC;YACnF,OAAO;SACV;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,MAAyC,CAAC;QAC9C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;YAC9B,MAAM,GAAG,MAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,MAAM,0CAAE,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SAC7D;QACD,IAAI,KAAK,KAAK,eAAe,CAAC,SAAS,EAAE;YACrC,MAAM,CAAC,QAAQ,KAAK,eAAe,CAAC,UAAU,EAC1C,KAAK,CAAC,oDAAoD,CAAC,CAAC;YAChE,yEAAyE;YACzE,IAAI,MAAM,KAAK,SAAS,EAAE;gBACtB,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;aAChC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC;SACzC;aAAM,IAAI,KAAK,KAAK,eAAe,CAAC,YAAY,EAAE;YAC/C,2DAA2D;YAC3D,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;YAElC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;aAC1B;YAED,wGAAwG;YACxG,oDAAoD;YACpD,qGAAqG;YACrG,sGAAsG;YACtG,IAAI,MAAM,KAAK,SAAS;mBACjB,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;mBACpC,CAAC,IAAI,CAAC,iBAAiB,CAAC,6CAA6C;cAC1E;gBACE,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;aACtC;iBAAM;gBACH,2FAA2F;gBAC3F,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBACnC,SAAS,EAAE,sBAAsB;oBACjC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,QAAQ,EAAE,IAAI,CAAC,SAAS;wBACxB,QAAQ,EAAE,MAAM,KAAK,SAAS;wBAC9B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;wBACzC,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;qBAC1D,CAAC;iBACL,CAAC,CAAC;aACN;SACJ;QAED,4DAA4D;QAC5D,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjF,CAAC;IAED,uDAAuD;IACvD,2DAA2D;IAC3D,8DAA8D;IAC9D,IAAc,UAAU;;QACpB,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,MAAM,CAAC;IACjC,CAAC;IAEM,YAAY,CAAC,QAA0B;;QAC1C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,MAAA,IAAI,CAAC,UAAU,0CAAE,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC1C,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAA,IAAI,CAAC,UAAU,0CAAE,EAAE,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC7C,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,6FAA6F;QAC7F,mDAAmD;QACnD,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE;YACtC,oEAAoE;YACpE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,eAAgB,CAAC,CAAC;SACtD;QAED,wFAAwF;QACxF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;YAC9D,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;SACtC;IACL,CAAC;IAES,SAAS,CAAC,QAAiB;;QACjC,OAAO,CAAA,MAAA,IAAI,CAAC,UAAU,0CAAE,SAAS,CAAC,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,EAAE,CAAC,MAAK,SAAS,CAAC;IACpE,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger, ITelemetryProperties } from \"@fluidframework/common-definitions\";\nimport { assert, Timer } from \"@fluidframework/common-utils\";\nimport { IConnectionDetails, IDeltaManager } from \"@fluidframework/container-definitions\";\nimport { ILocalSequencedClient } from \"@fluidframework/protocol-base\";\nimport { ConnectionMode } from \"@fluidframework/protocol-definitions\";\nimport { PerformanceEvent, loggerToMonitoringContext } from \"@fluidframework/telemetry-utils\";\nimport { ConnectionState } from \"./connectionState\";\nimport { CatchUpMonitor, ICatchUpMonitor } from \"./catchUpMonitor\";\nimport { IProtocolHandler } from \"./protocol\";\n\nconst JoinOpTimeoutMs = 45000;\n\n/** Constructor parameter type for passing in dependencies needed by the ConnectionStateHandler */\nexport interface IConnectionStateHandlerInputs {\n    logger: ITelemetryLogger;\n    /** Log to telemetry any change in state, included to Connecting */\n    connectionStateChanged:\n        (value: ConnectionState, oldState: ConnectionState, reason?: string | undefined) => void;\n    /** Whether to expect the client to join in write mode on next connection */\n    shouldClientJoinWrite: () => boolean;\n    /** (Optional) How long should we wait on our previous client's Leave op before transitioning to Connected again */\n    maxClientLeaveWaitTime: number | undefined;\n    /** Log an issue encountered while in the Connecting state. details will be logged as a JSON string */\n    logConnectionIssue: (eventName: string, details?: ITelemetryProperties) => void;\n}\n\n/**\n * interface that connection state handler implements\n */\nexport interface IConnectionStateHandler {\n    readonly connectionState: ConnectionState;\n    readonly pendingClientId: string | undefined;\n\n    containerSaved(): void;\n    dispose(): void;\n    initProtocol(protocol: IProtocolHandler): void;\n    receivedConnectEvent(connectionMode: ConnectionMode, details: IConnectionDetails): void;\n    receivedDisconnectEvent(reason: string): void;\n}\n\nexport function createConnectionStateHandler(\n    inputs: IConnectionStateHandlerInputs,\n    deltaManager: IDeltaManager<any, any>,\n    clientId?: string,\n) {\n    const mc = loggerToMonitoringContext(inputs.logger);\n    return createConnectionStateHandlerCore(\n        mc.config.getBoolean(\"Fluid.Container.CatchUpBeforeDeclaringConnected\") === true,\n        inputs,\n        deltaManager,\n        clientId,\n    );\n}\n\nexport function createConnectionStateHandlerCore(\n    wait: boolean,\n    inputs: IConnectionStateHandlerInputs,\n    deltaManager: IDeltaManager<any, any>,\n    clientId?: string,\n) {\n    if (!wait) {\n        return new ConnectionStateHandler(inputs, clientId);\n    }\n    return new ConnectionStateCatchup(\n        inputs,\n        (handler: IConnectionStateHandlerInputs) => new ConnectionStateHandler(handler, clientId),\n        deltaManager);\n}\n\n/**\n * Class that can be used as a base class for building IConnectionStateHandler adapters / pipeline.\n * It implements both ends of communication interfaces and passes data back and forward\n */\nclass ConnectionStateHandlerPassThrough implements IConnectionStateHandler, IConnectionStateHandlerInputs {\n    protected readonly pimpl: IConnectionStateHandler;\n\n    constructor(\n        protected readonly inputs: IConnectionStateHandlerInputs,\n        pimplFactory: (handler: IConnectionStateHandlerInputs) => IConnectionStateHandler,\n    ) {\n        this.pimpl = pimplFactory(this);\n    }\n\n    /**\n     * IConnectionStateHandler\n     */\n    public get connectionState() { return this.pimpl.connectionState; }\n    public get pendingClientId() { return this.pimpl.pendingClientId; }\n\n    public containerSaved() { return this.pimpl.containerSaved(); }\n    public dispose() { return this.pimpl.dispose(); }\n    public initProtocol(protocol: IProtocolHandler) { return this.pimpl.initProtocol(protocol); }\n    public receivedDisconnectEvent(reason: string) { return this.pimpl.receivedDisconnectEvent(reason); }\n\n    public receivedConnectEvent(connectionMode: ConnectionMode, details: IConnectionDetails) {\n        return this.pimpl.receivedConnectEvent(connectionMode, details);\n    }\n\n    /**\n     * IConnectionStateHandlerInputs\n     */\n\n    public get logger() { return this.inputs.logger; }\n    public connectionStateChanged(\n        value: ConnectionState,\n        oldState: ConnectionState,\n        reason?: string | undefined,\n    ) {\n        return this.inputs.connectionStateChanged(value, oldState, reason);\n    }\n    public shouldClientJoinWrite() { return this.inputs.shouldClientJoinWrite(); }\n    public get maxClientLeaveWaitTime() { return this.inputs.maxClientLeaveWaitTime; }\n    public logConnectionIssue(eventName: string, details?: ITelemetryProperties) {\n        return this.inputs.logConnectionIssue(eventName, details);\n    }\n}\n\n/**\n * Implementation of IConnectionStateHandler pass-through adapter that waits for specific sequence number\n * before raising connected event\n */\nclass ConnectionStateCatchup extends ConnectionStateHandlerPassThrough {\n    private catchUpMonitor: ICatchUpMonitor | undefined;\n\n    constructor(\n        inputs: IConnectionStateHandlerInputs,\n        pimplFactory: (handler: IConnectionStateHandlerInputs) => IConnectionStateHandler,\n        private readonly deltaManager: IDeltaManager<any, any>,\n    ) {\n        super(inputs, pimplFactory);\n        this._connectionState = this.pimpl.connectionState;\n    }\n\n    private _connectionState: ConnectionState;\n    public get connectionState() {\n        return this._connectionState;\n    }\n\n    public connectionStateChanged(value: ConnectionState, oldState: ConnectionState, reason?: string | undefined) {\n        switch (value) {\n            case ConnectionState.Connected:\n                assert(this._connectionState === ConnectionState.CatchingUp, 0x3e1 /* connectivity transitions */);\n                // Create catch-up monitor here (not earlier), as we might get more exact info by now about how far\n                // client is behind through join signal. This is only true if base layer uses signals (i.e. audience,\n                // not quorum, including for \"rea\" connections) to make decisions about moving to \"connected\" state.\n                // In addition to that, in its current form, doing this in ConnectionState.CatchingUp is dangerous as\n                // we might get callback right away, and it will screw up state transition (as code outside of switch\n                // statement will overwrite current state).\n                assert(this.catchUpMonitor === undefined, 0x3eb /* catchUpMonitor should be gone */);\n                this.catchUpMonitor = new CatchUpMonitor(this.deltaManager, this.transitionToConnectedState);\n                return;\n            case ConnectionState.Disconnected:\n                this.catchUpMonitor?.dispose();\n                this.catchUpMonitor = undefined;\n                break;\n            case ConnectionState.CatchingUp:\n                assert(this._connectionState === ConnectionState.Disconnected, 0x3e3 /* connectivity transitions */);\n                break;\n            default:\n        }\n        this._connectionState = value;\n        this.inputs.connectionStateChanged(value, oldState, reason);\n    }\n\n    private readonly transitionToConnectedState = () => {\n        // Defensive measure, we should always be in Connecting state when this is called.\n        const state = this.pimpl.connectionState;\n        assert(state === ConnectionState.Connected, 0x3e5 /* invariant broken */);\n        assert(this._connectionState === ConnectionState.CatchingUp, 0x3e6 /* invariant broken */);\n        this._connectionState = ConnectionState.Connected;\n        this.inputs.connectionStateChanged(ConnectionState.Connected, ConnectionState.CatchingUp, \"caught up\");\n    };\n}\n\n/**\n * In the lifetime of a container, the connection will likely disconnect and reconnect periodically.\n * This class ensures that any ops sent by this container instance on previous connection are either\n * sequenced or blocked by the server before emitting the new \"connected\" event and allowing runtime to resubmit ops.\n *\n * Each connection is assigned a clientId by the service, and the connection is book-ended by a Join and a Leave op\n * generated by the service. Due to the distributed nature of the Relay Service, in the case of reconnect we cannot\n * make any assumptions about ordering of operations between the old and new connections - i.e. new Join op could\n * be sequenced before old Leave op (and some acks from pending ops that were in flight when we disconnected).\n *\n * The job of this class is to encapsulate the transition period during reconnect, which is identified by\n * ConnectionState.CatchingUp. Specifically, before moving to Connected state with the new clientId, it ensures that:\n *\n * a. We process the Leave op for the previous clientId. This allows us to properly handle any acks from in-flight ops\n * that got sequenced with the old clientId (we'll recognize them as local ops). After the Leave op, any other\n * pending ops can safely be submitted with the new clientId without fear of duplication in the sequenced op stream.\n *\n * b. We process the Join op for the new clientId (identified when the underlying connection was first established),\n * indicating the service is ready to sequence ops sent with the new clientId.\n *\n * c. We process all ops known at the time the underlying connection was established (so we are \"caught up\")\n *\n * For (a) we give up waiting after some time (same timeout as server uses), and go ahead and transition to Connected.\n *\n * For (b) we log telemetry if it takes too long, but still only transition to Connected when the Join op is processed\n * and we are added to the Quorum.\n *\n * For (c) this is optional behavior, controlled by the parameters of receivedConnectEvent\n */\nclass ConnectionStateHandler implements IConnectionStateHandler {\n    private _connectionState = ConnectionState.Disconnected;\n    private _pendingClientId: string | undefined;\n    private readonly prevClientLeftTimer: Timer;\n    private readonly joinOpTimer: Timer;\n    private protocol?: IProtocolHandler;\n\n    private waitEvent: PerformanceEvent | undefined;\n\n    public get connectionState(): ConnectionState {\n        return this._connectionState;\n    }\n\n    private get clientId(): string | undefined {\n        return this._clientId;\n    }\n\n    public get pendingClientId(): string | undefined {\n        return this._pendingClientId;\n    }\n\n    constructor(\n        private readonly handler: IConnectionStateHandlerInputs,\n        private _clientId?: string,\n    ) {\n        this.prevClientLeftTimer = new Timer(\n            // Default is 5 min for which we are going to wait for its own \"leave\" message. This is same as\n            // the max time on server after which leave op is sent.\n            this.handler.maxClientLeaveWaitTime ?? 300000,\n            () => {\n                assert(this.connectionState !== ConnectionState.Connected,\n                    0x2ac /* \"Connected when timeout waiting for leave from previous session fired!\" */);\n                this.applyForConnectedState(\"timeout\");\n            },\n        );\n\n        // Based on recent data, it looks like majority of cases where we get stuck are due to really slow or\n        // timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so\n        // if retrying fixes the problem, we should not see these events.\n        this.joinOpTimer = new Timer(\n            JoinOpTimeoutMs,\n            () => {\n                // I've observed timer firing within couple ms from disconnect event, looks like\n                // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.\n                if (this.connectionState !== ConnectionState.CatchingUp) {\n                    return;\n                }\n                const details = {\n                    protocolInitialized: this.protocol !== undefined,\n                    pendingClientId: this.pendingClientId,\n                    clientJoined: this.hasMember(this.pendingClientId),\n                    waitingForLeaveOp: this.waitingForLeaveOp,\n                };\n                this.handler.logConnectionIssue(\"NoJoinOp\", details);\n            },\n        );\n    }\n\n    private startJoinOpTimer() {\n        assert(!this.joinOpTimer.hasTimer, 0x234 /* \"has joinOpTimer\" */);\n        this.joinOpTimer.start();\n    }\n\n    private stopJoinOpTimer() {\n        assert(this.joinOpTimer.hasTimer, 0x235 /* \"no joinOpTimer\" */);\n        this.joinOpTimer.clear();\n    }\n\n    private get waitingForLeaveOp() {\n        return this.prevClientLeftTimer.hasTimer;\n    }\n\n    public dispose() {\n        assert(!this.joinOpTimer.hasTimer, 0x2a5 /* \"join timer\" */);\n        this.prevClientLeftTimer.clear();\n    }\n\n    public containerSaved() {\n        // If we were waiting for moving to Connected state, then only apply for state change. Since the container\n        // is now saved and we don't have any ops to roundtrip, we can clear the timer and apply for connected state.\n        if (this.waitingForLeaveOp) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"containerSaved\");\n        }\n    }\n\n    private receivedAddMemberEvent(clientId: string) {\n        // This is the only one that requires the pending client ID\n        if (clientId === this.pendingClientId) {\n            if (this.joinOpTimer.hasTimer) {\n                this.stopJoinOpTimer();\n            } else {\n                // timer has already fired, meaning it took too long to get join on.\n                // Record how long it actually took to recover.\n                this.handler.logConnectionIssue(\"ReceivedJoinOp\");\n            }\n            // Start the event in case we are waiting for leave or timeout.\n            if (this.waitingForLeaveOp) {\n                this.waitEvent = PerformanceEvent.start(this.handler.logger, {\n                    eventName: \"WaitBeforeClientLeave\",\n                    details: JSON.stringify({\n                        waitOnClientId: this._clientId,\n                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),\n                    }),\n                });\n            }\n            this.applyForConnectedState(\"addMemberEvent\");\n        }\n    }\n\n    private applyForConnectedState(source: \"removeMemberEvent\" | \"addMemberEvent\" | \"timeout\" | \"containerSaved\") {\n        assert(this.protocol !== undefined, 0x236 /* \"In all cases it should be already installed\" */);\n\n        assert(!this.waitingForLeaveOp || this.hasMember(this.clientId),\n            0x2e2 /* \"Must only wait for leave message when clientId in quorum\" */);\n\n        // Move to connected state only if we are in Connecting state, we have seen our join op\n        // and there is no timer running which means we are not waiting for previous client to leave\n        // or timeout has occurred while doing so.\n        if (this.pendingClientId !== this.clientId\n            && this.hasMember(this.pendingClientId)\n            && !this.waitingForLeaveOp\n        ) {\n            this.waitEvent?.end({ source });\n            this.setConnectionState(ConnectionState.Connected);\n        } else {\n            // Adding this event temporarily so that we can get help debugging if something goes wrong.\n            this.handler.logger.sendTelemetryEvent({\n                eventName: \"connectedStateRejected\",\n                category: source === \"timeout\" ? \"error\" : \"generic\",\n                details: JSON.stringify({\n                    source,\n                    pendingClientId: this.pendingClientId,\n                    clientId: this.clientId,\n                    waitingForLeaveOp: this.waitingForLeaveOp,\n                    clientJoined: this.hasMember(this.pendingClientId),\n                }),\n            });\n        }\n    }\n\n    private receivedRemoveMemberEvent(clientId: string) {\n        // If the client which has left was us, then finish the timer.\n        if (this.clientId === clientId) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"removeMemberEvent\");\n        }\n    }\n\n    public receivedDisconnectEvent(reason: string) {\n        this.setConnectionState(ConnectionState.Disconnected, reason);\n    }\n\n    /**\n     * The \"connect\" event indicates the connection to the Relay Service is live.\n     * However, some additional conditions must be met before we can fully transition to\n     * \"Connected\" state. This function handles that interim period, known as \"Connecting\" state.\n     * @param connectionMode - Read or Write connection\n     * @param details - Connection details returned from the Relay Service\n     * @param deltaManager - DeltaManager to be used for delaying Connected transition until caught up.\n     * If it's undefined, then don't delay and transition to Connected as soon as Leave/Join op are accounted for\n     */\n    public receivedConnectEvent(\n        connectionMode: ConnectionMode,\n        details: IConnectionDetails,\n    ) {\n        const oldState = this._connectionState;\n        this._connectionState = ConnectionState.CatchingUp;\n\n        const writeConnection = connectionMode === \"write\";\n\n        // The following checks are wrong. They are only valid if user has write access to a file.\n        // If user lost such access mid-session, user will not be able to get \"write\" connection.\n        // assert(!this.handler.shouldClientJoinWrite() || writeConnection,\n        //    0x30a /* shouldClientJoinWrite should imply this is a writeConnection */);\n        // assert(!this.waitingForLeaveOp || writeConnection,\n        //    0x2a6 /* \"waitingForLeaveOp should imply writeConnection (we need to be ready to flush pending ops)\" */);\n\n        // Stash the clientID to detect when transitioning from connecting (socket.io channel open) to connected\n        // (have received the join message for the client ID)\n        // This is especially important in the reconnect case. It's possible there could be outstanding\n        // ops sent by this client, so we should keep the old client id until we see our own client's\n        // join message. after we see the join message for our new connection with our new client id,\n        // we know there can no longer be outstanding ops that we sent with the previous client id.\n        this._pendingClientId = details.clientId;\n\n        // IMPORTANT: Report telemetry after we set _pendingClientId, but before transitioning to Connected state\n        this.handler.connectionStateChanged(ConnectionState.CatchingUp, oldState);\n\n        // For write connections, this pending clientId could be in the quorum already (i.e. join op already processed).\n        // We are fetching ops from storage in parallel to connecting to Relay Service,\n        // and given async processes, it's possible that we have already processed our own join message before\n        // connection was fully established.\n        // If protocol is not initialized yet, we expect it will process the join op after it's initialized.\n        const waitingForJoinOp = writeConnection && !this.hasMember(this._pendingClientId);\n\n        if (waitingForJoinOp) {\n            // Previous client left, and we are waiting for our own join op. When it is processed we'll join the quorum\n            // and attempt to transition to Connected state via receivedAddMemberEvent.\n            this.startJoinOpTimer();\n        } else if (!this.waitingForLeaveOp) {\n            // We're not waiting for Join or Leave op (if read-only connection those don't even apply),\n            // go ahead and declare the state to be Connected!\n            // If we are waiting for Leave op still, do nothing for now, we will transition to Connected later.\n            this.setConnectionState(ConnectionState.Connected);\n        }\n        // else - We are waiting for Leave op still, do nothing for now, we will transition to Connected later\n    }\n\n    private setConnectionState(value: ConnectionState.Disconnected, reason: string): void;\n    private setConnectionState(value: ConnectionState.Connected): void;\n    private setConnectionState(value: ConnectionState.Disconnected | ConnectionState.Connected, reason?: string): void {\n        if (this.connectionState === value) {\n            // Already in the desired state - exit early\n            this.handler.logger.sendErrorEvent({ eventName: \"setConnectionStateSame\", value });\n            return;\n        }\n\n        const oldState = this._connectionState;\n        this._connectionState = value;\n        let client: ILocalSequencedClient | undefined;\n        if (this._clientId !== undefined) {\n            client = this.protocol?.quorum?.getMember(this._clientId);\n        }\n        if (value === ConnectionState.Connected) {\n            assert(oldState === ConnectionState.CatchingUp,\n                0x1d8 /* \"Should only transition from Connecting state\" */);\n            // Mark our old client should have left in the quorum if it's still there\n            if (client !== undefined) {\n                client.shouldHaveLeft = true;\n            }\n            this._clientId = this.pendingClientId;\n        } else if (value === ConnectionState.Disconnected) {\n            // Clear pending state immediately to prepare for reconnect\n            this._pendingClientId = undefined;\n\n            if (this.joinOpTimer.hasTimer) {\n                this.stopJoinOpTimer();\n            }\n\n            // Only wait for \"leave\" message if the connected client exists in the quorum and had some non-acked ops\n            // Also check if the timer is not already running as\n            // we could receive \"Disconnected\" event multiple times without getting connected and in that case we\n            // don't want to reset the timer as we still want to wait on original client which started this timer.\n            if (client !== undefined\n                && this.handler.shouldClientJoinWrite()\n                && !this.waitingForLeaveOp // same as !this.prevClientLeftTimer.hasTimer\n            ) {\n                this.prevClientLeftTimer.restart();\n            } else {\n                // Adding this event temporarily so that we can get help debugging if something goes wrong.\n                this.handler.logger.sendTelemetryEvent({\n                    eventName: \"noWaitOnDisconnected\",\n                    details: JSON.stringify({\n                        clientId: this._clientId,\n                        inQuorum: client !== undefined,\n                        waitingForLeaveOp: this.waitingForLeaveOp,\n                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),\n                    }),\n                });\n            }\n        }\n\n        // Report transition before we propagate event across layers\n        this.handler.connectionStateChanged(this._connectionState, oldState, reason);\n    }\n\n    // Helper method to switch between quorum and audience.\n    // Old design was checking only quorum for \"write\" clients.\n    // Latest change checks audience for all types of connections.\n    protected get membership() {\n        return this.protocol?.quorum;\n    }\n\n    public initProtocol(protocol: IProtocolHandler) {\n        this.protocol = protocol;\n\n        this.membership?.on(\"addMember\", (clientId) => {\n            this.receivedAddMemberEvent(clientId);\n        });\n\n        this.membership?.on(\"removeMember\", (clientId) => {\n            this.receivedRemoveMemberEvent(clientId);\n        });\n\n        // Very unlikely race condition, but theoretically can happen - our new connection is already\n        // summarized and we are loading from such summary.\n        if (this.hasMember(this.pendingClientId)) {\n            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n            this.receivedAddMemberEvent(this.pendingClientId!);\n        }\n\n        // if we have a clientId from a previous container we need to wait for its leave message\n        if (this.clientId !== undefined && this.hasMember(this.clientId)) {\n            this.prevClientLeftTimer.restart();\n        }\n    }\n\n    protected hasMember(clientId?: string) {\n        return this.membership?.getMember(clientId ?? \"\") !== undefined;\n    }\n}\n"]}
+{"version":3,"file":"connectionStateHandler.js","sourceRoot":"","sources":["../src/connectionStateHandler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAI7D,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAC9F,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAmB,MAAM,kBAAkB,CAAC;AAGnE,qGAAqG;AACrG,kGAAkG;AAClG,iEAAiE;AACjE,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,2DAA2D;AAC3D,MAAM,mBAAmB,GAAG,IAAI,CAAC;AA8BjC,MAAM,UAAU,4BAA4B,CACxC,MAAqC,EACrC,YAAqC,EACrC,QAAiB;IAEjB,MAAM,EAAE,GAAG,yBAAyB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpD,OAAO,gCAAgC,CACnC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iDAAiD,CAAC,KAAK,IAAI,EAAE,8BAA8B;IAChH,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,uCAAuC,CAAC,KAAK,IAAI,EAAE,+BAA+B;IACvG,MAAM,EACN,YAAY,EACZ,QAAQ,CACX,CAAC;AACN,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC5C,2BAAoC,EACpC,4BAAqC,EACrC,MAAqC,EACrC,YAAqC,EACrC,QAAiB;IAEjB,IAAI,CAAC,2BAA2B,EAAE;QAC9B,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,4BAA4B,EAAE,QAAQ,CAAC,CAAC;KACrF;IACD,OAAO,IAAI,sBAAsB,CAC7B,MAAM,EACN,CAAC,OAAsC,EAAE,EAAE,CAAC,IAAI,sBAAsB,CAClE,OAAO,EACP,4BAA4B,EAC5B,QAAQ,CAAC,EACb,YAAY,CAAC,CAAC;AACtB,CAAC;AAYD;;;GAGG;AACH,MAAM,iCAAiC;IAGnC,YACuB,MAAqC,EACxD,YAAiF;QAD9D,WAAM,GAAN,MAAM,CAA+B;QAGxD,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,IAAW,eAAe,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;IACnE,IAAW,eAAe,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;IAE5D,cAAc,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACxD,OAAO,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC1C,YAAY,CAAC,QAA0B,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACtF,uBAAuB,CAAC,MAAc,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE9F,oBAAoB,CAAC,OAA2B;QACnD,OAAO,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IAEH,IAAW,MAAM,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3C,sBAAsB,CACzB,KAAsB,EACtB,QAAyB,EACzB,MAA2B;QAE3B,OAAO,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvE,CAAC;IACM,qBAAqB,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAW,sBAAsB,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAC3E,kBAAkB,CAAC,SAAiB,EAAE,OAA8B;QACvE,OAAO,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;CACJ;AAED;;;GAGG;AACH,MAAM,sBAAuB,SAAQ,iCAAiC;IAGlE,YACI,MAAqC,EACrC,YAAiF,EAChE,YAAqC;QAEtD,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAFX,iBAAY,GAAZ,YAAY,CAAyB;QAqCzC,+BAA0B,GAAG,GAAG,EAAE;YAC/C,kFAAkF;YAClF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;YACzC,MAAM,CAAC,KAAK,KAAK,eAAe,CAAC,SAAS,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC3F,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC,SAAS,CAAC;YAClD,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,eAAe,CAAC,SAAS,EAAE,eAAe,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC3G,CAAC,CAAC;QAzCE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;IACvD,CAAC;IAGD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEM,sBAAsB,CAAC,KAAsB,EAAE,QAAyB,EAAE,MAA2B;;QACxG,QAAQ,KAAK,EAAE;YACX,KAAK,eAAe,CAAC,SAAS;gBAC1B,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;gBACnG,mGAAmG;gBACnG,qGAAqG;gBACrG,oGAAoG;gBACpG,qGAAqG;gBACrG,qGAAqG;gBACrG,2CAA2C;gBAC3C,MAAM,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBACrF,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBAC7F,OAAO;YACX,KAAK,eAAe,CAAC,YAAY;gBAC7B,MAAA,IAAI,CAAC,cAAc,0CAAE,OAAO,EAAE,CAAC;gBAC/B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;gBAChC,MAAM;YACV,KAAK,eAAe,CAAC,UAAU;gBAC3B,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,eAAe,CAAC,YAAY,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;gBACrG,MAAM;YACV,QAAQ;SACX;QACD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAChE,CAAC;CAUJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,sBAAsB;IAuBxB,YACqB,OAAsC,EACtC,4BAAqC,EACtD,yBAAkC;;QAFjB,YAAO,GAAP,OAAO,CAA+B;QACtC,iCAA4B,GAA5B,4BAA4B,CAAS;QAxBlD,qBAAgB,GAAG,eAAe,CAAC,YAAY,CAAC;QA2BpD,IAAI,CAAC,SAAS,GAAG,yBAAyB,CAAC;QAC3C,IAAI,CAAC,mBAAmB,GAAG,IAAI,KAAK;QAChC,+FAA+F;QAC/F,uDAAuD;QACvD,MAAA,IAAI,CAAC,OAAO,CAAC,sBAAsB,mCAAI,MAAM,EAC7C,GAAG,EAAE;YACD,MAAM,CAAC,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,SAAS,EACrD,KAAK,CAAC,6EAA6E,CAAC,CAAC;YACzF,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC,CACJ,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,KAAK,CACxB,CAAC,EAAE,6EAA6E;QAChF,GAAG,EAAE;YACD,gFAAgF;YAChF,iGAAiG;YACjG,IAAI,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;gBACrD,OAAO;aACV;YACD,MAAM,OAAO,GAAG;gBACZ,mBAAmB,EAAE,IAAI,CAAC,QAAQ,KAAK,SAAS;gBAChD,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC;gBAClD,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;aAC5C,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC,CACJ,CAAC;IACN,CAAC;IA9CD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,IAAY,QAAQ;QAChB,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAsCO,gBAAgB;QACpB,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAClE,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACnE,IAAI,CAAC,WAAW,CAAC,KAAK,CAClB,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAC3E,CAAC;IACN,CAAC;IAEO,eAAe;QACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAEM,OAAO;QACV,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC7D,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAEM,cAAc;QACjB,0GAA0G;QAC1G,6GAA6G;QAC7G,IAAI,IAAI,CAAC,iBAAiB,EAAE;YACxB,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,QAAgB;QAC3C,2DAA2D;QAC3D,IAAI,QAAQ,KAAK,IAAI,CAAC,eAAe,EAAE;YACnC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;aAC1B;iBAAM,IAAI,IAAI,CAAC,uBAAuB,EAAE,EAAE;gBACvC,2EAA2E;gBAC3E,+CAA+C;gBAC/C,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;aACrD;YACD,+DAA+D;YAC/D,IAAI,IAAI,CAAC,iBAAiB,EAAE;gBACxB,IAAI,CAAC,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;oBACzD,SAAS,EAAE,uBAAuB;oBAClC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,cAAc,EAAE,IAAI,CAAC,SAAS;wBAC9B,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;qBAC1D,CAAC;iBACL,CAAC,CAAC;aACN;YACD,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,MAA6E;;QACxG,MAAM,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAE/F,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAC3D,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAE5E,uFAAuF;QACvF,4FAA4F;QAC5F,0CAA0C;QAC1C,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC,QAAQ;eACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC;eACpC,CAAC,IAAI,CAAC,iBAAiB,EAC5B;YACE,MAAA,IAAI,CAAC,SAAS,0CAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;aAAM;YACH,2FAA2F;YAC3F,wFAAwF;YACxF,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,YAAY,CAAC;YAC5F,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBACnC,SAAS,EAAE,wBAAwB;gBACnC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;gBACrC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,MAAM;oBACN,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;oBACzC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC;iBACrD,CAAC;aACL,CAAC,CAAC;SACN;IACL,CAAC;IAEO,yBAAyB,CAAC,QAAgB;QAC9C,8DAA8D;QAC9D,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAC5B,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,mBAAmB,CAAC,CAAC;SACpD;IACL,CAAC;IAEM,uBAAuB,CAAC,MAAc;QACzC,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IAEO,uBAAuB;QAC3B,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,kDAAkD,CAAC,CAAC;QAChG,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,4BAA4B,CAAC;IACjF,CAAC;IAED;;;;;;;OAOG;IACI,oBAAoB,CACvB,OAA2B;QAE3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;QAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC,UAAU,CAAC;QAEnD,0FAA0F;QAC1F,yFAAyF;QACzF,EAAE;QACF,oDAAoD;QACpD,mEAAmE;QACnE,gFAAgF;QAChF,qDAAqD;QACrD,+GAA+G;QAE/G,wGAAwG;QACxG,qDAAqD;QACrD,+FAA+F;QAC/F,6FAA6F;QAC7F,6FAA6F;QAC7F,2FAA2F;QAC3F,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEzC,yGAAyG;QACzG,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAE1E,6GAA6G;QAC7G,sFAAsF;QACtF,kFAAkF;QAClF,yGAAyG;QACzG,uCAAuC;QACvC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,uBAAuB,EAAE,EAAE;YAC1E,oEAAoE;YACpE,oFAAoF;YACpF,IAAI,CAAC,gBAAgB,EAAE,CAAC;SAC3B;aAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAChC,2FAA2F;YAC3F,kDAAkD;YAClD,mGAAmG;YACnG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;QACD,sGAAsG;IAC1G,CAAC;IAIO,kBAAkB,CAAC,KAA+D,EAAE,MAAe;;QACvG,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE;YAChC,4CAA4C;YAC5C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,wBAAwB,EAAE,KAAK,EAAE,CAAC,CAAC;YACnF,OAAO;SACV;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAE9B,sFAAsF;QACtF,oGAAoG;QACpG,oCAAoC;QACpC,IAAI,MAAyC,CAAC;QAC9C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;YAC9B,MAAM,GAAG,MAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,MAAM,0CAAE,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SAC7D;QACD,IAAI,KAAK,KAAK,eAAe,CAAC,SAAS,EAAE;YACrC,MAAM,CAAC,QAAQ,KAAK,eAAe,CAAC,UAAU,EAC1C,KAAK,CAAC,oDAAoD,CAAC,CAAC;YAChE,yEAAyE;YACzE,IAAI,MAAM,KAAK,SAAS,EAAE;gBACtB,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;aAChC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC;SACzC;aAAM,IAAI,KAAK,KAAK,eAAe,CAAC,YAAY,EAAE;YAC/C,2DAA2D;YAC3D,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;YAElC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;aAC1B;YAED,wGAAwG;YACxG,oDAAoD;YACpD,qGAAqG;YACrG,sGAAsG;YACtG,IAAI,MAAM,KAAK,SAAS;mBACjB,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;mBACpC,CAAC,IAAI,CAAC,iBAAiB,CAAC,6CAA6C;cAC1E;gBACE,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;aACtC;iBAAM;gBACH,2FAA2F;gBAC3F,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBACnC,SAAS,EAAE,sBAAsB;oBACjC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,QAAQ,EAAE,IAAI,CAAC,SAAS;wBACxB,QAAQ,EAAE,MAAM,KAAK,SAAS;wBAC9B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;wBACzC,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;qBAC1D,CAAC;iBACL,CAAC,CAAC;aACN;SACJ;QAED,4DAA4D;QAC5D,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjF,CAAC;IAED,uDAAuD;IACvD,2DAA2D;IAC3D,8DAA8D;IAC9D,IAAc,UAAU;;QACpB,kFAAkF;QAClF,qEAAqE;QACrE,yGAAyG;QACzG,yGAAyG;QACzG,oBAAoB;QACpB,OAAO,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC,MAAA,IAAI,CAAC,QAAQ,0CAAE,QAAQ,CAAC,CAAC,CAAC,MAAA,IAAI,CAAC,QAAQ,0CAAE,MAAM,CAAC;IAC/F,CAAC;IAEM,YAAY,CAAC,QAA0B;;QAC1C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,MAAA,IAAI,CAAC,UAAU,0CAAE,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE;YACnD,MAAM,CAAE,OAAmB,CAAC,IAAI,KAAK,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,SAAS,EAC5F,KAAK,CAAC,kCAAkC,CAAC,CAAC;YAC9C,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAA,IAAI,CAAC,UAAU,0CAAE,EAAE,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACpG,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH;;;;;;UAME;QACF,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE;YACtC,oEAAoE;YACpE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,eAAgB,CAAC,CAAC;SACtD;QAED,wFAAwF;QACxF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;YAC9D,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;SACtC;IACL,CAAC;IAES,SAAS,CAAC,QAAiB;;QACjC,OAAO,CAAA,MAAA,IAAI,CAAC,UAAU,0CAAE,SAAS,CAAC,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,EAAE,CAAC,MAAK,SAAS,CAAC;IACpE,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger, ITelemetryProperties } from \"@fluidframework/common-definitions\";\nimport { assert, Timer } from \"@fluidframework/common-utils\";\nimport { IConnectionDetails, IDeltaManager } from \"@fluidframework/container-definitions\";\nimport { ILocalSequencedClient } from \"@fluidframework/protocol-base\";\nimport { ISequencedClient, IClient } from \"@fluidframework/protocol-definitions\";\nimport { PerformanceEvent, loggerToMonitoringContext } from \"@fluidframework/telemetry-utils\";\nimport { ConnectionState } from \"./connectionState\";\nimport { CatchUpMonitor, ICatchUpMonitor } from \"./catchUpMonitor\";\nimport { IProtocolHandler } from \"./protocol\";\n\n// Based on recent data, it looks like majority of cases where we get stuck are due to really slow or\n// timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so\n// if retrying fixes the problem, we should not see these events.\nconst JoinOpTimeoutMs = 45000;\n\n// Timeout waiting for \"self\" join signal, before giving up\nconst JoinSignalTimeoutMs = 5000;\n\n/** Constructor parameter type for passing in dependencies needed by the ConnectionStateHandler */\nexport interface IConnectionStateHandlerInputs {\n    logger: ITelemetryLogger;\n    /** Log to telemetry any change in state, included to Connecting */\n    connectionStateChanged:\n        (value: ConnectionState, oldState: ConnectionState, reason?: string | undefined) => void;\n    /** Whether to expect the client to join in write mode on next connection */\n    shouldClientJoinWrite: () => boolean;\n    /** (Optional) How long should we wait on our previous client's Leave op before transitioning to Connected again */\n    maxClientLeaveWaitTime: number | undefined;\n    /** Log an issue encountered while in the Connecting state. details will be logged as a JSON string */\n    logConnectionIssue: (eventName: string, details?: ITelemetryProperties) => void;\n}\n\n/**\n * interface that connection state handler implements\n */\nexport interface IConnectionStateHandler {\n    readonly connectionState: ConnectionState;\n    readonly pendingClientId: string | undefined;\n\n    containerSaved(): void;\n    dispose(): void;\n    initProtocol(protocol: IProtocolHandler): void;\n    receivedConnectEvent(details: IConnectionDetails): void;\n    receivedDisconnectEvent(reason: string): void;\n}\n\nexport function createConnectionStateHandler(\n    inputs: IConnectionStateHandlerInputs,\n    deltaManager: IDeltaManager<any, any>,\n    clientId?: string,\n) {\n    const mc = loggerToMonitoringContext(inputs.logger);\n    return createConnectionStateHandlerCore(\n        mc.config.getBoolean(\"Fluid.Container.CatchUpBeforeDeclaringConnected\") === true, // connectedRaisedWhenCaughtUp\n        mc.config.getBoolean(\"Fluid.Container.DisableJoinSignalWait\") !== true, // readClientsWaitForJoinSignal\n        inputs,\n        deltaManager,\n        clientId,\n    );\n}\n\nexport function createConnectionStateHandlerCore(\n    connectedRaisedWhenCaughtUp: boolean,\n    readClientsWaitForJoinSignal: boolean,\n    inputs: IConnectionStateHandlerInputs,\n    deltaManager: IDeltaManager<any, any>,\n    clientId?: string,\n) {\n    if (!connectedRaisedWhenCaughtUp) {\n        return new ConnectionStateHandler(inputs, readClientsWaitForJoinSignal, clientId);\n    }\n    return new ConnectionStateCatchup(\n        inputs,\n        (handler: IConnectionStateHandlerInputs) => new ConnectionStateHandler(\n            handler,\n            readClientsWaitForJoinSignal,\n            clientId),\n        deltaManager);\n}\n\n/**\n * Helper internal interface to abstract away Audience & Quorum\n */\ninterface IMembership {\n    on(\n        eventName: \"addMember\" | \"removeMember\",\n        listener: (clientId: string, details: IClient | ISequencedClient) => void);\n    getMember(clientId: string): undefined | unknown;\n}\n\n/**\n * Class that can be used as a base class for building IConnectionStateHandler adapters / pipeline.\n * It implements both ends of communication interfaces and passes data back and forward\n */\nclass ConnectionStateHandlerPassThrough implements IConnectionStateHandler, IConnectionStateHandlerInputs {\n    protected readonly pimpl: IConnectionStateHandler;\n\n    constructor(\n        protected readonly inputs: IConnectionStateHandlerInputs,\n        pimplFactory: (handler: IConnectionStateHandlerInputs) => IConnectionStateHandler,\n    ) {\n        this.pimpl = pimplFactory(this);\n    }\n\n    /**\n     * IConnectionStateHandler\n     */\n    public get connectionState() { return this.pimpl.connectionState; }\n    public get pendingClientId() { return this.pimpl.pendingClientId; }\n\n    public containerSaved() { return this.pimpl.containerSaved(); }\n    public dispose() { return this.pimpl.dispose(); }\n    public initProtocol(protocol: IProtocolHandler) { return this.pimpl.initProtocol(protocol); }\n    public receivedDisconnectEvent(reason: string) { return this.pimpl.receivedDisconnectEvent(reason); }\n\n    public receivedConnectEvent(details: IConnectionDetails) {\n        return this.pimpl.receivedConnectEvent(details);\n    }\n\n    /**\n     * IConnectionStateHandlerInputs\n     */\n\n    public get logger() { return this.inputs.logger; }\n    public connectionStateChanged(\n        value: ConnectionState,\n        oldState: ConnectionState,\n        reason?: string | undefined,\n    ) {\n        return this.inputs.connectionStateChanged(value, oldState, reason);\n    }\n    public shouldClientJoinWrite() { return this.inputs.shouldClientJoinWrite(); }\n    public get maxClientLeaveWaitTime() { return this.inputs.maxClientLeaveWaitTime; }\n    public logConnectionIssue(eventName: string, details?: ITelemetryProperties) {\n        return this.inputs.logConnectionIssue(eventName, details);\n    }\n}\n\n/**\n * Implementation of IConnectionStateHandler pass-through adapter that waits for specific sequence number\n * before raising connected event\n */\nclass ConnectionStateCatchup extends ConnectionStateHandlerPassThrough {\n    private catchUpMonitor: ICatchUpMonitor | undefined;\n\n    constructor(\n        inputs: IConnectionStateHandlerInputs,\n        pimplFactory: (handler: IConnectionStateHandlerInputs) => IConnectionStateHandler,\n        private readonly deltaManager: IDeltaManager<any, any>,\n    ) {\n        super(inputs, pimplFactory);\n        this._connectionState = this.pimpl.connectionState;\n    }\n\n    private _connectionState: ConnectionState;\n    public get connectionState() {\n        return this._connectionState;\n    }\n\n    public connectionStateChanged(value: ConnectionState, oldState: ConnectionState, reason?: string | undefined) {\n        switch (value) {\n            case ConnectionState.Connected:\n                assert(this._connectionState === ConnectionState.CatchingUp, 0x3e1 /* connectivity transitions */);\n                // Create catch-up monitor here (not earlier), as we might get more exact info by now about how far\n                // client is behind through join signal. This is only true if base layer uses signals (i.e. audience,\n                // not quorum, including for \"rea\" connections) to make decisions about moving to \"connected\" state.\n                // In addition to that, in its current form, doing this in ConnectionState.CatchingUp is dangerous as\n                // we might get callback right away, and it will screw up state transition (as code outside of switch\n                // statement will overwrite current state).\n                assert(this.catchUpMonitor === undefined, 0x3eb /* catchUpMonitor should be gone */);\n                this.catchUpMonitor = new CatchUpMonitor(this.deltaManager, this.transitionToConnectedState);\n                return;\n            case ConnectionState.Disconnected:\n                this.catchUpMonitor?.dispose();\n                this.catchUpMonitor = undefined;\n                break;\n            case ConnectionState.CatchingUp:\n                assert(this._connectionState === ConnectionState.Disconnected, 0x3e3 /* connectivity transitions */);\n                break;\n            default:\n        }\n        this._connectionState = value;\n        this.inputs.connectionStateChanged(value, oldState, reason);\n    }\n\n    private readonly transitionToConnectedState = () => {\n        // Defensive measure, we should always be in Connecting state when this is called.\n        const state = this.pimpl.connectionState;\n        assert(state === ConnectionState.Connected, 0x3e5 /* invariant broken */);\n        assert(this._connectionState === ConnectionState.CatchingUp, 0x3e6 /* invariant broken */);\n        this._connectionState = ConnectionState.Connected;\n        this.inputs.connectionStateChanged(ConnectionState.Connected, ConnectionState.CatchingUp, \"caught up\");\n    };\n}\n\n/**\n * In the lifetime of a container, the connection will likely disconnect and reconnect periodically.\n * This class ensures that any ops sent by this container instance on previous connection are either\n * sequenced or blocked by the server before emitting the new \"connected\" event and allowing runtime to resubmit ops.\n *\n * Each connection is assigned a clientId by the service, and the connection is book-ended by a Join and a Leave op\n * generated by the service. Due to the distributed nature of the Relay Service, in the case of reconnect we cannot\n * make any assumptions about ordering of operations between the old and new connections - i.e. new Join op could\n * be sequenced before old Leave op (and some acks from pending ops that were in flight when we disconnected).\n *\n * The job of this class is to encapsulate the transition period during reconnect, which is identified by\n * ConnectionState.CatchingUp. Specifically, before moving to Connected state with the new clientId, it ensures that:\n *\n * a. We process the Leave op for the previous clientId. This allows us to properly handle any acks from in-flight ops\n * that got sequenced with the old clientId (we'll recognize them as local ops). After the Leave op, any other\n * pending ops can safely be submitted with the new clientId without fear of duplication in the sequenced op stream.\n *\n * b. We process the Join op for the new clientId (identified when the underlying connection was first established),\n * indicating the service is ready to sequence ops sent with the new clientId.\n *\n * c. We process all ops known at the time the underlying connection was established (so we are \"caught up\")\n *\n * For (a) we give up waiting after some time (same timeout as server uses), and go ahead and transition to Connected.\n *\n * For (b) we log telemetry if it takes too long, but still only transition to Connected when the Join op/signal is\n * processed.\n *\n * For (c) this is optional behavior, controlled by the parameters of receivedConnectEvent\n */\nclass ConnectionStateHandler implements IConnectionStateHandler {\n    private _connectionState = ConnectionState.Disconnected;\n    private _pendingClientId: string | undefined;\n    private readonly prevClientLeftTimer: Timer;\n    private readonly joinOpTimer: Timer;\n    private protocol?: IProtocolHandler;\n    private connection?: IConnectionDetails;\n    private _clientId?: string;\n\n    private waitEvent: PerformanceEvent | undefined;\n\n    public get connectionState(): ConnectionState {\n        return this._connectionState;\n    }\n\n    private get clientId(): string | undefined {\n        return this._clientId;\n    }\n\n    public get pendingClientId(): string | undefined {\n        return this._pendingClientId;\n    }\n\n    constructor(\n        private readonly handler: IConnectionStateHandlerInputs,\n        private readonly readClientsWaitForJoinSignal: boolean,\n        clientIdFromPausedSession?: string,\n    ) {\n        this._clientId = clientIdFromPausedSession;\n        this.prevClientLeftTimer = new Timer(\n            // Default is 5 min for which we are going to wait for its own \"leave\" message. This is same as\n            // the max time on server after which leave op is sent.\n            this.handler.maxClientLeaveWaitTime ?? 300000,\n            () => {\n                assert(this.connectionState !== ConnectionState.Connected,\n                    0x2ac /* \"Connected when timeout waiting for leave from previous session fired!\" */);\n                this.applyForConnectedState(\"timeout\");\n            },\n        );\n\n        this.joinOpTimer = new Timer(\n            0, // default value is not used - startJoinOpTimer() explicitly provides timeout\n            () => {\n                // I've observed timer firing within couple ms from disconnect event, looks like\n                // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.\n                if (this.connectionState !== ConnectionState.CatchingUp) {\n                    return;\n                }\n                const details = {\n                    protocolInitialized: this.protocol !== undefined,\n                    pendingClientId: this.pendingClientId,\n                    clientJoined: this.hasMember(this.pendingClientId),\n                    waitingForLeaveOp: this.waitingForLeaveOp,\n                };\n                this.handler.logConnectionIssue(\"NoJoinOp\", details);\n            },\n        );\n    }\n\n    private startJoinOpTimer() {\n        assert(!this.joinOpTimer.hasTimer, 0x234 /* \"has joinOpTimer\" */);\n        assert(this.connection !== undefined, 0x4b3 /* have connection */);\n        this.joinOpTimer.start(\n            this.connection.mode === \"write\" ? JoinOpTimeoutMs : JoinSignalTimeoutMs,\n        );\n    }\n\n    private stopJoinOpTimer() {\n        assert(this.joinOpTimer.hasTimer, 0x235 /* \"no joinOpTimer\" */);\n        this.joinOpTimer.clear();\n    }\n\n    private get waitingForLeaveOp() {\n        return this.prevClientLeftTimer.hasTimer;\n    }\n\n    public dispose() {\n        assert(!this.joinOpTimer.hasTimer, 0x2a5 /* \"join timer\" */);\n        this.prevClientLeftTimer.clear();\n    }\n\n    public containerSaved() {\n        // If we were waiting for moving to Connected state, then only apply for state change. Since the container\n        // is now saved and we don't have any ops to roundtrip, we can clear the timer and apply for connected state.\n        if (this.waitingForLeaveOp) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"containerSaved\");\n        }\n    }\n\n    private receivedAddMemberEvent(clientId: string) {\n        // This is the only one that requires the pending client ID\n        if (clientId === this.pendingClientId) {\n            if (this.joinOpTimer.hasTimer) {\n                this.stopJoinOpTimer();\n            } else if (this.shouldWaitForJoinSignal()) {\n                // timer has already fired, meaning it took too long to get join op/signal.\n                // Record how long it actually took to recover.\n                this.handler.logConnectionIssue(\"ReceivedJoinOp\");\n            }\n            // Start the event in case we are waiting for leave or timeout.\n            if (this.waitingForLeaveOp) {\n                this.waitEvent = PerformanceEvent.start(this.handler.logger, {\n                    eventName: \"WaitBeforeClientLeave\",\n                    details: JSON.stringify({\n                        waitOnClientId: this._clientId,\n                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),\n                    }),\n                });\n            }\n            this.applyForConnectedState(\"addMemberEvent\");\n        }\n    }\n\n    private applyForConnectedState(source: \"removeMemberEvent\" | \"addMemberEvent\" | \"timeout\" | \"containerSaved\") {\n        assert(this.protocol !== undefined, 0x236 /* \"In all cases it should be already installed\" */);\n\n        assert(!this.waitingForLeaveOp || this.hasMember(this.clientId),\n            0x2e2 /* \"Must only wait for leave message when clientId in quorum\" */);\n\n        // Move to connected state only if we are in Connecting state, we have seen our join op\n        // and there is no timer running which means we are not waiting for previous client to leave\n        // or timeout has occurred while doing so.\n        if (this.pendingClientId !== this.clientId\n            && this.hasMember(this.pendingClientId)\n            && !this.waitingForLeaveOp\n        ) {\n            this.waitEvent?.end({ source });\n            this.setConnectionState(ConnectionState.Connected);\n        } else {\n            // Adding this event temporarily so that we can get help debugging if something goes wrong.\n            // We may not see any ops due to being disconnected all that time - that's not an error!\n            const error = source === \"timeout\" && this.connectionState !== ConnectionState.Disconnected;\n            this.handler.logger.sendTelemetryEvent({\n                eventName: \"connectedStateRejected\",\n                category: error ? \"error\" : \"generic\",\n                details: JSON.stringify({\n                    source,\n                    pendingClientId: this.pendingClientId,\n                    clientId: this.clientId,\n                    waitingForLeaveOp: this.waitingForLeaveOp,\n                    clientJoined: this.hasMember(this.pendingClientId),\n                }),\n            });\n        }\n    }\n\n    private receivedRemoveMemberEvent(clientId: string) {\n        // If the client which has left was us, then finish the timer.\n        if (this.clientId === clientId) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"removeMemberEvent\");\n        }\n    }\n\n    public receivedDisconnectEvent(reason: string) {\n        this.connection = undefined;\n        this.setConnectionState(ConnectionState.Disconnected, reason);\n    }\n\n    private shouldWaitForJoinSignal() {\n        assert(this.connection !== undefined, 0x4b4 /* all callers call here with active connection */);\n        return this.connection.mode === \"write\" || this.readClientsWaitForJoinSignal;\n    }\n\n    /**\n     * The \"connect\" event indicates the connection to the Relay Service is live.\n     * However, some additional conditions must be met before we can fully transition to\n     * \"Connected\" state. This function handles that interim period, known as \"Connecting\" state.\n     * @param details - Connection details returned from the Relay Service\n     * @param deltaManager - DeltaManager to be used for delaying Connected transition until caught up.\n     * If it's undefined, then don't delay and transition to Connected as soon as Leave/Join op are accounted for\n     */\n    public receivedConnectEvent(\n        details: IConnectionDetails,\n    ) {\n        this.connection = details;\n\n        const oldState = this._connectionState;\n        this._connectionState = ConnectionState.CatchingUp;\n\n        // The following checks are wrong. They are only valid if user has write access to a file.\n        // If user lost such access mid-session, user will not be able to get \"write\" connection.\n        //\n        // const writeConnection = details.mode === \"write\";\n        // assert(!this.handler.shouldClientJoinWrite() || writeConnection,\n        //    0x30a /* shouldClientJoinWrite should imply this is a writeConnection */);\n        // assert(!this.waitingForLeaveOp || writeConnection,\n        //    0x2a6 /* \"waitingForLeaveOp should imply writeConnection (we need to be ready to flush pending ops)\" */);\n\n        // Stash the clientID to detect when transitioning from connecting (socket.io channel open) to connected\n        // (have received the join message for the client ID)\n        // This is especially important in the reconnect case. It's possible there could be outstanding\n        // ops sent by this client, so we should keep the old client id until we see our own client's\n        // join message. after we see the join message for our new connection with our new client id,\n        // we know there can no longer be outstanding ops that we sent with the previous client id.\n        this._pendingClientId = details.clientId;\n\n        // IMPORTANT: Report telemetry after we set _pendingClientId, but before transitioning to Connected state\n        this.handler.connectionStateChanged(ConnectionState.CatchingUp, oldState);\n\n        // Check if we need to wait for join op/signal, and if we need to wait for leave op from previous connection.\n        // Pending clientId could have joined already (i.e. join op/signal already processed):\n        //    We are fetching ops from storage in parallel to connecting to Relay Service,\n        //    and given async processes, it's possible that we have already processed our own join message before\n        //    connection was fully established.\n        if (!this.hasMember(this._pendingClientId) && this.shouldWaitForJoinSignal()) {\n            // We are waiting for our own join op / signal. When it is processed\n            // we'll attempt to transition to Connected state via receivedAddMemberEvent() flow.\n            this.startJoinOpTimer();\n        } else if (!this.waitingForLeaveOp) {\n            // We're not waiting for Join or Leave op (if read-only connection those don't even apply),\n            // go ahead and declare the state to be Connected!\n            // If we are waiting for Leave op still, do nothing for now, we will transition to Connected later.\n            this.setConnectionState(ConnectionState.Connected);\n        }\n        // else - We are waiting for Leave op still, do nothing for now, we will transition to Connected later\n    }\n\n    private setConnectionState(value: ConnectionState.Disconnected, reason: string): void;\n    private setConnectionState(value: ConnectionState.Connected): void;\n    private setConnectionState(value: ConnectionState.Disconnected | ConnectionState.Connected, reason?: string): void {\n        if (this.connectionState === value) {\n            // Already in the desired state - exit early\n            this.handler.logger.sendErrorEvent({ eventName: \"setConnectionStateSame\", value });\n            return;\n        }\n\n        const oldState = this._connectionState;\n        this._connectionState = value;\n\n        // This is the only place in code that deals with quorum. The rest works with audience\n        // The code below ensures that we do not send ops until we know that old \"write\" client's disconnect\n        // produced (and sequenced) leave op\n        let client: ILocalSequencedClient | undefined;\n        if (this._clientId !== undefined) {\n            client = this.protocol?.quorum?.getMember(this._clientId);\n        }\n        if (value === ConnectionState.Connected) {\n            assert(oldState === ConnectionState.CatchingUp,\n                0x1d8 /* \"Should only transition from Connecting state\" */);\n            // Mark our old client should have left in the quorum if it's still there\n            if (client !== undefined) {\n                client.shouldHaveLeft = true;\n            }\n            this._clientId = this.pendingClientId;\n        } else if (value === ConnectionState.Disconnected) {\n            // Clear pending state immediately to prepare for reconnect\n            this._pendingClientId = undefined;\n\n            if (this.joinOpTimer.hasTimer) {\n                this.stopJoinOpTimer();\n            }\n\n            // Only wait for \"leave\" message if the connected client exists in the quorum and had some non-acked ops\n            // Also check if the timer is not already running as\n            // we could receive \"Disconnected\" event multiple times without getting connected and in that case we\n            // don't want to reset the timer as we still want to wait on original client which started this timer.\n            if (client !== undefined\n                && this.handler.shouldClientJoinWrite()\n                && !this.waitingForLeaveOp // same as !this.prevClientLeftTimer.hasTimer\n            ) {\n                this.prevClientLeftTimer.restart();\n            } else {\n                // Adding this event temporarily so that we can get help debugging if something goes wrong.\n                this.handler.logger.sendTelemetryEvent({\n                    eventName: \"noWaitOnDisconnected\",\n                    details: JSON.stringify({\n                        clientId: this._clientId,\n                        inQuorum: client !== undefined,\n                        waitingForLeaveOp: this.waitingForLeaveOp,\n                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),\n                    }),\n                });\n            }\n        }\n\n        // Report transition before we propagate event across layers\n        this.handler.connectionStateChanged(this._connectionState, oldState, reason);\n    }\n\n    // Helper method to switch between quorum and audience.\n    // Old design was checking only quorum for \"write\" clients.\n    // Latest change checks audience for all types of connections.\n    protected get membership(): IMembership | undefined {\n        // We could always use audience here, and in practice it will probably be correct.\n        // (including case when this.readClientsWaitForJoinSignal === false).\n        // But only if it's superset of quorum, i.e. when filtered to \"write\" clients, they are always identical!\n        // It's safer to assume that we have bugs and engaging kill-bit switch should bring us back to well-known\n        // and tested state!\n        return this.readClientsWaitForJoinSignal ? this.protocol?.audience : this.protocol?.quorum;\n    }\n\n    public initProtocol(protocol: IProtocolHandler) {\n        this.protocol = protocol;\n\n        this.membership?.on(\"addMember\", (clientId, details) => {\n            assert((details as IClient).mode === \"read\" || protocol.quorum.getMember(clientId) !== undefined,\n                0x4b5 /* Audience is subset of quorum */);\n            this.receivedAddMemberEvent(clientId);\n        });\n\n        this.membership?.on(\"removeMember\", (clientId) => {\n            assert(protocol.quorum.getMember(clientId) === undefined, 0x4b6 /* Audience is subset of quorum */);\n            this.receivedRemoveMemberEvent(clientId);\n        });\n\n        /* There is a tiny tiny race possible, where these events happen in this order:\n          1. A connection is established (no \"cached\" mode is used, so it happens in parallel / faster than other steps)\n          2. Some other client produces a summary\n          3. We get \"lucky\" and load from that summary as our initial snapshot\n          4. ConnectionStateHandler.initProtocol is called, \"self\" is already in the quorum.\n        We could avoid this sequence (and delete test case for it) if we move connection lower in Container.load()\n        */\n        if (this.hasMember(this.pendingClientId)) {\n            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n            this.receivedAddMemberEvent(this.pendingClientId!);\n        }\n\n        // if we have a clientId from a previous container we need to wait for its leave message\n        if (this.clientId !== undefined && this.hasMember(this.clientId)) {\n            this.prevClientLeftTimer.restart();\n        }\n    }\n\n    protected hasMember(clientId?: string) {\n        return this.membership?.getMember(clientId ?? \"\") !== undefined;\n    }\n}\n"]}

package/lib/container.d.ts

@@ -2,6 +2,7 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
+import { ITelemetryLogger, ITelemetryProperties } from "@fluidframework/common-definitions";
 import { IRequest, IResponse, IFluidRouter } from "@fluidframework/core-interfaces";
 import { IAudience, IContainer, IContainerEvents, IDeltaManager, ICriticalContainerError, AttachState, ReadOnlyInfo, IContainerLoadMode, IFluidCodeDetails } from "@fluidframework/container-definitions";
 import { IDocumentStorageService, IFluidResolvedUrl, IResolvedUrl } from "@fluidframework/driver-definitions";
@@ -58,6 +59,13 @@ export interface IContainerConfig {
  * @throws an error beginning with `"Container closed"` if the container is closed before it catches up.
  */
 export declare function waitContainerToCatchUp(container: IContainer): Promise<boolean>;
+/**
+ * Helper function to report to telemetry cases where operation takes longer than expected (1s)
+ * @param logger - logger to use
+ * @param eventName - event name
+ * @param action - functor to call and measure
+ */
+export declare function ReportIfTooLong(logger: ITelemetryLogger, eventName: string, action: () => Promise<ITelemetryProperties>): Promise<void>;
 /**
  * State saved by a container at close time, to be used to load a new instance
  * of the container to the same state

package/lib/container.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"container.d.ts","sourceRoot":"","sources":["../src/container.ts"],"names":[],"mappings":"AAAA;;;GAGG;AASH,OAAO,EACH,QAAQ,EACR,SAAS,EACT,YAAY,EACf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACH,SAAS,EAET,UAAU,EACV,gBAAgB,EAChB,aAAa,EACb,uBAAuB,EAEvB,WAAW,EAEX,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EAGpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,EAEH,uBAAuB,EACvB,iBAAiB,EACjB,YAAY,EACf,MAAM,oCAAoC,CAAC;AAW5C,OAAO,EAEH,oBAAoB,EACpB,cAAc,EAGd,gBAAgB,EAChB,cAAc,EACd,cAAc,EAGd,yBAAyB,EAMzB,QAAQ,EAGX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAEH,6BAA6B,EAG7B,eAAe,EAOlB,MAAM,iCAAiC,CAAC;AAMzC,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,MAAM,UAAU,CAAC;AAWlE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAGH,sBAAsB,EACzB,MAAM,YAAY,CAAC;AAOpB,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC,WAAW,EAAE,iBAAiB,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC7B,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC;;OAEG;IACH,wBAAwB,CAAC,EAAE,sBAAsB,CAAC;CACrD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,sBAAsB,CAAC,SAAS,EAAE,UAAU,oBAmEjE;AAwBD;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,cAAc,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,qBAAa,SAAU,SAAQ,6BAA6B,CAAC,gBAAgB,CAAE,YAAW,UAAU;IA+R5F,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IAhS5C,OAAc,OAAO,SAAY;IAEjC;;OAEG;WACiB,IAAI,CACpB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,qBAAqB,EAClC,iBAAiB,CAAC,EAAE,sBAAsB,EAC1C,sBAAsB,CAAC,EAAE,sBAAsB,GAChD,OAAO,CAAC,SAAS,CAAC;IAmDrB;;OAEG;WACiB,cAAc,CAC9B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,iBAAiB,EAC9B,sBAAsB,CAAC,EAAE,sBAAsB,GAChD,OAAO,CAAC,SAAS,CAAC;IAgBrB;;;OAGG;WACiB,6BAA6B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,sBAAsB,CAAC,EAAE,sBAAsB,GAChD,OAAO,CAAC,SAAS,CAAC;IAiBd,SAAS,EAAE,eAAe,CAAC;IAIlC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAiB;IAE/C,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,eAAe,CAA0D;IAEjF,OAAO,CAAC,SAAS;IAUjB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED,OAAO,CAAC,YAAY,CAAwB;IAE5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0B;IACzD,IAAW,OAAO,IAAI,uBAAuB,CAE5C;IAED,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAA6B;IACnE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,OAAO,CAAC,OAAO,CAA+B;IAE9C,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,KAAK,OAAO,GAKlB;IACD,OAAO,CAAC,gBAAgB,CAA+B;IACvD,OAAO,KAAK,eAAe,GAK1B;IAED,gHAAgH;IAChH,OAAO,CAAC,0BAA0B,CAAQ;IAC1C,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAgB;IAC1D,OAAO,CAAC,8BAA8B,CAAa;IACnD,OAAO,CAAC,kBAAkB,CAAuB;IACjD,OAAO,CAAC,YAAY,CAAgC;IACpD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAEhC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA2B;IAClE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA0B;IAEjE,OAAO,CAAC,oBAAoB,CAAqB;IAEjD,OAAO,CAAC,mBAAmB,CAAkC;IAE7D,OAAO,KAAK,cAAc,GAAkE;IAE5F,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,WAAW,IAAI,YAAY,GAAG,SAAS,CAEjD;IAED,IAAW,iBAAiB,IAAI,QAAQ,GAAG,SAAS,CAEnD;IAED,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED,IAAW,WAAW,IAAI,WAAW,CAEpC;IAED;;OAEG;IACI,aAAa,CAAC,QAAQ,EAAE,OAAO;IAItC,IAAW,YAAY,IAAI,aAAa,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,CAEpF;IAED,IAAW,eAAe,IAAI,eAAe,CAE5C;IAED,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED;;;OAGG;IACH,IAAW,oBAAoB,IAAI,oBAAoB,GAAG,SAAS,CAElE;IAED,OAAO,CAAC,SAAS,CAAqB;IAEtC;;;OAGG;IACH,IAAW,QAAQ,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;OAGG;IACH,IAAW,MAAM,IAAI,MAAM,EAAE,GAAG,SAAS,CAExC;IAED,IAAW,aAAa,IAAI,cAAc,CAEzC;IAED;;;OAGG;IACI,uBAAuB,IAAI,iBAAiB,GAAG,SAAS;IAI/D;;;;OAIG;IACI,oBAAoB,IAAI,iBAAiB,GAAG,SAAS;IAI5D;;OAEG;IACH,IAAW,QAAQ,IAAI,SAAS,CAE/B;IAED;;;;OAIG;IACH,IAAW,OAAO,YAEjB;IAED,OAAO,KAAK,cAAc,GAA0D;IACpF,OAAO,KAAK,WAAW,GAA+C;IACtE,SAAgB,OAAO,EAAE,cAAc,CAAC;IACxC,OAAO,KAAK,KAAK,GAAyC;IAC1D,OAAO,KAAK,UAAU,GAA8C;gBAG/C,MAAM,EAAE,MAAM,EAC/B,MAAM,EAAE,gBAAgB,EACP,sBAAsB,CAAC,oCAAwB;IAyKpE;;OAEG;IACI,SAAS,IAAI,cAAc;IAI3B,KAAK,CAAC,KAAK,CAAC,EAAE,uBAAuB;IAY5C,OAAO,CAAC,SAAS;IA6CV,4BAA4B,IAAI,MAAM;IAyB7C,IAAW,WAAW,IAAI,WAAW,CAEpC;IAEM,SAAS,IAAI,MAAM;IAab,MAAM,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAkHxC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;IASxD,OAAO,CAAC,wBAAwB;IAqBzB,OAAO;IAad,OAAO,CAAC,eAAe;IAahB,UAAU;IAQjB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,cAAc;IAcT,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAWhE,kBAAkB,CAAC,WAAW,EAAE,iBAAiB;YAmBhD,mBAAmB;YAiBnB,UAAU;IAKxB,OAAO,CAAC,sBAAsB;IAM9B,OAAO,CAAC,oBAAoB;IAW5B;;;;OAIG;YACW,IAAI;YA0JJ,cAAc;YA4Bd,6BAA6B;YAqC7B,qBAAqB;YA2BrB,mCAAmC;IAuBjD,OAAO,CAAC,uBAAuB;IAiD/B,OAAO,CAAC,sBAAsB;IA2B9B,OAAO,CAAC,wBAAwB;IAQhC,OAAO,KAAK,MAAM,GAkBjB;IAED;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAKxB,OAAO,CAAC,kBAAkB;YAsDZ,2BAA2B;IAgBzC,OAAO,CAAC,iCAAiC;IAsDzC,OAAO,CAAC,wBAAwB;IAgChC,OAAO,CAAC,sBAAsB;IAkB9B,+DAA+D;IAC/D,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,oBAAoB;IAmC5B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,aAAa;IAUrB;;;;OAIG;YACW,iBAAiB;YAiBjB,0BAA0B;YAgB1B,kBAAkB;IAkChC,OAAO,CAAC,yBAAyB;IAQjC;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;CAWnC"}
+{"version":3,"file":"container.d.ts","sourceRoot":"","sources":["../src/container.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EACH,gBAAgB,EAAE,oBAAoB,EACzC,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EACH,QAAQ,EACR,SAAS,EACT,YAAY,EACf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACH,SAAS,EAET,UAAU,EACV,gBAAgB,EAChB,aAAa,EACb,uBAAuB,EAEvB,WAAW,EAEX,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EAGpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,EAEH,uBAAuB,EACvB,iBAAiB,EACjB,YAAY,EACf,MAAM,oCAAoC,CAAC;AAW5C,OAAO,EAEH,oBAAoB,EACpB,cAAc,EAGd,gBAAgB,EAChB,cAAc,EACd,cAAc,EAGd,yBAAyB,EAMzB,QAAQ,EAGX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAEH,6BAA6B,EAG7B,eAAe,EAOlB,MAAM,iCAAiC,CAAC;AAMzC,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,MAAM,UAAU,CAAC;AAWlE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAGH,sBAAsB,EACzB,MAAM,YAAY,CAAC;AAOpB,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC,WAAW,EAAE,iBAAiB,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC7B,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC;;OAEG;IACH,wBAAwB,CAAC,EAAE,sBAAsB,CAAC;CACrD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,sBAAsB,CAAC,SAAS,EAAE,UAAU,oBAmEjE;AAMD;;;;;GAKG;AACH,wBAAsB,eAAe,CACjC,MAAM,EAAE,gBAAgB,EACxB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,OAAO,CAAC,oBAAoB,CAAC,iBAO9C;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,cAAc,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,qBAAa,SAAU,SAAQ,6BAA6B,CAAC,gBAAgB,CAAE,YAAW,UAAU;IA+R5F,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IAhS5C,OAAc,OAAO,SAAY;IAEjC;;OAEG;WACiB,IAAI,CACpB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,qBAAqB,EAClC,iBAAiB,CAAC,EAAE,sBAAsB,EAC1C,sBAAsB,CAAC,EAAE,sBAAsB,GAChD,OAAO,CAAC,SAAS,CAAC;IAmDrB;;OAEG;WACiB,cAAc,CAC9B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,iBAAiB,EAC9B,sBAAsB,CAAC,EAAE,sBAAsB,GAChD,OAAO,CAAC,SAAS,CAAC;IAgBrB;;;OAGG;WACiB,6BAA6B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,sBAAsB,CAAC,EAAE,sBAAsB,GAChD,OAAO,CAAC,SAAS,CAAC;IAiBd,SAAS,EAAE,eAAe,CAAC;IAIlC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAiB;IAE/C,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,eAAe,CAA0D;IAEjF,OAAO,CAAC,SAAS;IAUjB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED,OAAO,CAAC,YAAY,CAAwB;IAE5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0B;IACzD,IAAW,OAAO,IAAI,uBAAuB,CAE5C;IAED,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAA6B;IACnE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,OAAO,CAAC,OAAO,CAA+B;IAE9C,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,KAAK,OAAO,GAKlB;IACD,OAAO,CAAC,gBAAgB,CAA+B;IACvD,OAAO,KAAK,eAAe,GAK1B;IAED,gHAAgH;IAChH,OAAO,CAAC,0BAA0B,CAAQ;IAC1C,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAgB;IAC1D,OAAO,CAAC,8BAA8B,CAAa;IACnD,OAAO,CAAC,kBAAkB,CAAuB;IACjD,OAAO,CAAC,YAAY,CAAgC;IACpD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAEhC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA2B;IAClE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA0B;IAEjE,OAAO,CAAC,oBAAoB,CAAqB;IAEjD,OAAO,CAAC,mBAAmB,CAAkC;IAE7D,OAAO,KAAK,cAAc,GAAkE;IAE5F,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,WAAW,IAAI,YAAY,GAAG,SAAS,CAEjD;IAED,IAAW,iBAAiB,IAAI,QAAQ,GAAG,SAAS,CAEnD;IAED,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED,IAAW,WAAW,IAAI,WAAW,CAEpC;IAED;;OAEG;IACI,aAAa,CAAC,QAAQ,EAAE,OAAO;IAItC,IAAW,YAAY,IAAI,aAAa,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,CAEpF;IAED,IAAW,eAAe,IAAI,eAAe,CAE5C;IAED,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED;;;OAGG;IACH,IAAW,oBAAoB,IAAI,oBAAoB,GAAG,SAAS,CAElE;IAED,OAAO,CAAC,SAAS,CAAqB;IAEtC;;;OAGG;IACH,IAAW,QAAQ,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;OAGG;IACH,IAAW,MAAM,IAAI,MAAM,EAAE,GAAG,SAAS,CAExC;IAED,IAAW,aAAa,IAAI,cAAc,CAEzC;IAED;;;OAGG;IACI,uBAAuB,IAAI,iBAAiB,GAAG,SAAS;IAI/D;;;;OAIG;IACI,oBAAoB,IAAI,iBAAiB,GAAG,SAAS;IAI5D;;OAEG;IACH,IAAW,QAAQ,IAAI,SAAS,CAE/B;IAED;;;;OAIG;IACH,IAAW,OAAO,YAEjB;IAED,OAAO,KAAK,cAAc,GAA0D;IACpF,OAAO,KAAK,WAAW,GAA+C;IACtE,SAAgB,OAAO,EAAE,cAAc,CAAC;IACxC,OAAO,KAAK,KAAK,GAAyC;IAC1D,OAAO,KAAK,UAAU,GAA8C;gBAG/C,MAAM,EAAE,MAAM,EAC/B,MAAM,EAAE,gBAAgB,EACP,sBAAsB,CAAC,oCAAwB;IAwLpE;;OAEG;IACI,SAAS,IAAI,cAAc;IAI3B,KAAK,CAAC,KAAK,CAAC,EAAE,uBAAuB;IAY5C,OAAO,CAAC,SAAS;IA6CV,4BAA4B,IAAI,MAAM;IAyB7C,IAAW,WAAW,IAAI,WAAW,CAEpC;IAEM,SAAS,IAAI,MAAM;IAab,MAAM,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAkHxC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;IASxD,OAAO,CAAC,wBAAwB;IAqBzB,OAAO;IAad,OAAO,CAAC,eAAe;IAahB,UAAU;IAQjB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,cAAc;IAcT,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAWhE,kBAAkB,CAAC,WAAW,EAAE,iBAAiB;YAmBhD,mBAAmB;YAiBnB,UAAU;IAKxB,OAAO,CAAC,sBAAsB;IAM9B,OAAO,CAAC,oBAAoB;IAW5B;;;;OAIG;YACW,IAAI;YA0JJ,cAAc;YA4Bd,6BAA6B;YAqC7B,qBAAqB;YA2BrB,mCAAmC;IAuBjD,OAAO,CAAC,uBAAuB;IAiD/B,OAAO,CAAC,sBAAsB;IA2B9B,OAAO,CAAC,wBAAwB;IAQhC,OAAO,KAAK,MAAM,GAkBjB;IAED;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAKxB,OAAO,CAAC,kBAAkB;YAsDZ,2BAA2B;IAgBzC,OAAO,CAAC,iCAAiC;IAsDzC,OAAO,CAAC,wBAAwB;IAgChC,OAAO,CAAC,sBAAsB;IAkB9B,+DAA+D;IAC/D,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,aAAa;IAerB,OAAO,CAAC,oBAAoB;IAmC5B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,aAAa;IAUrB;;;;OAIG;YACW,iBAAiB;YAiBjB,0BAA0B;YAgB1B,kBAAkB;IAkChC,OAAO,CAAC,yBAAyB;IAQjC;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;CAWnC"}

package/lib/container.js

@@ -111,10 +111,10 @@ const getCodeProposal =
  * @param eventName - event name
  * @param action - functor to call and measure
  */
-async function ReportIfTooLong(logger, eventName, action) {
+export async function ReportIfTooLong(logger, eventName, action) {
     const event = PerformanceEvent.start(logger, { eventName });
     const props = await action();
-    if (event.duration > 1000) {
+    if (event.duration > 200) {
         event.end(props);
     }
 }
@@ -202,9 +202,23 @@ export class Container extends EventEmitterWithErrorHandling {
             shouldClientJoinWrite: () => this._deltaManager.connectionManager.shouldJoinWrite(),
             maxClientLeaveWaitTime: this.loader.services.options.maxClientLeaveWaitTime,
             logConnectionIssue: (eventName, details) => {
+                const mode = this.connectionMode;
                 // We get here when socket does not receive any ops on "write" connection, including
                 // its own join op. Attempt recovery option.
-                this._deltaManager.logConnectionIssue(Object.assign({ eventName, duration: performance.now() - this.connectionTransitionTimes[ConnectionState.CatchingUp] }, (details === undefined ? {} : { details: JSON.stringify(details) })));
+                this._deltaManager.logConnectionIssue(Object.assign({ eventName,
+                    mode, duration: performance.now() - this.connectionTransitionTimes[ConnectionState.CatchingUp] }, (details === undefined ? {} : { details: JSON.stringify(details) })));
+                // If this is "write" connection, it took too long to receive join op. But in most cases that's due
+                // to very slow op fetches and we will eventually get there.
+                // For "read" connections, we get here due to self join signal not arriving on time. We will need to
+                // better understand when and why it may happen.
+                // For now, attempt to recover by reconnecting. In future, maybe we can query relay service for
+                // current state of audience.
+                // Other possible recovery path - move to connected state (i.e. ConnectionStateHandler.joinOpTimer
+                // to call this.applyForConnectedState("addMemberEvent") for "read" connections)
+                if (mode === "read") {
+                    this.disconnect();
+                    this.connect();
+                }
             },
         }, this.deltaManager, this._clientId);
         this.on(savedContainerEvent, () => {
@@ -1025,7 +1039,8 @@ export class Container extends EventEmitterWithErrorHandling {
         // eslint-disable-next-line @typescript-eslint/no-floating-promises
         deltaManager.inboundSignal.pause();
         deltaManager.on("connect", (details, _opsBehind) => {
-            this.connectionStateHandler.receivedConnectEvent(this.connectionMode, details);
+            assert(this.connectionMode === details.mode, 0x4b7 /* mismatch */);
+            this.connectionStateHandler.receivedConnectEvent(details);
         });
         deltaManager.on("disconnect", (reason) => {
             var _a;
@@ -1138,7 +1153,7 @@ export class Container extends EventEmitterWithErrorHandling {
         let clientSequenceNumber = -1;
         for (const message of batch) {
             clientSequenceNumber = this.submitMessage(MessageType.Operation, message.contents, true, // batch
-            message.metadata);
+            message.metadata, message.compression);
         }
         this._deltaManager.flush();
         return clientSequenceNumber;
@@ -1155,7 +1170,7 @@ export class Container extends EventEmitterWithErrorHandling {
             this.options.summarizeProtocolTree === true;
         return this.submitMessage(MessageType.Summarize, JSON.stringify(summary), false /* batch */);
     }
-    submitMessage(type, contents, batch, metadata) {
+    submitMessage(type, contents, batch, metadata, compression) {
         var _a;
         if (this.connectionState !== ConnectionState.Connected) {
             this.mc.logger.sendErrorEvent({ eventName: "SubmitMessageWithNoConnection", type });
@@ -1163,7 +1178,7 @@ export class Container extends EventEmitterWithErrorHandling {
         }
         this.messageCountAfterDisconnection += 1;
         (_a = this.collabWindowTracker) === null || _a === void 0 ? void 0 : _a.stopSequenceNumberUpdate();
-        return this._deltaManager.submit(type, contents, batch, metadata);
+        return this._deltaManager.submit(type, contents, batch, metadata, compression);
     }
     processRemoteMessage(message) {
         const local = this.clientId === message.clientId;