@fluidframework/container-loader 1.0.1 → 1.1.0-75972

package/lib/connectionStateHandler.js

@@ -5,7 +5,29 @@
 import { assert, Timer } from "@fluidframework/common-utils";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { ConnectionState } from "./connectionState";
-const JoinOpTimer = 45000;
+const JoinOpTimeoutMs = 45000;
+/**
+ * In the lifetime of a container, the connection will likely disconnect and reconnect periodically.
+ * This class ensures that any ops sent by this container instance on previous connection are either
+ * sequenced or blocked by the server before emitting the new "connected" event and allowing runtime to resubmit ops.
+ *
+ * Each connection is assigned a clientId by the service, and the connection is book-ended by a Join and a Leave op
+ * generated by the service. Due to the distributed nature of the ordering service, in the case of reconnect we cannot
+ * make any assumptions about ordering of operations between the old and new connections - i.e. new Join op could
+ * be sequenced before old Leave op (and some acks from pending ops that were in flight when we disconnected).
+ *
+ * The job of this class is to encapsulate the transition period during reconnect, which is identified by
+ * ConnectionState.CatchingUp. Specifically, before moving to Connected state with the new clientId, it ensures that:
+ * (A) We process the Leave op for the previous clientId. This allows us to properly handle any acks from in-flight ops
+ *     that got sequenced with the old clientId (we'll recognize them as local ops). After the Leave op, any other
+ *     pending ops can safely be submitted with the new clientId without fear of duplication in the sequenced op stream.
+ * (B) We process the Join op for the new clientId (identified when the underlying connection was first established),
+ *     indicating the service is ready to sequence ops sent with the new clientId.
+ *
+ * For (A) we give up waiting after some time (same timeout as server uses), and go ahead and transition to Connected.
+ * For (B) we log telemetry if it takes too long, but still only transition to Connected when the Join op is processed
+ * and we are added to the Quorum.
+ */
 export class ConnectionStateHandler {
     constructor(handler, logger, _clientId) {
         var _a;
@@ -23,12 +45,21 @@ export class ConnectionStateHandler {
         // Based on recent data, it looks like majority of cases where we get stuck are due to really slow or
         // timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so
         // if retrying fixes the problem, we should not see these events.
-        this.joinOpTimer = new Timer(JoinOpTimer, () => {
+        this.joinOpTimer = new Timer(JoinOpTimeoutMs, () => {
+            var _a;
             // I've observed timer firing within couple ms from disconnect event, looks like
             // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.
-            if (this.connectionState === ConnectionState.CatchingUp) {
-                this.handler.logConnectionIssue("NoJoinOp");
+            if (this.connectionState !== ConnectionState.CatchingUp) {
+                return;
             }
+            const quorumClients = this.handler.quorumClients();
+            const details = {
+                quorumInitialized: quorumClients !== undefined,
+                hasPendingClientId: this.pendingClientId !== undefined,
+                inQuorum: (quorumClients === null || quorumClients === void 0 ? void 0 : quorumClients.getMember((_a = this.pendingClientId) !== null && _a !== void 0 ? _a : "")) !== undefined,
+                waitingForLeaveOp: this.waitingForLeaveOp,
+            };
+            this.handler.logConnectionIssue("NoJoinOp", details);
         });
     }
     get connectionState() {
@@ -51,6 +82,9 @@ export class ConnectionStateHandler {
         assert(this.joinOpTimer.hasTimer, 0x235 /* "no joinOpTimer" */);
         this.joinOpTimer.clear();
     }
+    get waitingForLeaveOp() {
+        return this.prevClientLeftTimer.hasTimer;
+    }
     dispose() {
         assert(!this.joinOpTimer.hasTimer, 0x2a5 /* "join timer" */);
         this.prevClientLeftTimer.clear();
@@ -58,7 +92,7 @@ export class ConnectionStateHandler {
     containerSaved() {
         // If we were waiting for moving to Connected state, then only apply for state change. Since the container
         // is now saved and we don't have any ops to roundtrip, we can clear the timer and apply for connected state.
-        if (this.prevClientLeftTimer.hasTimer) {
+        if (this.waitingForLeaveOp) {
             this.prevClientLeftTimer.clear();
             this.applyForConnectedState("containerSaved");
         }
@@ -75,29 +109,31 @@ export class ConnectionStateHandler {
                 this.handler.logConnectionIssue("ReceivedJoinOp");
             }
             // Start the event in case we are waiting for leave or timeout.
-            if (this.prevClientLeftTimer.hasTimer) {
+            if (this.waitingForLeaveOp) {
                 this.waitEvent = PerformanceEvent.start(this.logger, {
                     eventName: "WaitBeforeClientLeave",
-                    waitOnClientId: this._clientId,
-                    hadOutstandingOps: this.handler.shouldClientJoinWrite(),
+                    details: JSON.stringify({
+                        waitOnClientId: this._clientId,
+                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),
+                    }),
                 });
             }
             this.applyForConnectedState("addMemberEvent");
         }
     }
     applyForConnectedState(source) {
-        var _a;
+        var _a, _b;
         const quorumClients = this.handler.quorumClients();
         assert(quorumClients !== undefined, 0x236 /* "In all cases it should be already installed" */);
-        assert(this.prevClientLeftTimer.hasTimer === false ||
+        assert(this.waitingForLeaveOp === false ||
             (this.clientId !== undefined && quorumClients.getMember(this.clientId) !== undefined), 0x2e2 /* "Must only wait for leave message when clientId in quorum" */);
         // Move to connected state only if we are in Connecting state, we have seen our join op
         // and there is no timer running which means we are not waiting for previous client to leave
-        // or timeout has occured while doing so.
+        // or timeout has occurred while doing so.
         if (this.pendingClientId !== this.clientId
             && this.pendingClientId !== undefined
             && quorumClients.getMember(this.pendingClientId) !== undefined
-            && !this.prevClientLeftTimer.hasTimer) {
+            && !this.waitingForLeaveOp) {
             (_a = this.waitEvent) === null || _a === void 0 ? void 0 : _a.end({ source });
             this.setConnectionState(ConnectionState.Connected);
         }
@@ -106,12 +142,13 @@ export class ConnectionStateHandler {
             this.logger.sendTelemetryEvent({
                 eventName: "connectedStateRejected",
                 category: source === "timeout" ? "error" : "generic",
-                source,
-                pendingClientId: this.pendingClientId,
-                clientId: this.clientId,
-                hasTimer: this.prevClientLeftTimer.hasTimer,
-                inQuorum: quorumClients !== undefined && this.pendingClientId !== undefined
-                    && quorumClients.getMember(this.pendingClientId) !== undefined,
+                details: JSON.stringify({
+                    source,
+                    pendingClientId: this.pendingClientId,
+                    clientId: this.clientId,
+                    waitingForLeaveOp: this.waitingForLeaveOp,
+                    inQuorum: (quorumClients === null || quorumClients === void 0 ? void 0 : quorumClients.getMember((_b = this.pendingClientId) !== null && _b !== void 0 ? _b : "")) !== undefined,
+                }),
             });
         }
     }
@@ -128,32 +165,48 @@ export class ConnectionStateHandler {
         }
         this.setConnectionState(ConnectionState.Disconnected, reason);
     }
+    /**
+     * The "connect" event indicates the connection to the Relay Service is live.
+     * However, some additional conditions must be met before we can fully transition to
+     * "Connected" state. This function handles that interim period, known as "Connecting" state.
+     * @param connectionMode - Read or Write connection
+     * @param details - Connection details returned from the ordering service
+     */
     receivedConnectEvent(connectionMode, details) {
         const oldState = this._connectionState;
         this._connectionState = ConnectionState.CatchingUp;
+        const writeConnection = connectionMode === "write";
+        assert(writeConnection || !this.handler.shouldClientJoinWrite(), 0x30a /* shouldClientJoinWrite should imply this is a writeConnection */);
+        assert(writeConnection || !this.waitingForLeaveOp, 0x2a6 /* "waitingForLeaveOp should imply writeConnection (we need to be ready to flush pending ops)" */);
+        // Note that this may be undefined since the connection is established proactively on load
+        // and the quorum may still be under initialization.
+        const quorumClients = this.handler.quorumClients();
         // Stash the clientID to detect when transitioning from connecting (socket.io channel open) to connected
         // (have received the join message for the client ID)
         // This is especially important in the reconnect case. It's possible there could be outstanding
         // ops sent by this client, so we should keep the old client id until we see our own client's
-        // join message. after we see the join message for out new connection with our new client id,
+        // join message. after we see the join message for our new connection with our new client id,
         // we know there can no longer be outstanding ops that we sent with the previous client id.
         this._pendingClientId = details.clientId;
-        // Report telemetry after we set client id, but before transitioning to Connected state below!
+        // IMPORTANT: Report telemetry after we set _pendingClientId, but before transitioning to Connected state
         this.handler.logConnectionStateChangeTelemetry(ConnectionState.CatchingUp, oldState);
-        const quorumClients = this.handler.quorumClients();
-        // Check if we already processed our own join op through delta storage!
-        // we are fetching ops from storage in parallel to connecting to ordering service
-        // Given async processes, it's possible that we have already processed our own join message before
+        // For write connections, this pending clientId could be in the quorum already (i.e. join op already processed).
+        // We are fetching ops from storage in parallel to connecting to Relay Service,
+        // and given async processes, it's possible that we have already processed our own join message before
         // connection was fully established.
-        // Note that we might be still initializing quorum - connection is established proactively on load!
-        if ((quorumClients === null || quorumClients === void 0 ? void 0 : quorumClients.getMember(details.clientId)) !== undefined
-            || connectionMode === "read") {
-            assert(!this.prevClientLeftTimer.hasTimer, 0x2a6 /* "there should be no timer for 'read' connections" */);
-            this.setConnectionState(ConnectionState.Connected);
-        }
-        else if (connectionMode === "write") {
+        // If quorumClients itself is undefined, we expect it will process the join op after it's initialized.
+        const waitingForJoinOp = writeConnection && (quorumClients === null || quorumClients === void 0 ? void 0 : quorumClients.getMember(this._pendingClientId)) === undefined;
+        if (waitingForJoinOp) {
+            // Previous client left, and we are waiting for our own join op. When it is processed we'll join the quorum
+            // and attempt to transition to Connected state via receivedAddMemberEvent.
             this.startJoinOpTimer();
         }
+        else if (!this.waitingForLeaveOp) {
+            // We're not waiting for Join or Leave op (if read-only connection those don't even apply),
+            // go ahead and declare the state to be Connected!
+            // If we are waiting for Leave op still, do nothing for now, we will transition to Connected later.
+            this.setConnectionState(ConnectionState.Connected);
+        }
     }
     setConnectionState(value, reason) {
         if (this.connectionState === value) {
@@ -194,9 +247,11 @@ export class ConnectionStateHandler {
                 // Adding this event temporarily so that we can get help debugging if something goes wrong.
                 this.logger.sendTelemetryEvent({
                     eventName: "noWaitOnDisconnected",
-                    inQuorum: client !== undefined,
-                    hasTimer: this.prevClientLeftTimer.hasTimer,
-                    shouldClientJoinWrite: this.handler.shouldClientJoinWrite(),
+                    details: JSON.stringify({
+                        inQuorum: client !== undefined,
+                        waitingForLeaveOp: this.waitingForLeaveOp,
+                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),
+                    }),
                 });
             }
         }
@@ -206,7 +261,7 @@ export class ConnectionStateHandler {
         this.handler.connectionStateChanged();
     }
     initProtocol(protocol) {
-        protocol.quorum.on("addMember", (clientId, details) => {
+        protocol.quorum.on("addMember", (clientId, _details) => {
             this.receivedAddMemberEvent(clientId);
         });
         protocol.quorum.on("removeMember", (clientId) => {

package/lib/connectionStateHandler.js.map

@@ -1 +1 @@
-{"version":3,"file":"connectionStateHandler.js","sourceRoot":"","sources":["../src/connectionStateHandler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAI7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAmBpD,MAAM,WAAW,GAAG,KAAK,CAAC;AAE1B,MAAM,OAAO,sBAAsB;IAwB/B,YACqB,OAAgC,EAChC,MAAwB,EACjC,SAAkB;;QAFT,YAAO,GAAP,OAAO,CAAyB;QAChC,WAAM,GAAN,MAAM,CAAkB;QACjC,cAAS,GAAT,SAAS,CAAS;QA1BtB,qBAAgB,GAAG,eAAe,CAAC,YAAY,CAAC;QA4BpD,IAAI,CAAC,mBAAmB,GAAG,IAAI,KAAK;QAChC,+FAA+F;QAC/F,uDAAuD;QACvD,MAAA,IAAI,CAAC,OAAO,CAAC,sBAAsB,mCAAI,MAAM,EAC7C,GAAG,EAAE;YACD,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,EAClB,KAAK,CAAC,6EAA6E,CAAC,CAAC;YACzF,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC,CACJ,CAAC;QAEF,qGAAqG;QACrG,kGAAkG;QAClG,iEAAiE;QACjE,IAAI,CAAC,WAAW,GAAG,IAAI,KAAK,CACxB,WAAW,EACX,GAAG,EAAE;YACD,gFAAgF;YAChF,iGAAiG;YACjG,IAAI,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;gBACrD,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;aAC/C;QACL,CAAC,CACJ,CAAC;IACN,CAAC;IA7CD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,SAAS,CAAC;IAC9D,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAiCO,gBAAgB;QACpB,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAEO,eAAe;QACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAEM,OAAO;QACV,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC7D,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAEM,cAAc;QACjB,0GAA0G;QAC1G,6GAA6G;QAC7G,IAAI,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YACnC,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,QAAgB;QAC3C,2DAA2D;QAC3D,IAAI,QAAQ,KAAK,IAAI,CAAC,eAAe,EAAE;YACnC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;aAC1B;iBAAM;gBACH,oEAAoE;gBACpE,+CAA+C;gBAC/C,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;aACrD;YACD,+DAA+D;YAC/D,IAAI,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE;gBACnC,IAAI,CAAC,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE;oBACjD,SAAS,EAAE,uBAAuB;oBAClC,cAAc,EAAE,IAAI,CAAC,SAAS;oBAC9B,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;iBAC1D,CAAC,CAAC;aACN;YACD,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,MAA6E;;QACxG,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACnD,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAE/F,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,QAAQ,KAAK,KAAK;YAC9C,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,EACrF,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAE5E,uFAAuF;QACvF,4FAA4F;QAC5F,yCAAyC;QACzC,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC,QAAQ;eACnC,IAAI,CAAC,eAAe,KAAK,SAAS;eAClC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,SAAS;eAC3D,CAAC,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EACvC;YACE,MAAA,IAAI,CAAC,SAAS,0CAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;aAAM;YACH,2FAA2F;YAC3F,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBAC3B,SAAS,EAAE,wBAAwB;gBACnC,QAAQ,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;gBACpD,MAAM;gBACN,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,QAAQ;gBAC3C,QAAQ,EAAE,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS;uBACpE,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,SAAS;aACrE,CAAC,CAAC;SACN;IACL,CAAC;IAEO,yBAAyB,CAAC,QAAgB;QAC9C,8DAA8D;QAC9D,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAC5B,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,mBAAmB,CAAC,CAAC;SACpD;IACL,CAAC;IAEM,uBAAuB,CAAC,MAAc;QACzC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;YAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;SAC1B;QACD,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IAEM,oBAAoB,CACvB,cAA8B,EAC9B,OAA2B;QAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC,UAAU,CAAC;QAEnD,wGAAwG;QACxG,qDAAqD;QACrD,+FAA+F;QAC/F,6FAA6F;QAC7F,6FAA6F;QAC7F,2FAA2F;QAC3F,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEzC,8FAA8F;QAC9F,IAAI,CAAC,OAAO,CAAC,iCAAiC,CAAC,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAErF,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACnD,uEAAuE;QACvE,iFAAiF;QACjF,kGAAkG;QAClG,oCAAoC;QACpC,mGAAmG;QACnG,IAAI,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAK,SAAS;eACrD,cAAc,KAAK,MAAM,EAC9B;YACE,MAAM,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,uDAAuD,CAAC,CAAC;YAC1G,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;aAAM,IAAI,cAAc,KAAK,OAAO,EAAE;YACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;SAC3B;IACL,CAAC;IAIO,kBAAkB,CAAC,KAAsB,EAAE,MAAe;QAC9D,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE;YAChC,4CAA4C;YAC5C,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,wBAAwB,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3E,OAAO;SACV;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACnD,IAAI,MAAyC,CAAC;QAC9C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;YAC9B,MAAM,GAAG,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrD;QACD,IAAI,KAAK,KAAK,eAAe,CAAC,SAAS,EAAE;YACrC,MAAM,CAAC,QAAQ,KAAK,eAAe,CAAC,UAAU,EAC1C,KAAK,CAAC,oDAAoD,CAAC,CAAC;YAChE,yEAAyE;YACzE,IAAI,MAAM,KAAK,SAAS,EAAE;gBACtB,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;aAChC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC;SACzC;aAAM,IAAI,KAAK,KAAK,eAAe,CAAC,YAAY,EAAE;YAC/C,4EAA4E;YAC5E,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;YAClC,oGAAoG;YACpG,sGAAsG;YACtG,mCAAmC;YACnC,uGAAuG;YACvG,qGAAqG;YACrG,sGAAsG;YACtG,IAAI,MAAM,KAAK,SAAS;mBACjB,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;mBACpC,IAAI,CAAC,mBAAmB,CAAC,QAAQ,KAAK,KAAK,EAChD;gBACE,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;aACtC;iBAAM;gBACH,2FAA2F;gBAC3F,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC3B,SAAS,EAAE,sBAAsB;oBACjC,QAAQ,EAAE,MAAM,KAAK,SAAS;oBAC9B,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,QAAQ;oBAC3C,qBAAqB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;iBAC9D,CAAC,CAAC;aACN;SACJ;QAED,4DAA4D;QAC5D,IAAI,CAAC,OAAO,CAAC,iCAAiC,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAExF,gCAAgC;QAChC,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAC1C,CAAC;IAEM,YAAY,CAAC,QAA2B;QAC3C,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE;YAClD,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC5C,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,wFAAwF;QACxF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE;YACvF,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;SACtC;IACL,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, Timer } from \"@fluidframework/common-utils\";\nimport { IConnectionDetails } from \"@fluidframework/container-definitions\";\nimport { ProtocolOpHandler } from \"@fluidframework/protocol-base\";\nimport { ConnectionMode, IQuorumClients, ISequencedClient } from \"@fluidframework/protocol-definitions\";\nimport { PerformanceEvent } from \"@fluidframework/telemetry-utils\";\nimport { ConnectionState } from \"./connectionState\";\n\nexport interface IConnectionStateHandler {\n    quorumClients: () => IQuorumClients | undefined;\n    logConnectionStateChangeTelemetry: (\n        value: ConnectionState,\n        oldState: ConnectionState,\n        reason?: string | undefined\n    ) => void;\n    shouldClientJoinWrite: () => boolean;\n    maxClientLeaveWaitTime: number | undefined;\n    logConnectionIssue: (eventName: string) => void;\n    connectionStateChanged: () => void;\n}\n\nexport interface ILocalSequencedClient extends ISequencedClient {\n    shouldHaveLeft?: boolean;\n}\n\nconst JoinOpTimer = 45000;\n\nexport class ConnectionStateHandler {\n    private _connectionState = ConnectionState.Disconnected;\n    private _pendingClientId: string | undefined;\n    private readonly prevClientLeftTimer: Timer;\n    private readonly joinOpTimer: Timer;\n\n    private waitEvent: PerformanceEvent | undefined;\n\n    public get connectionState(): ConnectionState {\n        return this._connectionState;\n    }\n\n    public get connected(): boolean {\n        return this.connectionState === ConnectionState.Connected;\n    }\n\n    public get clientId(): string | undefined {\n        return this._clientId;\n    }\n\n    public get pendingClientId(): string | undefined {\n        return this._pendingClientId;\n    }\n\n    constructor(\n        private readonly handler: IConnectionStateHandler,\n        private readonly logger: ITelemetryLogger,\n        private _clientId?: string,\n    ) {\n        this.prevClientLeftTimer = new Timer(\n            // Default is 5 min for which we are going to wait for its own \"leave\" message. This is same as\n            // the max time on server after which leave op is sent.\n            this.handler.maxClientLeaveWaitTime ?? 300000,\n            () => {\n                assert(!this.connected,\n                    0x2ac /* \"Connected when timeout waiting for leave from previous session fired!\" */);\n                this.applyForConnectedState(\"timeout\");\n            },\n        );\n\n        // Based on recent data, it looks like majority of cases where we get stuck are due to really slow or\n        // timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so\n        // if retrying fixes the problem, we should not see these events.\n        this.joinOpTimer = new Timer(\n            JoinOpTimer,\n            () => {\n                // I've observed timer firing within couple ms from disconnect event, looks like\n                // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.\n                if (this.connectionState === ConnectionState.CatchingUp) {\n                    this.handler.logConnectionIssue(\"NoJoinOp\");\n                }\n            },\n        );\n    }\n\n    private startJoinOpTimer() {\n        assert(!this.joinOpTimer.hasTimer, 0x234 /* \"has joinOpTimer\" */);\n        this.joinOpTimer.start();\n    }\n\n    private stopJoinOpTimer() {\n        assert(this.joinOpTimer.hasTimer, 0x235 /* \"no joinOpTimer\" */);\n        this.joinOpTimer.clear();\n    }\n\n    public dispose() {\n        assert(!this.joinOpTimer.hasTimer, 0x2a5 /* \"join timer\" */);\n        this.prevClientLeftTimer.clear();\n    }\n\n    public containerSaved() {\n        // If we were waiting for moving to Connected state, then only apply for state change. Since the container\n        // is now saved and we don't have any ops to roundtrip, we can clear the timer and apply for connected state.\n        if (this.prevClientLeftTimer.hasTimer) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"containerSaved\");\n        }\n    }\n\n    private receivedAddMemberEvent(clientId: string) {\n        // This is the only one that requires the pending client ID\n        if (clientId === this.pendingClientId) {\n            if (this.joinOpTimer.hasTimer) {\n                this.stopJoinOpTimer();\n            } else {\n                // timer has already fired, meaning it took too long to get join on.\n                // Record how long it actually took to recover.\n                this.handler.logConnectionIssue(\"ReceivedJoinOp\");\n            }\n            // Start the event in case we are waiting for leave or timeout.\n            if (this.prevClientLeftTimer.hasTimer) {\n                this.waitEvent = PerformanceEvent.start(this.logger, {\n                    eventName: \"WaitBeforeClientLeave\",\n                    waitOnClientId: this._clientId,\n                    hadOutstandingOps: this.handler.shouldClientJoinWrite(),\n                });\n            }\n            this.applyForConnectedState(\"addMemberEvent\");\n        }\n    }\n\n    private applyForConnectedState(source: \"removeMemberEvent\" | \"addMemberEvent\" | \"timeout\" | \"containerSaved\") {\n        const quorumClients = this.handler.quorumClients();\n        assert(quorumClients !== undefined, 0x236 /* \"In all cases it should be already installed\" */);\n\n        assert(this.prevClientLeftTimer.hasTimer === false ||\n            (this.clientId !== undefined && quorumClients.getMember(this.clientId) !== undefined),\n            0x2e2 /* \"Must only wait for leave message when clientId in quorum\" */);\n\n        // Move to connected state only if we are in Connecting state, we have seen our join op\n        // and there is no timer running which means we are not waiting for previous client to leave\n        // or timeout has occured while doing so.\n        if (this.pendingClientId !== this.clientId\n            && this.pendingClientId !== undefined\n            && quorumClients.getMember(this.pendingClientId) !== undefined\n            && !this.prevClientLeftTimer.hasTimer\n        ) {\n            this.waitEvent?.end({ source });\n            this.setConnectionState(ConnectionState.Connected);\n        } else {\n            // Adding this event temporarily so that we can get help debugging if something goes wrong.\n            this.logger.sendTelemetryEvent({\n                eventName: \"connectedStateRejected\",\n                category: source === \"timeout\" ? \"error\" : \"generic\",\n                source,\n                pendingClientId: this.pendingClientId,\n                clientId: this.clientId,\n                hasTimer: this.prevClientLeftTimer.hasTimer,\n                inQuorum: quorumClients !== undefined && this.pendingClientId !== undefined\n                    && quorumClients.getMember(this.pendingClientId) !== undefined,\n            });\n        }\n    }\n\n    private receivedRemoveMemberEvent(clientId: string) {\n        // If the client which has left was us, then finish the timer.\n        if (this.clientId === clientId) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"removeMemberEvent\");\n        }\n    }\n\n    public receivedDisconnectEvent(reason: string) {\n        if (this.joinOpTimer.hasTimer) {\n            this.stopJoinOpTimer();\n        }\n        this.setConnectionState(ConnectionState.Disconnected, reason);\n    }\n\n    public receivedConnectEvent(\n        connectionMode: ConnectionMode,\n        details: IConnectionDetails,\n    ) {\n        const oldState = this._connectionState;\n        this._connectionState = ConnectionState.CatchingUp;\n\n        // Stash the clientID to detect when transitioning from connecting (socket.io channel open) to connected\n        // (have received the join message for the client ID)\n        // This is especially important in the reconnect case. It's possible there could be outstanding\n        // ops sent by this client, so we should keep the old client id until we see our own client's\n        // join message. after we see the join message for out new connection with our new client id,\n        // we know there can no longer be outstanding ops that we sent with the previous client id.\n        this._pendingClientId = details.clientId;\n\n        // Report telemetry after we set client id, but before transitioning to Connected state below!\n        this.handler.logConnectionStateChangeTelemetry(ConnectionState.CatchingUp, oldState);\n\n        const quorumClients = this.handler.quorumClients();\n        // Check if we already processed our own join op through delta storage!\n        // we are fetching ops from storage in parallel to connecting to ordering service\n        // Given async processes, it's possible that we have already processed our own join message before\n        // connection was fully established.\n        // Note that we might be still initializing quorum - connection is established proactively on load!\n        if (quorumClients?.getMember(details.clientId) !== undefined\n            || connectionMode === \"read\"\n        ) {\n            assert(!this.prevClientLeftTimer.hasTimer, 0x2a6 /* \"there should be no timer for 'read' connections\" */);\n            this.setConnectionState(ConnectionState.Connected);\n        } else if (connectionMode === \"write\") {\n            this.startJoinOpTimer();\n        }\n    }\n\n    private setConnectionState(value: ConnectionState.Disconnected, reason: string);\n    private setConnectionState(value: ConnectionState.Connected);\n    private setConnectionState(value: ConnectionState, reason?: string) {\n        if (this.connectionState === value) {\n            // Already in the desired state - exit early\n            this.logger.sendErrorEvent({ eventName: \"setConnectionStateSame\", value });\n            return;\n        }\n\n        const oldState = this._connectionState;\n        this._connectionState = value;\n        const quorumClients = this.handler.quorumClients();\n        let client: ILocalSequencedClient | undefined;\n        if (this._clientId !== undefined) {\n            client = quorumClients?.getMember(this._clientId);\n        }\n        if (value === ConnectionState.Connected) {\n            assert(oldState === ConnectionState.CatchingUp,\n                0x1d8 /* \"Should only transition from Connecting state\" */);\n            // Mark our old client should have left in the quorum if it's still there\n            if (client !== undefined) {\n                client.shouldHaveLeft = true;\n            }\n            this._clientId = this.pendingClientId;\n        } else if (value === ConnectionState.Disconnected) {\n            // Important as we process our own joinSession message through delta request\n            this._pendingClientId = undefined;\n            // Only wait for \"leave\" message if the connected client exists in the quorum because only the write\n            // client will exist in the quorum and only for those clients we will receive \"removeMember\" event and\n            // the client has some unacked ops.\n            // Also server would not accept ops from read client. Also check if the timer is not already running as\n            // we could receive \"Disconnected\" event multiple times without getting connected and in that case we\n            // don't want to reset the timer as we still want to wait on original client which started this timer.\n            if (client !== undefined\n                && this.handler.shouldClientJoinWrite()\n                && this.prevClientLeftTimer.hasTimer === false\n            ) {\n                this.prevClientLeftTimer.restart();\n            } else {\n                // Adding this event temporarily so that we can get help debugging if something goes wrong.\n                this.logger.sendTelemetryEvent({\n                    eventName: \"noWaitOnDisconnected\",\n                    inQuorum: client !== undefined,\n                    hasTimer: this.prevClientLeftTimer.hasTimer,\n                    shouldClientJoinWrite: this.handler.shouldClientJoinWrite(),\n                });\n            }\n        }\n\n        // Report transition before we propagate event across layers\n        this.handler.logConnectionStateChangeTelemetry(this._connectionState, oldState, reason);\n\n        // Propagate event across layers\n        this.handler.connectionStateChanged();\n    }\n\n    public initProtocol(protocol: ProtocolOpHandler) {\n        protocol.quorum.on(\"addMember\", (clientId, details) => {\n            this.receivedAddMemberEvent(clientId);\n        });\n\n        protocol.quorum.on(\"removeMember\", (clientId) => {\n            this.receivedRemoveMemberEvent(clientId);\n        });\n\n        // if we have a clientId from a previous container we need to wait for its leave message\n        if (this.clientId !== undefined && protocol.quorum.getMember(this.clientId) !== undefined) {\n            this.prevClientLeftTimer.restart();\n        }\n    }\n}\n"]}
+{"version":3,"file":"connectionStateHandler.js","sourceRoot":"","sources":["../src/connectionStateHandler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAI7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAmBpD,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,OAAO,sBAAsB;IAwB/B,YACqB,OAAsC,EACtC,MAAwB,EACjC,SAAkB;;QAFT,YAAO,GAAP,OAAO,CAA+B;QACtC,WAAM,GAAN,MAAM,CAAkB;QACjC,cAAS,GAAT,SAAS,CAAS;QA1BtB,qBAAgB,GAAG,eAAe,CAAC,YAAY,CAAC;QA4BpD,IAAI,CAAC,mBAAmB,GAAG,IAAI,KAAK;QAChC,+FAA+F;QAC/F,uDAAuD;QACvD,MAAA,IAAI,CAAC,OAAO,CAAC,sBAAsB,mCAAI,MAAM,EAC7C,GAAG,EAAE;YACD,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,EAClB,KAAK,CAAC,6EAA6E,CAAC,CAAC;YACzF,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC,CACJ,CAAC;QAEF,qGAAqG;QACrG,kGAAkG;QAClG,iEAAiE;QACjE,IAAI,CAAC,WAAW,GAAG,IAAI,KAAK,CACxB,eAAe,EACf,GAAG,EAAE;;YACD,gFAAgF;YAChF,iGAAiG;YACjG,IAAI,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;gBACrD,OAAO;aACV;YACD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;YACnD,MAAM,OAAO,GAAG;gBACZ,iBAAiB,EAAE,aAAa,KAAK,SAAS;gBAC9C,kBAAkB,EAAE,IAAI,CAAC,eAAe,KAAK,SAAS;gBACtD,QAAQ,EAAE,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,SAAS,CAAC,MAAA,IAAI,CAAC,eAAe,mCAAI,EAAE,CAAC,MAAK,SAAS;gBAC5E,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;aAC5C,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC,CACJ,CAAC;IACN,CAAC;IArDD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC,SAAS,CAAC;IAC9D,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAyCO,gBAAgB;QACpB,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAEO,eAAe;QACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAEM,OAAO;QACV,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC7D,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAEM,cAAc;QACjB,0GAA0G;QAC1G,6GAA6G;QAC7G,IAAI,IAAI,CAAC,iBAAiB,EAAE;YACxB,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,QAAgB;QAC3C,2DAA2D;QAC3D,IAAI,QAAQ,KAAK,IAAI,CAAC,eAAe,EAAE;YACnC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;aAC1B;iBAAM;gBACH,oEAAoE;gBACpE,+CAA+C;gBAC/C,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;aACrD;YACD,+DAA+D;YAC/D,IAAI,IAAI,CAAC,iBAAiB,EAAE;gBACxB,IAAI,CAAC,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE;oBACjD,SAAS,EAAE,uBAAuB;oBAClC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,cAAc,EAAE,IAAI,CAAC,SAAS;wBAC9B,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;qBAC1D,CAAC;iBACL,CAAC,CAAC;aACN;YACD,IAAI,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;SACjD;IACL,CAAC;IAEO,sBAAsB,CAAC,MAA6E;;QACxG,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACnD,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAE/F,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,KAAK;YACnC,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,EACrF,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAE5E,uFAAuF;QACvF,4FAA4F;QAC5F,0CAA0C;QAC1C,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC,QAAQ;eACnC,IAAI,CAAC,eAAe,KAAK,SAAS;eAClC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,SAAS;eAC3D,CAAC,IAAI,CAAC,iBAAiB,EAC5B;YACE,MAAA,IAAI,CAAC,SAAS,0CAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;aAAM;YACH,2FAA2F;YAC3F,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBAC3B,SAAS,EAAE,wBAAwB;gBACnC,QAAQ,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;gBACpD,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,MAAM;oBACN,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;oBACzC,QAAQ,EAAE,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,SAAS,CAAC,MAAA,IAAI,CAAC,eAAe,mCAAI,EAAE,CAAC,MAAK,SAAS;iBAC/E,CAAC;aACL,CAAC,CAAC;SACN;IACL,CAAC;IAEO,yBAAyB,CAAC,QAAgB;QAC9C,8DAA8D;QAC9D,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAC5B,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,CAAC,mBAAmB,CAAC,CAAC;SACpD;IACL,CAAC;IAEM,uBAAuB,CAAC,MAAc;QACzC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;YAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;SAC1B;QACD,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;OAMG;IACI,oBAAoB,CACvB,cAA8B,EAC9B,OAA2B;QAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC,UAAU,CAAC;QAEnD,MAAM,eAAe,GAAG,cAAc,KAAK,OAAO,CAAC;QACnD,MAAM,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAC3D,KAAK,CAAC,kEAAkE,CAAC,CAAC;QAC9E,MAAM,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAC7C,KAAK,CAAC,iGAAiG,CAAC,CAAC;QAE7G,0FAA0F;QAC1F,oDAAoD;QACpD,MAAM,aAAa,GAA+B,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QAE/E,wGAAwG;QACxG,qDAAqD;QACrD,+FAA+F;QAC/F,6FAA6F;QAC7F,6FAA6F;QAC7F,2FAA2F;QAC3F,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEzC,yGAAyG;QACzG,IAAI,CAAC,OAAO,CAAC,iCAAiC,CAAC,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAErF,gHAAgH;QAChH,+EAA+E;QAC/E,sGAAsG;QACtG,oCAAoC;QACpC,sGAAsG;QACtG,MAAM,gBAAgB,GAAG,eAAe,IAAI,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAK,SAAS,CAAC;QAE1G,IAAI,gBAAgB,EAAE;YAClB,2GAA2G;YAC3G,2EAA2E;YAC3E,IAAI,CAAC,gBAAgB,EAAE,CAAC;SAC3B;aAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAChC,2FAA2F;YAC3F,kDAAkD;YAClD,mGAAmG;YACnG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACtD;IACL,CAAC;IAIO,kBAAkB,CAAC,KAAsB,EAAE,MAAe;QAC9D,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE;YAChC,4CAA4C;YAC5C,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,wBAAwB,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3E,OAAO;SACV;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACnD,IAAI,MAAyC,CAAC;QAC9C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;YAC9B,MAAM,GAAG,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrD;QACD,IAAI,KAAK,KAAK,eAAe,CAAC,SAAS,EAAE;YACrC,MAAM,CAAC,QAAQ,KAAK,eAAe,CAAC,UAAU,EAC1C,KAAK,CAAC,oDAAoD,CAAC,CAAC;YAChE,yEAAyE;YACzE,IAAI,MAAM,KAAK,SAAS,EAAE;gBACtB,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;aAChC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC;SACzC;aAAM,IAAI,KAAK,KAAK,eAAe,CAAC,YAAY,EAAE;YAC/C,4EAA4E;YAC5E,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;YAClC,oGAAoG;YACpG,sGAAsG;YACtG,mCAAmC;YACnC,uGAAuG;YACvG,qGAAqG;YACrG,sGAAsG;YACtG,IAAI,MAAM,KAAK,SAAS;mBACjB,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;mBACpC,IAAI,CAAC,mBAAmB,CAAC,QAAQ,KAAK,KAAK,EAChD;gBACE,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;aACtC;iBAAM;gBACH,2FAA2F;gBAC3F,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC3B,SAAS,EAAE,sBAAsB;oBACjC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,QAAQ,EAAE,MAAM,KAAK,SAAS;wBAC9B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;wBACzC,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;qBAC1D,CAAC;iBACL,CAAC,CAAC;aACN;SACJ;QAED,4DAA4D;QAC5D,IAAI,CAAC,OAAO,CAAC,iCAAiC,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAExF,gCAAgC;QAChC,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAC1C,CAAC;IAEM,YAAY,CAAC,QAA0B;QAC1C,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;YACnD,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC5C,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,wFAAwF;QACxF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE;YACvF,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;SACtC;IACL,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger, ITelemetryProperties } from \"@fluidframework/common-definitions\";\nimport { assert, Timer } from \"@fluidframework/common-utils\";\nimport { IConnectionDetails } from \"@fluidframework/container-definitions\";\nimport { ILocalSequencedClient, IProtocolHandler } from \"@fluidframework/protocol-base\";\nimport { ConnectionMode, IQuorumClients } from \"@fluidframework/protocol-definitions\";\nimport { PerformanceEvent } from \"@fluidframework/telemetry-utils\";\nimport { ConnectionState } from \"./connectionState\";\n\n/** Constructor parameter type for passing in dependencies needed by the ConnectionStateHandler */\nexport interface IConnectionStateHandlerInputs {\n    /** Provides access to the clients currently in the quorum */\n    quorumClients: () => IQuorumClients | undefined;\n    /** Log to telemetry any change in state, included to Connecting */\n    logConnectionStateChangeTelemetry:\n        (value: ConnectionState, oldState: ConnectionState, reason?: string | undefined) => void;\n    /** Whether to expect the client to join in write mode on next connection */\n    shouldClientJoinWrite: () => boolean;\n    /** (Optional) How long should we wait on our previous client's Leave op before transitioning to Connected again */\n    maxClientLeaveWaitTime: number | undefined;\n    /** Log an issue encountered while in the Connecting state. details will be logged as a JSON string */\n    logConnectionIssue: (eventName: string, details?: ITelemetryProperties) => void;\n    /** Callback whenever the ConnectionState changes between Disconnected and Connected */\n    connectionStateChanged: () => void;\n}\n\nconst JoinOpTimeoutMs = 45000;\n\n/**\n * In the lifetime of a container, the connection will likely disconnect and reconnect periodically.\n * This class ensures that any ops sent by this container instance on previous connection are either\n * sequenced or blocked by the server before emitting the new \"connected\" event and allowing runtime to resubmit ops.\n *\n * Each connection is assigned a clientId by the service, and the connection is book-ended by a Join and a Leave op\n * generated by the service. Due to the distributed nature of the ordering service, in the case of reconnect we cannot\n * make any assumptions about ordering of operations between the old and new connections - i.e. new Join op could\n * be sequenced before old Leave op (and some acks from pending ops that were in flight when we disconnected).\n *\n * The job of this class is to encapsulate the transition period during reconnect, which is identified by\n * ConnectionState.CatchingUp. Specifically, before moving to Connected state with the new clientId, it ensures that:\n * (A) We process the Leave op for the previous clientId. This allows us to properly handle any acks from in-flight ops\n *     that got sequenced with the old clientId (we'll recognize them as local ops). After the Leave op, any other\n *     pending ops can safely be submitted with the new clientId without fear of duplication in the sequenced op stream.\n * (B) We process the Join op for the new clientId (identified when the underlying connection was first established),\n *     indicating the service is ready to sequence ops sent with the new clientId.\n *\n * For (A) we give up waiting after some time (same timeout as server uses), and go ahead and transition to Connected.\n * For (B) we log telemetry if it takes too long, but still only transition to Connected when the Join op is processed\n * and we are added to the Quorum.\n */\nexport class ConnectionStateHandler {\n    private _connectionState = ConnectionState.Disconnected;\n    private _pendingClientId: string | undefined;\n    private readonly prevClientLeftTimer: Timer;\n    private readonly joinOpTimer: Timer;\n\n    private waitEvent: PerformanceEvent | undefined;\n\n    public get connectionState(): ConnectionState {\n        return this._connectionState;\n    }\n\n    public get connected(): boolean {\n        return this.connectionState === ConnectionState.Connected;\n    }\n\n    public get clientId(): string | undefined {\n        return this._clientId;\n    }\n\n    public get pendingClientId(): string | undefined {\n        return this._pendingClientId;\n    }\n\n    constructor(\n        private readonly handler: IConnectionStateHandlerInputs,\n        private readonly logger: ITelemetryLogger,\n        private _clientId?: string,\n    ) {\n        this.prevClientLeftTimer = new Timer(\n            // Default is 5 min for which we are going to wait for its own \"leave\" message. This is same as\n            // the max time on server after which leave op is sent.\n            this.handler.maxClientLeaveWaitTime ?? 300000,\n            () => {\n                assert(!this.connected,\n                    0x2ac /* \"Connected when timeout waiting for leave from previous session fired!\" */);\n                this.applyForConnectedState(\"timeout\");\n            },\n        );\n\n        // Based on recent data, it looks like majority of cases where we get stuck are due to really slow or\n        // timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so\n        // if retrying fixes the problem, we should not see these events.\n        this.joinOpTimer = new Timer(\n            JoinOpTimeoutMs,\n            () => {\n                // I've observed timer firing within couple ms from disconnect event, looks like\n                // queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.\n                if (this.connectionState !== ConnectionState.CatchingUp) {\n                    return;\n                }\n                const quorumClients = this.handler.quorumClients();\n                const details = {\n                    quorumInitialized: quorumClients !== undefined,\n                    hasPendingClientId: this.pendingClientId !== undefined,\n                    inQuorum: quorumClients?.getMember(this.pendingClientId ?? \"\") !== undefined,\n                    waitingForLeaveOp: this.waitingForLeaveOp,\n                };\n                this.handler.logConnectionIssue(\"NoJoinOp\", details);\n            },\n        );\n    }\n\n    private startJoinOpTimer() {\n        assert(!this.joinOpTimer.hasTimer, 0x234 /* \"has joinOpTimer\" */);\n        this.joinOpTimer.start();\n    }\n\n    private stopJoinOpTimer() {\n        assert(this.joinOpTimer.hasTimer, 0x235 /* \"no joinOpTimer\" */);\n        this.joinOpTimer.clear();\n    }\n\n    private get waitingForLeaveOp() {\n        return this.prevClientLeftTimer.hasTimer;\n    }\n\n    public dispose() {\n        assert(!this.joinOpTimer.hasTimer, 0x2a5 /* \"join timer\" */);\n        this.prevClientLeftTimer.clear();\n    }\n\n    public containerSaved() {\n        // If we were waiting for moving to Connected state, then only apply for state change. Since the container\n        // is now saved and we don't have any ops to roundtrip, we can clear the timer and apply for connected state.\n        if (this.waitingForLeaveOp) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"containerSaved\");\n        }\n    }\n\n    private receivedAddMemberEvent(clientId: string) {\n        // This is the only one that requires the pending client ID\n        if (clientId === this.pendingClientId) {\n            if (this.joinOpTimer.hasTimer) {\n                this.stopJoinOpTimer();\n            } else {\n                // timer has already fired, meaning it took too long to get join on.\n                // Record how long it actually took to recover.\n                this.handler.logConnectionIssue(\"ReceivedJoinOp\");\n            }\n            // Start the event in case we are waiting for leave or timeout.\n            if (this.waitingForLeaveOp) {\n                this.waitEvent = PerformanceEvent.start(this.logger, {\n                    eventName: \"WaitBeforeClientLeave\",\n                    details: JSON.stringify({\n                        waitOnClientId: this._clientId,\n                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),\n                    }),\n                });\n            }\n            this.applyForConnectedState(\"addMemberEvent\");\n        }\n    }\n\n    private applyForConnectedState(source: \"removeMemberEvent\" | \"addMemberEvent\" | \"timeout\" | \"containerSaved\") {\n        const quorumClients = this.handler.quorumClients();\n        assert(quorumClients !== undefined, 0x236 /* \"In all cases it should be already installed\" */);\n\n        assert(this.waitingForLeaveOp === false ||\n            (this.clientId !== undefined && quorumClients.getMember(this.clientId) !== undefined),\n            0x2e2 /* \"Must only wait for leave message when clientId in quorum\" */);\n\n        // Move to connected state only if we are in Connecting state, we have seen our join op\n        // and there is no timer running which means we are not waiting for previous client to leave\n        // or timeout has occurred while doing so.\n        if (this.pendingClientId !== this.clientId\n            && this.pendingClientId !== undefined\n            && quorumClients.getMember(this.pendingClientId) !== undefined\n            && !this.waitingForLeaveOp\n        ) {\n            this.waitEvent?.end({ source });\n            this.setConnectionState(ConnectionState.Connected);\n        } else {\n            // Adding this event temporarily so that we can get help debugging if something goes wrong.\n            this.logger.sendTelemetryEvent({\n                eventName: \"connectedStateRejected\",\n                category: source === \"timeout\" ? \"error\" : \"generic\",\n                details: JSON.stringify({\n                    source,\n                    pendingClientId: this.pendingClientId,\n                    clientId: this.clientId,\n                    waitingForLeaveOp: this.waitingForLeaveOp,\n                    inQuorum: quorumClients?.getMember(this.pendingClientId ?? \"\") !== undefined,\n                }),\n            });\n        }\n    }\n\n    private receivedRemoveMemberEvent(clientId: string) {\n        // If the client which has left was us, then finish the timer.\n        if (this.clientId === clientId) {\n            this.prevClientLeftTimer.clear();\n            this.applyForConnectedState(\"removeMemberEvent\");\n        }\n    }\n\n    public receivedDisconnectEvent(reason: string) {\n        if (this.joinOpTimer.hasTimer) {\n            this.stopJoinOpTimer();\n        }\n        this.setConnectionState(ConnectionState.Disconnected, reason);\n    }\n\n    /**\n     * The \"connect\" event indicates the connection to the Relay Service is live.\n     * However, some additional conditions must be met before we can fully transition to\n     * \"Connected\" state. This function handles that interim period, known as \"Connecting\" state.\n     * @param connectionMode - Read or Write connection\n     * @param details - Connection details returned from the ordering service\n     */\n    public receivedConnectEvent(\n        connectionMode: ConnectionMode,\n        details: IConnectionDetails,\n    ) {\n        const oldState = this._connectionState;\n        this._connectionState = ConnectionState.CatchingUp;\n\n        const writeConnection = connectionMode === \"write\";\n        assert(writeConnection || !this.handler.shouldClientJoinWrite(),\n            0x30a /* shouldClientJoinWrite should imply this is a writeConnection */);\n        assert(writeConnection || !this.waitingForLeaveOp,\n            0x2a6 /* \"waitingForLeaveOp should imply writeConnection (we need to be ready to flush pending ops)\" */);\n\n        // Note that this may be undefined since the connection is established proactively on load\n        // and the quorum may still be under initialization.\n        const quorumClients: IQuorumClients | undefined = this.handler.quorumClients();\n\n        // Stash the clientID to detect when transitioning from connecting (socket.io channel open) to connected\n        // (have received the join message for the client ID)\n        // This is especially important in the reconnect case. It's possible there could be outstanding\n        // ops sent by this client, so we should keep the old client id until we see our own client's\n        // join message. after we see the join message for our new connection with our new client id,\n        // we know there can no longer be outstanding ops that we sent with the previous client id.\n        this._pendingClientId = details.clientId;\n\n        // IMPORTANT: Report telemetry after we set _pendingClientId, but before transitioning to Connected state\n        this.handler.logConnectionStateChangeTelemetry(ConnectionState.CatchingUp, oldState);\n\n        // For write connections, this pending clientId could be in the quorum already (i.e. join op already processed).\n        // We are fetching ops from storage in parallel to connecting to Relay Service,\n        // and given async processes, it's possible that we have already processed our own join message before\n        // connection was fully established.\n        // If quorumClients itself is undefined, we expect it will process the join op after it's initialized.\n        const waitingForJoinOp = writeConnection && quorumClients?.getMember(this._pendingClientId) === undefined;\n\n        if (waitingForJoinOp) {\n            // Previous client left, and we are waiting for our own join op. When it is processed we'll join the quorum\n            // and attempt to transition to Connected state via receivedAddMemberEvent.\n            this.startJoinOpTimer();\n        } else if (!this.waitingForLeaveOp) {\n            // We're not waiting for Join or Leave op (if read-only connection those don't even apply),\n            // go ahead and declare the state to be Connected!\n            // If we are waiting for Leave op still, do nothing for now, we will transition to Connected later.\n            this.setConnectionState(ConnectionState.Connected);\n        }\n    }\n\n    private setConnectionState(value: ConnectionState.Disconnected, reason: string): void;\n    private setConnectionState(value: ConnectionState.Connected): void;\n    private setConnectionState(value: ConnectionState, reason?: string): void {\n        if (this.connectionState === value) {\n            // Already in the desired state - exit early\n            this.logger.sendErrorEvent({ eventName: \"setConnectionStateSame\", value });\n            return;\n        }\n\n        const oldState = this._connectionState;\n        this._connectionState = value;\n        const quorumClients = this.handler.quorumClients();\n        let client: ILocalSequencedClient | undefined;\n        if (this._clientId !== undefined) {\n            client = quorumClients?.getMember(this._clientId);\n        }\n        if (value === ConnectionState.Connected) {\n            assert(oldState === ConnectionState.CatchingUp,\n                0x1d8 /* \"Should only transition from Connecting state\" */);\n            // Mark our old client should have left in the quorum if it's still there\n            if (client !== undefined) {\n                client.shouldHaveLeft = true;\n            }\n            this._clientId = this.pendingClientId;\n        } else if (value === ConnectionState.Disconnected) {\n            // Important as we process our own joinSession message through delta request\n            this._pendingClientId = undefined;\n            // Only wait for \"leave\" message if the connected client exists in the quorum because only the write\n            // client will exist in the quorum and only for those clients we will receive \"removeMember\" event and\n            // the client has some unacked ops.\n            // Also server would not accept ops from read client. Also check if the timer is not already running as\n            // we could receive \"Disconnected\" event multiple times without getting connected and in that case we\n            // don't want to reset the timer as we still want to wait on original client which started this timer.\n            if (client !== undefined\n                && this.handler.shouldClientJoinWrite()\n                && this.prevClientLeftTimer.hasTimer === false\n            ) {\n                this.prevClientLeftTimer.restart();\n            } else {\n                // Adding this event temporarily so that we can get help debugging if something goes wrong.\n                this.logger.sendTelemetryEvent({\n                    eventName: \"noWaitOnDisconnected\",\n                    details: JSON.stringify({\n                        inQuorum: client !== undefined,\n                        waitingForLeaveOp: this.waitingForLeaveOp,\n                        hadOutstandingOps: this.handler.shouldClientJoinWrite(),\n                    }),\n                });\n            }\n        }\n\n        // Report transition before we propagate event across layers\n        this.handler.logConnectionStateChangeTelemetry(this._connectionState, oldState, reason);\n\n        // Propagate event across layers\n        this.handler.connectionStateChanged();\n    }\n\n    public initProtocol(protocol: IProtocolHandler) {\n        protocol.quorum.on(\"addMember\", (clientId, _details) => {\n            this.receivedAddMemberEvent(clientId);\n        });\n\n        protocol.quorum.on(\"removeMember\", (clientId) => {\n            this.receivedRemoveMemberEvent(clientId);\n        });\n\n        // if we have a clientId from a previous container we need to wait for its leave message\n        if (this.clientId !== undefined && protocol.quorum.getMember(this.clientId) !== undefined) {\n            this.prevClientLeftTimer.restart();\n        }\n    }\n}\n"]}

package/lib/container.d.ts

@@ -83,8 +83,7 @@ export declare class Container extends EventEmitterWithErrorHandling<IContainerE
     private readonly _canReconnect;
     private readonly mc;
     private _lifecycleState;
-    private get loaded();
-    private set loaded(value);
+    private setLoaded;
     get closed(): boolean;
     private _attachState;
     private readonly _storage;
@@ -173,6 +172,7 @@ export declare class Container extends EventEmitterWithErrorHandling<IContainerE
      */
     getQuorum(): IQuorumClients;
     close(error?: ICriticalContainerError): void;
+    private closeCore;
     closeAndGetPendingLocalState(): string;
     get attachState(): AttachState;
     serialize(): string;

package/lib/container.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"container.d.ts","sourceRoot":"","sources":["../src/container.ts"],"names":[],"mappings":"AAAA;;;GAGG;AASH,OAAO,EACH,QAAQ,EACR,SAAS,EACT,YAAY,EACf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACH,SAAS,EAET,UAAU,EACV,gBAAgB,EAChB,aAAa,EACb,uBAAuB,EAEvB,WAAW,EAEX,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EAEpB,MAAM,uCAAuC,CAAC;AAO/C,OAAO,EAEH,uBAAuB,EACvB,iBAAiB,EACjB,YAAY,EACf,MAAM,oCAAoC,CAAC;AAc5C,OAAO,EAEH,oBAAoB,EACpB,cAAc,EAGd,gBAAgB,EAEhB,cAAc,EACd,cAAc,EAGd,yBAAyB,EAOzB,QAAQ,EAGX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAEH,6BAA6B,EAG7B,eAAe,EAOlB,MAAM,iCAAiC,CAAC;AAMzC,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,MAAM,UAAU,CAAC;AAUlE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAOpD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC,WAAW,EAAE,iBAAiB,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC7B,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC;;OAEG;IACH,wBAAwB,CAAC,EAAE,sBAAsB,CAAC;CACrD;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,sBAAsB,CAAC,SAAS,EAAE,UAAU,oBA6DjE;AAMD;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,cAAc,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,qBAAa,SAAU,SAAQ,6BAA6B,CAAC,gBAAgB,CAAE,YAAW,UAAU;IAsS5F,OAAO,CAAC,QAAQ,CAAC,MAAM;IArS3B,OAAc,OAAO,SAAY;IAEjC;;OAEG;WACiB,IAAI,CACpB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,qBAAqB,EAClC,iBAAiB,CAAC,EAAE,sBAAsB,GAC3C,OAAO,CAAC,SAAS,CAAC;IAmDrB;;OAEG;WACiB,cAAc,CAC9B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,iBAAiB,GAC/B,OAAO,CAAC,SAAS,CAAC;IAgBrB;;;OAGG;WACiB,6BAA6B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,SAAS,CAAC;IAgBd,SAAS,EAAE,eAAe,CAAC;IAIlC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAiB;IAE/C,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,eAAe,CAAsE;IAE7F,OAAO,KAAK,MAAM,GAEjB;IAED,OAAO,KAAK,MAAM,QASjB;IAED,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED,OAAO,CAAC,YAAY,CAAwB;IAE5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0B;IACnD,IAAW,OAAO,IAAI,uBAAuB,CAE5C;IAED,OAAO,CAAC,eAAe,CAAoD;IAC3E,OAAO,KAAK,cAAc,GAKzB;IAED,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAA6B;IACnE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IAErC,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,KAAK,OAAO,GAKlB;IACD,OAAO,CAAC,gBAAgB,CAAgC;IACxD,OAAO,KAAK,eAAe,GAK1B;IAED,OAAO,CAAC,4BAA4B,CAAS;IAC7C,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAgB;IAC1D,OAAO,CAAC,8BAA8B,CAAa;IACnD,OAAO,CAAC,kBAAkB,CAAuB;IACjD,OAAO,CAAC,YAAY,CAAgC;IACpD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAEhC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA2B;IAClE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAyB;IAEhE,OAAO,CAAC,oBAAoB,CAAqB;IAEjD,OAAO,CAAC,mBAAmB,CAAkC;IAE7D,OAAO,KAAK,cAAc,GAAkE;IAE5F,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,WAAW,IAAI,YAAY,GAAG,SAAS,CAEjD;IAED,IAAW,iBAAiB,IAAI,QAAQ,GAAG,SAAS,CAEnD;IAED,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED,IAAW,WAAW,IAAI,WAAW,CAEpC;IAED;;OAEG;IACI,aAAa,CAAC,QAAQ,EAAE,OAAO;IAItC,IAAW,YAAY,IAAI,aAAa,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,CAEpF;IAED,IAAW,eAAe,IAAI,eAAe,CAE5C;IAED,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED;;;OAGG;IACH,IAAW,oBAAoB,IAAI,oBAAoB,GAAG,SAAS,CAElE;IAED;;;OAGG;IACH,IAAW,QAAQ,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;OAGG;IACH,IAAW,MAAM,IAAI,MAAM,EAAE,GAAG,SAAS,CAExC;IAED,IAAW,aAAa,IAAI,cAAc,CAEzC;IAED;;;OAGG;IACI,uBAAuB,IAAI,iBAAiB,GAAG,SAAS;IAI/D;;;;OAIG;IACI,oBAAoB,IAAI,iBAAiB,GAAG,SAAS;IAI5D;;OAEG;IACH,IAAW,QAAQ,IAAI,SAAS,CAE/B;IAED;;;;OAIG;IACH,IAAW,OAAO,YAEjB;IAED,OAAO,KAAK,cAAc,GAA0D;IACpF,OAAO,KAAK,WAAW,GAA+C;IACtE,SAAgB,OAAO,EAAE,cAAc,CAAC;IACxC,OAAO,KAAK,KAAK,GAAyC;IAC1D,OAAO,KAAK,UAAU,GAA8C;gBAG/C,MAAM,EAAE,MAAM,EAC/B,MAAM,EAAE,gBAAgB;IAuK5B;;OAEG;IACI,SAAS,IAAI,cAAc;IAI3B,KAAK,CAAC,KAAK,CAAC,EAAE,uBAAuB;IAkDrC,4BAA4B,IAAI,MAAM;IAsB7C,IAAW,WAAW,IAAI,WAAW,CAEpC;IAEM,SAAS,IAAI,MAAM;IAab,MAAM,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAoHxC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;IASxD,OAAO,CAAC,wBAAwB;IAqBzB,OAAO;IAad,OAAO,CAAC,eAAe;IAahB,UAAU;IAQjB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,cAAc;IAcT,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAWhE,kBAAkB,CAAC,WAAW,EAAE,iBAAiB;YAmBhD,mBAAmB;YAiBnB,UAAU;IAKxB,OAAO,CAAC,sBAAsB;IAM9B,OAAO,CAAC,oBAAoB;IAW5B;;;;;;OAMG;YACW,IAAI;YAkJJ,cAAc;YA4Bd,6BAA6B;YAqC7B,qBAAqB;YAsBrB,qBAAqB;YA2BrB,mCAAmC;YA2BnC,uBAAuB;IAkDrC,OAAO,CAAC,sBAAsB;IAmC9B,OAAO,CAAC,wBAAwB;IAQhC,OAAO,KAAK,MAAM,GAkBjB;IAED;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAKxB,OAAO,CAAC,kBAAkB;YA4DZ,2BAA2B;IAgBzC,OAAO,CAAC,iCAAiC;IA0DzC,OAAO,CAAC,wBAAwB;IAuBhC,OAAO,CAAC,sBAAsB;IA4B9B,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,oBAAoB;IA2D5B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,aAAa;IAiBrB;;;;OAIG;YACW,iBAAiB;YAiBjB,0BAA0B;YAgB1B,kBAAkB;IAgChC,OAAO,CAAC,yBAAyB;IAQjC,OAAO,CAAC,iBAAiB;CAG5B"}
+{"version":3,"file":"container.d.ts","sourceRoot":"","sources":["../src/container.ts"],"names":[],"mappings":"AAAA;;;GAGG;AASH,OAAO,EACH,QAAQ,EACR,SAAS,EACT,YAAY,EACf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACH,SAAS,EAET,UAAU,EACV,gBAAgB,EAChB,aAAa,EACb,uBAAuB,EAEvB,WAAW,EAEX,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EAEpB,MAAM,uCAAuC,CAAC;AAO/C,OAAO,EAEH,uBAAuB,EACvB,iBAAiB,EACjB,YAAY,EACf,MAAM,oCAAoC,CAAC;AAe5C,OAAO,EAEH,oBAAoB,EACpB,cAAc,EAGd,gBAAgB,EAEhB,cAAc,EACd,cAAc,EAGd,yBAAyB,EAOzB,QAAQ,EAGX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAEH,6BAA6B,EAG7B,eAAe,EAOlB,MAAM,iCAAiC,CAAC;AAMzC,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,MAAM,UAAU,CAAC;AAUlE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAOpD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC,WAAW,EAAE,iBAAiB,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC7B,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC;IACvC;;OAEG;IACH,wBAAwB,CAAC,EAAE,sBAAsB,CAAC;CACrD;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,sBAAsB,CAAC,SAAS,EAAE,UAAU,oBA6DjE;AAMD;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,cAAc,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,qBAAa,SAAU,SAAQ,6BAA6B,CAAC,gBAAgB,CAAE,YAAW,UAAU;IA8R5F,OAAO,CAAC,QAAQ,CAAC,MAAM;IA7R3B,OAAc,OAAO,SAAY;IAEjC;;OAEG;WACiB,IAAI,CACpB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,qBAAqB,EAClC,iBAAiB,CAAC,EAAE,sBAAsB,GAC3C,OAAO,CAAC,SAAS,CAAC;IAkDrB;;OAEG;WACiB,cAAc,CAC9B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,iBAAiB,GAC/B,OAAO,CAAC,SAAS,CAAC;IAerB;;;OAGG;WACiB,6BAA6B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,SAAS,CAAC;IAed,SAAS,EAAE,eAAe,CAAC;IAIlC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAiB;IAE/C,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,eAAe,CAA0D;IAEjF,OAAO,CAAC,SAAS;IAUjB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED,OAAO,CAAC,YAAY,CAAwB;IAE5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0B;IACnD,IAAW,OAAO,IAAI,uBAAuB,CAE5C;IAED,OAAO,CAAC,eAAe,CAAoD;IAC3E,OAAO,KAAK,cAAc,GAKzB;IAED,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAA6B;IACnE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IAErC,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,KAAK,OAAO,GAKlB;IACD,OAAO,CAAC,gBAAgB,CAA+B;IACvD,OAAO,KAAK,eAAe,GAK1B;IAED,OAAO,CAAC,4BAA4B,CAAS;IAC7C,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAgB;IAC1D,OAAO,CAAC,8BAA8B,CAAa;IACnD,OAAO,CAAC,kBAAkB,CAAuB;IACjD,OAAO,CAAC,YAAY,CAAgC;IACpD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAEhC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA2B;IAClE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAyB;IAEhE,OAAO,CAAC,oBAAoB,CAAqB;IAEjD,OAAO,CAAC,mBAAmB,CAAkC;IAE7D,OAAO,KAAK,cAAc,GAAkE;IAE5F,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,WAAW,IAAI,YAAY,GAAG,SAAS,CAEjD;IAED,IAAW,iBAAiB,IAAI,QAAQ,GAAG,SAAS,CAEnD;IAED,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED,IAAW,WAAW,IAAI,WAAW,CAEpC;IAED;;OAEG;IACI,aAAa,CAAC,QAAQ,EAAE,OAAO;IAItC,IAAW,YAAY,IAAI,aAAa,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,CAEpF;IAED,IAAW,eAAe,IAAI,eAAe,CAE5C;IAED,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED;;;OAGG;IACH,IAAW,oBAAoB,IAAI,oBAAoB,GAAG,SAAS,CAElE;IAED;;;OAGG;IACH,IAAW,QAAQ,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;OAGG;IACH,IAAW,MAAM,IAAI,MAAM,EAAE,GAAG,SAAS,CAExC;IAED,IAAW,aAAa,IAAI,cAAc,CAEzC;IAED;;;OAGG;IACI,uBAAuB,IAAI,iBAAiB,GAAG,SAAS;IAI/D;;;;OAIG;IACI,oBAAoB,IAAI,iBAAiB,GAAG,SAAS;IAI5D;;OAEG;IACH,IAAW,QAAQ,IAAI,SAAS,CAE/B;IAED;;;;OAIG;IACH,IAAW,OAAO,YAEjB;IAED,OAAO,KAAK,cAAc,GAA0D;IACpF,OAAO,KAAK,WAAW,GAA+C;IACtE,SAAgB,OAAO,EAAE,cAAc,CAAC;IACxC,OAAO,KAAK,KAAK,GAAyC;IAC1D,OAAO,KAAK,UAAU,GAA8C;gBAG/C,MAAM,EAAE,MAAM,EAC/B,MAAM,EAAE,gBAAgB;IAyK5B;;OAEG;IACI,SAAS,IAAI,cAAc;IAI3B,KAAK,CAAC,KAAK,CAAC,EAAE,uBAAuB;IAY5C,OAAO,CAAC,SAAS;IA6CV,4BAA4B,IAAI,MAAM;IAwB7C,IAAW,WAAW,IAAI,WAAW,CAEpC;IAEM,SAAS,IAAI,MAAM;IAab,MAAM,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAoHxC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;IASxD,OAAO,CAAC,wBAAwB;IAqBzB,OAAO;IAad,OAAO,CAAC,eAAe;IAahB,UAAU;IAQjB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,cAAc;IAcT,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAWhE,kBAAkB,CAAC,WAAW,EAAE,iBAAiB;YAmBhD,mBAAmB;YAiBnB,UAAU;IAKxB,OAAO,CAAC,sBAAsB;IAM9B,OAAO,CAAC,oBAAoB;IAW5B;;;;;;OAMG;YACW,IAAI;YA+IJ,cAAc;YA0Bd,6BAA6B;YAmC7B,qBAAqB;YAsBrB,qBAAqB;YA2BrB,mCAAmC;YA2BnC,uBAAuB;IAkDrC,OAAO,CAAC,sBAAsB;IA2B9B,OAAO,CAAC,wBAAwB;IAQhC,OAAO,KAAK,MAAM,GAkBjB;IAED;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAKxB,OAAO,CAAC,kBAAkB;YA4DZ,2BAA2B;IAgBzC,OAAO,CAAC,iCAAiC;IA0DzC,OAAO,CAAC,wBAAwB;IAyBhC,OAAO,CAAC,sBAAsB;IA4B9B,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,oBAAoB;IA6C5B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,aAAa;IAiBrB;;;;OAIG;YACW,iBAAiB;YAiBjB,0BAA0B;YAgB1B,kBAAkB;IAgChC,OAAO,CAAC,yBAAyB;IAQjC,OAAO,CAAC,iBAAiB;CAG5B"}