@fluidframework/container-runtime 0.56.5 → 0.56.8

package/lib/containerRuntime.js

@@ -1,7 +1,3 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
 import { assert, Trace, TypedEventEmitter, unreachableCase, performance, } from "@fluidframework/common-utils";
 import { ChildLogger, raiseConnectedEvent, PerformanceEvent, normalizeError, TaggedLoggerAdapter, loggerToMonitoringContext, } from "@fluidframework/telemetry-utils";
@@ -59,6 +55,7 @@ const DefaultSummaryConfiguration = {
 ;
 // Local storage key to set the default flush mode to TurnBased
 const turnBasedFlushModeKey = "Fluid.ContainerRuntime.FlushModeTurnBased";
+const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
 export var RuntimeMessage;
 (function (RuntimeMessage) {
     RuntimeMessage["FluidDataStoreOp"] = "component";
@@ -332,7 +329,7 @@ export function getDeviceSpec() {
  */
 export class ContainerRuntime extends TypedEventEmitter {
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, requestHandler, _storage) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c, _d, _e, _f;
         super();
         this.context = context;
         this.registry = registry;
@@ -341,10 +338,12 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.logger = logger;
         this.requestHandler = requestHandler;
         this._storage = _storage;
+        this.defaultMaxConsecutiveReconnects = 15;
         this._orderSequentiallyCalls = 0;
         this.needsFlush = false;
         this.flushTrigger = false;
         this.paused = false;
+        this.consecutiveReconnects = 0;
         this._disposed = false;
         this.dirtyContainer = false;
         this.emitDirtyDocumentEvent = true;
@@ -435,6 +434,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const timestamp = client === null || client === void 0 ? void 0 : client.timestamp;
             return (_c = (_b = (_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp) !== null && _b !== void 0 ? _b : timestamp) !== null && _c !== void 0 ? _c : Date.now();
         };
+        this.maxConsecutiveReconnects = (_c = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _c !== void 0 ? _c : this.defaultMaxConsecutiveReconnects;
         this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), getCurrentTimestamp, this.closeFn, context.baseSnapshot, async (id) => readAndParse(this.storage, id), this.mc.logger, existing, metadata);
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"), 
@@ -476,8 +476,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             const orderedClientLogger = ChildLogger.create(this.logger, "OrderedClientElection");
             const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
             const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
-            const summarizerClientElectionEnabled = (_c = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _c !== void 0 ? _c : ((_d = this.runtimeOptions.summaryOptions) === null || _d === void 0 ? void 0 : _d.summarizerClientElection) === true;
-            const maxOpsSinceLastSummary = (_e = this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary) !== null && _e !== void 0 ? _e : 7000;
+            const summarizerClientElectionEnabled = (_d = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _d !== void 0 ? _d : ((_e = this.runtimeOptions.summaryOptions) === null || _e === void 0 ? void 0 : _e.summarizerClientElection) === true;
+            const maxOpsSinceLastSummary = (_f = this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary) !== null && _f !== void 0 ? _f : 7000;
             this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, maxOpsSinceLastSummary, summarizerClientElectionEnabled);
             if (this.context.clientDetails.type === summarizerClientType) {
                 this._summarizer = new Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime));
@@ -883,6 +883,37 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
+    // Track how many times the container tries to reconnect with pending messages.
+    // This happens when the connection state is changed and we reset the counter
+    // when we are able to process a local op or when there are no pending messages.
+    // If this counter reaches a max, it's a good indicator that the container
+    // is not making progress and it is stuck in a retry loop.
+    shouldContinueReconnecting() {
+        if (this.maxConsecutiveReconnects <= 0) {
+            // Feature disabled, we never stop reconnecting
+            return true;
+        }
+        if (!this.pendingStateManager.hasPendingMessages()) {
+            // If there are no pending messages, we can always reconnect
+            this.resetReconnectCount();
+            return true;
+        }
+        this.consecutiveReconnects++;
+        if (this.consecutiveReconnects === Math.floor(this.maxConsecutiveReconnects / 2)) {
+            // If we're halfway through the max reconnects, send an event in order
+            // to better identify false positives, if any. If the rate of this event
+            // matches `MaxReconnectsWithNoProgress`, we can safely cut down
+            // maxConsecutiveReconnects to half.
+            this.mc.logger.sendTelemetryEvent({
+                eventName: "ReconnectsWithNoProgress",
+                attempts: this.consecutiveReconnects,
+            });
+        }
+        return this.consecutiveReconnects < this.maxConsecutiveReconnects;
+    }
+    resetReconnectCount() {
+        this.consecutiveReconnects = 0;
+    }
     replayPendingStates() {
         // We need to be able to send ops to replay states
         if (!this.canSendOps()) {
@@ -935,6 +966,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (changeOfState) {
             this.deltaManager.off("op", this.onOp);
             this.context.pendingLocalState = undefined;
+            if (!this.shouldContinueReconnecting()) {
+                this.closeFn(new GenericError("MaxReconnectsWithNoProgress", undefined, // error
+                { attempts: this.consecutiveReconnects }));
+                return;
+            }
             this.replayPendingStates();
         }
         this.dataStores.setConnectionState(connected, clientId);
@@ -987,6 +1023,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             this.emit("op", message);
             this.scheduleManager.afterOpProcessing(undefined, message);
+            if (local) {
+                // If we have processed a local op, this means that the container is
+                // making progress and we can reset the counter for how many times
+                // we have consecutively replayed the pending states
+                this.resetReconnectCount();
+            }
         }
         catch (e) {
             this.scheduleManager.afterOpProcessing(e, message);