@fluidframework/container-runtime 0.56.6 → 0.56.7

package/lib/packageVersion.d.ts

@@ -5,5 +5,5 @@
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
  */
 export declare const pkgName = "@fluidframework/container-runtime";
-export declare const pkgVersion = "0.56.6";
+export declare const pkgVersion = "0.56.7";
 //# sourceMappingURL=packageVersion.d.ts.map

package/lib/packageVersion.js

@@ -5,5 +5,5 @@
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
  */
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "0.56.6";
+export const pkgVersion = "0.56.7";
 //# sourceMappingURL=packageVersion.js.map

package/lib/packageVersion.js.map

@@ -1 +1 @@
-{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,QAAQ,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"0.56.6\";\n"]}
+{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,QAAQ,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"0.56.7\";\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "0.56.6",
+  "version": "0.56.7",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": "https://github.com/microsoft/FluidFramework",
@@ -59,26 +59,26 @@
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^0.32.1",
     "@fluidframework/container-definitions": "^0.45.0",
-    "@fluidframework/container-runtime-definitions": "^0.56.6",
-    "@fluidframework/container-utils": "^0.56.6",
+    "@fluidframework/container-runtime-definitions": "^0.56.7",
+    "@fluidframework/container-utils": "^0.56.7",
     "@fluidframework/core-interfaces": "^0.42.0",
-    "@fluidframework/datastore": "^0.56.6",
+    "@fluidframework/datastore": "^0.56.7",
     "@fluidframework/driver-definitions": "^0.44.0",
-    "@fluidframework/driver-utils": "^0.56.6",
-    "@fluidframework/garbage-collector": "^0.56.6",
+    "@fluidframework/driver-utils": "^0.56.7",
+    "@fluidframework/garbage-collector": "^0.56.7",
     "@fluidframework/protocol-base": "^0.1034.0",
     "@fluidframework/protocol-definitions": "^0.1026.0",
-    "@fluidframework/runtime-definitions": "^0.56.6",
-    "@fluidframework/runtime-utils": "^0.56.6",
-    "@fluidframework/telemetry-utils": "^0.56.6",
+    "@fluidframework/runtime-definitions": "^0.56.7",
+    "@fluidframework/runtime-utils": "^0.56.7",
+    "@fluidframework/telemetry-utils": "^0.56.7",
     "double-ended-queue": "^2.1.0-0",
     "uuid": "^8.3.1"
   },
   "devDependencies": {
     "@fluidframework/build-common": "^0.23.0",
     "@fluidframework/eslint-config-fluid": "^0.25.0",
-    "@fluidframework/mocha-test-setup": "^0.56.6",
-    "@fluidframework/test-runtime-utils": "^0.56.6",
+    "@fluidframework/mocha-test-setup": "^0.56.7",
+    "@fluidframework/test-runtime-utils": "^0.56.7",
     "@microsoft/api-extractor": "^7.16.1",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/double-ended-queue": "^2.1.0",

package/src/containerRuntime.ts

@@ -2,7 +2,8 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-
+// See #9219
+/* eslint-disable max-lines */
 import { EventEmitter } from "events";
 import { ITelemetryBaseLogger, ITelemetryGenericEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
 import {
@@ -301,6 +302,7 @@ interface OldContainerContextWithLogger extends IContainerContext {
 
 // Local storage key to set the default flush mode to TurnBased
 const turnBasedFlushModeKey = "Fluid.ContainerRuntime.FlushModeTurnBased";
+const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
 
 export enum RuntimeMessage {
     FluidDataStoreOp = "component",
@@ -854,6 +856,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
     private readonly summarizerNode: IRootSummarizerNodeWithGC;
 
+    private readonly maxConsecutiveReconnects: number;
+    private readonly defaultMaxConsecutiveReconnects = 15;
+
     private _orderSequentiallyCalls: number = 0;
     private _flushMode: FlushMode;
     private needsFlush = false;
@@ -863,6 +868,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
     private paused: boolean = false;
 
+    private consecutiveReconnects = 0;
+
     public get connected(): boolean {
         return this._connected;
     }
@@ -990,6 +997,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             const timestamp = client?.timestamp;
             return this.deltaManager.lastMessage?.timestamp ?? timestamp ?? Date.now();
         };
+
+        this.maxConsecutiveReconnects =
+            this.mc.config.getNumber(maxConsecutiveReconnectsKey) ?? this.defaultMaxConsecutiveReconnects;
+
         this.garbageCollector = GarbageCollector.create(
             this,
             this.runtimeOptions.gcOptions,
@@ -1415,6 +1426,42 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         }
     }
 
+    // Track how many times the container tries to reconnect with pending messages.
+    // This happens when the connection state is changed and we reset the counter
+    // when we are able to process a local op or when there are no pending messages.
+    // If this counter reaches a max, it's a good indicator that the container
+    // is not making progress and it is stuck in a retry loop.
+    private shouldContinueReconnecting(): boolean {
+        if (this.maxConsecutiveReconnects <= 0) {
+            // Feature disabled, we never stop reconnecting
+            return true;
+        }
+
+        if (!this.pendingStateManager.hasPendingMessages()) {
+            // If there are no pending messages, we can always reconnect
+            this.resetReconnectCount();
+            return true;
+        }
+
+        this.consecutiveReconnects++;
+        if (this.consecutiveReconnects === Math.floor(this.maxConsecutiveReconnects / 2)) {
+            // If we're halfway through the max reconnects, send an event in order
+            // to better identify false positives, if any. If the rate of this event
+            // matches `MaxReconnectsWithNoProgress`, we can safely cut down
+            // maxConsecutiveReconnects to half.
+            this.mc.logger.sendTelemetryEvent({
+                eventName: "ReconnectsWithNoProgress",
+                attempts: this.consecutiveReconnects,
+            });
+        }
+
+        return this.consecutiveReconnects < this.maxConsecutiveReconnects;
+    }
+
+    private resetReconnectCount() {
+        this.consecutiveReconnects = 0;
+    }
+
     private replayPendingStates() {
         // We need to be able to send ops to replay states
         if (!this.canSendOps()) { return; }
@@ -1495,6 +1542,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         if (changeOfState) {
             this.deltaManager.off("op", this.onOp);
             this.context.pendingLocalState = undefined;
+            if (!this.shouldContinueReconnecting()) {
+                this.closeFn(new GenericError(
+                    "MaxReconnectsWithNoProgress",
+                    undefined, // error
+                    { attempts: this.consecutiveReconnects }));
+                return;
+            }
+
             this.replayPendingStates();
         }
 
@@ -1558,6 +1613,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
             this.emit("op", message);
             this.scheduleManager.afterOpProcessing(undefined, message);
+
+            if (local) {
+                // If we have processed a local op, this means that the container is
+                // making progress and we can reset the counter for how many times
+                // we have consecutively replayed the pending states
+                this.resetReconnectCount();
+            }
         } catch (e) {
             this.scheduleManager.afterOpProcessing(e, message);
             throw e;

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "0.56.6";
+export const pkgVersion = "0.56.7";