@fluidframework/container-runtime 0.59.1001-62246 → 0.59.2000

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "0.59.1001-62246",
+  "version": "0.59.2000",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -16,7 +16,7 @@
   "types": "dist/index.d.ts",
   "scripts": {
     "build": "npm run build:genver && concurrently npm:build:compile npm:lint && npm run build:docs",
-    "build:commonjs": "npm run tsc && npm run build:test",
+    "build:commonjs": "npm run tsc && npm run typetests:gen && npm run build:test",
     "build:compile": "concurrently npm:build:commonjs npm:build:esnext",
     "build:docs": "api-extractor run --local --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
     "build:esnext": "tsc --project ./tsconfig.esnext.json",
@@ -38,8 +38,7 @@
     "tsc:watch": "tsc --watch",
     "tsfmt": "tsfmt --verify",
     "tsfmt:fix": "tsfmt --replace",
-    "typetests:gen": "fluid-type-validator -d .",
-    "typetests:prepare": "fluid-type-validator -d . -p"
+    "typetests:gen": "fluid-type-validator -g -d ."
   },
   "nyc": {
     "all": true,
@@ -65,29 +64,29 @@
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^0.32.1",
     "@fluidframework/container-definitions": "^0.48.1000",
-    "@fluidframework/container-runtime-definitions": "0.59.1001-62246",
-    "@fluidframework/container-utils": "0.59.1001-62246",
+    "@fluidframework/container-runtime-definitions": "^0.59.2000",
+    "@fluidframework/container-utils": "^0.59.2000",
     "@fluidframework/core-interfaces": "^0.43.1000",
-    "@fluidframework/datastore": "0.59.1001-62246",
+    "@fluidframework/datastore": "^0.59.2000",
     "@fluidframework/driver-definitions": "^0.46.1000",
-    "@fluidframework/driver-utils": "0.59.1001-62246",
-    "@fluidframework/garbage-collector": "0.59.1001-62246",
+    "@fluidframework/driver-utils": "^0.59.2000",
+    "@fluidframework/garbage-collector": "^0.59.2000",
     "@fluidframework/protocol-base": "^0.1036.1000",
     "@fluidframework/protocol-definitions": "^0.1028.1000",
-    "@fluidframework/runtime-definitions": "0.59.1001-62246",
-    "@fluidframework/runtime-utils": "0.59.1001-62246",
-    "@fluidframework/telemetry-utils": "0.59.1001-62246",
+    "@fluidframework/runtime-definitions": "^0.59.2000",
+    "@fluidframework/runtime-utils": "^0.59.2000",
+    "@fluidframework/telemetry-utils": "^0.59.2000",
     "double-ended-queue": "^2.1.0-0",
     "uuid": "^8.3.1"
   },
   "devDependencies": {
     "@fluidframework/build-common": "^0.23.0",
-    "@fluidframework/build-tools": "^0.2.61288",
-    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@^0.58.0",
+    "@fluidframework/build-tools": "^0.2.61804",
+    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@0.59.1000",
     "@fluidframework/eslint-config-fluid": "^0.28.1000",
-    "@fluidframework/mocha-test-setup": "0.59.1001-62246",
-    "@fluidframework/test-runtime-utils": "0.59.1001-62246",
-    "@microsoft/api-extractor": "^7.16.1",
+    "@fluidframework/mocha-test-setup": "^0.59.2000",
+    "@fluidframework/test-runtime-utils": "^0.59.2000",
+    "@microsoft/api-extractor": "^7.22.2",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/double-ended-queue": "^2.1.0",
     "@types/mocha": "^8.2.2",
@@ -103,8 +102,11 @@
     "eslint-plugin-editorconfig": "~3.2.0",
     "eslint-plugin-eslint-comments": "~3.2.0",
     "eslint-plugin-import": "~2.25.4",
-    "eslint-plugin-no-null": "~1.0.2",
+    "eslint-plugin-jest": "~26.1.3",
+    "eslint-plugin-mocha": "~10.0.3",
+    "eslint-plugin-promise": "~6.0.0",
     "eslint-plugin-react": "~7.28.0",
+    "eslint-plugin-tsdoc": "~0.2.14",
     "eslint-plugin-unicorn": "~40.0.0",
     "mocha": "^8.4.0",
     "nyc": "^15.0.0",
@@ -114,42 +116,16 @@
     "typescript-formatter": "7.1.0"
   },
   "typeValidation": {
-    "version": "0.59.1000",
+    "version": "0.59.2000",
     "broken": {
-      "0.58.2002": {
-        "InterfaceDeclaration_IBaseSummarizeResult": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_IGenerateSummaryTreeResult": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_ISubmitSummaryOpResult": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_IUploadSummaryResult": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_SubmitSummaryResult": {
-          "forwardCompat": false
-        },
-        "TypeAliasDeclaration_SubmitSummaryResult": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_IGeneratedSummaryStats": {
-          "forwardCompat": false
-        },
-        "ClassDeclaration_ContainerRuntime": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_IGarbageCollectionRuntime": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_IGCStats": {
-          "forwardCompat": false
-        },
-        "InterfaceDeclaration_IRootSummaryTreeWithStats": {
-          "forwardCompat": false
-        }
+      "ClassDeclaration_Summarizer": {
+        "forwardCompat": false
+      },
+      "InterfaceDeclaration_IProvideSummarizer": {
+        "forwardCompat": false
+      },
+      "InterfaceDeclaration_ISummarizer": {
+        "forwardCompat": false
       }
     }
   }

package/src/connectionTelemetry.ts

@@ -33,7 +33,7 @@ interface IOpPerfTelemetryProperties {
     /** Measure time between (3) and (4) - Time between DM's inbound "push" event until DM's "op" event */
     durationInboundToProcessing: number;
     /** Length of the DeltaManager's inbound queue at the time of the DM's inbound "push" event (3) */
-    lenghtInboundQueue: number;
+    lengthInboundQueue: number;
 }
 
 /**
@@ -135,7 +135,7 @@ class OpPerfTelemetry {
                 this.opPerfData.durationInboundQueue = this.opProcessingTimes.opStartTimeInboundPushEvent
                                      - this.opProcessingTimes.opStartTimeSittingInboundQueue;
                 this.opProcessingTimes.opStartTimeSittingInboundQueue = undefined;
-                this.opPerfData.lenghtInboundQueue = this.deltaManager.inbound.length;
+                this.opPerfData.lengthInboundQueue = this.deltaManager.inbound.length;
             }
         });
 

package/src/containerRuntime.ts

@@ -194,17 +194,17 @@ export interface ContainerRuntimeMessage {
 const IdleDetectionTime = 5000;
 
 const DefaultSummaryConfiguration: ISummaryConfiguration = {
-    idleTime: IdleDetectionTime,
+    idleTime: IdleDetectionTime * 3,
 
     maxTime: IdleDetectionTime * 12,
 
-    // Snapshot if 1000 ops received since last snapshot.
+    // Summarize if 1000 ops received since last snapshot.
     maxOps: 1000,
 
-    // Wait 2 minutes for summary ack
+    // Wait 10 minutes for summary ack
     // this is less than maxSummarizeAckWaitTime
     // the min of the two will be chosen
-    maxAckWaitTime: 120000,
+    maxAckWaitTime: 600000,
 };
 
 export interface IGCRuntimeOptions {
@@ -967,8 +967,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         return {
             // the defaults
             ... DefaultSummaryConfiguration,
-            // the server provided values
-            ... this.context?.serviceConfiguration?.summary,
             // the runtime configuration overrides
             ... this.runtimeOptions.summaryOptions?.summaryConfigOverrides,
         };

package/src/garbageCollection.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLogger, ITelemetryPerformanceEvent } from "@fluidframework/common-definitions";
 import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
@@ -66,6 +66,8 @@ const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
 // Feature gate key to expire a session after a set period of time.
 const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
+// Feature gate key to log error messages if GC reference validation fails.
+const logUnknownOutboundReferencesKey = "Fluid.GarbageCollection.LogUnknownOutboundReferences";
 
 const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
 export const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
@@ -227,6 +229,19 @@ class UnreferencedStateTracker {
 /**
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
  * its state across summaries.
+ *
+ * Node - represented as nodeId, it's a node on the GC graph
+ * Outbound Route - a path from one node to another node, think `nodeA` -> `nodeB`
+ * Graph - all nodes with their respective routes
+ *             GC Graph
+ *
+ *               Node
+ *        NodeId = "datastore1"
+ *           /             \
+ *    OutboundRoute   OutboundRoute
+ *         /                 \
+ *       Node               Node
+ *  NodeId = "dds1"     NodeId = "dds2"
  */
 export class GarbageCollector implements IGarbageCollector {
     public static create(
@@ -320,10 +335,10 @@ export class GarbageCollector implements IGarbageCollector {
     private latestSummaryGCVersion: GCVersion;
 
     // Keeps track of the GC state from the last run.
-    private gcDataFromLastRun: IGarbageCollectionData | undefined;
+    private previousGCDataFromLastRun: IGarbageCollectionData | undefined;
     // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
     // outbound routes from that node.
-    private readonly referencesSinceLastRun: Map<string, string[]> = new Map();
+    private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
 
     // Promise when resolved initializes the base state of the nodes from the base summary state.
     private readonly initializeBaseStateP: Promise<void>;
@@ -513,7 +528,7 @@ export class GarbageCollector implements IGarbageCollector {
                 }
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
-            this.gcDataFromLastRun = { gcNodes };
+            this.previousGCDataFromLastRun = { gcNodes };
         });
 
         // Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
@@ -637,12 +652,12 @@ export class GarbageCollector implements IGarbageCollector {
      * blobs. All the blob keys should start with `gcBlobPrefix`.
      */
     public summarize(): ISummaryTreeWithStats | undefined {
-        if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
+        if (!this.shouldRunGC || this.previousGCDataFromLastRun === undefined) {
             return;
         }
 
         const gcState: IGarbageCollectionState = { gcNodes: {} };
-        for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
+        for (const [nodeId, outboundRoutes] of Object.entries(this.previousGCDataFromLastRun.gcNodes)) {
             gcState.gcNodes[nodeId] = {
                 outboundRoutes,
                 unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
@@ -729,9 +744,9 @@ export class GarbageCollector implements IGarbageCollector {
             return;
         }
 
-        const outboundRoutes = this.referencesSinceLastRun.get(fromNodePath) ?? [];
+        const outboundRoutes = this.newReferencesSinceLastRun.get(fromNodePath) ?? [];
         outboundRoutes.push(toNodePath);
-        this.referencesSinceLastRun.set(fromNodePath, outboundRoutes);
+        this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
 
         // If the node that got referenced is inactive, log an event as that may indicate use-after-delete.
         this.logIfInactive(
@@ -772,8 +787,8 @@ export class GarbageCollector implements IGarbageCollector {
         gcResult: IGCResult,
         currentReferenceTimestampMs?: number,
     ) {
-        this.gcDataFromLastRun = cloneGCData(gcData);
-        this.referencesSinceLastRun.clear();
+        this.previousGCDataFromLastRun = cloneGCData(gcData);
+        this.newReferencesSinceLastRun.clear();
 
         // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
         for (const nodeId of gcResult.referencedNodeIds) {
@@ -826,13 +841,37 @@ export class GarbageCollector implements IGarbageCollector {
      * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
      */
     private updateStateSinceLastRun(currentGCData: IGarbageCollectionData) {
-        // If we haven't run GC before or no references were added since the last run, there is nothing to do.
-        if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
+        // If we haven't run GC before there is nothing to do.
+        if (this.previousGCDataFromLastRun === undefined) {
             return;
         }
 
-        // Validate that we have identified all references correctly.
-        this.validateReferenceCorrectness(currentGCData);
+        // Find any references that haven't been identified correctly.
+        const missingExplicitReferences = this.findMissingExplicitReferences(
+            currentGCData,
+            this.previousGCDataFromLastRun,
+            this.newReferencesSinceLastRun,
+        );
+
+        // The following log will be enabled once this issue is resolved:
+        // https://github.com/microsoft/FluidFramework/issues/8878.
+        if(this.mc.config.getBoolean(logUnknownOutboundReferencesKey) === true
+            && missingExplicitReferences.length > 0) {
+            missingExplicitReferences.forEach((missingExplicitReference) => {
+                const event: ITelemetryPerformanceEvent = {
+                    eventName: "gcUnknownOutboundReferences",
+                    gcNodeId: missingExplicitReference[0],
+                    gcRoutes: JSON.stringify(missingExplicitReference[1]),
+                };
+                this.mc.logger.sendPerformanceEvent(event);
+            });
+        }
+
+        // No references were added since the last run so we don't have to update reference states of any unreferenced
+        // nodes
+        if (this.newReferencesSinceLastRun.size === 0) {
+            return;
+        }
 
         /**
          * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
@@ -849,8 +888,8 @@ export class GarbageCollector implements IGarbageCollector {
          *      which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
          *    - A new data store may have "root" DDSs already created and we don't detect them today.
          */
-        const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
-        this.referencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
+        const gcDataSuperSet = concatGarbageCollectionData(this.previousGCDataFromLastRun, currentGCData);
+        this.newReferencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
             if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
                 gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
             } else {
@@ -876,59 +915,55 @@ export class GarbageCollector implements IGarbageCollector {
     }
 
     /**
-     * Validates that all new references are correctly identified and processed. The basic principle for validation is
-     * that we should not have new references in the reference graph (GC data) that have not been notified to the
-     * garbage collector via `referenceAdded`.
-     * We validate that the references in the current reference graph should be a subset of the references in the last
-     * run's reference graph + references since the last run.
+     * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
+     * The principle is that every new reference or outbound route must be notified to GC via the
+     * addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
+     *
+     * In more simple terms:
+     * Missing Explicit References = Current References - Previous References - Explicitly Added References;
+     *
      * @param currentGCData - The GC data (reference graph) from the current GC run.
+     * @param previousGCData - The GC data (reference graph) from the previous GC run.
+     * @param explicitReferences - New references added explicity between the previous and the current run.
+     * @returns - a list of missing explicit references
      */
-    private validateReferenceCorrectness(currentGCData: IGarbageCollectionData) {
+    private findMissingExplicitReferences(
+        currentGCData: IGarbageCollectionData,
+        previousGCData: IGarbageCollectionData,
+        explicitReferences: Map<string, string[]>,
+    ): [string, string[]][] {
         assert(
-            this.gcDataFromLastRun !== undefined,
+            previousGCData !== undefined,
             0x2b7, /* "Can't validate correctness without GC data from last run" */
         );
 
-        // Get a list of all the outbound routes (or references) in the current GC data.
-        const currentReferences: string[] = [];
-        for (const [nodeId, outboundRoutes] of Object.entries(currentGCData.gcNodes)) {
-            /**
-             * Remove routes from a child node to its parent which is added implicitly by the runtime. For instance,
-             * each adds its data store as an outbound route to mark it as referenced if the DDS is referenced.
-             * We won't get any explicit notification for these references so they must be removed before validation.
-             */
-            const explicitRoutes = outboundRoutes.filter((route) => !nodeId.startsWith(route));
-            currentReferences.push(...explicitRoutes);
-        }
-
-        // Get a list of outbound routes (or references) from the last run's GC data plus references added since the
-        // last run that were notified via `referenceAdded`.
-        const explicitReferences: string[] = [];
-        for (const [, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
-            explicitReferences.push(...outboundRoutes);
-        }
-        this.referencesSinceLastRun.forEach((outboundRoutes: string[]) => {
-            explicitReferences.push(...outboundRoutes);
-        });
-
-        // Validate that the current reference graph doesn't have references that we are not already aware of. If this
-        // happens, it might indicate data corruption since we may delete objects prematurely.
-        currentReferences.forEach((route: string) => {
-            // Validate references for data stores only. Currently, layers below data stores don't have GC implemented
-            // so there is no guarantee their references will be notified.
-            if (this.runtime.getNodeType(route) === GCNodeType.DataStore && !explicitReferences.includes(route)) {
-                /**
-                 * The following log will be enabled once this issue is resolved:
-                 * https://github.com/microsoft/FluidFramework/issues/8878.
-                 */
-                // We should ideally throw a data corruption error here. However, send an error for now until we have
-                // implemented sweep and have reasonable confidence in the sweep process.
-                // this.mc.logger.sendErrorEvent({
-                //     eventName: "gcUnknownOutboundRoute",
-                //     route,
-                // });
+        const currentGraph = Object.entries(currentGCData.gcNodes);
+        const missingExplicitReferences: [string, string[]][] = [];
+        currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
+            const previousRoutes = previousGCData.gcNodes[nodeId] ?? [];
+            const explicitRoutes = explicitReferences.get(nodeId) ?? [];
+            const missingExplicitRoutes: string[] = [];
+            currentOutboundRoutes.forEach((route) => {
+                const isBlobOrDataStoreRoute =
+                    this.runtime.getNodeType(route) === GCNodeType.Blob ||
+                    this.runtime.getNodeType(route) === GCNodeType.DataStore;
+                // Ignore implicitly added DDS routes to their parent datastores
+                const notRouteFromDDSToParentDataStore = !nodeId.startsWith(route);
+                if (
+                    isBlobOrDataStoreRoute &&
+                    notRouteFromDDSToParentDataStore &&
+                    (!previousRoutes.includes(route) && !explicitRoutes.includes(route))
+                ) {
+                    missingExplicitRoutes.push(route);
+                }
+            });
+            if (missingExplicitRoutes.length > 0) {
+                missingExplicitReferences.push([nodeId, missingExplicitRoutes]);
             }
         });
+
+        // Ideally missingExplicitReferences should always have a size 0
+        return missingExplicitReferences;
     }
 
     /**