@fluidframework/runtime-utils 2.0.0-internal.2.2.0 → 2.0.0-internal.2.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/runtime-utils",
-  "version": "2.0.0-internal.2.2.0",
+  "version": "2.0.0-internal.2.3.0",
   "description": "Collection of utility functions for Fluid Runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -64,23 +64,23 @@
   "dependencies": {
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^1.0.0",
-    "@fluidframework/container-definitions": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0",
-    "@fluidframework/container-runtime-definitions": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0",
-    "@fluidframework/core-interfaces": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0",
-    "@fluidframework/datastore-definitions": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0",
-    "@fluidframework/garbage-collector": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0",
+    "@fluidframework/container-definitions": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0",
+    "@fluidframework/container-runtime-definitions": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0",
+    "@fluidframework/core-interfaces": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0",
+    "@fluidframework/datastore-definitions": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0",
+    "@fluidframework/garbage-collector": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0",
     "@fluidframework/protocol-base": "^0.1038.2000",
     "@fluidframework/protocol-definitions": "^1.1.0",
-    "@fluidframework/runtime-definitions": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0",
-    "@fluidframework/telemetry-utils": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0"
+    "@fluidframework/runtime-definitions": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0",
+    "@fluidframework/telemetry-utils": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0"
   },
   "devDependencies": {
     "@fluid-tools/build-cli": "^0.7.0",
     "@fluidframework/build-common": "^1.1.0",
     "@fluidframework/build-tools": "^0.7.0",
-    "@fluidframework/eslint-config-fluid": "^1.2.0",
-    "@fluidframework/mocha-test-setup": ">=2.0.0-internal.2.2.0 <2.0.0-internal.3.0.0",
-    "@fluidframework/runtime-utils-previous": "npm:@fluidframework/runtime-utils@2.0.0-internal.2.1.0",
+    "@fluidframework/eslint-config-fluid": "^2.0.0",
+    "@fluidframework/mocha-test-setup": ">=2.0.0-internal.2.3.0 <2.0.0-internal.3.0.0",
+    "@fluidframework/runtime-utils-previous": "npm:@fluidframework/runtime-utils@2.0.0-internal.2.2.0",
     "@microsoft/api-extractor": "^7.22.2",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/mocha": "^9.1.1",
@@ -98,9 +98,9 @@
     "typescript": "~4.5.5"
   },
   "typeValidation": {
-    "version": "2.0.0-internal.2.2.0",
-    "baselineRange": ">=2.0.0-internal.2.1.0 <2.0.0-internal.2.2.0",
-    "baselineVersion": "2.0.0-internal.2.1.0",
+    "version": "2.0.0-internal.2.3.0",
+    "baselineRange": ">=2.0.0-internal.2.2.0 <2.0.0-internal.2.3.0",
+    "baselineVersion": "2.0.0-internal.2.2.0",
     "broken": {}
   }
 }

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/runtime-utils";
-export const pkgVersion = "2.0.0-internal.2.2.0";
+export const pkgVersion = "2.0.0-internal.2.3.0";

package/src/summarizerNode/summarizerNode.ts

@@ -384,7 +384,7 @@ export class SummarizerNode implements IRootSummarizerNode {
         return this._latestSummary;
     }
 
-    private readonly canReuseHandle: boolean;
+    protected readonly canReuseHandle: boolean;
 
     /**
      * Do not call constructor directly.

package/src/summarizerNode/summarizerNodeWithGc.ts

@@ -5,11 +5,11 @@
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, LazyPromise } from "@fluidframework/common-utils";
-import { cloneGCData } from "@fluidframework/garbage-collector";
+import { cloneGCData, getGCDataFromSnapshot, runGarbageCollection, unpackChildNodesGCDetails } from "@fluidframework/garbage-collector";
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
 import {
     CreateChildSummarizerNodeParam,
-    gcBlobKey,
+    gcTreeKey,
     IGarbageCollectionData,
     IGarbageCollectionDetailsBase,
     ISummarizeInternalResult,
@@ -26,6 +26,7 @@ import {
     ICreateChildDetails,
     IInitialSummary,
     ISummarizerNodeRootContract,
+    parseSummaryForSubtrees,
     SummaryNode,
 } from "./summarizerNodeUtils";
 
@@ -127,22 +128,28 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
      * - gcData: The garbage collection data of this node that is required for running GC.
      */
     private async loadBaseGCDetails() {
+        if (this.baseGCDetailsLoaded) {
+            return;
+        }
         const baseGCDetails = await this.baseGCDetailsP;
 
-        // Possible race - If there were parallel calls to loadBaseGCDetails, we want to make sure that we only update
+        // Possible race - If there were parallel calls to loadBaseGCDetails, we want to make sure that we update
         // the state from the base details only once.
         if (this.baseGCDetailsLoaded) {
             return;
         }
         this.baseGCDetailsLoaded = true;
 
+        // Update GC data, used routes and reference used routes. The used routes are sorted because they are compared
+        // across GC runs to check if they changed. Sorting ensures that the elements are in the same order.
         // If the GC details has GC data, initialize our GC data from it.
         if (baseGCDetails.gcData !== undefined) {
             this.gcData = cloneGCData(baseGCDetails.gcData);
         }
-        // Sort the used routes because we compare them with the current used routes to check if they changed between
-        // summaries. Both are sorted so that the order of elements is the same.
-        this.referenceUsedRoutes = baseGCDetails.usedRoutes?.sort();
+        if (baseGCDetails.usedRoutes !== undefined) {
+            this.usedRoutes = Array.from(baseGCDetails.usedRoutes).sort();
+            this.referenceUsedRoutes = Array.from(baseGCDetails.usedRoutes).sort()
+        }
     }
 
     public async summarize(
@@ -177,10 +184,11 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
         // called and the node's data has not changed since last summary, the GC data in initial details is returned.
         await this.loadBaseGCDetails();
 
-        // If there is no new data since last summary and we have GC data from the previous run, return it. We may not
-        // have data from previous GC run for clients with older summary format before GC was added. They won't have
-        // GC details in their initial summary.
-        if (!fullGC && !this.hasDataChanged() && this.gcData !== undefined) {
+        // If there is no new data since last summary and we have GC data from the previous run, return it. The previous
+        // GC data may not be available if loaded from a snapshot with either GC disabled or before GC was added.
+        // Note - canReuseHandle is checked to be consistent with summarize - generate GC data for nodes for which
+        // summary must be generated.
+        if (this.canReuseHandle && !fullGC && !this.hasDataChanged() && this.gcData !== undefined) {
             return cloneGCData(this.gcData);
         }
 
@@ -259,8 +267,7 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
     }
 
     /**
-     * Called when we need to upload the reference state from the given summary. Read the GC blob and get the state
-     * to upload from it.
+     * Called when we need to upload the reference state from the given summary.
      */
     protected async refreshLatestSummaryFromSnapshot(
         referenceSequenceNumber: number,
@@ -270,21 +277,7 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
         correlatedSummaryLogger: ITelemetryLogger,
         readAndParseBlob: ReadAndParseBlob,
     ): Promise<void> {
-        // If GC is disabled, skip setting referenced used routes since we are not tracking GC state.
-        if (!this.gcDisabled) {
-            const gcDetailsBlob = snapshotTree.blobs[gcBlobKey];
-            if (gcDetailsBlob !== undefined) {
-                const gcDetails = await readAndParseBlob<IGarbageCollectionDetailsBase>(gcDetailsBlob);
-
-                // Possible re-entrancy. If we have already seen a summary later than this one, ignore it.
-                if (this.referenceSequenceNumber >= referenceSequenceNumber) {
-                    return;
-                }
-
-                this.referenceUsedRoutes = gcDetails.usedRoutes;
-            }
-        }
-
+        await this.refreshGCStateFromSnapshot(referenceSequenceNumber, snapshotTree, readAndParseBlob);
         return super.refreshLatestSummaryFromSnapshot(
             referenceSequenceNumber,
             snapshotTree,
@@ -295,6 +288,85 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
         );
     }
 
+    /**
+     * Updates GC state from the given snapshot if GC is enabled and the snapshot is newer than the one this node
+     * is tracking.
+     */
+    private async refreshGCStateFromSnapshot(
+        referenceSequenceNumber: number,
+        snapshotTree: ISnapshotTree,
+        readAndParseBlob: ReadAndParseBlob,
+    ): Promise<void> {
+        // If GC is disabled or we have seen a newer summary, skip updating GC state.
+        if (this.gcDisabled || this.referenceSequenceNumber >= referenceSequenceNumber) {
+            return;
+        }
+
+        // Load the base GC details before proceeding because if that happens later it can overwrite the GC details
+        // written by the following code.
+        await this.loadBaseGCDetails();
+
+        // Possible re-entrancy. We may already have processed this while loading base GC details.
+        if (this.referenceSequenceNumber >= referenceSequenceNumber) {
+            return;
+        }
+
+        /**
+         * GC data is written at root of the snapshot tree under "gc" sub-tree. This data needs to be propagated to
+         * all the nodes in the container.
+         * The root summarizer node reads the GC data from the "gc" sub-tree, runs GC on it to get used routes in
+         * the container and updates its GC data and referenced used routes. It then gets the GC data and used
+         * routes of all its children and adds it to their snapshot tree.
+         * All the other nodes gets the GC data and used routes from their snapshot tree and updates their state.
+         * They get the GC data and used routes of their children and add it to their snapshot tree and so on.
+         *
+         * Note that if the snapshot does not have GC tree, GC data will be set to undefined and used routes will be
+         * set to self-route (meaning referenced) for all nodes. This is important because the GC data needs to be
+         * regenerated in the next summary.
+         */
+        let gcDetails: IGarbageCollectionDetailsBase | undefined;
+        const gcSnapshotTree = snapshotTree.trees[gcTreeKey];
+        if (gcSnapshotTree !== undefined) {
+            // If there is a GC tree in the snapshot, this is the root summarizer node. Read GC data from the tree
+            // process it as explained above.
+            const gcSnapshotData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
+
+            const gcNodes: { [id: string]: string[]; } = {};
+            for (const [nodeId, nodeData] of Object.entries(gcSnapshotData.gcState.gcNodes)) {
+                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
+            }
+            // Run GC on the nodes in the snapshot to get the used routes for each node in the container.
+            const usedRoutes = runGarbageCollection(gcNodes, ["/"]).referencedNodeIds;
+            gcDetails = { gcData: { gcNodes }, usedRoutes };
+        } else {
+            // If there is a GC blob in the snapshot, it's a non-root summarizer nodes - The root summarizer node
+            // writes GC blob in the snapshot of child nodes. Get  GC data and used routes from the blob.
+            const gcDetailsBlob = snapshotTree.blobs[gcTreeKey];
+            if (gcDetailsBlob !== undefined) {
+                gcDetails = JSON.parse(gcDetailsBlob) as IGarbageCollectionDetailsBase;
+            }
+        }
+
+        // Update this node to the same GC state it was when the ack corresponding to this summary was processed.
+        this.gcData = gcDetails?.gcData !== undefined ? cloneGCData(gcDetails.gcData) : undefined;
+        this.referenceUsedRoutes = gcDetails?.usedRoutes !== undefined ? Array.from(gcDetails.usedRoutes) : undefined;
+        // If there are no used routes in the GC details, set it to have self route which will make the node
+        // referenced. This scenario can only happen if the snapshot is from a client where GC was not run or
+        // disabled. In both the cases, the node should be referenced.
+        this.usedRoutes = gcDetails?.usedRoutes !== undefined ? Array.from(gcDetails.usedRoutes) : [""];
+
+        if (gcDetails === undefined) {
+            return;
+        }
+
+        // Generate the GC data and used routes of children GC nodes and add it to their snapshot tree.
+        const gcDetailsMap = unpackChildNodesGCDetails(gcDetails);
+        const { childrenTree } = parseSummaryForSubtrees(snapshotTree);
+        gcDetailsMap.forEach((childGCDetails: IGarbageCollectionDetailsBase, childId: string) => {
+            childrenTree.trees[childId].blobs[gcTreeKey] = JSON.stringify(childGCDetails);
+        });
+    }
+
     /**
      * Override the createChild method to return an instance of SummarizerNodeWithGC.
      */
@@ -315,6 +387,20 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
     ): ISummarizerNodeWithGC {
         assert(!this.children.has(id), 0x1b6 /* "Create SummarizerNode child already exists" */);
 
+        /**
+          * Update the child node's base GC details from this node's current GC details instead of updating from the base
+          * GC details of this node. This will handle scenarios where the GC details was updated during refresh from
+          * snapshot and the child node wasn't created then. If a child is created after that, its GC details should be
+          * the one from the downloaded snapshot and not the base GC details.
+          */
+        const getChildBaseGCDetailsP = new LazyPromise<IGarbageCollectionDetailsBase>(async () => {
+            // Ensure that the base GC details is loaded because a child can be created before GC runs which is when
+            // base GC details is usually loaded.
+            await this.loadBaseGCDetails();
+            const childBaseGCDetails = unpackChildNodesGCDetails({ gcData: this.gcData, usedRoutes: this.usedRoutes });
+            return childBaseGCDetails.get(id) ?? {};
+        });
+
         const createDetails: ICreateChildDetails = this.getCreateDetailsForChild(id, createParam);
         const child = new SummarizerNodeWithGC(
             this.defaultLogger,
@@ -329,7 +415,7 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
             createDetails.initialSummary,
             this.wipSummaryLogger,
             getGCDataFn,
-            getBaseGCDetailsFn,
+            async () => getChildBaseGCDetailsP,
         );
 
         // There may be additional state that has to be updated in this child. For example, if a summary is being