@fluidframework/container-runtime 2.0.0-internal.7.4.6 → 2.0.0-internal.7.4.7

package/lib/gc/garbageCollection.js

@@ -2,10 +2,11 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { LazyPromise, Timer } from "@fluidframework/core-utils";
+import { assert, LazyPromise, Timer } from "@fluidframework/core-utils";
 import { gcTreeKey, } from "@fluidframework/runtime-definitions";
 import { createResponseError, responseToException } from "@fluidframework/runtime-utils";
 import { createChildLogger, createChildMonitoringContext, DataProcessingError, PerformanceEvent, } from "@fluidframework/telemetry-utils";
+import { BlobManager } from "../blobManager";
 import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey, } from "../containerRuntime";
 import { ClientSessionExpiredError } from "../error";
 import { ContainerMessageType } from "../messageTypes";
@@ -80,7 +81,6 @@ export class GarbageCollector {
         this.isSummarizerClient = createParams.isSummarizerClient;
         this.getNodePackagePath = createParams.getNodePackagePath;
         this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
-        this.activeConnection = createParams.activeConnection;
         this.submitMessage = createParams.submitMessage;
         const baseSnapshot = createParams.baseSnapshot;
         const readAndParseBlob = createParams.readAndParseBlob;
@@ -141,28 +141,14 @@ export class GarbageCollector {
             }
         });
         /**
-         * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
-         * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
-         * GC state and updates their inactive or sweep-ready state.
+         * Set up the initializer which initializes the GC state from the data in base snapshot. It sets up GC data
+         * from the base GC state and starts tracking the state of unreferenced nodes.
+         *
+         * Must only be called if there is a current reference timestamp.
          */
         this.initializeGCStateFromBaseSnapshotP = new LazyPromise(async () => {
-            /**
-             * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
-             * how long objects have been unreferenced and if they can be deleted.
-             *
-             * Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
-             * for this container and it is in read mode. In this scenario, there is no point in running GC anyway
-             * because references in the container do not change without any ops, i.e., there is nothing to collect.
-             */
             const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
-            if (currentReferenceTimestampMs === undefined) {
-                // Log an event so we can evaluate how often we run into this scenario.
-                this.mc.logger.sendErrorEvent({
-                    eventName: "GarbageCollectorInitializedWithoutTimestamp",
-                    gcConfigs: JSON.stringify(this.configs),
-                });
-                return;
-            }
+            assert(currentReferenceTimestampMs !== undefined, "Trying to initialize GC state without current timestamp");
             /**
              * The base snapshot data will not be present if the container is loaded from:
              * 1. The first summary created by the detached container.
@@ -170,11 +156,20 @@ export class GarbageCollector {
              * 3. A summary that was generated before GC even existed.
              */
             const baseSnapshotData = await this.baseSnapshotDataP;
-            if (baseSnapshotData === undefined) {
+            this.summaryStateTracker.initializeBaseState(baseSnapshotData);
+            if (baseSnapshotData?.gcState === undefined) {
                 return;
             }
-            this.updateStateFromSnapshotData(baseSnapshotData, currentReferenceTimestampMs);
-            this.summaryStateTracker.initializeBaseState(baseSnapshotData);
+            // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
+            // to the GC data from the snapshot data.
+            const gcNodes = {};
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
+                if (nodeData.unreferencedTimestampMs !== undefined) {
+                    this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
+                }
+                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
+            }
+            this.gcDataFromLastRun = { gcNodes };
         });
         // Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
         // used to initialize the GC state of all the nodes in the container.
@@ -203,8 +198,10 @@ export class GarbageCollector {
         });
     }
     /**
-     * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
-     * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
+     * Called during container initialization. Initializes the tombstone and deleted nodes state from the base snapshot.
+     * Also, initializes the GC state including unreferenced nodes tracking if a current reference timestamp exists.
+     * Note that if there is any GC state in the base snapshot, then there will definitely be a reference timestamp
+     * to work with - The GC state would have been generated using a timestamp which is part of the snapshot.
      */
     async initializeBaseState() {
         const baseSnapshotData = await this.baseSnapshotDataP;
@@ -229,98 +226,51 @@ export class GarbageCollector {
             this.tombstones = Array.from(baseSnapshotData.tombstones);
             this.runtime.updateTombstonedRoutes(this.tombstones);
         }
+        await this.initializeOrUpdateGCState();
     }
     /**
-     * Update state from the given snapshot data. This is done during load and during refreshing state from a snapshot.
-     * All current tracking is reset and updated from the data in the snapshot.
-     * @param snapshotData - The snapshot data to update state from. If this is undefined, all GC state and tracking
-     * is reset.
-     * @param currentReferenceTimestampMs - The current reference timestamp for marking unreferenced nodes' unreferenced
-     * timestamp.
+     * Initialize the GC state if not already initialized. If GC state is already initialized, update the unreferenced
+     * state tracking as per the current reference timestamp.
      */
-    updateStateFromSnapshotData(snapshotData, currentReferenceTimestampMs) {
-        /**
-         * Note: "newReferencesSinceLastRun" is not reset here. This is done because there may be references since the
-         * snapshot that we are updating state from. For example, this client may have processed ops till seq#1000 and
-         * its refreshing state from a summary that happened at seq#900. In this case, there may be references between
-         * seq#901 and seq#1000 that we don't want to reset.
-         * Unfortunately, there is no way to track the seq# of ops that add references, so we choose to not reset any
-         * references here. This should be fine because, in the worst case, we may end up updating the unreferenced
-         * timestamp of a node which will delay its deletion. Although not ideal, this will only happen in rare
-         * scenarios, so it should be okay.
-         */
-        // Clear all existing unreferenced state tracking.
-        for (const [, nodeStateTracker] of this.unreferencedNodesState) {
-            nodeStateTracker.stopTracking();
-        }
-        this.unreferencedNodesState.clear();
-        // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
-        // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
-        // tombstones.
-        // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
-        // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
-        if (this.configs.shouldRunSweep) {
-            const snapshotDeletedNodes = snapshotData?.deletedNodes
-                ? new Set(snapshotData.deletedNodes)
-                : undefined;
-            // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
-            // delete them.
-            if (snapshotDeletedNodes !== undefined) {
-                const newDeletedNodes = [];
-                for (const nodeId of snapshotDeletedNodes) {
-                    if (!this.deletedNodes.has(nodeId)) {
-                        newDeletedNodes.push(nodeId);
-                    }
-                }
-                if (newDeletedNodes.length > 0) {
-                    // Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
-                }
-            }
-        }
-        else if (this.configs.tombstoneMode) {
-            // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
-            // notify the runtime to update its state as well.
-            this.tombstones = snapshotData?.tombstones ? Array.from(snapshotData.tombstones) : [];
-            this.runtime.updateTombstonedRoutes(this.tombstones);
+    async initializeOrUpdateGCState() {
+        const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
+        if (currentReferenceTimestampMs === undefined) {
+            return;
         }
-        // If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
-        if (snapshotData?.gcState === undefined) {
-            this.gcDataFromLastRun = undefined;
+        // If the GC state hasn't been initialized yet, initialize it and return.
+        if (this.gcDataFromLastRun === undefined) {
+            await this.initializeGCStateFromBaseSnapshotP;
             return;
         }
-        // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
-        // to the GC data from the snapshot data.
-        const gcNodes = {};
-        for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
-            if (nodeData.unreferencedTimestampMs !== undefined) {
-                this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
-            }
-            gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
+        // If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
+        // reference timestamp.
+        for (const [, nodeStateTracker] of this.unreferencedNodesState) {
+            nodeStateTracker.updateTracking(currentReferenceTimestampMs);
         }
-        this.gcDataFromLastRun = { gcNodes };
     }
     /**
      * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
-     * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
+     * to initialize or update the unreference state tracking.
      * @param connected - Whether the runtime connected / disconnected.
      * @param clientId - The clientId of this runtime.
      */
     setConnectionState(connected, clientId) {
         /**
-         * For all clients, initialize the base state when the container becomes active, i.e., it transitions
-         * to "write" mode. This will ensure that the container's own join op is processed and there is a recent
-         * reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
-         * could affect the GC state will have been processed.
+         * When the client connects (or reconnects), attempt to initialize or update the GC state. This will keep
+         * the unreferenced state tracking updated as per the reference timestamp at the time of connection.
          *
-         * If GC is up-to-date for the client and the summarizing client, there will be an doubling of both
-         * InactiveObject_Loaded and SweepReady_Loaded errors, as there will be one from the sending client and one from
-         * the receiving summarizer client.
-         *
-         * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
-         * sweep in phases and we want to track when inactive and sweep-ready objects are used in any client.
+         * During GC initialization and during connections in read mode, it is possible that either no ops are
+         * processed or only trailing ops are processed. This means that the GC state is not initialized or initialized
+         * with an older reference timestamp. So, doing this on every connection will keep the unreferenced state
+         * tracking up-to-date.
          */
-        if (this.activeConnection() && this.configs.shouldRunGC) {
-            this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
+        if (connected && this.configs.shouldRunGC) {
+            this.initializeOrUpdateGCState().catch((error) => {
+                this.mc.logger.sendErrorEvent({
+                    eventName: "GCInitializationOrUpdateFailed",
+                    gcConfigs: JSON.stringify(this.configs),
+                }, error);
+            });
         }
     }
     /**
@@ -377,8 +327,11 @@ export class GarbageCollector {
             const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
             event.end({
                 ...gcStats,
-                timestamp: currentReferenceTimestampMs,
-                sweep: this.configs.shouldRunSweep,
+                details: {
+                    timestamp: currentReferenceTimestampMs,
+                    sweep: this.configs.shouldRunSweep,
+                    tombstone: this.configs.throwOnTombstoneLoad,
+                },
             });
             /** Post-GC steps */
             // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
@@ -420,8 +373,9 @@ export class GarbageCollector {
         // It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
         const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
         // 4. Run the Sweep phase.
-        // It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
-        // sweep op. All clients, including this one, will delete these nodes once it processes the op.
+        // It will initiate the deletion (sending the GC Sweep op) of any sweep-ready nodes that are
+        // allowed to be deleted per config, and tombstone the rest along with the tombstone-ready nodes.
+        // Note that no nodes will be deleted until the GC Sweep op is processed.
         this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
         this.gcDataFromLastRun = cloneGCData(gcData);
         // 5. Get the sweep phase stats.
@@ -501,25 +455,41 @@ export class GarbageCollector {
             this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
             return;
         }
-        // If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
+        // We'll build up the lists of nodes to be either Tombstoned or Deleted
+        // based on the configuration and the nodes' current state.
+        // We must Tombstone any sweep-ready node that Sweep won't run for.
         // This is important because a container may never load during a node's Sweep Grace Period,
         // so that node would directly become sweep-ready skipping over tombstone-ready state,
         // but should be Tombstoned since Sweep is disabled.
-        const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
-            ? {
-                nodesToTombstone: [...tombstoneReadyNodes],
-                nodesToDelete: [...sweepReadyNodes],
-            }
-            : {
-                nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
-                nodesToDelete: [],
-            };
+        const { nodesToTombstone, nodesToDelete } = {
+            nodesToTombstone: [...tombstoneReadyNodes],
+            nodesToDelete: [],
+        };
+        switch (this.configs.shouldRunSweep) {
+            case "YES":
+                nodesToDelete.push(...sweepReadyNodes);
+                break;
+            case "ONLY_BLOBS":
+                sweepReadyNodes.forEach((nodeId) => {
+                    const nodeType = this.runtime.getNodeType(nodeId);
+                    if (nodeType === GCNodeType.Blob) {
+                        nodesToDelete.push(nodeId);
+                    }
+                    else {
+                        nodesToTombstone.push(nodeId);
+                    }
+                });
+                break;
+            default: // case "NO":
+                nodesToTombstone.push(...sweepReadyNodes);
+                break;
+        }
         if (this.configs.tombstoneMode) {
             this.tombstones = nodesToTombstone;
             // If we are running in GC tombstone mode, update tombstoned routes.
             this.runtime.updateTombstonedRoutes(this.tombstones);
         }
-        if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
+        if (nodesToDelete.length > 0) {
             // Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
             // to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
             // local state when processing the op.
@@ -675,6 +645,10 @@ export class GarbageCollector {
     /**
      * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
      * tracking for nodes that are actually deleted by the runtime.
+     *
+     * Note that this doesn't check any configuration around whether Sweep is enabled.
+     * That happens before the op is submitted, and from that point, any client should execute the delete.
+     *
      * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
      */
     deleteSweepReadyNodes(sweepReadyNodeIds) {
@@ -855,8 +829,9 @@ export class GarbageCollector {
     }
     /**
      * Generates the stats of a garbage collection sweep phase run.
-     * @param deletedNodes - The nodes that have been deleted until this run.
-     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
+     * @param deletedNodes - The nodes that have already been deleted even before this run.
+     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run. These will be deleted but are not deleted yet,
+     * due to either sweep not being enabled or the Sweep Op needing to roundtrip before the delete is executed.
      * @param markPhaseStats - The stats of the mark phase run.
      * @returns the stats of the sweep phase run.
      */
@@ -871,9 +846,24 @@ export class GarbageCollector {
             deletedDataStoreCount: 0,
             deletedAttachmentBlobCount: 0,
         };
+        // The runtime can't reliably identify the type of deleted nodes. So, get the type here. This should
+        // be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
+        const getDeletedNodeType = (nodeId) => {
+            const pathParts = nodeId.split("/");
+            if (pathParts[1] === BlobManager.basePath) {
+                return GCNodeType.Blob;
+            }
+            if (pathParts.length === 2) {
+                return GCNodeType.DataStore;
+            }
+            if (pathParts.length === 3) {
+                return GCNodeType.SubDataStore;
+            }
+            return GCNodeType.Other;
+        };
         for (const nodeId of deletedNodes) {
             sweepPhaseStats.deletedNodeCount++;
-            const nodeType = this.runtime.getNodeType(nodeId);
+            const nodeType = getDeletedNodeType(nodeId);
             if (nodeType === GCNodeType.DataStore) {
                 sweepPhaseStats.deletedDataStoreCount++;
             }
@@ -881,17 +871,17 @@ export class GarbageCollector {
                 sweepPhaseStats.deletedAttachmentBlobCount++;
             }
         }
-        // If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
+        // The counts from the mark phase stats do not include nodes that were
         // deleted in previous runs. So, add the deleted node counts to life time stats.
         sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
         sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
         sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
-        if (this.configs.shouldRunSweep) {
-            return sweepPhaseStats;
-        }
-        // If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
-        // is the final state the node will be in.
-        // If sweep is enabled, this will happen in the run after the GC op round trips back.
+        // These stats are used to estimate the impact of GC in terms of how much garbage is/will be cleaned up.
+        // So we include the current sweep-ready node stats since these nodes will be deleted eventually.
+        // - If sweep is enabled, this will happen in the run after the GC op round trips back
+        //   (they'll be in deletedNodes that time).
+        // - If sweep is not enabled, we still want to include these nodes since they
+        //   _will be_ deleted once it is enabled.
         for (const nodeId of sweepReadyNodes) {
             sweepPhaseStats.deletedNodeCount++;
             const nodeType = this.runtime.getNodeType(nodeId);