@fluidframework/container-runtime 2.0.0-internal.4.2.1 → 2.0.0-internal.4.4.0

package/lib/gc/garbageCollection.js

@@ -2,29 +2,18 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-var __rest = (this && this.__rest) || function (s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
-                t[p[i]] = s[p[i]];
-        }
-    return t;
-};
-import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
+import { LazyPromise, Timer } from "@fluidframework/common-utils";
 import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
 import { gcTreeKey, } from "@fluidframework/runtime-definitions";
-import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
-import { ChildLogger, generateStack, loggerToMonitoringContext, PerformanceEvent, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
+import { ChildLogger, loggerToMonitoringContext, PerformanceEvent, } from "@fluidframework/telemetry-utils";
 import { RuntimeHeaders } from "../containerRuntime";
 import { generateGCConfigs } from "./gcConfigs";
-import { disableSweepLogKey, GCNodeType, UnreferencedState, } from "./gcDefinitions";
-import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot, sendGCUnexpectedUsageEvent, } from "./gcHelpers";
+import { GCNodeType, UnreferencedState, } from "./gcDefinitions";
+import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot } from "./gcHelpers";
 import { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
 import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
 import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
+import { GCTelemetryTracker } from "./gcTelemetry";
 /**
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
  * its state across summaries.
@@ -58,16 +47,10 @@ export class GarbageCollector {
         this.deletedNodes = new Set();
         // Map of node ids to their unreferenced state tracker.
         this.unreferencedNodesState = new Map();
-        // Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
-        // per event per node.
-        this.loggedUnreferencedEvents = new Set();
-        // Queue for unreferenced events that should be logged the next time GC runs.
-        this.pendingEventsQueue = [];
         // The number of times GC has successfully completed on this instance of GarbageCollector.
         this.completedRuns = 0;
         this.runtime = createParams.runtime;
         this.isSummarizerClient = createParams.isSummarizerClient;
-        this.createContainerMetadata = createParams.createContainerMetadata;
         this.getNodePackagePath = createParams.getNodePackagePath;
         this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
         this.activeConnection = createParams.activeConnection;
@@ -88,6 +71,7 @@ export class GarbageCollector {
             this.sessionExpiryTimer.start();
         }
         this.summaryStateTracker = new GCSummaryStateTracker(this.configs, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */);
+        this.telemetryTracker = new GCTelemetryTracker(this.mc, this.configs, this.isSummarizerClient, this.runtime.gcTombstoneEnforcementAllowed, createParams.createContainerMetadata, (nodeId) => this.runtime.getNodeType(nodeId), (nodeId) => this.unreferencedNodesState.get(nodeId), this.getNodePackagePath);
         // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
         // it involves fetching blobs from storage which is expensive.
         this.baseSnapshotDataP = new LazyPromise(async () => {
@@ -107,8 +91,7 @@ export class GarbageCollector {
                 // in the snapshot cannot be interpreted correctly. Set everything to undefined except for
                 // deletedNodes because irrespective of GC versions, these nodes have been deleted and cannot be
                 // brought back. The deletedNodes info is needed to identify when these nodes are used.
-                if (this.configs.gcVersionInBaseSnapshot !==
-                    this.summaryStateTracker.currentGCVersion) {
+                if (this.configs.gcVersionInEffect !== this.configs.gcVersionInBaseSnapshot) {
                     return {
                         gcState: undefined,
                         tombstones: undefined,
@@ -197,6 +180,10 @@ export class GarbageCollector {
     get summaryStateNeedsReset() {
         return this.summaryStateTracker.doesSummaryStateNeedReset;
     }
+    /** Returns the count of data stores whose GC state updated since the last summary. */
+    get updatedDSCountSinceLastSummary() {
+        return this.summaryStateTracker.updatedDSCountSinceLastSummary;
+    }
     /**
      * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
      * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
@@ -318,6 +305,13 @@ export class GarbageCollector {
             this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
         }
     }
+    /**
+     * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
+     * in the container.
+     */
+    async getBaseGCDetails() {
+        return this.baseGCDetailsP;
+    }
     /**
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
      * @returns stats of the GC run or undefined if GC did not run.
@@ -353,200 +347,67 @@ export class GarbageCollector {
             return undefined;
         }
         return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
-            await this.runPreGCSteps();
-            // Get the runtime's GC data and run GC on the reference graph in it.
-            const gcData = await this.runtime.getGCData(fullGC);
-            const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
-            const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
+            /** Pre-GC steps */
+            // Ensure that state has been initialized from the base snapshot data.
+            await this.initializeGCStateFromBaseSnapshotP;
+            // Let the runtime update its pending state before GC runs.
+            await this.runtime.updateStateBeforeGC();
+            /** GC step */
+            const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
             event.end(Object.assign(Object.assign({}, gcStats), { timestamp: currentReferenceTimestampMs }));
+            /** Post-GC steps */
+            // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
+            // updates its state so that we don't send false positives based on intermediate state. For example, we may get
+            // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
+            await this.telemetryTracker.logPendingEvents(logger);
+            // Update the state of summary state tracker from this run's stats.
+            this.summaryStateTracker.updateStateFromGCRunStats(gcStats);
+            this.newReferencesSinceLastRun.clear();
             this.completedRuns++;
             return gcStats;
         }, { end: true, cancel: "error" });
     }
-    async runPreGCSteps() {
-        // Ensure that state has been initialized from the base snapshot data.
-        await this.initializeGCStateFromBaseSnapshotP;
-        // Let the runtime update its pending state before GC runs.
-        await this.runtime.updateStateBeforeGC();
-    }
-    async runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs) {
-        // Generate statistics from the current run. This is done before updating the current state because it
-        // generates some of its data based on previous state of the system.
-        const gcStats = this.generateStats(gcResult);
-        // Update the current mark state and update the runtime of all used routes or ids that used as per the GC run.
-        const sweepReadyNodes = this.updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger);
-        this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
-        // Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
-        // delete these objects here instead.
-        this.logSweepEvents(logger, currentReferenceTimestampMs);
-        let updatedGCData = gcData;
-        if (this.configs.shouldRunSweep) {
-            updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
-        }
-        else if (this.configs.testMode) {
-            // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
-            // involving access to deleted data.
-            this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
-        }
-        else if (this.configs.tombstoneMode) {
-            this.tombstones = sweepReadyNodes;
-            // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
-            // involving access to "deleted" data without actually deleting the data from summaries.
-            // Note: we will not tombstone in test mode.
-            this.runtime.updateTombstonedRoutes(this.tombstones);
-        }
-        this.gcDataFromLastRun = cloneGCData(updatedGCData);
-        // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
-        // updates its state so that we don't send false positives based on intermediate state. For example, we may get
-        // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
-        await this.logUnreferencedEvents(logger);
-        return gcStats;
-    }
     /**
-     * Summarizes the GC data and returns it as a summary tree.
-     * We current write the entire GC state in a single blob. This can be modified later to write multiple
-     * blobs. All the blob keys should start with `gcBlobPrefix`.
+     * Runs garbage collection. It does the following:
+     * 1. It generates / analyzes the runtime's reference graph.
+     * 2. Generates stats for the GC run based on previous / current GC state.
+     * 3. Runs Mark phase.
+     * 4. Runs Sweep phase.
      */
-    summarize(fullTree, trackState, telemetryContext) {
+    async runGC(fullGC, currentReferenceTimestampMs, logger) {
         var _a;
-        if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
-            return;
-        }
-        const gcState = { gcNodes: {} };
-        for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
-            gcState.gcNodes[nodeId] = {
-                outboundRoutes,
-                unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
-            };
-        }
-        return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
-    }
-    getMetadata() {
-        return {
-            /**
-             * If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
-             * into the metadata blob. If GC is disabled, the gcFeature is 0.
-             */
-            gcFeature: this.configs.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
-            gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
-            sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
-            sweepEnabled: false,
-            sweepTimeoutMs: this.configs.sweepTimeoutMs,
-        };
-    }
-    /**
-     * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
-     * in the container.
-     */
-    async getBaseGCDetails() {
-        return this.baseGCDetailsP;
-    }
-    /**
-     * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
-     * is downloaded and should be used to update the state.
-     */
-    async refreshLatestSummary(proposalHandle, result, readAndParseBlob) {
-        const latestSnapshotData = await this.summaryStateTracker.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
-        // If the latest summary was updated but it was not tracked by this client, our state needs to be updated from
-        // this snapshot data.
-        if (this.shouldRunGC && result.latestSummaryUpdated && !result.wasSummaryTracked) {
-            // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
-            // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
-            const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
-            if (currentReferenceTimestampMs === undefined) {
-                throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
-                    proposalHandle,
-                    summaryRefSeq: result.summaryRefSeq,
-                    gcConfigs: JSON.stringify(this.configs),
-                });
-            }
-            this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
-        }
-    }
-    /**
-     * Called when a node with the given id is updated. If the node is inactive, log an error.
-     * @param nodePath - The id of the node that changed.
-     * @param reason - Whether the node was loaded or changed.
-     * @param timestampMs - The timestamp when the node changed.
-     * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
-     * @param requestHeaders - If the node was loaded via request path, the headers in the request.
-     */
-    nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
-        if (!this.configs.shouldRunGC) {
-            return;
-        }
-        const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
-        if (nodeStateTracker && nodeStateTracker.state !== UnreferencedState.Active) {
-            this.inactiveNodeUsed(reason, nodePath, nodeStateTracker, undefined /* fromNodeId */, packagePath, timestampMs, requestHeaders);
-        }
+        // 1. Generate / analyze the runtime's reference graph.
+        // Get the reference graph (gcData) and run GC algorithm to get referenced / unreferenced nodes.
+        const gcData = await this.runtime.getGCData(fullGC);
+        const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
+        // Get all referenced nodes - References in this run + references between the previous and current runs.
+        const allReferencedNodeIds = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
+        // 2. Generate stats based on the previous / current GC state.
+        // Must happen before running Mark / Sweep phase because previous GC state will be updated in these stages.
+        const gcStats = this.generateStats(gcResult);
+        // 3. Run the Mark phase.
+        // It will mark nodes as referenced / unreferenced and return a list of node ids that are ready to be swept.
+        const sweepReadyNodeIds = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
+        // 4. Run the Sweep phase.
+        // It will delete sweep ready nodes and return a list of deleted node ids.
+        const deletedNodeIds = this.runSweepPhase(gcResult, sweepReadyNodeIds, currentReferenceTimestampMs, logger);
+        this.gcDataFromLastRun = cloneGCData(gcData, (id) => deletedNodeIds.includes(id) /* filter out deleted nodes */);
+        return gcStats;
     }
     /**
-     * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
-     * referenced between summaries so that their unreferenced timestamp can be reset.
+     * Runs the GC Mark phase. It does the following:
+     * 1. Marks all referenced nodes in this run by clearing tracking for them.
+     * 2. Marks unreferenced nodes in this run by starting tracking for them.
+     * 3. Calls the runtime to update nodes that were marked referenced.
      *
-     * @param fromNodePath - The node from which the reference is added.
-     * @param toNodePath - The node to which the reference is added.
-     */
-    addedOutboundReference(fromNodePath, toNodePath) {
-        var _a;
-        if (!this.configs.shouldRunGC) {
-            return;
-        }
-        const outboundRoutes = (_a = this.newReferencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
-        outboundRoutes.push(toNodePath);
-        this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
-        const nodeStateTracker = this.unreferencedNodesState.get(toNodePath);
-        if (nodeStateTracker && nodeStateTracker.state !== UnreferencedState.Active) {
-            this.inactiveNodeUsed("Revived", toNodePath, nodeStateTracker, fromNodePath);
-        }
-        if (this.tombstones.includes(toNodePath)) {
-            const nodeType = this.runtime.getNodeType(toNodePath);
-            let eventName = "GC_Tombstone_SubDatastore_Revived";
-            if (nodeType === GCNodeType.DataStore) {
-                eventName = "GC_Tombstone_Datastore_Revived";
-            }
-            else if (nodeType === GCNodeType.Blob) {
-                eventName = "GC_Tombstone_Blob_Revived";
-            }
-            sendGCUnexpectedUsageEvent(this.mc, {
-                eventName,
-                category: "generic",
-                url: toNodePath,
-                nodeType,
-                gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
-            }, undefined /* packagePath */);
-        }
-    }
-    /**
-     * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
-     * cases where objects are used after they are deleted and throw / log errors accordingly.
-     */
-    isNodeDeleted(nodePath) {
-        return this.deletedNodes.has(nodePath);
-    }
-    dispose() {
-        var _a;
-        (_a = this.sessionExpiryTimer) === null || _a === void 0 ? void 0 : _a.clear();
-        this.sessionExpiryTimer = undefined;
-    }
-    /**
-     * Updates the state of the system as per the current GC run. It does the following:
-     * 1. Sets up the current GC state as per the gcData.
-     * 2. Starts tracking for nodes that have become unreferenced in this run.
-     * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
-     * @param gcData - The data representing the reference graph on which GC is run.
      * @param gcResult - The result of the GC run on the gcData.
+     * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
      * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
-     * @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
+     * @returns - A list of sweep ready nodes, i.e., nodes that ready to be deleted.
      */
-    updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger) {
-        var _a;
-        // Get references from the current GC run + references between previous and current run and then update each
-        // node's state
-        const allNodesReferencedBetweenGCs = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
-        this.newReferencesSinceLastRun.clear();
-        // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
-        for (const nodeId of allNodesReferencedBetweenGCs) {
+    runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
+        // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
+        for (const nodeId of allReferencedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker !== undefined) {
                 // Stop tracking so as to clear out any running timers.
@@ -555,38 +416,72 @@ export class GarbageCollector {
                 this.unreferencedNodesState.delete(nodeId);
             }
         }
-        /**
-         * If a node became unreferenced in this run, start tracking it.
-         * If a node was already unreferenced, update its tracking information. Since the current reference time is
-         * from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
-         *
-         * If a node is sweep ready, store and then return it.
-         */
-        const sweepReadyNodes = [];
+        // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
+        const sweepReadyNodeIds = [];
         for (const nodeId of gcResult.deletedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker === undefined) {
                 this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
             }
             else {
+                // If a node was already unreferenced, update its tracking information. Since the current reference time
+                // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
                 nodeStateTracker.updateTracking(currentReferenceTimestampMs);
+                // If a node is sweep ready, store it so it can be returned.
                 if (nodeStateTracker.state === UnreferencedState.SweepReady) {
-                    sweepReadyNodes.push(nodeId);
+                    sweepReadyNodeIds.push(nodeId);
                 }
             }
         }
-        return sweepReadyNodes;
+        // 3. Call the runtime to update referenced nodes in this run.
+        this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
+        return sweepReadyNodeIds;
     }
     /**
-     * Deletes nodes from both the runtime and garbage collection
-     * @param sweepReadyNodes - nodes that are ready to be deleted
+     * Runs the GC Sweep phase. It does the following:
+     * 1. Calls the runtime to delete nodes that are sweep ready.
+     * 2. Clears tracking for deleted nodes.
+     *
+     * @param gcResult - The result of the GC run on the gcData.
+     * @param sweepReadyNodes - List of nodes that are sweep ready.
+     * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
+     * @param logger - The logger to be used to log any telemetry.
+     * @returns - A list of nodes that have been deleted.
      */
-    runSweepPhase(sweepReadyNodes, gcData) {
-        // TODO: GC:Validation - validate that removed routes are not double deleted
-        // TODO: GC:Validation - validate that the child routes of removed routes are deleted as well
-        const sweptRoutes = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
-        const updatedGCData = this.deleteSweptRoutes(sweptRoutes, gcData);
-        for (const nodeId of sweptRoutes) {
+    runSweepPhase(gcResult, sweepReadyNodes, currentReferenceTimestampMs, logger) {
+        // Log events for objects that are ready to be deleted by sweep. This will give us data on sweep when
+        // its not enabled.
+        this.telemetryTracker.logSweepEvents(logger, currentReferenceTimestampMs, this.unreferencedNodesState, this.completedRuns, this.getLastSummaryTimestampMs());
+        /**
+         * Currently, there are 3 modes for sweep:
+         * Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
+         * Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted.
+         * Sweep mode - Sweep ready modes are deleted.
+         *
+         * These modes serve as staging for applications that want to enable sweep by providing an incremental
+         * way to test and validate sweep works as expected.
+         */
+        if (this.configs.testMode) {
+            // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
+            this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
+            return [];
+        }
+        if (this.configs.tombstoneMode) {
+            this.tombstones = sweepReadyNodes;
+            // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
+            // involving access to "deleted" data without actually deleting the data from summaries.
+            this.runtime.updateTombstonedRoutes(this.tombstones);
+            return [];
+        }
+        if (!this.configs.shouldRunSweep) {
+            return [];
+        }
+        // 1. Call the runtime to delete sweep ready nodes. The runtime returns a list of nodes it deleted.
+        // TODO: GC:Validation - validate that removed routes are not double delete and that the child routes of
+        // removed routes are deleted as well.
+        const deletedNodeIds = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
+        // 2. Clear unreferenced state tracking for deleted nodes.
+        for (const nodeId of deletedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             // TODO: GC:Validation - assert that the nodeStateTracker is defined
             if (nodeStateTracker !== undefined) {
@@ -598,23 +493,7 @@ export class GarbageCollector {
             // TODO: GC:Validation - assert that the deleted node is not a duplicate
             this.deletedNodes.add(nodeId);
         }
-        return updatedGCData;
-    }
-    /**
-     * @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
-     */
-    deleteSweptRoutes(sweptRoutes, gcData) {
-        const sweptRoutesSet = new Set(sweptRoutes);
-        const gcNodes = {};
-        for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
-            if (!sweptRoutesSet.has(id)) {
-                gcNodes[id] = Array.from(outboundRoutes);
-            }
-        }
-        // TODO: GC:Validation - assert that the nodeId is in gcData
-        return {
-            gcNodes,
-        };
+        return deletedNodeIds;
     }
     /**
      * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
@@ -637,17 +516,11 @@ export class GarbageCollector {
         if (previousGCData === undefined) {
             return undefined;
         }
-        // Find any references that haven't been identified correctly.
-        const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun);
-        if (missingExplicitReferences.length > 0) {
-            missingExplicitReferences.forEach((missingExplicitReference) => {
-                logger.sendErrorEvent({
-                    eventName: "gcUnknownOutboundReferences",
-                    gcNodeId: missingExplicitReference[0],
-                    gcRoutes: JSON.stringify(missingExplicitReference[1]),
-                });
-            });
-        }
+        /**
+         * If there are references that were not explicitly notified to GC, log an error because this should never happen.
+         * If it does, this may result in the unreferenced timestamps of these nodes not updated when they were referenced.
+         */
+        this.telemetryTracker.logIfMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun, logger);
         // No references were added since the last run so we don't have to update reference states of any unreferenced
         // nodes. There is no in between state at this point.
         if (this.newReferencesSinceLastRun.size === 0) {
@@ -694,50 +567,119 @@ export class GarbageCollector {
         return gcResult.referencedNodeIds;
     }
     /**
-     * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
-     * The principle is that every new reference or outbound route must be notified to GC via the
-     * addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
-     *
-     * In more simple terms:
-     * Missing Explicit References = Current References - Previous References - Explicitly Added References;
-     *
-     * @param currentGCData - The GC data (reference graph) from the current GC run.
-     * @param previousGCData - The GC data (reference graph) from the previous GC run.
-     * @param explicitReferences - New references added explicity between the previous and the current run.
-     * @returns - a list of missing explicit references
+     * Summarizes the GC data and returns it as a summary tree.
+     * We current write the entire GC state in a single blob. This can be modified later to write multiple
+     * blobs. All the blob keys should start with `gcBlobPrefix`.
      */
-    findMissingExplicitReferences(currentGCData, previousGCData, explicitReferences) {
-        assert(previousGCData !== undefined, 0x2b7 /* "Can't validate correctness without GC data from last run" */);
-        const currentGraph = Object.entries(currentGCData.gcNodes);
-        const missingExplicitReferences = [];
-        currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
-            var _a, _b;
-            const previousRoutes = (_a = previousGCData.gcNodes[nodeId]) !== null && _a !== void 0 ? _a : [];
-            const explicitRoutes = (_b = explicitReferences.get(nodeId)) !== null && _b !== void 0 ? _b : [];
-            const missingExplicitRoutes = [];
+    summarize(fullTree, trackState, telemetryContext) {
+        var _a;
+        if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
+            return;
+        }
+        const gcState = { gcNodes: {} };
+        for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
+            gcState.gcNodes[nodeId] = {
+                outboundRoutes,
+                unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
+            };
+        }
+        return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
+    }
+    getMetadata() {
+        return {
             /**
-             * 1. For routes in the current GC data, routes that were not present in previous GC data and did not have
-             * explicit references should be added to missing explicit routes list.
-             * 2. Only include data store and blob routes since GC only works for these two.
-             * Note: Due to a bug with de-duped blobs, only adding data store routes for now.
-             * 3. Ignore DDS routes to their parent datastores since those were added implicitly. So, there won't be
-             * explicit routes to them.
+             * If GC is enabled, the GC data is written using the GC version in effect and that is the gcFeature that goes
+             * into the metadata blob. If GC is disabled, the gcFeature is 0.
              */
-            currentOutboundRoutes.forEach((route) => {
-                const nodeType = this.runtime.getNodeType(route);
-                if ((nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) &&
-                    !nodeId.startsWith(route) &&
-                    !previousRoutes.includes(route) &&
-                    !explicitRoutes.includes(route)) {
-                    missingExplicitRoutes.push(route);
-                }
-            });
-            if (missingExplicitRoutes.length > 0) {
-                missingExplicitReferences.push([nodeId, missingExplicitRoutes]);
+            gcFeature: this.configs.gcEnabled ? this.configs.gcVersionInEffect : 0,
+            gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
+            sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
+            sweepEnabled: false,
+            sweepTimeoutMs: this.configs.sweepTimeoutMs,
+        };
+    }
+    /**
+     * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
+     * is downloaded and should be used to update the state.
+     */
+    async refreshLatestSummary(proposalHandle, result, readAndParseBlob) {
+        const latestSnapshotData = await this.summaryStateTracker.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
+        // If the latest summary was updated but it was not tracked by this client, our state needs to be updated from
+        // this snapshot data.
+        if (this.shouldRunGC && result.latestSummaryUpdated && !result.wasSummaryTracked) {
+            // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
+            // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
+            const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
+            if (currentReferenceTimestampMs === undefined) {
+                throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
+                    proposalHandle,
+                    summaryRefSeq: result.summaryRefSeq,
+                    gcConfigs: JSON.stringify(this.configs),
+                });
             }
+            this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
+        }
+    }
+    /**
+     * Called when a node with the given id is updated. If the node is inactive, log an error.
+     * @param nodePath - The id of the node that changed.
+     * @param reason - Whether the node was loaded or changed.
+     * @param timestampMs - The timestamp when the node changed.
+     * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
+     * @param requestHeaders - If the node was loaded via request path, the headers in the request.
+     */
+    nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
+        if (!this.configs.shouldRunGC) {
+            return;
+        }
+        this.telemetryTracker.nodeUsed({
+            id: nodePath,
+            usageType: reason,
+            currentReferenceTimestampMs: timestampMs !== null && timestampMs !== void 0 ? timestampMs : this.runtime.getCurrentReferenceTimestampMs(),
+            packagePath,
+            completedGCRuns: this.completedRuns,
+            isTombstoned: this.tombstones.includes(nodePath),
+            lastSummaryTime: this.getLastSummaryTimestampMs(),
+            viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle],
+        });
+    }
+    /**
+     * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
+     * referenced between summaries so that their unreferenced timestamp can be reset.
+     *
+     * @param fromNodePath - The node from which the reference is added.
+     * @param toNodePath - The node to which the reference is added.
+     */
+    addedOutboundReference(fromNodePath, toNodePath) {
+        var _a;
+        if (!this.configs.shouldRunGC) {
+            return;
+        }
+        const outboundRoutes = (_a = this.newReferencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
+        outboundRoutes.push(toNodePath);
+        this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
+        this.telemetryTracker.nodeUsed({
+            id: toNodePath,
+            usageType: "Revived",
+            currentReferenceTimestampMs: this.runtime.getCurrentReferenceTimestampMs(),
+            packagePath: undefined,
+            completedGCRuns: this.completedRuns,
+            isTombstoned: this.tombstones.includes(toNodePath),
+            lastSummaryTime: this.getLastSummaryTimestampMs(),
+            fromId: fromNodePath,
         });
-        // Ideally missingExplicitReferences should always have a size 0
-        return missingExplicitReferences;
+    }
+    /**
+     * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
+     * cases where objects are used after they are deleted and throw / log errors accordingly.
+     */
+    isNodeDeleted(nodePath) {
+        return this.deletedNodes.has(nodePath);
+    }
+    dispose() {
+        var _a;
+        (_a = this.sessionExpiryTimer) === null || _a === void 0 ? void 0 : _a.clear();
+        this.sessionExpiryTimer = undefined;
     }
     /**
      * Generates the stats of a garbage collection run from the given results of the run.
@@ -795,128 +737,5 @@ export class GarbageCollector {
         }
         return gcStats;
     }
-    /**
-     * For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
-     * this will give us a view into how much deleted content a container has.
-     */
-    logSweepEvents(logger, currentReferenceTimestampMs) {
-        if (this.mc.config.getBoolean(disableSweepLogKey) === true ||
-            this.configs.sweepTimeoutMs === undefined) {
-            return;
-        }
-        this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
-            if (nodeStateTracker.state !== UnreferencedState.SweepReady) {
-                return;
-            }
-            const nodeType = this.runtime.getNodeType(nodeId);
-            if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
-                return;
-            }
-            // Log deleted event for each node only once to reduce noise in telemetry.
-            const uniqueEventId = `Deleted-${nodeId}`;
-            if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
-                return;
-            }
-            this.loggedUnreferencedEvents.add(uniqueEventId);
-            logger.sendTelemetryEvent({
-                eventName: "GCObjectDeleted",
-                id: nodeId,
-                type: nodeType,
-                age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
-                timeout: this.configs.sweepTimeoutMs,
-                completedGCRuns: this.completedRuns,
-                lastSummaryTime: this.getLastSummaryTimestampMs(),
-            });
-        });
-    }
-    /**
-     * Called when an inactive node is used after. Queue up an event that will be logged next time GC runs.
-     */
-    inactiveNodeUsed(usageType, nodeId, nodeStateTracker, fromNodeId, packagePath, currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs(), requestHeaders) {
-        // If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
-        // logging as nothing interesting would have happened worth logging.
-        // If the node is active, skip logging.
-        if (currentReferenceTimestampMs === undefined ||
-            nodeStateTracker.state === UnreferencedState.Active) {
-            return;
-        }
-        // We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
-        // as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
-        const nodeType = this.runtime.getNodeType(nodeId);
-        if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
-            return;
-        }
-        const state = nodeStateTracker.state;
-        const uniqueEventId = `${state}-${nodeId}-${usageType}`;
-        if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
-            return;
-        }
-        this.loggedUnreferencedEvents.add(uniqueEventId);
-        const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === UnreferencedState.Inactive
-                ? this.configs.inactiveTimeoutMs
-                : this.configs.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
-        // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
-        // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
-        // but it's a good signal nonetheless and we can consume it with a grain of salt.
-        // Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
-        // SweepReady errors are usages of Objects that will be deleted by GC Sweep!
-        if (this.isSummarizerClient) {
-            this.pendingEventsQueue.push(Object.assign(Object.assign({}, propsToLog), { usageType, state }));
-        }
-        else {
-            // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
-            // summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
-            // Events generated:
-            // InactiveObject_Loaded, SweepReadyObject_Loaded
-            if (usageType === "Loaded") {
-                const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: packagePathToTelemetryProperty(packagePath), stack: generateStack() });
-                // Do not log the inactive object x events as error events as they are not the best signal for
-                // detecting something wrong with GC either from the partner or from the runtime itself.
-                if (state === UnreferencedState.Inactive) {
-                    this.mc.logger.sendTelemetryEvent(event);
-                }
-                else {
-                    this.mc.logger.sendErrorEvent(event);
-                }
-            }
-        }
-    }
-    async logUnreferencedEvents(logger) {
-        // Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
-        // summary time they are then logged.
-        // Events generated:
-        // InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
-        // SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
-        for (const eventProps of this.pendingEventsQueue) {
-            const { usageType, state } = eventProps, propsToLog = __rest(eventProps, ["usageType", "state"]);
-            /**
-             * Revived event is logged only if the node is active. If the node is not active, the reference to it was
-             * from another unreferenced node and this scenario is not interesting to log.
-             * Loaded and Changed events are logged only if the node is not active. If the node is active, it was
-             * revived and a Revived event will be logged for it.
-             */
-            const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
-            const active = nodeStateTracker === undefined ||
-                nodeStateTracker.state === UnreferencedState.Active;
-            if ((usageType === "Revived") === active) {
-                const pkg = await this.getNodePackagePath(eventProps.id);
-                const fromPkg = eventProps.fromId
-                    ? await this.getNodePackagePath(eventProps.fromId)
-                    : undefined;
-                const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: pkg
-                        ? { value: pkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
-                        : undefined, fromPkg: fromPkg
-                        ? { value: fromPkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
-                        : undefined });
-                if (state === UnreferencedState.Inactive) {
-                    logger.sendTelemetryEvent(event);
-                }
-                else {
-                    logger.sendErrorEvent(event);
-                }
-            }
-        }
-        this.pendingEventsQueue = [];
-    }
 }
 //# sourceMappingURL=garbageCollection.js.map