@fluidframework/container-runtime 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.224419

package/lib/gc/garbageCollection.mjs

@@ -0,0 +1,857 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { assert, LazyPromise, Timer } from "@fluidframework/core-utils";
+import { gcTreeKey, } from "@fluidframework/runtime-definitions";
+import { createResponseError, responseToException } from "@fluidframework/runtime-utils";
+import { createChildLogger, createChildMonitoringContext, DataProcessingError, PerformanceEvent, } from "@fluidframework/telemetry-utils";
+import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey, } from "../containerRuntime.mjs";
+import { ClientSessionExpiredError } from "../error.mjs";
+import { ContainerMessageType } from "../messageTypes.mjs";
+import { generateGCConfigs } from "./gcConfigs.mjs";
+import { GCNodeType, UnreferencedState, GarbageCollectionMessageType, } from "./gcDefinitions.mjs";
+import { cloneGCData, compatBehaviorAllowsGCMessageType, concatGarbageCollectionData, getGCDataFromSnapshot, } from "./gcHelpers.mjs";
+import { runGarbageCollection } from "./gcReferenceGraphAlgorithm.mjs";
+import { GCSummaryStateTracker } from "./gcSummaryStateTracker.mjs";
+import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker.mjs";
+import { GCTelemetryTracker } from "./gcTelemetry.mjs";
+/**
+ * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
+ * its state across summaries.
+ *
+ * Node - represented as nodeId, it's a node on the GC graph
+ *
+ * Outbound Route - a path from one node to another node, think `nodeA` -\> `nodeB`
+ *
+ * Graph - all nodes with their respective routes
+ *
+ * ```
+ *			 GC Graph
+ *
+ *			   Node
+ *		NodeId = "datastore1"
+ *		   /			 \\
+ *	OutboundRoute   OutboundRoute
+ *		 /				 \\
+ *	   Node			   Node
+ *  NodeId = "dds1"	 NodeId = "dds2"
+ * ```
+ */
+export class GarbageCollector {
+    static create(createParams) {
+        return new GarbageCollector(createParams);
+    }
+    get shouldRunGC() {
+        return this.configs.shouldRunGC;
+    }
+    /** If false, loading or using a Tombstoned object should merely log, not fail */
+    get tombstoneEnforcementAllowed() {
+        return this.configs.sweepEnabled;
+    }
+    /** If true, throw an error when a tombstone data store is retrieved */
+    get throwOnTombstoneLoad() {
+        return this.configs.throwOnTombstoneLoad;
+    }
+    /** If true, throw an error when a tombstone data store is used */
+    get throwOnTombstoneUsage() {
+        return this.configs.throwOnTombstoneUsage;
+    }
+    get summaryStateNeedsReset() {
+        return this.summaryStateTracker.doesSummaryStateNeedReset;
+    }
+    /** Returns the count of data stores whose GC state updated since the last summary. */
+    get updatedDSCountSinceLastSummary() {
+        return this.summaryStateTracker.updatedDSCountSinceLastSummary;
+    }
+    constructor(createParams) {
+        // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
+        // outbound routes from that node.
+        this.newReferencesSinceLastRun = new Map();
+        // A list of nodes that have been tombstoned.
+        this.tombstones = [];
+        // A list of nodes that have been deleted during sweep phase.
+        this.deletedNodes = new Set();
+        // Map of node ids to their unreferenced state tracker.
+        this.unreferencedNodesState = new Map();
+        // The number of times GC has successfully completed on this instance of GarbageCollector.
+        this.completedRuns = 0;
+        this.runtime = createParams.runtime;
+        this.isSummarizerClient = createParams.isSummarizerClient;
+        this.getNodePackagePath = createParams.getNodePackagePath;
+        this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
+        this.submitMessage = createParams.submitMessage;
+        const baseSnapshot = createParams.baseSnapshot;
+        const readAndParseBlob = createParams.readAndParseBlob;
+        this.mc = createChildMonitoringContext({
+            logger: createParams.baseLogger,
+            namespace: "GarbageCollector",
+            properties: {
+                all: { completedGCRuns: () => this.completedRuns },
+            },
+        });
+        this.configs = generateGCConfigs(this.mc, createParams);
+        // If session expiry is enabled, we need to close the container when the session expiry timeout expires.
+        if (this.configs.sessionExpiryTimeoutMs !== undefined) {
+            // If Test Override config is set, override Session Expiry timeout.
+            const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
+            const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.configs.sessionExpiryTimeoutMs;
+            this.sessionExpiryTimer = new Timer(timeoutMs, () => {
+                this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
+            });
+            this.sessionExpiryTimer.start();
+        }
+        this.summaryStateTracker = new GCSummaryStateTracker(this.configs, baseSnapshot?.trees[gcTreeKey] !== undefined /* wasGCRunInBaseSnapshot */);
+        this.telemetryTracker = new GCTelemetryTracker(this.mc, this.configs, this.isSummarizerClient, createParams.createContainerMetadata, (nodeId) => this.runtime.getNodeType(nodeId), (nodeId) => this.unreferencedNodesState.get(nodeId), this.getNodePackagePath);
+        // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
+        // it involves fetching blobs from storage which is expensive.
+        this.baseSnapshotDataP = new LazyPromise(async () => {
+            if (baseSnapshot === undefined) {
+                return undefined;
+            }
+            try {
+                // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
+                const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
+                if (gcSnapshotTree === undefined) {
+                    // back-compat - Older documents get their gc data reset for simplicity as there are few of them
+                    // incremental gc summary will not work with older gc data as well
+                    return undefined;
+                }
+                const snapshotData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
+                // If the GC version in base snapshot does not match the GC version currently in effect, the GC data
+                // in the snapshot cannot be interpreted correctly. Set everything to undefined except for
+                // deletedNodes because irrespective of GC versions, these nodes have been deleted and cannot be
+                // brought back. The deletedNodes info is needed to identify when these nodes are used.
+                if (this.configs.gcVersionInEffect !== this.configs.gcVersionInBaseSnapshot) {
+                    return {
+                        gcState: undefined,
+                        tombstones: undefined,
+                        deletedNodes: snapshotData.deletedNodes,
+                    };
+                }
+                return snapshotData;
+            }
+            catch (error) {
+                const dpe = DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
+                dpe.addTelemetryProperties({
+                    gcConfigs: JSON.stringify(this.configs),
+                });
+                throw dpe;
+            }
+        });
+        /**
+         * Set up the initializer which initializes the GC state from the data in base snapshot. It sets up GC data
+         * from the base GC state and starts tracking the state of unreferenced nodes.
+         *
+         * Must only be called if there is a current reference timestamp.
+         */
+        this.initializeGCStateFromBaseSnapshotP = new LazyPromise(async () => {
+            const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
+            assert(currentReferenceTimestampMs !== undefined, "Trying to initialize GC state without current timestamp");
+            /**
+             * The base snapshot data will not be present if the container is loaded from:
+             * 1. The first summary created by the detached container.
+             * 2. A summary that was generated with GC disabled.
+             * 3. A summary that was generated before GC even existed.
+             */
+            const baseSnapshotData = await this.baseSnapshotDataP;
+            this.summaryStateTracker.initializeBaseState(baseSnapshotData);
+            if (baseSnapshotData?.gcState === undefined) {
+                return;
+            }
+            // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
+            // to the GC data from the snapshot data.
+            const gcNodes = {};
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
+                if (nodeData.unreferencedTimestampMs !== undefined) {
+                    this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
+                }
+                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
+            }
+            this.gcDataFromLastRun = { gcNodes };
+        });
+        // Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
+        // used to initialize the GC state of all the nodes in the container.
+        this.baseGCDetailsP = new LazyPromise(async () => {
+            const baseSnapshotData = await this.baseSnapshotDataP;
+            if (baseSnapshotData?.gcState === undefined) {
+                return {};
+            }
+            const gcNodes = {};
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
+                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
+            }
+            // Run GC on the nodes in the base summary to get the routes used in each node in the container.
+            // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
+            // each node in the summary.
+            const usedRoutes = runGarbageCollection(gcNodes, ["/"]).referencedNodeIds;
+            return { gcData: { gcNodes }, usedRoutes };
+        });
+        // Log all the GC options and the state determined by the garbage collector.
+        // This is useful even for interactive clients since they track unreferenced nodes and log errors.
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "GarbageCollectorLoaded",
+            gcConfigs: JSON.stringify(this.configs),
+            gcOptions: JSON.stringify(createParams.gcOptions),
+            ...createParams.createContainerMetadata,
+        });
+    }
+    /**
+     * Called during container initialization. Initializes the tombstone and deleted nodes state from the base snapshot.
+     * Also, initializes the GC state including unreferenced nodes tracking if a current reference timestamp exists.
+     * Note that if there is any GC state in the base snapshot, then there will definitely be a reference timestamp
+     * to work with - The GC state would have been generated using a timestamp which is part of the snapshot.
+     */
+    async initializeBaseState() {
+        const baseSnapshotData = await this.baseSnapshotDataP;
+        /**
+         * The base snapshot data will not be present if the container is loaded from:
+         * 1. The first summary created by the detached container.
+         * 2. A summary that was generated with GC disabled.
+         * 3. A summary that was generated before GC even existed.
+         */
+        if (baseSnapshotData === undefined) {
+            return;
+        }
+        // Initialize the deleted nodes from the snapshot. This is done irrespective of whether sweep is enabled or not
+        // to identify deleted nodes' usage.
+        if (baseSnapshotData.deletedNodes !== undefined) {
+            this.deletedNodes = new Set(baseSnapshotData.deletedNodes);
+        }
+        // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
+        // tombstone routes.
+        if (this.configs.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
+            // Create a copy since we are writing from a source we don't control
+            this.tombstones = Array.from(baseSnapshotData.tombstones);
+            this.runtime.updateTombstonedRoutes(this.tombstones);
+        }
+        await this.initializeOrUpdateGCState();
+    }
+    /**
+     * Initialize the GC state if not already initialized. If GC state is already initialized, update the unreferenced
+     * state tracking as per the current reference timestamp.
+     */
+    async initializeOrUpdateGCState() {
+        const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
+        if (currentReferenceTimestampMs === undefined) {
+            return;
+        }
+        // If the GC state hasn't been initialized yet, initialize it and return.
+        if (this.gcDataFromLastRun === undefined) {
+            await this.initializeGCStateFromBaseSnapshotP;
+            return;
+        }
+        // If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
+        // reference timestamp.
+        for (const [, nodeStateTracker] of this.unreferencedNodesState) {
+            nodeStateTracker.updateTracking(currentReferenceTimestampMs);
+        }
+    }
+    /**
+     * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
+     * to initialize or update the unreference state tracking.
+     * @param connected - Whether the runtime connected / disconnected.
+     * @param clientId - The clientId of this runtime.
+     */
+    setConnectionState(connected, clientId) {
+        /**
+         * When the client connects (or reconnects), attempt to initialize or update the GC state. This will keep
+         * the unreferenced state tracking updated as per the reference timestamp at the time of connection.
+         *
+         * During GC initialization and during connections in read mode, it is possible that either no ops are
+         * processed or only trailing ops are processed. This means that the GC state is not initialized or initialized
+         * with an older reference timestamp. So, doing this on every connection will keep the unreferenced state
+         * tracking up-to-date.
+         */
+        if (connected && this.configs.shouldRunGC) {
+            this.initializeOrUpdateGCState().catch((error) => {
+                this.mc.logger.sendErrorEvent({
+                    eventName: "GCInitializationOrUpdateFailed",
+                    gcConfigs: JSON.stringify(this.configs),
+                }, error);
+            });
+        }
+    }
+    /**
+     * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
+     * in the container.
+     */
+    async getBaseGCDetails() {
+        return this.baseGCDetailsP;
+    }
+    /**
+     * Runs garbage collection and updates the reference / used state of the nodes in the container.
+     * @returns stats of the GC run or undefined if GC did not run.
+     */
+    async collectGarbage(options, telemetryContext) {
+        const fullGC = options.fullGC ??
+            (this.configs.runFullGC === true || this.summaryStateTracker.doesSummaryStateNeedReset);
+        // Add the options that are used to run GC to the telemetry context.
+        telemetryContext?.setMultiple("fluid_GC", "Options", {
+            fullGC,
+            runSweep: options.runSweep,
+        });
+        const logger = options.logger
+            ? createChildLogger({
+                logger: options.logger,
+                properties: {
+                    all: { completedGCRuns: () => this.completedRuns },
+                },
+            })
+            : this.mc.logger;
+        /**
+         * If there is no current reference timestamp, skip running GC. We need the current timestamp to track
+         * how long objects have been unreferenced and if they should be deleted.
+         *
+         * Note that the only scenario where GC is called and there is no reference timestamp is when no ops have ever
+         * been processed for this container and it is in read mode. In this scenario, there is no point in running GC
+         * anyway because references in the container do not change without any ops, i.e., there is nothing to collect.
+         */
+        const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
+        if (currentReferenceTimestampMs === undefined) {
+            // Log an event so we can evaluate how often we run into this scenario.
+            logger.sendErrorEvent({
+                eventName: "CollectGarbageCalledWithoutTimestamp",
+                gcConfigs: JSON.stringify(this.configs),
+            });
+            return undefined;
+        }
+        return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
+            /** Pre-GC steps */
+            // Ensure that state has been initialized from the base snapshot data.
+            await this.initializeGCStateFromBaseSnapshotP;
+            // Let the runtime update its pending state before GC runs.
+            await this.runtime.updateStateBeforeGC();
+            /** GC step */
+            const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
+            event.end({
+                ...gcStats,
+                timestamp: currentReferenceTimestampMs,
+                sweep: this.configs.shouldRunSweep,
+            });
+            /** Post-GC steps */
+            // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
+            // updates its state so that we don't send false positives based on intermediate state. For example, we may get
+            // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
+            await this.telemetryTracker.logPendingEvents(logger);
+            // Update the state of summary state tracker from this run's stats.
+            this.summaryStateTracker.updateStateFromGCRunStats(gcStats);
+            this.newReferencesSinceLastRun.clear();
+            this.completedRuns++;
+            return gcStats;
+        }, { end: true, cancel: "error" });
+    }
+    /**
+     * Runs garbage collection. It does the following:
+     *
+     * 1. It generates / analyzes the runtime's reference graph.
+     *
+     * 2. Generates mark phase stats.
+     *
+     * 3. Runs Mark phase.
+     *
+     * 4. Runs Sweep phase.
+     *
+     * 5. Generates sweep phase stats.
+     */
+    async runGC(fullGC, currentReferenceTimestampMs, logger) {
+        // 1. Generate / analyze the runtime's reference graph.
+        // Get the reference graph (gcData) and run GC algorithm to get referenced / unreferenced nodes.
+        const gcData = await this.runtime.getGCData(fullGC);
+        const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
+        // Get all referenced nodes - References in this run + references between the previous and current runs.
+        const allReferencedNodeIds = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
+            gcResult.referencedNodeIds;
+        // 2. Get the mark phase stats based on the previous / current GC state.
+        // This is done before running mark phase because we need the previous GC state before it is updated.
+        const markPhaseStats = this.getMarkPhaseStats(gcResult);
+        // 3. Run the Mark phase.
+        // It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
+        const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
+        // 4. Run the Sweep phase.
+        // It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
+        // sweep op. All clients, including this one, will delete these nodes once it processes the op.
+        this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
+        this.gcDataFromLastRun = cloneGCData(gcData);
+        // 5. Get the sweep phase stats.
+        const sweepPhaseStats = this.getSweepPhaseStats(this.deletedNodes, sweepReadyNodeIds, markPhaseStats);
+        return { ...markPhaseStats, ...sweepPhaseStats };
+    }
+    /**
+     * Runs the GC Mark phase. It does the following:
+     *
+     * 1. Marks all referenced nodes in this run by clearing tracking for them.
+     *
+     * 2. Marks unreferenced nodes in this run by starting tracking for them.
+     *
+     * 3. Calls the runtime to update nodes that were marked referenced.
+     *
+     * @param gcResult - The result of the GC run on the gcData.
+     * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
+     * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
+     * @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
+     */
+    runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
+        // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
+        for (const nodeId of allReferencedNodeIds) {
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker !== undefined) {
+                // Stop tracking so as to clear out any running timers.
+                nodeStateTracker.stopTracking();
+                // Delete the node as we don't need to track it any more.
+                this.unreferencedNodesState.delete(nodeId);
+            }
+        }
+        // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
+        const tombstoneReadyNodeIds = new Set();
+        const sweepReadyNodeIds = new Set();
+        for (const nodeId of gcResult.deletedNodeIds) {
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker === undefined) {
+                this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
+            }
+            else {
+                // If a node was already unreferenced, update its tracking information. Since the current reference time
+                // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
+                nodeStateTracker.updateTracking(currentReferenceTimestampMs);
+                // If a node is tombstone or sweep-ready, store it so it can be returned.
+                if (nodeStateTracker.state === UnreferencedState.TombstoneReady) {
+                    tombstoneReadyNodeIds.add(nodeId);
+                }
+                if (nodeStateTracker.state === UnreferencedState.SweepReady) {
+                    sweepReadyNodeIds.add(nodeId);
+                }
+            }
+        }
+        // 3. Call the runtime to update referenced nodes in this run.
+        this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
+        return { tombstoneReadyNodeIds, sweepReadyNodeIds };
+    }
+    /**
+     * Runs the GC Sweep phase. It does the following:
+     *
+     * 1. Marks tombstone-ready nodes as tombstones.
+     *
+     * 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
+     *
+     * @param gcResult - The result of the GC run on the gcData.
+     * @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
+     * @param sweepReadyNodes - List of nodes that are sweep-ready.
+     */
+    runSweepPhase(gcResult, tombstoneReadyNodes, sweepReadyNodes) {
+        /**
+         * Under "Test Mode", unreferenced nodes are immediately deleted without waiting for them to be sweep-ready.
+         *
+         * Otherwise, depending on how long it's been since the node was unreferenced, it will either be
+         * marked as Tombstone, or deleted by Sweep.
+         */
+        if (this.configs.testMode) {
+            // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
+            this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
+            return;
+        }
+        // If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
+        // This is important because a container may never load during a node's Sweep Grace Period,
+        // so that node would directly become sweep-ready skipping over tombstone-ready state,
+        // but should be Tombstoned since Sweep is disabled.
+        const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
+            ? {
+                nodesToTombstone: [...tombstoneReadyNodes],
+                nodesToDelete: [...sweepReadyNodes],
+            }
+            : {
+                nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
+                nodesToDelete: [],
+            };
+        if (this.configs.tombstoneMode) {
+            this.tombstones = nodesToTombstone;
+            // If we are running in GC tombstone mode, update tombstoned routes.
+            this.runtime.updateTombstonedRoutes(this.tombstones);
+        }
+        if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
+            // Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
+            // to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
+            // local state when processing the op.
+            const sweepReadyDSAndBlobs = nodesToDelete.filter((nodeId) => {
+                const nodeType = this.runtime.getNodeType(nodeId);
+                return nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob;
+            });
+            const contents = {
+                type: GarbageCollectionMessageType.Sweep,
+                deletedNodeIds: sweepReadyDSAndBlobs,
+            };
+            // Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
+            // is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
+            // accidental usage. The clients will sync without the delete op too but it may take longer.
+            const containerGCMessage = {
+                type: ContainerMessageType.GC,
+                contents,
+                compatDetails: { behavior: "Ignore" },
+            };
+            this.submitMessage(containerGCMessage);
+            return;
+        }
+    }
+    /**
+     * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
+     * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
+     * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
+     * these objects while there can be in-memory referenced to it:
+     * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
+     * added, the object may have been accessed and in-memory reference to it added.
+     * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
+     * unreferenced, they could have been accessed and in-memory reference to them added.
+     *
+     * This function identifies nodes that were referenced since the last run.
+     * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
+     *
+     * @returns A list of all nodes referenced from the last local summary until now.
+     */
+    findAllNodesReferencedBetweenGCs(currentGCData, previousGCData, logger) {
+        // If we haven't run GC before there is nothing to do.
+        // No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
+        if (previousGCData === undefined) {
+            return undefined;
+        }
+        /**
+         * If there are references that were not explicitly notified to GC, log an error because this should never happen.
+         * If it does, this may result in the unreferenced timestamps of these nodes not updated when they were referenced.
+         */
+        this.telemetryTracker.logIfMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun, logger);
+        // No references were added since the last run so we don't have to update reference states of any unreferenced
+        // nodes. There is no in between state at this point.
+        if (this.newReferencesSinceLastRun.size === 0) {
+            return undefined;
+        }
+        /**
+         * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
+         * edges that have been added since then. To do this, combine the GC data from the last run and the current
+         * run, and then add the references since last run.
+         *
+         * Note on why we need to combine the data from previous run, current run and all references in between -
+         * 1. We need data from last run because some of its references may have been deleted since then. If those
+         * references added new outbound references before they were deleted, we need to detect them.
+         *
+         * 2. We need new outbound references since last run because some of them may have been deleted later. If those
+         * references added new outbound references before they were deleted, we need to detect them.
+         *
+         * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
+         * - We don't require DDSes handles to be stored in a referenced DDS.
+         * - A new data store may have "root" DDSes already created and we don't detect them today.
+         */
+        const gcDataSuperSet = concatGarbageCollectionData(previousGCData, currentGCData);
+        const newOutboundRoutesSinceLastRun = [];
+        this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
+            if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
+                gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
+            }
+            else {
+                gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
+            }
+            newOutboundRoutesSinceLastRun.push(...outboundRoutes);
+        });
+        /**
+         * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
+         * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
+         * unreferenced, stop tracking them and remove from unreferenced list.
+         * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
+         * unreferenced and add unreferenced state.
+         */
+        const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [
+            "/",
+            ...newOutboundRoutesSinceLastRun,
+        ]);
+        return gcResult.referencedNodeIds;
+    }
+    /**
+     * Summarizes the GC data and returns it as a summary tree.
+     * We current write the entire GC state in a single blob. This can be modified later to write multiple
+     * blobs. All the blob keys should start with `gcBlobPrefix`.
+     */
+    summarize(fullTree, trackState, telemetryContext) {
+        if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
+            return;
+        }
+        const gcState = { gcNodes: {} };
+        for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
+            gcState.gcNodes[nodeId] = {
+                outboundRoutes,
+                unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
+            };
+        }
+        return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
+    }
+    getMetadata() {
+        return {
+            /**
+             * If GC is enabled, the GC data is written using the GC version in effect and that is the gcFeature that goes
+             * into the metadata blob. If GC is disabled, the gcFeature is 0.
+             */
+            gcFeature: this.configs.gcEnabled ? this.configs.gcVersionInEffect : 0,
+            gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
+            sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
+            sweepEnabled: false,
+            sweepTimeoutMs: this.configs.sweepTimeoutMs,
+        };
+    }
+    /**
+     * Called to refresh the latest summary state. This happens when either a pending summary is acked.
+     */
+    async refreshLatestSummary(result) {
+        return this.summaryStateTracker.refreshLatestSummary(result);
+    }
+    /**
+     * Process a GC message.
+     * @param message - The GC message from the container runtime.
+     * @param local - Whether it was send by this client.
+     */
+    processMessage(message, local) {
+        switch (message.contents.type) {
+            case "Sweep": {
+                // Delete the nodes whose ids are present in the contents.
+                this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
+                break;
+            }
+            default: {
+                if (!compatBehaviorAllowsGCMessageType(message.contents.type, message.compatDetails?.behavior)) {
+                    const error = DataProcessingError.create(`Garbage collection message of unknown type ${message.contents.type}`, "processMessage");
+                    throw error;
+                }
+                break;
+            }
+        }
+    }
+    /**
+     * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
+     * tracking for nodes that are actually deleted by the runtime.
+     * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyNodeIds) {
+        // Use a set for lookup because its much faster than array or map.
+        const sweepReadyNodesSet = new Set(sweepReadyNodeIds);
+        // The ids in the sweep-ready nodes do not contain DDS node ids. This is an optimization to reduce the size
+        // of the GC op. Since GC applies to data store only, all its DDSes are deleted along with it. So, get the
+        // DDS nodes ID from the unreferenced nodes state.
+        const allSweepReadyNodeIds = Array.from(sweepReadyNodeIds);
+        for (const [id] of this.unreferencedNodesState) {
+            // Ignore data store nodes since they would already be in the list.
+            const pathParts = id.split("/");
+            if (pathParts.length <= 2) {
+                continue;
+            }
+            // Get the data store id part. Note that this may include blobs but that's okay since the part would just
+            // be "_blobs" and it won't be found.
+            const dsId = `/${pathParts[1]}`;
+            if (sweepReadyNodesSet.has(dsId)) {
+                allSweepReadyNodeIds.push(id);
+            }
+        }
+        const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
+        // Clear unreferenced state tracking for deleted nodes.
+        for (const nodeId of deletedNodeIds) {
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker !== undefined) {
+                // Stop tracking so as to clear out any running timers.
+                nodeStateTracker.stopTracking();
+                // Delete the node as we don't need to track it any more.
+                this.unreferencedNodesState.delete(nodeId);
+            }
+            this.deletedNodes.add(nodeId);
+        }
+    }
+    /**
+     * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
+     * or throw an error if failing on incorrect usage is configured.
+     * @param nodePath - The path of the node that changed.
+     * @param reason - Whether the node was loaded or changed.
+     * @param timestampMs - The timestamp when the node changed.
+     * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
+     * @param request - The original request for loads to preserve it in telemetry.
+     * @param requestHeaders - If the node was loaded via request path, the headers in the request.
+     */
+    nodeUpdated(nodePath, reason, timestampMs, packagePath, request, headerData) {
+        if (!this.configs.shouldRunGC) {
+            return;
+        }
+        const isTombstoned = this.tombstones.includes(nodePath);
+        // This will log if appropriate
+        this.telemetryTracker.nodeUsed({
+            id: nodePath,
+            usageType: reason,
+            currentReferenceTimestampMs: timestampMs ?? this.runtime.getCurrentReferenceTimestampMs(),
+            packagePath,
+            completedGCRuns: this.completedRuns,
+            isTombstoned,
+            lastSummaryTime: this.getLastSummaryTimestampMs(),
+            headers: headerData,
+        });
+        const nodeType = this.runtime.getNodeType(nodePath);
+        // Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
+        if (reason !== "Loaded" || ![GCNodeType.Blob, GCNodeType.DataStore].includes(nodeType)) {
+            return;
+        }
+        const errorRequest = request ?? { url: nodePath };
+        // If the object is tombstoned and tombstone enforcement is configured, throw an error.
+        if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
+            // The requested data store is removed by gc. Create a 404 gc response exception.
+            throw responseToException(createResponseError(404, `${nodeType} was tombstoned`, errorRequest, {
+                [TombstoneResponseHeaderKey]: true,
+            }), errorRequest);
+        }
+        // If the object is inactive and inactive enforcement is configured, throw an error.
+        if (this.unreferencedNodesState.get(nodePath)?.state === "Inactive") {
+            const shouldThrowOnInactiveLoad = !this.isSummarizerClient &&
+                this.configs.throwOnInactiveLoad === true &&
+                headerData?.allowInactive !== true;
+            if (shouldThrowOnInactiveLoad) {
+                throw responseToException(createResponseError(404, `${nodeType} is inactive`, errorRequest, {
+                    [InactiveResponseHeaderKey]: true,
+                }), errorRequest);
+            }
+        }
+    }
+    /**
+     * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
+     * referenced between summaries so that their unreferenced timestamp can be reset.
+     *
+     * @param fromNodePath - The node from which the reference is added.
+     * @param toNodePath - The node to which the reference is added.
+     */
+    addedOutboundReference(fromNodePath, toNodePath) {
+        if (!this.configs.shouldRunGC) {
+            return;
+        }
+        const outboundRoutes = this.newReferencesSinceLastRun.get(fromNodePath) ?? [];
+        outboundRoutes.push(toNodePath);
+        this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
+        this.telemetryTracker.nodeUsed({
+            id: toNodePath,
+            usageType: "Revived",
+            currentReferenceTimestampMs: this.runtime.getCurrentReferenceTimestampMs(),
+            packagePath: undefined,
+            completedGCRuns: this.completedRuns,
+            isTombstoned: this.tombstones.includes(toNodePath),
+            lastSummaryTime: this.getLastSummaryTimestampMs(),
+            fromId: fromNodePath,
+        });
+    }
+    /**
+     * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
+     * cases where objects are used after they are deleted and throw / log errors accordingly.
+     */
+    isNodeDeleted(nodePath) {
+        return this.deletedNodes.has(nodePath);
+    }
+    dispose() {
+        this.sessionExpiryTimer?.clear();
+        this.sessionExpiryTimer = undefined;
+    }
+    /**
+     * Generates the stats of a garbage collection mark phase run.
+     * @param gcResult - The result of the current GC run.
+     * @returns the stats of the mark phase run.
+     */
+    getMarkPhaseStats(gcResult) {
+        const markPhaseStats = {
+            nodeCount: 0,
+            dataStoreCount: 0,
+            attachmentBlobCount: 0,
+            unrefNodeCount: 0,
+            unrefDataStoreCount: 0,
+            unrefAttachmentBlobCount: 0,
+            updatedNodeCount: 0,
+            updatedDataStoreCount: 0,
+            updatedAttachmentBlobCount: 0,
+        };
+        const updateNodeStats = (nodeId, referenced) => {
+            markPhaseStats.nodeCount++;
+            // If there is no previous GC data, every node's state is generated and is considered as updated.
+            // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
+            const stateUpdated = this.gcDataFromLastRun === undefined ||
+                this.unreferencedNodesState.has(nodeId) === referenced;
+            if (stateUpdated) {
+                markPhaseStats.updatedNodeCount++;
+            }
+            if (!referenced) {
+                markPhaseStats.unrefNodeCount++;
+            }
+            if (this.runtime.getNodeType(nodeId) === GCNodeType.DataStore) {
+                markPhaseStats.dataStoreCount++;
+                if (stateUpdated) {
+                    markPhaseStats.updatedDataStoreCount++;
+                }
+                if (!referenced) {
+                    markPhaseStats.unrefDataStoreCount++;
+                }
+            }
+            if (this.runtime.getNodeType(nodeId) === GCNodeType.Blob) {
+                markPhaseStats.attachmentBlobCount++;
+                if (stateUpdated) {
+                    markPhaseStats.updatedAttachmentBlobCount++;
+                }
+                if (!referenced) {
+                    markPhaseStats.unrefAttachmentBlobCount++;
+                }
+            }
+        };
+        for (const nodeId of gcResult.referencedNodeIds) {
+            updateNodeStats(nodeId, true /* referenced */);
+        }
+        for (const nodeId of gcResult.deletedNodeIds) {
+            updateNodeStats(nodeId, false /* referenced */);
+        }
+        return markPhaseStats;
+    }
+    /**
+     * Generates the stats of a garbage collection sweep phase run.
+     * @param deletedNodes - The nodes that have been deleted until this run.
+     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
+     * @param markPhaseStats - The stats of the mark phase run.
+     * @returns the stats of the sweep phase run.
+     */
+    getSweepPhaseStats(deletedNodes, sweepReadyNodes, markPhaseStats) {
+        // Initialize the life time node counts to the mark phase node counts. If sweep is not enabled,
+        // these will be the life time node count for this container.
+        const sweepPhaseStats = {
+            lifetimeNodeCount: markPhaseStats.nodeCount,
+            lifetimeDataStoreCount: markPhaseStats.dataStoreCount,
+            lifetimeAttachmentBlobCount: markPhaseStats.attachmentBlobCount,
+            deletedNodeCount: 0,
+            deletedDataStoreCount: 0,
+            deletedAttachmentBlobCount: 0,
+        };
+        for (const nodeId of deletedNodes) {
+            sweepPhaseStats.deletedNodeCount++;
+            const nodeType = this.runtime.getNodeType(nodeId);
+            if (nodeType === GCNodeType.DataStore) {
+                sweepPhaseStats.deletedDataStoreCount++;
+            }
+            else if (nodeType === GCNodeType.Blob) {
+                sweepPhaseStats.deletedAttachmentBlobCount++;
+            }
+        }
+        // If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
+        // deleted in previous runs. So, add the deleted node counts to life time stats.
+        sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
+        sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
+        sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
+        if (this.configs.shouldRunSweep) {
+            return sweepPhaseStats;
+        }
+        // If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
+        // is the final state the node will be in.
+        // If sweep is enabled, this will happen in the run after the GC op round trips back.
+        for (const nodeId of sweepReadyNodes) {
+            sweepPhaseStats.deletedNodeCount++;
+            const nodeType = this.runtime.getNodeType(nodeId);
+            if (nodeType === GCNodeType.DataStore) {
+                sweepPhaseStats.deletedDataStoreCount++;
+            }
+            else if (nodeType === GCNodeType.Blob) {
+                sweepPhaseStats.deletedAttachmentBlobCount++;
+            }
+        }
+        return sweepPhaseStats;
+    }
+}
+//# sourceMappingURL=garbageCollection.mjs.map