@fluidframework/container-runtime 0.52.0 → 0.54.0-47413

package/lib/garbageCollection.js

@@ -2,36 +2,97 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { runGarbageCollection } from "@fluidframework/garbage-collector";
-import { ChildLogger, PerformanceEvent } from "@fluidframework/telemetry-utils";
-import { getLocalStorageFeatureGate } from "./localStorageFeatureGates";
-import { getGCVersion, metadataBlobName, } from "./summaryFormat";
+import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
+import { cloneGCData, concatGarbageCollectionStates, concatGarbageCollectionData, runGarbageCollection, unpackChildNodesGCDetails, } from "@fluidframework/garbage-collector";
+import { gcBlobKey, } from "@fluidframework/runtime-definitions";
+import { SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
+import { ChildLogger, loggerToMonitoringContext, PerformanceEvent, } from "@fluidframework/telemetry-utils";
+import { getSummaryForDatastores } from "./dataStores";
+import { getGCVersion, metadataBlobName, dataStoreAttributesBlobName, } from "./summaryFormat";
 /** This is the current version of garbage collection. */
 const GCVersion = 1;
+// The key for the GC tree in summary.
+export const gcTreeKey = "gc";
+// They prefix for GC blobs in the GC tree in summary.
+export const gcBlobPrefix = "__gc";
 // Local storage key to turn GC on / off.
-const runGCKey = "FluidRunGC";
+const runGCKey = "Fluid.GarbageCollection.RunGC";
 // Local storage key to turn GC test mode on / off.
-const gcTestModeKey = "FluidGCTestMode";
+const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
 // Local storage key to turn GC sweep on / off.
-const runSweepKey = "FluidRunSweep";
+const runSweepKey = "Fluid.GarbageCollection.RunSweep";
+const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
+/**
+ * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced. It also sets
+ * the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).
+ */
+class UnreferencedStateTracker {
+    constructor(unreferencedTimestampMs, inactiveTimeoutMs) {
+        this.unreferencedTimestampMs = unreferencedTimestampMs;
+        this.inactive = false;
+        // Keeps track of all inactive events that are logged. This is used to limit the log generation for each event to 1
+        // so that it is not noisy.
+        this.inactiveEventsLogged = new Set();
+        // If the timeout has already expired, the node should become inactive immediately. Otherwise, start a timer of
+        // inactiveTimeoutMs after which the node will become inactive.
+        if (inactiveTimeoutMs <= 0) {
+            this.inactive = true;
+        }
+        else {
+            this.timer = new Timer(inactiveTimeoutMs, () => { this.inactive = true; });
+            this.timer.start();
+        }
+    }
+    /** Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state. */
+    stopTracking() {
+        var _a;
+        (_a = this.timer) === null || _a === void 0 ? void 0 : _a.clear();
+        this.inactive = false;
+    }
+    /** Logs an error with the given properties if the node  is inactive. */
+    logIfInactive(logger, eventName, currentTimestampMs, deleteTimeoutMs, inactiveNodeId) {
+        if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
+            logger.sendErrorEvent({
+                eventName,
+                age: currentTimestampMs - this.unreferencedTimestampMs,
+                timeout: deleteTimeoutMs,
+                id: inactiveNodeId,
+            });
+            this.inactiveEventsLogged.add(eventName);
+        }
+    }
+}
 /**
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
  * its state across summaries.
  */
 export class GarbageCollector {
     constructor(provider, gcOptions, 
-    /**
-     * After GC has run, called to delete objects in the runtime whose routes are unused. This is not part of the
-     * provider because its specific to this garbage collector implementation and is not part of the contract.
-     */
-    deleteUnusedRoutes, baseLogger, existing, metadata) {
-        var _a, _b, _c;
+    /** After GC has run, called to delete objects in the runtime whose routes are unused. */
+    deleteUnusedRoutes, 
+    /** Returns the current timestamp to be assigned to nodes that become unreferenced. */
+    getCurrentTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
+        var _a, _b, _c, _d;
         this.provider = provider;
         this.gcOptions = gcOptions;
         this.deleteUnusedRoutes = deleteUnusedRoutes;
+        this.getCurrentTimestampMs = getCurrentTimestampMs;
+        /**
+         * Tells whether the GC data should be written to the root of the summary tree. We do this under 2 conditions:
+         * 1. If `writeDataAtRoot` GC option is enabled.
+         * 2. If the base summary has the GC data written at the root. This is to support forward compatibility where when
+         *    we start writing the GC data at root, older versions can detect that and write at root too.
+         */
+        this._writeDataAtRoot = false;
         // The current GC version that this container is running.
         this.currentGCVersion = GCVersion;
-        this.logger = ChildLogger.create(baseLogger, "GarbageCollector");
+        // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
+        // outbound routes from that node.
+        this.referencesSinceLastRun = new Map();
+        // Map of node ids to their unreferenced state tracker.
+        this.unreferencedNodesState = new Map();
+        this.mc = loggerToMonitoringContext(ChildLogger.create(baseLogger, "GarbageCollector"));
+        this.deleteTimeoutMs = (_a = this.gcOptions.deleteTimeoutMs) !== null && _a !== void 0 ? _a : defaultDeleteTimeoutMs;
         let prevSummaryGCVersion;
         // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing
         // documents, we get this information from the metadata blob.
@@ -49,7 +110,7 @@ export class GarbageCollector {
         // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
         this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
         // Whether GC should run or not. Can override with localStorage flag.
-        this.shouldRunGC = (_a = getLocalStorageFeatureGate(runGCKey)) !== null && _a !== void 0 ? _a : (
+        this.shouldRunGC = (_b = this.mc.config.getBoolean(runGCKey)) !== null && _b !== void 0 ? _b : (
         // GC must be enabled for the document.
         this.gcEnabled
             // GC must not be disabled via GC options.
@@ -57,16 +118,118 @@ export class GarbageCollector {
         // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
         // localStorage flag.
         this.shouldRunSweep = this.shouldRunGC &&
-            ((_b = getLocalStorageFeatureGate(runSweepKey)) !== null && _b !== void 0 ? _b : gcOptions.runSweep === true);
+            ((_c = this.mc.config.getBoolean(runSweepKey)) !== null && _c !== void 0 ? _c : gcOptions.runSweep === true);
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
-        this.testMode = (_c = getLocalStorageFeatureGate(gcTestModeKey)) !== null && _c !== void 0 ? _c : gcOptions.runGCInTestMode === true;
+        this.testMode = (_d = this.mc.config.getBoolean(gcTestModeKey)) !== null && _d !== void 0 ? _d : gcOptions.runGCInTestMode === true;
+        // If `writeDataAtRoot` GC option is true, we should write the GC data into the root of the summary tree. This
+        // GC option is used for testing only. It will be removed once we start writing GC data into root by default.
+        this._writeDataAtRoot = this.gcOptions.writeDataAtRoot === true;
+        // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
+        // this once since it involves fetching blobs from storage which is expensive.
+        const baseSummaryStateP = new LazyPromise(async () => {
+            var _a;
+            if (baseSnapshot === undefined) {
+                return undefined;
+            }
+            // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
+            const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
+            if (gcSnapshotTree !== undefined) {
+                // forward-compat - If a newer version has written the GC tree at root, we should also do the same.
+                this._writeDataAtRoot = true;
+                return getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);
+            }
+            // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
+            // consolidate into IGarbageCollectionState format.
+            // Add a node for the root node that is not present in older snapshot format.
+            const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
+            const dataStoreSnaphotTree = getSummaryForDatastores(baseSnapshot, metadata);
+            assert(dataStoreSnaphotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
+            for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnaphotTree.trees)) {
+                const blobId = dsSnapshotTree.blobs[gcBlobKey];
+                if (blobId === undefined) {
+                    continue;
+                }
+                const gcSummaryDetails = await readAndParseBlob(blobId);
+                // If there are no nodes for this data store, skip it.
+                if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
+                    continue;
+                }
+                const dsRootId = `/${dsId}`;
+                // Since we used to write GC data at data store level, we won't have an entry for the root ("/").
+                // Construct that entry by adding root data store ids to its outbound routes.
+                const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[dataStoreAttributesBlobName]);
+                if (initialSnapshotDetails.isRootDataStore) {
+                    gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
+                }
+                for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
+                    // Prefix the data store id to the GC node ids to make them relative to the root from being
+                    // relative to the data store. Similar to how its done in DataStore::getGCData.
+                    const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
+                    gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
+                }
+                assert(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);
+                gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
+            }
+            // If there is only one node (root node just added above), either GC is disabled or we are loading from the
+            // very first summary generated by detached container. In both cases, GC was not run - return undefined.
+            return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
+        });
+        // Set up the initializer which initializes the base GC state from the base snapshot. Use lazy promise because
+        // we only do this once - the very first time we run GC.
+        this.initializeBaseStateP = new LazyPromise(async () => {
+            const currentTimestampMs = this.getCurrentTimestampMs();
+            const baseState = await baseSummaryStateP;
+            if (baseState === undefined) {
+                return;
+            }
+            const gcNodes = {};
+            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+                const unreferencedTimestampMs = nodeData.unreferencedTimestampMs;
+                if (unreferencedTimestampMs !== undefined) {
+                    // Get how long it has been since the node was unreferenced. Start a timeout for the remaining time
+                    // left for it to be eligible for deletion.
+                    const unreferencedDurationMs = currentTimestampMs - unreferencedTimestampMs;
+                    this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs - unreferencedDurationMs));
+                }
+                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
+            }
+            this.gcDataFromLastRun = { gcNodes };
+        });
+        // Get the GC details for each data store from the GC state in the base summary. This is returned in
+        // getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.
+        this.dataStoreGCDetailsP = new LazyPromise(async () => {
+            const baseState = await baseSummaryStateP;
+            if (baseState === undefined) {
+                return new Map();
+            }
+            const gcNodes = {};
+            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
+            }
+            // Run GC on the nodes in the base summary to get the routes used in each node in the container.
+            // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
+            // each node in the summary.
+            const usedRoutes = runGarbageCollection(gcNodes, ["/"], this.mc.logger).referencedNodeIds;
+            const dataStoreGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
+            // Currently, the data stores write the GC data. So, we need to update it's base GC details with the
+            // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
+            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+                if (nodeData.unreferencedTimestampMs !== undefined) {
+                    const dataStoreGCDetails = dataStoreGCDetailsMap.get(nodeId.slice(1));
+                    if (dataStoreGCDetails !== undefined) {
+                        dataStoreGCDetails.unrefTimestamp = nodeData.unreferencedTimestampMs;
+                    }
+                }
+            }
+            return dataStoreGCDetailsMap;
+        });
     }
-    static create(provider, gcOptions, deleteUnusedRoutes, baseLogger, existing, metadata) {
-        return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, baseLogger, existing, metadata);
+    static create(provider, gcOptions, deleteUnusedRoutes, getCurrentTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
+        return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, getCurrentTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata);
     }
     /**
      * This tracks two things:
-     * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is > 0, GC is enabled.
+     * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.
      * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.
      */
     get gcSummaryFeatureVersion() {
@@ -81,37 +244,71 @@ export class GarbageCollector {
         // 2. This client's latest summary was updated from a snapshot that has a different GC version.
         return this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion;
     }
+    get writeDataAtRoot() {
+        return this._writeDataAtRoot;
+    }
     /**
      * Runs garbage collection and udpates the reference / used state of the nodes in the container.
      * @returns the number of data stores that have been marked as unreferenced.
      */
     async collectGarbage(options) {
-        const { logger = this.logger, runSweep = this.shouldRunSweep, fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged, } = options;
+        const { logger = this.mc.logger, runSweep = this.shouldRunSweep, fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged, } = options;
         return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
+            await this.initializeBaseStateP;
             const gcStats = {};
             // Get the runtime's GC data and run GC on the reference graph in it.
             const gcData = await this.provider.getGCData(fullGC);
-            const { referencedNodeIds, deletedNodeIds } = runGarbageCollection(gcData.gcNodes, ["/"], logger);
-            // Remove this node's route ("/") and notify data stores of routes that are used in it.
-            const usedRoutes = referencedNodeIds.filter((id) => { return id !== "/"; });
-            const dataStoreUsedStateStats = this.provider.updateUsedRoutes(usedRoutes);
+            this.updateStateSinceLatestRun(gcData);
+            const gcResult = runGarbageCollection(gcData.gcNodes, ["/"], logger);
+            const currentTimestampMs = this.getCurrentTimestampMs();
+            // Update the current state of the system based on the GC run.
+            this.updateCurrentState(gcData, gcResult, currentTimestampMs);
+            const dataStoreUsedStateStats = this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);
             if (runSweep) {
                 // Placeholder for running sweep logic.
             }
             // Update stats to be reported in the peformance event.
-            gcStats.deletedNodes = deletedNodeIds.length;
-            gcStats.totalNodes = referencedNodeIds.length + deletedNodeIds.length;
+            gcStats.deletedNodes = gcResult.deletedNodeIds.length;
+            gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;
             gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;
             gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;
             // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
             // involving access to deleted data.
             if (this.testMode) {
-                this.deleteUnusedRoutes(deletedNodeIds);
+                this.deleteUnusedRoutes(gcResult.deletedNodeIds);
             }
             event.end(gcStats);
             return gcStats;
         }, { end: true, cancel: "error" });
     }
+    /**
+     * Summarizes the GC data and returns it as a summary tree.
+     * We current write the entire GC state in a single blob. This can be modified later to write multiple
+     * blobs. All the blob keys should start with `gcBlobPrefix`.
+     */
+    summarize() {
+        var _a;
+        if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
+            return;
+        }
+        const gcState = { gcNodes: {} };
+        for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
+            gcState.gcNodes[nodeId] = {
+                outboundRoutes,
+                unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
+            };
+        }
+        const builder = new SummaryTreeBuilder();
+        builder.addBlob(`${gcBlobPrefix}_root`, JSON.stringify(gcState));
+        return builder.getSummaryTree();
+    }
+    /**
+     * Returns a map of data store ids to their base GC details generated from the base summary.This is used to
+     * initialize the data stores with their base GC state.
+     */
+    async getDataStoreBaseGCDetails() {
+        return this.dataStoreGCDetailsP;
+    }
     /**
      * Called when the latest summary of the system has been refreshed. This will be used to update the state of the
      * latest summary tracked.
@@ -130,6 +327,28 @@ export class GarbageCollector {
         // that is now the latest summary.
         await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);
     }
+    /**
+     * Called when a node with the given id is changed. If the node is inactive, log an error.
+     */
+    nodeChanged(id) {
+        var _a;
+        // Prefix "/" if needed to make it relative to the root.
+        const nodeId = id.startsWith("/") ? id : `/${id}`;
+        (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.logIfInactive(this.mc.logger, "inactiveObjectChanged", this.getCurrentTimestampMs(), this.deleteTimeoutMs, nodeId);
+    }
+    /**
+     * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
+     * referenced between summaries so that their unreferenced timestamp can be reset.
+     *
+     * @param fromNodeId - The node from which the reference is added.
+     * @param toNodeId - The node to which the reference is added.
+     */
+    addedOutboundReference(fromNodeId, toNodeId) {
+        var _a;
+        const outboundRoutes = (_a = this.referencesSinceLastRun.get(fromNodeId)) !== null && _a !== void 0 ? _a : [];
+        outboundRoutes.push(toNodeId);
+        this.referencesSinceLastRun.set(fromNodeId, outboundRoutes);
+    }
     /**
      * Update the latest summary GC version from the metadata blob in the given snapshot.
      */
@@ -140,5 +359,119 @@ export class GarbageCollector {
             this.latestSummaryGCVersion = getGCVersion(metadata);
         }
     }
+    /**
+     * Updates the state of the system as per the current GC run. It does the following:
+     * 1. Sets up the current GC state as per the gcData.
+     * 2. Starts tracking for nodes that have become unreferenced in this run.
+     * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
+     * @param gcData - The data representing the reference graph on which GC is run.
+     * @param gcResult - The result of the GC run on the gcData.
+     * @param currentTimestampMs - The current timestamp to be used for unreferenced nodes' timestamp.
+     */
+    updateCurrentState(gcData, gcResult, currentTimestampMs) {
+        this.gcDataFromLastRun = cloneGCData(gcData);
+        this.referencesSinceLastRun.clear();
+        // Iterate through the deleted nodes and start tracking if they became unreferenced in this run.
+        for (const nodeId of gcResult.deletedNodeIds) {
+            // The time when the node became unreferenced. This is added to the current GC state.
+            let unreferencedTimestampMs = currentTimestampMs;
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker !== undefined) {
+                unreferencedTimestampMs = nodeStateTracker.unreferencedTimestampMs;
+            }
+            else {
+                // Start tracking this node as it became unreferenced in this run.
+                this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs));
+            }
+        }
+        // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
+        for (const nodeId of gcResult.referencedNodeIds) {
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker !== undefined) {
+                // If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
+                // log an error as this may mean the deleteTimeoutMs is not long enough.
+                nodeStateTracker.logIfInactive(this.mc.logger, "inactiveObjectRevived", currentTimestampMs, this.deleteTimeoutMs, nodeId);
+                // Stop tracking so as to clear out any running timers.
+                nodeStateTracker.stopTracking();
+                // Delete the node as we don't need to track it any more.
+                this.unreferencedNodesState.delete(nodeId);
+            }
+        }
+    }
+    /**
+     * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
+     * time. It's possible that nodes transition from `unreferenced -> referenced -> unreferenced` between two runs. The
+     * unreferenced timestamp of such nodes needs to be reset as they may have been accessed when they were referenced.
+     *
+     * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
+     * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
+     */
+    updateStateSinceLatestRun(currentGCData) {
+        // If we haven't run GC before or no references were added since the last run, there is nothing to do.
+        if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
+            return;
+        }
+        /**
+         * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
+         * edges that have been added since then. To do this, combine the GC data from the last run and the current
+         * run, and then add the references since last run.
+         *
+         * Note on why we need to combine the data from previous run, current run and all references in between -
+         * 1. We need data from last run because some of its references may have been deleted since then. If those
+         *    references added new outbound references before getting deleted, we need to detect them.
+         * 2. We need new outbound references since last run because some of them may have been deleted later. If those
+         *    references added new outbound references before getting deleted, we need to detect them.
+         * 3. We need data from the current run because currently we may not detect when DDSs are referenced:
+         *    - We don't require DDSs handles to be stored in a referenced DDS. For this, we need GC at DDS level
+         *      which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
+         *    - A new data store may have "root" DDSs already created and we don't detect them today.
+         */
+        const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
+        this.referencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
+            if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
+                gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
+            }
+            else {
+                gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
+            }
+        });
+        /**
+         * Run GC on the above reference graph to find all nodes that are referenced. For each one, if they are
+         * unreferenced, stop tracking them and remove from unreferenced list.
+         * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
+         */
+        const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/"], this.mc.logger);
+        for (const nodeId of gcResult.referencedNodeIds) {
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker !== undefined) {
+                // Stop tracking so as to clear out any running timers.
+                nodeStateTracker.stopTracking();
+                // Delete the node as we don't need to track it any more.
+                this.unreferencedNodesState.delete(nodeId);
+            }
+        }
+    }
+}
+/**
+ * Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
+ * Merge the GC state from all such blobs and return the merged GC state.
+*/
+async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
+    let rootGCState = { gcNodes: {} };
+    for (const key of Object.keys(gcSnapshotTree.blobs)) {
+        // Skip blobs that do not stsart with the GC prefix.
+        if (!key.startsWith(gcBlobPrefix)) {
+            continue;
+        }
+        const blobId = gcSnapshotTree.blobs[key];
+        if (blobId === undefined) {
+            continue;
+        }
+        const gcState = await readAndParseBlob(blobId);
+        assert(gcState !== undefined, 0x2ad /* "GC blob missing from snapshot" */);
+        // Merge the GC state of this blob into the root GC state.
+        rootGCState = concatGarbageCollectionStates(rootGCState, gcState);
+    }
+    return rootGCState;
 }
 //# sourceMappingURL=garbageCollection.js.map

package/lib/garbageCollection.js.map

@@ -1 +1 @@
-{"version":3,"file":"garbageCollection.js","sourceRoot":"","sources":["../src/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAIzE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAGhF,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EACH,YAAY,EAGZ,gBAAgB,GACnB,MAAM,iBAAiB,CAAC;AAEzB,yDAAyD;AACzD,MAAM,SAAS,GAAG,CAAC,CAAC;AAEpB,yCAAyC;AACzC,MAAM,QAAQ,GAAG,YAAY,CAAC;AAC9B,mDAAmD;AACnD,MAAM,aAAa,GAAG,iBAAiB,CAAC;AACxC,+CAA+C;AAC/C,MAAM,WAAW,GAAG,eAAe,CAAC;AA4CpC;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IAkDzB,YACqB,QAAmC,EACnC,SAA4B;IAC7C;;;OAGG;IACc,kBAAoD,EACrE,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;;QATnB,aAAQ,GAAR,QAAQ,CAA2B;QACnC,cAAS,GAAT,SAAS,CAAmB;QAK5B,uBAAkB,GAAlB,kBAAkB,CAAkC;QAZzE,yDAAyD;QACxC,qBAAgB,GAAG,SAAS,CAAC;QAgB1C,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;QAEjE,IAAI,oBAAwC,CAAC;QAC7C,sGAAsG;QACtG,6DAA6D;QAC7D,IAAI,QAAQ,EAAE;YACV,oBAAoB,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;YAC9C,oGAAoG;YACpG,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,oBAAoB,GAAG,CAAC,CAAC;SAC7C;aAAM;YACH,0FAA0F;YAC1F,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC;SACjD;QACD,0GAA0G;QAC1G,+GAA+G;QAC/G,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,IAAI,CAAC,gBAAgB,CAAC;QAE5E,qEAAqE;QACrE,IAAI,CAAC,WAAW,SAAG,0BAA0B,CAAC,QAAQ,CAAC,mCAAI;QACvD,uCAAuC;QACvC,IAAI,CAAC,SAAS;YACd,0CAA0C;eACvC,CAAC,SAAS,CAAC,SAAS,CAC1B,CAAC;QAEF,2GAA2G;QAC3G,qBAAqB;QACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,WAAW;YAClC,OAAC,0BAA0B,CAAC,WAAW,CAAC,mCAAI,SAAS,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAE7E,iGAAiG;QACjG,IAAI,CAAC,QAAQ,SAAG,0BAA0B,CAAC,aAAa,CAAC,mCAAI,SAAS,CAAC,eAAe,KAAK,IAAI,CAAC;IACpG,CAAC;IA9FM,MAAM,CAAC,MAAM,CAChB,QAAmC,EACnC,SAA4B,EAC5B,kBAAoD,EACpD,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;QAEpC,OAAO,IAAI,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,kBAAkB,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzG,CAAC;IAOD;;;;OAIG;IACH,IAAW,uBAAuB;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,IAAW,mBAAmB;QAC1B,+EAA+E;QAC/E,+EAA+E;QAC/E,+FAA+F;QAC/F,OAAO,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,gBAAgB,CAAC;IACrF,CAAC;IA+DD;;;OAGG;IACI,KAAK,CAAC,cAAc,CACvB,OAOC;QAED,MAAM,EACF,MAAM,GAAG,IAAI,CAAC,MAAM,EACpB,QAAQ,GAAG,IAAI,CAAC,cAAc,EAC9B,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,mBAAmB,GACzE,GAAG,OAAO,CAAC;QAEZ,OAAO,gBAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,mBAAmB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC/F,MAAM,OAAO,GAKT,EAAE,CAAC;YAEP,qEAAqE;YACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,oBAAoB,CAC9D,MAAM,CAAC,OAAO,EACd,CAAE,GAAG,CAAE,EACP,MAAM,CACT,CAAC;YAEF,uFAAuF;YACvF,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,EAAU,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpF,MAAM,uBAAuB,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAE3E,IAAI,QAAQ,EAAE;gBACV,uCAAuC;aAC1C;YAED,uDAAuD;YACvD,OAAO,CAAC,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC;YAC7C,OAAO,CAAC,UAAU,GAAG,iBAAiB,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;YACtE,OAAO,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,eAAe,CAAC;YACpE,OAAO,CAAC,eAAe,GAAG,uBAAuB,CAAC,cAAc,CAAC;YAEjE,sGAAsG;YACtG,oCAAoC;YACpC,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC;aAC3C;YACD,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,OAAmB,CAAC;QAC/B,CAAC,EACD,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,2BAA2B,CACpC,MAA4B,EAC5B,gBAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE;YACnD,OAAO;SACV;QAED,2GAA2G;QAC3G,uDAAuD;QACvD,IAAI,MAAM,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACpD,OAAO;SACV;QACD,6GAA6G;QAC7G,kCAAkC;QAClC,MAAM,IAAI,CAAC,kCAAkC,CAAC,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kCAAkC,CAAC,QAAuB,EAAE,gBAAkC;QACxG,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACxD,IAAI,cAAc,EAAE;YAChB,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAA4B,cAAc,CAAC,CAAC;YACnF,IAAI,CAAC,sBAAsB,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;SACxD;IACL,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { runGarbageCollection } from \"@fluidframework/garbage-collector\";\nimport { ISnapshotTree } from \"@fluidframework/protocol-definitions\";\nimport { IGarbageCollectionData } from \"@fluidframework/runtime-definitions\";\nimport { ReadAndParseBlob, RefreshSummaryResult } from \"@fluidframework/runtime-utils\";\nimport { ChildLogger, PerformanceEvent } from \"@fluidframework/telemetry-utils\";\n\nimport { IGCRuntimeOptions } from \"./containerRuntime\";\nimport { getLocalStorageFeatureGate } from \"./localStorageFeatureGates\";\nimport {\n    getGCVersion,\n    GCVersion,\n    IContainerRuntimeMetadata,\n    metadataBlobName,\n} from \"./summaryFormat\";\n\n/** This is the current version of garbage collection. */\nconst GCVersion = 1;\n\n// Local storage key to turn GC on / off.\nconst runGCKey = \"FluidRunGC\";\n// Local storage key to turn GC test mode on / off.\nconst gcTestModeKey = \"FluidGCTestMode\";\n// Local storage key to turn GC sweep on / off.\nconst runSweepKey = \"FluidRunSweep\";\n\n/** The used state statistics of a node. */\nexport interface IUsedStateStats {\n    totalNodeCount: number;\n    unusedNodeCount: number;\n}\n\n/** The statistics of the system state after a garbage collection run. */\nexport interface IGCStats {\n    totalNodes: number;\n    deletedNodes: number;\n    totalDataStores: number;\n    deletedDataStores: number;\n}\n\n/** Defines the APIs for the runtime object to be passed to the garbage collector. */\nexport interface IGarbageCollectionRuntime {\n    /** Returns the garbage collection data of the runtime. */\n    getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;\n    /** After GC has run, called to notify the runtime of routes that are used in it. */\n    updateUsedRoutes(usedRoutes: string[]): IUsedStateStats;\n}\n\n/** Defines the contract for the garbage collector. */\nexport interface IGarbageCollector {\n    /** Tells whether GC should run or not. */\n    readonly shouldRunGC: boolean;\n    /**\n     * This tracks two things:\n     * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is > 0, GC is enabled.\n     * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n     */\n    readonly gcSummaryFeatureVersion: number;\n    /** Tells whether the GC version has changed compared to the version in the latest summary. */\n    readonly hasGCVersionChanged: boolean;\n    /** Run garbage collection and update the reference / used state of the system. */\n    collectGarbage(\n        options: { logger?: ITelemetryLogger, runGC?: boolean, runSweep?: boolean, fullGC?: boolean },\n    ): Promise<IGCStats>;\n    /** Called when the latest summary of the system has been refreshed. */\n    latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;\n}\n\n/**\n * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains\n * its state across summaries.\n */\nexport class GarbageCollector implements IGarbageCollector {\n    public static create(\n        provider: IGarbageCollectionRuntime,\n        gcOptions: IGCRuntimeOptions,\n        deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n        baseLogger: ITelemetryLogger,\n        existing: boolean,\n        metadata?: IContainerRuntimeMetadata,\n    ): IGarbageCollector {\n        return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, baseLogger, existing, metadata);\n    }\n\n    /**\n     * Tells whether GC should be run based on the GC options and local storage flags.\n     */\n    public readonly shouldRunGC: boolean;\n\n    /**\n     * This tracks two things:\n     * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is > 0, GC is enabled.\n     * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n     */\n    public get gcSummaryFeatureVersion(): number {\n        return this.gcEnabled ? this.currentGCVersion : 0;\n    }\n\n    /**\n     * Tells whether the GC version has changed compared to the version in the latest summary.\n     */\n    public get hasGCVersionChanged(): boolean {\n        // The current version can differ from the latest summary version in two cases:\n        // 1. The summary this client loaded with has data from a different GC version.\n        // 2. This client's latest summary was updated from a snapshot that has a different GC version.\n        return this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion;\n    }\n\n    /**\n     * Tracks if GC is enabled for this document. This is specified during document creation and doesn't change\n     * throughout its lifetime.\n     */\n    private readonly gcEnabled: boolean;\n    private readonly shouldRunSweep: boolean;\n    private readonly testMode: boolean;\n    private readonly logger: ITelemetryLogger;\n\n    // The current GC version that this container is running.\n    private readonly currentGCVersion = GCVersion;\n    // This is the version of GC data in the latest summary being tracked.\n    private latestSummaryGCVersion: GCVersion;\n\n    protected constructor(\n        private readonly provider: IGarbageCollectionRuntime,\n        private readonly gcOptions: IGCRuntimeOptions,\n        /**\n         * After GC has run, called to delete objects in the runtime whose routes are unused. This is not part of the\n         * provider because its specific to this garbage collector implementation and is not part of the contract.\n         */\n        private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n        baseLogger: ITelemetryLogger,\n        existing: boolean,\n        metadata?: IContainerRuntimeMetadata,\n    ) {\n        this.logger = ChildLogger.create(baseLogger, \"GarbageCollector\");\n\n        let prevSummaryGCVersion: number | undefined;\n        // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing\n        // documents, we get this information from the metadata blob.\n        if (existing) {\n            prevSummaryGCVersion = getGCVersion(metadata);\n            // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all\n            // other exsiting documents, GC is enabled.\n            this.gcEnabled = prevSummaryGCVersion > 0;\n        } else {\n            // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.\n            this.gcEnabled = gcOptions.gcAllowed === true;\n        }\n        // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the\n        // latest tracked GC version. For new documents, we will be writing the first summary with the current version.\n        this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;\n\n        // Whether GC should run or not. Can override with localStorage flag.\n        this.shouldRunGC = getLocalStorageFeatureGate(runGCKey) ?? (\n            // GC must be enabled for the document.\n            this.gcEnabled\n            // GC must not be disabled via GC options.\n            && !gcOptions.disableGC\n        );\n\n        // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with\n        // localStorage flag.\n        this.shouldRunSweep = this.shouldRunGC &&\n            (getLocalStorageFeatureGate(runSweepKey) ?? gcOptions.runSweep === true);\n\n        // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.\n        this.testMode = getLocalStorageFeatureGate(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;\n    }\n\n    /**\n     * Runs garbage collection and udpates the reference / used state of the nodes in the container.\n     * @returns the number of data stores that have been marked as unreferenced.\n     */\n    public async collectGarbage(\n        options: {\n            /** Logger to use for logging GC events */\n            logger?: ITelemetryLogger,\n            /** True to run GC sweep phase after the mark phase */\n            runSweep?: boolean,\n            /** True to generate full GC data */\n            fullGC?: boolean,\n        },\n    ): Promise<IGCStats> {\n        const {\n            logger = this.logger,\n            runSweep = this.shouldRunSweep,\n            fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged,\n        } = options;\n\n        return PerformanceEvent.timedExecAsync(logger, { eventName: \"GarbageCollection\" }, async (event) => {\n            const gcStats: {\n                deletedNodes?: number,\n                totalNodes?: number,\n                deletedDataStores?: number,\n                totalDataStores?: number,\n            } = {};\n\n            // Get the runtime's GC data and run GC on the reference graph in it.\n            const gcData = await this.provider.getGCData(fullGC);\n            const { referencedNodeIds, deletedNodeIds } = runGarbageCollection(\n                gcData.gcNodes,\n                [ \"/\" ],\n                logger,\n            );\n\n            // Remove this node's route (\"/\") and notify data stores of routes that are used in it.\n            const usedRoutes = referencedNodeIds.filter((id: string) => { return id !== \"/\"; });\n            const dataStoreUsedStateStats = this.provider.updateUsedRoutes(usedRoutes);\n\n            if (runSweep) {\n                // Placeholder for running sweep logic.\n            }\n\n            // Update stats to be reported in the peformance event.\n            gcStats.deletedNodes = deletedNodeIds.length;\n            gcStats.totalNodes = referencedNodeIds.length + deletedNodeIds.length;\n            gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;\n            gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;\n\n            // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios\n            // involving access to deleted data.\n            if (this.testMode) {\n                this.deleteUnusedRoutes(deletedNodeIds);\n            }\n            event.end(gcStats);\n            return gcStats as IGCStats;\n        },\n        { end: true, cancel: \"error\" });\n    }\n\n    /**\n     * Called when the latest summary of the system has been refreshed. This will be used to update the state of the\n     * latest summary tracked.\n     */\n    public async latestSummaryStateRefreshed(\n        result: RefreshSummaryResult,\n        readAndParseBlob: ReadAndParseBlob,\n    ): Promise<void> {\n        if (!this.shouldRunGC || !result.latestSummaryUpdated) {\n            return;\n        }\n\n        // If the summary was tracked by this client, it was the one that generated the summary in the first place.\n        // Basically, it was written in the current GC version.\n        if (result.wasSummaryTracked) {\n            this.latestSummaryGCVersion = this.currentGCVersion;\n            return;\n        }\n        // If the summary was not tracked by this client, update latest GC version from the snapshot in the result as\n        // that is now the latest summary.\n        await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);\n    }\n\n    /**\n     * Update the latest summary GC version from the metadata blob in the given snapshot.\n     */\n    private async updateSummaryGCVersionFromSnapshot(snapshot: ISnapshotTree, readAndParseBlob: ReadAndParseBlob) {\n        const metadataBlobId = snapshot.blobs[metadataBlobName];\n        if (metadataBlobId) {\n            const metadata = await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId);\n            this.latestSummaryGCVersion = getGCVersion(metadata);\n        }\n    }\n}\n"]}
+{"version":3,"file":"garbageCollection.js","sourceRoot":"","sources":["../src/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EACH,WAAW,EACX,6BAA6B,EAC7B,2BAA2B,EAE3B,oBAAoB,EACpB,yBAAyB,GAC5B,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EACH,SAAS,GAKZ,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAGH,kBAAkB,GACrB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACH,WAAW,EACX,yBAAyB,EAEzB,gBAAgB,GAClB,MAAM,iCAAiC,CAAC;AAG1C,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EACH,YAAY,EAGZ,gBAAgB,EAEhB,2BAA2B,GAC9B,MAAM,iBAAiB,CAAC;AAEzB,yDAAyD;AACzD,MAAM,SAAS,GAAG,CAAC,CAAC;AAEpB,sCAAsC;AACtC,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,CAAC;AAC9B,sDAAsD;AACtD,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,CAAC;AAEnC,yCAAyC;AACzC,MAAM,QAAQ,GAAG,+BAA+B,CAAC;AACjD,mDAAmD;AACnD,MAAM,aAAa,GAAG,oCAAoC,CAAC;AAC3D,+CAA+C;AAC/C,MAAM,WAAW,GAAG,kCAAkC,CAAC;AAEvD,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAsDjE;;;GAGG;AACH,MAAM,wBAAwB;IAO1B,YACoB,uBAA+B,EAC/C,iBAAyB;QADT,4BAAuB,GAAvB,uBAAuB,CAAQ;QAP3C,aAAQ,GAAY,KAAK,CAAC;QAClC,mHAAmH;QACnH,2BAA2B;QACV,yBAAoB,GAAgB,IAAI,GAAG,EAAE,CAAC;QAO3D,+GAA+G;QAC/G,+DAA+D;QAC/D,IAAI,iBAAiB,IAAI,CAAC,EAAE;YACxB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;SACxB;aAAM;YACH,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,iBAAiB,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3E,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;SACtB;IACL,CAAC;IAED,+FAA+F;IACxF,YAAY;;QACf,MAAA,IAAI,CAAC,KAAK,0CAAE,KAAK,GAAG;QACpB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,wEAAwE;IACjE,aAAa,CAChB,MAAwB,EACxB,SAAiB,EACjB,kBAA0B,EAC1B,eAAuB,EACvB,cAAsB;QAEtB,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC5D,MAAM,CAAC,cAAc,CAAC;gBAClB,SAAS;gBACT,GAAG,EAAE,kBAAkB,GAAG,IAAI,CAAC,uBAAuB;gBACtD,OAAO,EAAE,eAAe;gBACxB,EAAE,EAAE,cAAc;aACrB,CAAC,CAAC;YACH,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;SAC5C;IACL,CAAC;CACJ;AAED;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IAyFzB,YACqB,QAAmC,EACnC,SAA4B;IAC7C,yFAAyF;IACxE,kBAAoD;IACrE,sFAAsF;IACrE,qBAAmC,EACpD,YAAuC,EACvC,gBAAkC,EAClC,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;;QAVnB,aAAQ,GAAR,QAAQ,CAA2B;QACnC,cAAS,GAAT,SAAS,CAAmB;QAE5B,uBAAkB,GAAlB,kBAAkB,CAAkC;QAEpD,0BAAqB,GAArB,qBAAqB,CAAc;QArCxD;;;;;WAKG;QACK,qBAAgB,GAAY,KAAK,CAAC;QAK1C,yDAAyD;QACxC,qBAAgB,GAAG,SAAS,CAAC;QAM9C,6GAA6G;QAC7G,kCAAkC;QACjB,2BAAsB,GAA0B,IAAI,GAAG,EAAE,CAAC;QAQ3E,uDAAuD;QACtC,2BAAsB,GAA0C,IAAI,GAAG,EAAE,CAAC;QAevF,IAAI,CAAC,EAAE,GAAG,yBAAyB,CAC/B,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC,CAAC;QAExD,IAAI,CAAC,eAAe,SAAG,IAAI,CAAC,SAAS,CAAC,eAAe,mCAAI,sBAAsB,CAAC;QAEhF,IAAI,oBAAwC,CAAC;QAC7C,sGAAsG;QACtG,6DAA6D;QAC7D,IAAI,QAAQ,EAAE;YACV,oBAAoB,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;YAC9C,oGAAoG;YACpG,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,oBAAoB,GAAG,CAAC,CAAC;SAC7C;aAAM;YACH,0FAA0F;YAC1F,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC;SACjD;QACD,0GAA0G;QAC1G,+GAA+G;QAC/G,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,IAAI,CAAC,gBAAgB,CAAC;QAE5E,qEAAqE;QACrE,IAAI,CAAC,WAAW,SAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,mCAAI;QACtD,uCAAuC;QACvC,IAAI,CAAC,SAAS;YACd,0CAA0C;eACvC,CAAC,SAAS,CAAC,SAAS,CAC1B,CAAC;QAEF,2GAA2G;QAC3G,qBAAqB;QACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,WAAW;YAClC,OAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,mCAAI,SAAS,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAE5E,iGAAiG;QACjG,IAAI,CAAC,QAAQ,SAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,mCAAI,SAAS,CAAC,eAAe,KAAK,IAAI,CAAC;QAE/F,8GAA8G;QAC9G,6GAA6G;QAC7G,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,eAAe,KAAK,IAAI,CAAC;QAEhE,qGAAqG;QACrG,8EAA8E;QAC9E,MAAM,iBAAiB,GAAG,IAAI,WAAW,CAAsC,KAAK,IAAI,EAAE;;YACtF,IAAI,YAAY,KAAK,SAAS,EAAE;gBAC5B,OAAO,SAAS,CAAC;aACpB;YAED,6FAA6F;YAC7F,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,cAAc,KAAK,SAAS,EAAE;gBAC9B,mGAAmG;gBACnG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,OAAO,sBAAsB,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;aACnE;YAED,uGAAuG;YACvG,mDAAmD;YACnD,6EAA6E;YAC7E,MAAM,OAAO,GAA4B,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;YACtF,MAAM,oBAAoB,GAAG,uBAAuB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;YAC7E,MAAM,CAAC,oBAAoB,KAAK,SAAS,EACrC,KAAK,CAAC,0DAA0D,CAAC,CAAC;YACtE,KAAK,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;gBAC7E,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,MAAM,KAAK,SAAS,EAAE;oBACtB,SAAS;iBACZ;gBAED,MAAM,gBAAgB,GAAG,MAAM,gBAAgB,CAAmC,MAAM,CAAC,CAAC;gBAC1F,sDAAsD;gBACtD,IAAI,OAAA,gBAAgB,CAAC,MAAM,0CAAE,OAAO,MAAK,SAAS,EAAE;oBAChD,SAAS;iBACZ;gBAED,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC5B,iGAAiG;gBACjG,6EAA6E;gBAC7E,MAAM,sBAAsB,GAAG,MAAM,gBAAgB,CACjD,cAAc,CAAC,KAAK,CAAC,2BAA2B,CAAC,CACpD,CAAC;gBACF,IAAI,sBAAsB,CAAC,eAAe,EAAE;oBACxC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;iBACtD;gBAED,KAAK,MAAM,CAAC,EAAE,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;oBAChF,2FAA2F;oBAC3F,+EAA+E;oBAC/E,MAAM,MAAM,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,EAAE,EAAE,CAAC;oBAC1D,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;iBAC5E;gBACD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,SAAS,EAC1C,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBAClE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,uBAAuB,GAAG,gBAAgB,CAAC,cAAc,CAAC;aACvF;YAED,2GAA2G;YAC3G,wGAAwG;YACxG,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;QAC3E,CAAC,CAAC,CAAC;QAEH,8GAA8G;QAC9G,wDAAwD;QACxD,IAAI,CAAC,oBAAoB,GAAG,IAAI,WAAW,CAAO,KAAK,IAAI,EAAE;YACzD,MAAM,kBAAkB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACxD,MAAM,SAAS,GAAI,MAAM,iBAAiB,CAAC;YAC3C,IAAI,SAAS,KAAK,SAAS,EAAE;gBACzB,OAAO;aACV;YAED,MAAM,OAAO,GAAiC,EAAE,CAAC;YACjD,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;gBAChE,MAAM,uBAAuB,GAAG,QAAQ,CAAC,uBAAuB,CAAC;gBACjE,IAAI,uBAAuB,KAAK,SAAS,EAAE;oBACvC,mGAAmG;oBACnG,2CAA2C;oBAC3C,MAAM,sBAAsB,GAAG,kBAAkB,GAAG,uBAAuB,CAAC;oBAC5E,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAC3B,MAAM,EACN,IAAI,wBAAwB,CACxB,uBAAuB,EACvB,IAAI,CAAC,eAAe,GAAG,sBAAsB,CAChD,CACJ,CAAC;iBACL;gBACD,OAAO,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;aACzD;YACD,IAAI,CAAC,iBAAiB,GAAG,EAAE,OAAO,EAAE,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,oGAAoG;QACpG,yFAAyF;QACzF,IAAI,CAAC,mBAAmB,GAAG,IAAI,WAAW,CAAgD,KAAK,IAAI,EAAE;YACjG,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC;YAC1C,IAAI,SAAS,KAAK,SAAS,EAAE;gBACzB,OAAO,IAAI,GAAG,EAAE,CAAC;aACpB;YAED,MAAM,OAAO,GAAiC,EAAE,CAAC;YACjD,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;gBAChE,OAAO,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;aACzD;YACD,gGAAgG;YAChG,uGAAuG;YACvG,4BAA4B;YAC5B,MAAM,UAAU,GAAG,oBAAoB,CACnC,OAAO,EACP,CAAE,GAAG,CAAE,EACP,IAAI,CAAC,EAAE,CAAC,MAAM,CACjB,CAAC,iBAAiB,CAAC;YAEpB,MAAM,qBAAqB,GAAG,yBAAyB,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;YAC7F,oGAAoG;YACpG,oGAAoG;YACpG,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;gBAChE,IAAI,QAAQ,CAAC,uBAAuB,KAAK,SAAS,EAAE;oBAChD,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBACtE,IAAI,kBAAkB,KAAK,SAAS,EAAE;wBAClC,kBAAkB,CAAC,cAAc,GAAG,QAAQ,CAAC,uBAAuB,CAAC;qBACxE;iBACJ;aACJ;YACD,OAAO,qBAAqB,CAAC;QACjC,CAAC,CAAC,CAAC;IACP,CAAC;IAzQM,MAAM,CAAC,MAAM,CAChB,QAAmC,EACnC,SAA4B,EAC5B,kBAAoD,EACpD,qBAAmC,EACnC,YAAuC,EACvC,gBAAkC,EAClC,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;QAEpC,OAAO,IAAI,gBAAgB,CACvB,QAAQ,EACR,SAAS,EACT,kBAAkB,EAClB,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,UAAU,EACV,QAAQ,EACR,QAAQ,CACX,CAAC;IACN,CAAC;IAOD;;;;OAIG;IACH,IAAW,uBAAuB;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,IAAW,mBAAmB;QAC1B,+EAA+E;QAC/E,+EAA+E;QAC/E,+FAA+F;QAC/F,OAAO,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,gBAAgB,CAAC;IACrF,CAAC;IAkBD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAyMD;;;OAGG;IACI,KAAK,CAAC,cAAc,CACvB,OAOC;QAED,MAAM,EACF,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,EACvB,QAAQ,GAAG,IAAI,CAAC,cAAc,EAC9B,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,mBAAmB,GACzE,GAAG,OAAO,CAAC;QAEZ,OAAO,gBAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,mBAAmB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC/F,MAAM,IAAI,CAAC,oBAAoB,CAAC;YAEhC,MAAM,OAAO,GAKT,EAAE,CAAC;YAEP,qEAAqE;YACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAErD,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC;YAEvC,MAAM,QAAQ,GAAG,oBAAoB,CACjC,MAAM,CAAC,OAAO,EACd,CAAE,GAAG,CAAE,EACP,MAAM,CACT,CAAC;YAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACxD,8DAA8D;YAC9D,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;YAE9D,MAAM,uBAAuB,GACzB,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;YAEnF,IAAI,QAAQ,EAAE;gBACV,uCAAuC;aAC1C;YAED,uDAAuD;YACvD,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC;YACtD,OAAO,CAAC,UAAU,GAAG,QAAQ,CAAC,iBAAiB,CAAC,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC;YACxF,OAAO,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,eAAe,CAAC;YACpE,OAAO,CAAC,eAAe,GAAG,uBAAuB,CAAC,cAAc,CAAC;YAEjE,sGAAsG;YACtG,oCAAoC;YACpC,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;aACpD;YACD,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,OAAmB,CAAC;QAC/B,CAAC,EACD,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACI,SAAS;;QACZ,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,iBAAiB,KAAK,SAAS,EAAE;YAC3D,OAAO;SACV;QAED,MAAM,OAAO,GAA4B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACzD,KAAK,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE;YACnF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG;gBACtB,cAAc;gBACd,uBAAuB,QAAE,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,0CAAE,uBAAuB;aAC5F,CAAC;SACL;QAED,MAAM,OAAO,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,OAAO,CAAC,GAAG,YAAY,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,OAAO,OAAO,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,yBAAyB;QAClC,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,2BAA2B,CACpC,MAA4B,EAC5B,gBAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE;YACnD,OAAO;SACV;QAED,2GAA2G;QAC3G,uDAAuD;QACvD,IAAI,MAAM,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACpD,OAAO;SACV;QACD,6GAA6G;QAC7G,kCAAkC;QAClC,MAAM,IAAI,CAAC,kCAAkC,CAAC,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACI,WAAW,CAAC,EAAU;;QACzB,wDAAwD;QACxD,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClD,MAAA,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,0CAAE,aAAa,CAClD,IAAI,CAAC,EAAE,CAAC,MAAM,EACd,uBAAuB,EACvB,IAAI,CAAC,qBAAqB,EAAE,EAC5B,IAAI,CAAC,eAAe,EACpB,MAAM,EACR;IACN,CAAC;IAED;;;;;;OAMG;IACI,sBAAsB,CAAC,UAAkB,EAAE,QAAgB;;QAC9D,MAAM,cAAc,SAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAC,mCAAI,EAAE,CAAC;QACzE,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kCAAkC,CAAC,QAAuB,EAAE,gBAAkC;QACxG,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACxD,IAAI,cAAc,EAAE;YAChB,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAA4B,cAAc,CAAC,CAAC;YACnF,IAAI,CAAC,sBAAsB,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;SACxD;IACL,CAAC;IAED;;;;;;;;OAQG;IACK,kBAAkB,CAAC,MAA8B,EAAE,QAAmB,EAAE,kBAA0B;QACtG,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,CAAC;QAEpC,gGAAgG;QAChG,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,cAAc,EAAE;YAC1C,qFAAqF;YACrF,IAAI,uBAAuB,GAAW,kBAAkB,CAAC;YACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAChC,uBAAuB,GAAG,gBAAgB,CAAC,uBAAuB,CAAC;aACtE;iBAAM;gBACH,kEAAkE;gBAClE,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAC3B,MAAM,EACN,IAAI,wBAAwB,CAAC,uBAAuB,EAAE,IAAI,CAAC,eAAe,CAAC,CAC9E,CAAC;aACL;SACJ;QAED,2FAA2F;QAC3F,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,iBAAiB,EAAE;YAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAChC,8FAA8F;gBAC9F,wEAAwE;gBACxE,gBAAgB,CAAC,aAAa,CAC1B,IAAI,CAAC,EAAE,CAAC,MAAM,EACd,uBAAuB,EACvB,kBAAkB,EAClB,IAAI,CAAC,eAAe,EACpB,MAAM,CACT,CAAC;gBACF,uDAAuD;gBACvD,gBAAgB,CAAC,YAAY,EAAE,CAAC;gBAChC,yDAAyD;gBACzD,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aAC9C;SACJ;IACL,CAAC;IAED;;;;;;;OAOG;IACK,yBAAyB,CAAC,aAAqC;QACnE,sGAAsG;QACtG,IAAI,IAAI,CAAC,iBAAiB,KAAK,SAAS,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,KAAK,CAAC,EAAE;YAChF,OAAO;SACV;QAED;;;;;;;;;;;;;;WAcG;QACH,MAAM,cAAc,GAAG,2BAA2B,CAAC,IAAI,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;QAC1F,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,cAAwB,EAAE,YAAoB,EAAE,EAAE;YACnF,IAAI,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,SAAS,EAAE;gBACpD,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,cAAc,CAAC;aACzD;iBAAM;gBACH,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;aAChE;QACL,CAAC,CAAC,CAAC;QAEH;;;;WAIG;QACH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACrF,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,iBAAiB,EAAE;YAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAChC,uDAAuD;gBACvD,gBAAgB,CAAC,YAAY,EAAE,CAAC;gBAChC,yDAAyD;gBACzD,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aAC9C;SACJ;IACL,CAAC;CACJ;AAED;;;EAGE;AACF,KAAK,UAAU,sBAAsB,CACjC,cAA6B,EAC7B,gBAAkC;IAElC,IAAI,WAAW,GAA4B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC3D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE;QACjD,oDAAoD;QACpD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;YAC/B,SAAS;SACZ;QAED,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAA0B,MAAM,CAAC,CAAC;QACxE,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC3E,0DAA0D;QAC1D,WAAW,GAAG,6BAA6B,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;KACrE;IACD,OAAO,WAAW,CAAC;AACvB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, LazyPromise, Timer } from \"@fluidframework/common-utils\";\nimport {\n    cloneGCData,\n    concatGarbageCollectionStates,\n    concatGarbageCollectionData,\n    IGCResult,\n    runGarbageCollection,\n    unpackChildNodesGCDetails,\n} from \"@fluidframework/garbage-collector\";\nimport { ISnapshotTree } from \"@fluidframework/protocol-definitions\";\nimport {\n    gcBlobKey,\n    IGarbageCollectionData,\n    IGarbageCollectionState,\n    IGarbageCollectionSummaryDetails,\n    ISummaryTreeWithStats,\n} from \"@fluidframework/runtime-definitions\";\nimport {\n    ReadAndParseBlob,\n    RefreshSummaryResult,\n    SummaryTreeBuilder,\n} from \"@fluidframework/runtime-utils\";\nimport {\n    ChildLogger,\n    loggerToMonitoringContext,\n    MonitoringContext,\n    PerformanceEvent,\n } from \"@fluidframework/telemetry-utils\";\n\nimport { IGCRuntimeOptions } from \"./containerRuntime\";\nimport { getSummaryForDatastores } from \"./dataStores\";\nimport {\n    getGCVersion,\n    GCVersion,\n    IContainerRuntimeMetadata,\n    metadataBlobName,\n    ReadFluidDataStoreAttributes,\n    dataStoreAttributesBlobName,\n} from \"./summaryFormat\";\n\n/** This is the current version of garbage collection. */\nconst GCVersion = 1;\n\n// The key for the GC tree in summary.\nexport const gcTreeKey = \"gc\";\n// They prefix for GC blobs in the GC tree in summary.\nexport const gcBlobPrefix = \"__gc\";\n\n// Local storage key to turn GC on / off.\nconst runGCKey = \"Fluid.GarbageCollection.RunGC\";\n// Local storage key to turn GC test mode on / off.\nconst gcTestModeKey = \"Fluid.GarbageCollection.GCTestMode\";\n// Local storage key to turn GC sweep on / off.\nconst runSweepKey = \"Fluid.GarbageCollection.RunSweep\";\n\nconst defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days\n\n/** The used state statistics of a node. */\nexport interface IUsedStateStats {\n    totalNodeCount: number;\n    unusedNodeCount: number;\n}\n\n/** The statistics of the system state after a garbage collection run. */\nexport interface IGCStats {\n    totalNodes: number;\n    deletedNodes: number;\n    totalDataStores: number;\n    deletedDataStores: number;\n}\n\n/** Defines the APIs for the runtime object to be passed to the garbage collector. */\nexport interface IGarbageCollectionRuntime {\n    /** Returns the garbage collection data of the runtime. */\n    getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;\n    /** After GC has run, called to notify the runtime of routes that are used in it. */\n    updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats;\n}\n\n/** Defines the contract for the garbage collector. */\nexport interface IGarbageCollector {\n    /** Tells whether GC should run or not. */\n    readonly shouldRunGC: boolean;\n    /**\n     * This tracks two things:\n     * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.\n     * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n     */\n    readonly gcSummaryFeatureVersion: number;\n    /** Tells whether the GC version has changed compared to the version in the latest summary. */\n    readonly hasGCVersionChanged: boolean;\n    /** Tells whether GC data should be written to the root of the summary tree. */\n    readonly writeDataAtRoot: boolean;\n    /** Run garbage collection and update the reference / used state of the system. */\n    collectGarbage(\n        options: { logger?: ITelemetryLogger, runGC?: boolean, runSweep?: boolean, fullGC?: boolean },\n    ): Promise<IGCStats>;\n    /** Summarizes the GC data and returns it as a summary tree. */\n    summarize(): ISummaryTreeWithStats | undefined;\n    /** Returns a map of each data store id to its GC details in the base summary. */\n    getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>>;\n    /** Called when the latest summary of the system has been refreshed. */\n    latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;\n    /** Called when a node is changed. Used to detect and log when an inactive node is changed. */\n    nodeChanged(id: string): void;\n    /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */\n    addedOutboundReference(fromNodeId: string, toNodeId: string): void;\n}\n\n/**\n * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced. It also sets\n * the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).\n */\nclass UnreferencedStateTracker {\n    private inactive: boolean = false;\n    // Keeps track of all inactive events that are logged. This is used to limit the log generation for each event to 1\n    // so that it is not noisy.\n    private readonly inactiveEventsLogged: Set<string> = new Set();\n    private readonly timer: Timer | undefined;\n\n    constructor(\n        public readonly unreferencedTimestampMs: number,\n        inactiveTimeoutMs: number,\n    ) {\n        // If the timeout has already expired, the node should become inactive immediately. Otherwise, start a timer of\n        // inactiveTimeoutMs after which the node will become inactive.\n        if (inactiveTimeoutMs <= 0) {\n            this.inactive = true;\n        } else {\n            this.timer = new Timer(inactiveTimeoutMs, () => { this.inactive = true; });\n            this.timer.start();\n        }\n    }\n\n    /** Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state. */\n    public stopTracking() {\n        this.timer?.clear();\n        this.inactive = false;\n    }\n\n    /** Logs an error with the given properties if the node  is inactive. */\n    public logIfInactive(\n        logger: ITelemetryLogger,\n        eventName: string,\n        currentTimestampMs: number,\n        deleteTimeoutMs: number,\n        inactiveNodeId: string,\n    ) {\n        if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {\n            logger.sendErrorEvent({\n                eventName,\n                age: currentTimestampMs - this.unreferencedTimestampMs,\n                timeout: deleteTimeoutMs,\n                id: inactiveNodeId,\n            });\n            this.inactiveEventsLogged.add(eventName);\n        }\n    }\n}\n\n/**\n * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains\n * its state across summaries.\n */\nexport class GarbageCollector implements IGarbageCollector {\n    public static create(\n        provider: IGarbageCollectionRuntime,\n        gcOptions: IGCRuntimeOptions,\n        deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n        getCurrentTimestampMs: () => number,\n        baseSnapshot: ISnapshotTree | undefined,\n        readAndParseBlob: ReadAndParseBlob,\n        baseLogger: ITelemetryLogger,\n        existing: boolean,\n        metadata?: IContainerRuntimeMetadata,\n    ): IGarbageCollector {\n        return new GarbageCollector(\n            provider,\n            gcOptions,\n            deleteUnusedRoutes,\n            getCurrentTimestampMs,\n            baseSnapshot,\n            readAndParseBlob,\n            baseLogger,\n            existing,\n            metadata,\n        );\n    }\n\n    /**\n     * Tells whether GC should be run based on the GC options and local storage flags.\n     */\n    public readonly shouldRunGC: boolean;\n\n    /**\n     * This tracks two things:\n     * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.\n     * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n     */\n    public get gcSummaryFeatureVersion(): number {\n        return this.gcEnabled ? this.currentGCVersion : 0;\n    }\n\n    /**\n     * Tells whether the GC version has changed compared to the version in the latest summary.\n     */\n    public get hasGCVersionChanged(): boolean {\n        // The current version can differ from the latest summary version in two cases:\n        // 1. The summary this client loaded with has data from a different GC version.\n        // 2. This client's latest summary was updated from a snapshot that has a different GC version.\n        return this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion;\n    }\n\n    /**\n     * Tracks if GC is enabled for this document. This is specified during document creation and doesn't change\n     * throughout its lifetime.\n     */\n    private readonly gcEnabled: boolean;\n    private readonly shouldRunSweep: boolean;\n    private readonly testMode: boolean;\n    private readonly mc: MonitoringContext;\n\n    /**\n     * Tells whether the GC data should be written to the root of the summary tree. We do this under 2 conditions:\n     * 1. If `writeDataAtRoot` GC option is enabled.\n     * 2. If the base summary has the GC data written at the root. This is to support forward compatibility where when\n     *    we start writing the GC data at root, older versions can detect that and write at root too.\n     */\n    private _writeDataAtRoot: boolean = false;\n    public get writeDataAtRoot(): boolean {\n        return this._writeDataAtRoot;\n    }\n\n    // The current GC version that this container is running.\n    private readonly currentGCVersion = GCVersion;\n    // This is the version of GC data in the latest summary being tracked.\n    private latestSummaryGCVersion: GCVersion;\n\n    // Keeps track of the GC state from the last run.\n    private gcDataFromLastRun: IGarbageCollectionData | undefined;\n    // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of\n    // outbound routes from that node.\n    private readonly referencesSinceLastRun: Map<string, string[]> = new Map();\n\n    // Promise when resolved initializes the base state of the nodes from the base summary state.\n    private readonly initializeBaseStateP: Promise<void>;\n    // The map of data store ids to their GC details in the base summary returned in getDataStoreGCDetails().\n    private readonly dataStoreGCDetailsP: Promise<Map<string, IGarbageCollectionSummaryDetails>>;\n    // The time after which an unreferenced node can be deleted. Currently, we only set the node's state to expired.\n    private readonly deleteTimeoutMs: number;\n    // Map of node ids to their unreferenced state tracker.\n    private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();\n\n    protected constructor(\n        private readonly provider: IGarbageCollectionRuntime,\n        private readonly gcOptions: IGCRuntimeOptions,\n        /** After GC has run, called to delete objects in the runtime whose routes are unused. */\n        private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n        /** Returns the current timestamp to be assigned to nodes that become unreferenced. */\n        private readonly getCurrentTimestampMs: () => number,\n        baseSnapshot: ISnapshotTree | undefined,\n        readAndParseBlob: ReadAndParseBlob,\n        baseLogger: ITelemetryLogger,\n        existing: boolean,\n        metadata?: IContainerRuntimeMetadata,\n    ) {\n        this.mc = loggerToMonitoringContext(\n            ChildLogger.create(baseLogger, \"GarbageCollector\"));\n\n        this.deleteTimeoutMs = this.gcOptions.deleteTimeoutMs ?? defaultDeleteTimeoutMs;\n\n        let prevSummaryGCVersion: number | undefined;\n        // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing\n        // documents, we get this information from the metadata blob.\n        if (existing) {\n            prevSummaryGCVersion = getGCVersion(metadata);\n            // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all\n            // other exsiting documents, GC is enabled.\n            this.gcEnabled = prevSummaryGCVersion > 0;\n        } else {\n            // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.\n            this.gcEnabled = gcOptions.gcAllowed === true;\n        }\n        // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the\n        // latest tracked GC version. For new documents, we will be writing the first summary with the current version.\n        this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;\n\n        // Whether GC should run or not. Can override with localStorage flag.\n        this.shouldRunGC = this.mc.config.getBoolean(runGCKey) ?? (\n            // GC must be enabled for the document.\n            this.gcEnabled\n            // GC must not be disabled via GC options.\n            && !gcOptions.disableGC\n        );\n\n        // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with\n        // localStorage flag.\n        this.shouldRunSweep = this.shouldRunGC &&\n            (this.mc.config.getBoolean(runSweepKey) ?? gcOptions.runSweep === true);\n\n        // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.\n        this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;\n\n        // If `writeDataAtRoot` GC option is true, we should write the GC data into the root of the summary tree. This\n        // GC option is used for testing only. It will be removed once we start writing GC data into root by default.\n        this._writeDataAtRoot = this.gcOptions.writeDataAtRoot === true;\n\n        // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do\n        // this once since it involves fetching blobs from storage which is expensive.\n        const baseSummaryStateP = new LazyPromise<IGarbageCollectionState | undefined>(async () => {\n            if (baseSnapshot === undefined) {\n                return undefined;\n            }\n\n            // For newer documents, GC data should be present in the GC tree in the root of the snapshot.\n            const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];\n            if (gcSnapshotTree !== undefined) {\n                // forward-compat - If a newer version has written the GC tree at root, we should also do the same.\n                this._writeDataAtRoot = true;\n                return getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);\n            }\n\n            // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and\n            // consolidate into IGarbageCollectionState format.\n            // Add a node for the root node that is not present in older snapshot format.\n            const gcState: IGarbageCollectionState = { gcNodes: { \"/\": { outboundRoutes: [] } } };\n            const dataStoreSnaphotTree = getSummaryForDatastores(baseSnapshot, metadata);\n            assert(dataStoreSnaphotTree !== undefined,\n                0x2a8 /* \"Expected data store snapshot tree in base snapshot\" */);\n            for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnaphotTree.trees)) {\n                const blobId = dsSnapshotTree.blobs[gcBlobKey];\n                if (blobId === undefined) {\n                    continue;\n                }\n\n                const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionSummaryDetails>(blobId);\n                // If there are no nodes for this data store, skip it.\n                if (gcSummaryDetails.gcData?.gcNodes === undefined) {\n                    continue;\n                }\n\n                const dsRootId = `/${dsId}`;\n                // Since we used to write GC data at data store level, we won't have an entry for the root (\"/\").\n                // Construct that entry by adding root data store ids to its outbound routes.\n                const initialSnapshotDetails = await readAndParseBlob<ReadFluidDataStoreAttributes>(\n                    dsSnapshotTree.blobs[dataStoreAttributesBlobName],\n                );\n                if (initialSnapshotDetails.isRootDataStore) {\n                    gcState.gcNodes[\"/\"].outboundRoutes.push(dsRootId);\n                }\n\n                for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {\n                    // Prefix the data store id to the GC node ids to make them relative to the root from being\n                    // relative to the data store. Similar to how its done in DataStore::getGCData.\n                    const rootId = id === \"/\" ? dsRootId : `${dsRootId}${id}`;\n                    gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };\n                }\n                assert(gcState.gcNodes[dsRootId] !== undefined,\n                    0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);\n                gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;\n            }\n\n            // If there is only one node (root node just added above), either GC is disabled or we are loading from the\n            // very first summary generated by detached container. In both cases, GC was not run - return undefined.\n            return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;\n        });\n\n        // Set up the initializer which initializes the base GC state from the base snapshot. Use lazy promise because\n        // we only do this once - the very first time we run GC.\n        this.initializeBaseStateP = new LazyPromise<void>(async () => {\n            const currentTimestampMs = this.getCurrentTimestampMs();\n            const baseState =  await baseSummaryStateP;\n            if (baseState === undefined) {\n                return;\n            }\n\n            const gcNodes: { [ id: string ]: string[] } = {};\n            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {\n                const unreferencedTimestampMs = nodeData.unreferencedTimestampMs;\n                if (unreferencedTimestampMs !== undefined) {\n                    // Get how long it has been since the node was unreferenced. Start a timeout for the remaining time\n                    // left for it to be eligible for deletion.\n                    const unreferencedDurationMs = currentTimestampMs - unreferencedTimestampMs;\n                    this.unreferencedNodesState.set(\n                        nodeId,\n                        new UnreferencedStateTracker(\n                            unreferencedTimestampMs,\n                            this.deleteTimeoutMs - unreferencedDurationMs,\n                        ),\n                    );\n                }\n                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);\n            }\n            this.gcDataFromLastRun = { gcNodes };\n        });\n\n        // Get the GC details for each data store from the GC state in the base summary. This is returned in\n        // getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.\n        this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionSummaryDetails>>(async () => {\n            const baseState = await baseSummaryStateP;\n            if (baseState === undefined) {\n                return new Map();\n            }\n\n            const gcNodes: { [ id: string ]: string[] } = {};\n            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {\n                gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);\n            }\n            // Run GC on the nodes in the base summary to get the routes used in each node in the container.\n            // This is an optimization for space (vs performance) wherein we don't need to store the used routes of\n            // each node in the summary.\n            const usedRoutes = runGarbageCollection(\n                gcNodes,\n                [ \"/\" ],\n                this.mc.logger,\n            ).referencedNodeIds;\n\n            const dataStoreGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });\n            // Currently, the data stores write the GC data. So, we need to update it's base GC details with the\n            // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.\n            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {\n                if (nodeData.unreferencedTimestampMs !== undefined) {\n                    const dataStoreGCDetails = dataStoreGCDetailsMap.get(nodeId.slice(1));\n                    if (dataStoreGCDetails !== undefined) {\n                        dataStoreGCDetails.unrefTimestamp = nodeData.unreferencedTimestampMs;\n                    }\n                }\n            }\n            return dataStoreGCDetailsMap;\n        });\n    }\n\n    /**\n     * Runs garbage collection and udpates the reference / used state of the nodes in the container.\n     * @returns the number of data stores that have been marked as unreferenced.\n     */\n    public async collectGarbage(\n        options: {\n            /** Logger to use for logging GC events */\n            logger?: ITelemetryLogger,\n            /** True to run GC sweep phase after the mark phase */\n            runSweep?: boolean,\n            /** True to generate full GC data */\n            fullGC?: boolean,\n        },\n    ): Promise<IGCStats> {\n        const {\n            logger = this.mc.logger,\n            runSweep = this.shouldRunSweep,\n            fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged,\n        } = options;\n\n        return PerformanceEvent.timedExecAsync(logger, { eventName: \"GarbageCollection\" }, async (event) => {\n            await this.initializeBaseStateP;\n\n            const gcStats: {\n                deletedNodes?: number,\n                totalNodes?: number,\n                deletedDataStores?: number,\n                totalDataStores?: number,\n            } = {};\n\n            // Get the runtime's GC data and run GC on the reference graph in it.\n            const gcData = await this.provider.getGCData(fullGC);\n\n            this.updateStateSinceLatestRun(gcData);\n\n            const gcResult = runGarbageCollection(\n                gcData.gcNodes,\n                [ \"/\" ],\n                logger,\n            );\n\n            const currentTimestampMs = this.getCurrentTimestampMs();\n            // Update the current state of the system based on the GC run.\n            this.updateCurrentState(gcData, gcResult, currentTimestampMs);\n\n            const dataStoreUsedStateStats =\n                this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);\n\n            if (runSweep) {\n                // Placeholder for running sweep logic.\n            }\n\n            // Update stats to be reported in the peformance event.\n            gcStats.deletedNodes = gcResult.deletedNodeIds.length;\n            gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;\n            gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;\n            gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;\n\n            // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios\n            // involving access to deleted data.\n            if (this.testMode) {\n                this.deleteUnusedRoutes(gcResult.deletedNodeIds);\n            }\n            event.end(gcStats);\n            return gcStats as IGCStats;\n        },\n        { end: true, cancel: \"error\" });\n    }\n\n    /**\n     * Summarizes the GC data and returns it as a summary tree.\n     * We current write the entire GC state in a single blob. This can be modified later to write multiple\n     * blobs. All the blob keys should start with `gcBlobPrefix`.\n     */\n    public summarize(): ISummaryTreeWithStats | undefined {\n        if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {\n            return;\n        }\n\n        const gcState: IGarbageCollectionState = { gcNodes: {} };\n        for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {\n            gcState.gcNodes[nodeId] = {\n                outboundRoutes,\n                unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,\n            };\n        }\n\n        const builder = new SummaryTreeBuilder();\n        builder.addBlob(`${gcBlobPrefix}_root`, JSON.stringify(gcState));\n        return builder.getSummaryTree();\n    }\n\n    /**\n     * Returns a map of data store ids to their base GC details generated from the base summary.This is used to\n     * initialize the data stores with their base GC state.\n     */\n    public async getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>> {\n        return this.dataStoreGCDetailsP;\n    }\n\n    /**\n     * Called when the latest summary of the system has been refreshed. This will be used to update the state of the\n     * latest summary tracked.\n     */\n    public async latestSummaryStateRefreshed(\n        result: RefreshSummaryResult,\n        readAndParseBlob: ReadAndParseBlob,\n    ): Promise<void> {\n        if (!this.shouldRunGC || !result.latestSummaryUpdated) {\n            return;\n        }\n\n        // If the summary was tracked by this client, it was the one that generated the summary in the first place.\n        // Basically, it was written in the current GC version.\n        if (result.wasSummaryTracked) {\n            this.latestSummaryGCVersion = this.currentGCVersion;\n            return;\n        }\n        // If the summary was not tracked by this client, update latest GC version from the snapshot in the result as\n        // that is now the latest summary.\n        await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);\n    }\n\n    /**\n     * Called when a node with the given id is changed. If the node is inactive, log an error.\n     */\n    public nodeChanged(id: string) {\n        // Prefix \"/\" if needed to make it relative to the root.\n        const nodeId = id.startsWith(\"/\") ? id : `/${id}`;\n        this.unreferencedNodesState.get(nodeId)?.logIfInactive(\n            this.mc.logger,\n            \"inactiveObjectChanged\",\n            this.getCurrentTimestampMs(),\n            this.deleteTimeoutMs,\n            nodeId,\n        );\n    }\n\n    /**\n     * Called when an outbound reference is added to a node. This is used to identify all nodes that have been\n     * referenced between summaries so that their unreferenced timestamp can be reset.\n     *\n     * @param fromNodeId - The node from which the reference is added.\n     * @param toNodeId - The node to which the reference is added.\n     */\n    public addedOutboundReference(fromNodeId: string, toNodeId: string) {\n        const outboundRoutes = this.referencesSinceLastRun.get(fromNodeId) ?? [];\n        outboundRoutes.push(toNodeId);\n        this.referencesSinceLastRun.set(fromNodeId, outboundRoutes);\n    }\n\n    /**\n     * Update the latest summary GC version from the metadata blob in the given snapshot.\n     */\n    private async updateSummaryGCVersionFromSnapshot(snapshot: ISnapshotTree, readAndParseBlob: ReadAndParseBlob) {\n        const metadataBlobId = snapshot.blobs[metadataBlobName];\n        if (metadataBlobId) {\n            const metadata = await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId);\n            this.latestSummaryGCVersion = getGCVersion(metadata);\n        }\n    }\n\n    /**\n     * Updates the state of the system as per the current GC run. It does the following:\n     * 1. Sets up the current GC state as per the gcData.\n     * 2. Starts tracking for nodes that have become unreferenced in this run.\n     * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.\n     * @param gcData - The data representing the reference graph on which GC is run.\n     * @param gcResult - The result of the GC run on the gcData.\n     * @param currentTimestampMs - The current timestamp to be used for unreferenced nodes' timestamp.\n     */\n    private updateCurrentState(gcData: IGarbageCollectionData, gcResult: IGCResult, currentTimestampMs: number) {\n        this.gcDataFromLastRun = cloneGCData(gcData);\n        this.referencesSinceLastRun.clear();\n\n        // Iterate through the deleted nodes and start tracking if they became unreferenced in this run.\n        for (const nodeId of gcResult.deletedNodeIds) {\n            // The time when the node became unreferenced. This is added to the current GC state.\n            let unreferencedTimestampMs: number = currentTimestampMs;\n            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);\n            if (nodeStateTracker !== undefined) {\n                unreferencedTimestampMs = nodeStateTracker.unreferencedTimestampMs;\n            } else {\n                // Start tracking this node as it became unreferenced in this run.\n                this.unreferencedNodesState.set(\n                    nodeId,\n                    new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs),\n                );\n            }\n        }\n\n        // Iterate through the referenced nodes and stop tracking if they were unreferenced before.\n        for (const nodeId of gcResult.referencedNodeIds) {\n            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);\n            if (nodeStateTracker !== undefined) {\n                // If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,\n                // log an error as this may mean the deleteTimeoutMs is not long enough.\n                nodeStateTracker.logIfInactive(\n                    this.mc.logger,\n                    \"inactiveObjectRevived\",\n                    currentTimestampMs,\n                    this.deleteTimeoutMs,\n                    nodeId,\n                );\n                // Stop tracking so as to clear out any running timers.\n                nodeStateTracker.stopTracking();\n                // Delete the node as we don't need to track it any more.\n                this.unreferencedNodesState.delete(nodeId);\n            }\n        }\n    }\n\n    /**\n     * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in\n     * time. It's possible that nodes transition from `unreferenced -> referenced -> unreferenced` between two runs. The\n     * unreferenced timestamp of such nodes needs to be reset as they may have been accessed when they were referenced.\n     *\n     * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.\n     * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.\n     */\n    private updateStateSinceLatestRun(currentGCData: IGarbageCollectionData) {\n        // If we haven't run GC before or no references were added since the last run, there is nothing to do.\n        if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {\n            return;\n        }\n\n        /**\n         * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and\n         * edges that have been added since then. To do this, combine the GC data from the last run and the current\n         * run, and then add the references since last run.\n         *\n         * Note on why we need to combine the data from previous run, current run and all references in between -\n         * 1. We need data from last run because some of its references may have been deleted since then. If those\n         *    references added new outbound references before getting deleted, we need to detect them.\n         * 2. We need new outbound references since last run because some of them may have been deleted later. If those\n         *    references added new outbound references before getting deleted, we need to detect them.\n         * 3. We need data from the current run because currently we may not detect when DDSs are referenced:\n         *    - We don't require DDSs handles to be stored in a referenced DDS. For this, we need GC at DDS level\n         *      which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.\n         *    - A new data store may have \"root\" DDSs already created and we don't detect them today.\n         */\n        const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);\n        this.referencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {\n            if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {\n                gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;\n            } else {\n                gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);\n            }\n        });\n\n        /**\n         * Run GC on the above reference graph to find all nodes that are referenced. For each one, if they are\n         * unreferenced, stop tracking them and remove from unreferenced list.\n         * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.\n         */\n        const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [\"/\"], this.mc.logger);\n        for (const nodeId of gcResult.referencedNodeIds) {\n            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);\n            if (nodeStateTracker !== undefined) {\n                // Stop tracking so as to clear out any running timers.\n                nodeStateTracker.stopTracking();\n                // Delete the node as we don't need to track it any more.\n                this.unreferencedNodesState.delete(nodeId);\n            }\n        }\n    }\n}\n\n/**\n * Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.\n * Merge the GC state from all such blobs and return the merged GC state.\n*/\nasync function getGCStateFromSnapshot(\n    gcSnapshotTree: ISnapshotTree,\n    readAndParseBlob: ReadAndParseBlob,\n): Promise<IGarbageCollectionState> {\n    let rootGCState: IGarbageCollectionState = { gcNodes: {} };\n    for (const key of Object.keys(gcSnapshotTree.blobs)) {\n        // Skip blobs that do not stsart with the GC prefix.\n        if (!key.startsWith(gcBlobPrefix)) {\n            continue;\n        }\n\n        const blobId = gcSnapshotTree.blobs[key];\n        if (blobId === undefined) {\n            continue;\n        }\n        const gcState = await readAndParseBlob<IGarbageCollectionState>(blobId);\n        assert(gcState !== undefined, 0x2ad /* \"GC blob missing from snapshot\" */);\n        // Merge the GC state of this blob into the root GC state.\n        rootGCState = concatGarbageCollectionStates(rootGCState, gcState);\n    }\n    return rootGCState;\n}\n"]}