@fluidframework/container-runtime 2.0.0-dev.1.4.6.106135 → 2.0.0-dev.2.3.0.115467

package/dist/garbageCollection.js

@@ -15,7 +15,7 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = exports.defaultSessionExpiryDurationMs = exports.defaultInactiveTimeoutMs = exports.oneDayMs = exports.disableSweepLogKey = exports.trackGCStateKey = exports.disableSessionExpiryKey = exports.runSessionExpiryKey = exports.gcTestModeKey = exports.runSweepKey = exports.runGCKey = exports.gcBlobPrefix = exports.gcTreeKey = void 0;
+exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = void 0;
 const common_utils_1 = require("@fluidframework/common-utils");
 const container_utils_1 = require("@fluidframework/container-utils");
 const garbage_collector_1 = require("@fluidframework/garbage-collector");
@@ -25,34 +25,11 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const containerRuntime_1 = require("./containerRuntime");
 const dataStores_1 = require("./dataStores");
+const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
 const gcSweepReadyUsageDetection_1 = require("./gcSweepReadyUsageDetection");
 const summaryFormat_1 = require("./summaryFormat");
 /** This is the current version of garbage collection. */
 const GCVersion = 1;
-// The key for the GC tree in summary.
-exports.gcTreeKey = "gc";
-// They prefix for GC blobs in the GC tree in summary.
-exports.gcBlobPrefix = "__gc";
-// Feature gate key to turn GC on / off.
-exports.runGCKey = "Fluid.GarbageCollection.RunGC";
-// Feature gate key to turn GC sweep on / off.
-exports.runSweepKey = "Fluid.GarbageCollection.RunSweep";
-// Feature gate key to turn GC test mode on / off.
-exports.gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
-// Feature gate key to write GC data at the root of the summary tree.
-const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
-// Feature gate key to expire a session after a set period of time.
-exports.runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
-// Feature gate key to disable expiring session after a set period of time, even if expiry value is present.
-exports.disableSessionExpiryKey = "Fluid.GarbageCollection.DisableSessionExpiry";
-// Feature gate key to write the gc blob as a handle if the data is the same.
-exports.trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
-// Feature gate key to turn GC sweep log off.
-exports.disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
-// One day in milliseconds.
-exports.oneDayMs = 1 * 24 * 60 * 60 * 1000;
-exports.defaultInactiveTimeoutMs = 7 * exports.oneDayMs; // 7 days
-exports.defaultSessionExpiryDurationMs = 30 * exports.oneDayMs; // 30 days
 /** The types of GC nodes in the GC reference graph. */
 exports.GCNodeType = {
     // Nodes that are for data stores.
@@ -166,10 +143,6 @@ exports.UnreferencedStateTracker = UnreferencedStateTracker;
 class GarbageCollector {
     constructor(createParams) {
         var _a, _b, _c, _d, _e, _f, _g, _h;
-        /**
-         * Tells whether the GC data should be written to the root of the summary tree.
-         */
-        this._writeDataAtRoot = true;
         /**
          * Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
          *
@@ -188,6 +161,7 @@ class GarbageCollector {
         // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
         // outbound routes from that node.
         this.newReferencesSinceLastRun = new Map();
+        this.tombstones = [];
         // Map of node ids to their unreferenced state tracker.
         this.unreferencedNodesState = new Map();
         // Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
@@ -200,6 +174,7 @@ class GarbageCollector {
         this.runtime = createParams.runtime;
         this.isSummarizerClient = createParams.isSummarizerClient;
         this.gcOptions = createParams.gcOptions;
+        this.createContainerMetadata = createParams.createContainerMetadata;
         this.getNodePackagePath = createParams.getNodePackagePath;
         this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
         this.activeConnection = createParams.activeConnection;
@@ -209,6 +184,20 @@ class GarbageCollector {
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } }));
         this.sweepReadyUsageHandler = new gcSweepReadyUsageDetection_1.SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
         let prevSummaryGCVersion;
+        /**
+         * Sweep timeout is the time after which unreferenced content can be swept.
+         * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
+         *
+         * The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
+         * The buffer is added to account for any clock skew or other edge cases.
+         * We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
+         */
+        function computeSweepTimeout(sessionExpiryTimeoutMs) {
+            const maxSnapshotCacheExpiryMs = 5 * garbageCollectionConstants_1.oneDayMs;
+            const bufferMs = garbageCollectionConstants_1.oneDayMs;
+            return sessionExpiryTimeoutMs &&
+                (sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
+        }
         /**
          * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
          * 1. Whether running GC mark phase is allowed or not.
@@ -223,6 +212,8 @@ class GarbageCollector {
             this.gcEnabled = prevSummaryGCVersion > 0;
             this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
             this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
+            this.sweepTimeoutMs =
+                (_b = metadata === null || metadata === void 0 ? void 0 : metadata.sweepTimeoutMs) !== null && _b !== void 0 ? _b : computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
         }
         else {
             // Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
@@ -230,36 +221,28 @@ class GarbageCollector {
             if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
                 throw new container_utils_1.UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
             }
+            // This Test Override only applies for new containers
+            const testOverrideSweepTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
             // For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
             // flag in GC options to false.
             this.gcEnabled = this.gcOptions.gcAllowed !== false;
             // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
-            this.sweepEnabled = this.gcOptions.sweepAllowed === true;
+            // ...unless we're using the TestOverride
+            this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
             // Set the Session Expiry only if the flag is enabled and GC is enabled.
-            if (this.mc.config.getBoolean(exports.runSessionExpiryKey) && this.gcEnabled) {
-                this.sessionExpiryTimeoutMs = (_b = this.gcOptions.sessionExpiryTimeoutMs) !== null && _b !== void 0 ? _b : exports.defaultSessionExpiryDurationMs;
+            if (this.mc.config.getBoolean(garbageCollectionConstants_1.runSessionExpiryKey) && this.gcEnabled) {
+                this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : garbageCollectionConstants_1.defaultSessionExpiryDurationMs;
             }
+            this.sweepTimeoutMs =
+                testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
         }
         // If session expiry is enabled, we need to close the container when the session expiry timeout expires.
-        if (this.sessionExpiryTimeoutMs !== undefined && this.mc.config.getBoolean(exports.disableSessionExpiryKey) !== true) {
+        if (this.sessionExpiryTimeoutMs !== undefined) {
             // If Test Override config is set, override Session Expiry timeout.
             const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
             const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
             this.sessionExpiryTimer = new common_utils_1.Timer(timeoutMs, () => { this.runtime.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs)); });
             this.sessionExpiryTimer.start();
-            // TEMPORARY: Hardcode a default of 5 days which will be >= the policy's value in the ODSP driver.
-            // This unblocks the Sweep Log (see logSweepEvents function).
-            // This will be removed before sweep is fully implemented.
-            const snapshotCacheExpiryMs = (_c = createParams.snapshotCacheExpiryMs) !== null && _c !== void 0 ? _c : 5 * 24 * 60 * 60 * 1000;
-            /**
-             * Sweep timeout is the time after which unreferenced content can be swept.
-             * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer. The buffer is
-             * added to account for any clock skew. We use server timestamps throughout so the skew should be minimal
-             * but make it one day to be safe.
-             */
-            if (snapshotCacheExpiryMs !== undefined) {
-                this.sweepTimeoutMs = this.sessionExpiryTimeoutMs + snapshotCacheExpiryMs + exports.oneDayMs;
-            }
         }
         // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
         // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
@@ -273,7 +256,7 @@ class GarbageCollector {
          *
          * These conditions can be overridden via runGCKey feature flag.
          */
-        this.shouldRunGC = (_d = this.mc.config.getBoolean(exports.runGCKey)) !== null && _d !== void 0 ? _d : (
+        this.shouldRunGC = (_d = this.mc.config.getBoolean(garbageCollectionConstants_1.runGCKey)) !== null && _d !== void 0 ? _d : (
         // GC must be enabled for the document.
         this.gcEnabled
             // GC must not be disabled via GC options.
@@ -282,51 +265,43 @@ class GarbageCollector {
          * Whether sweep should run or not. The following conditions have to be met to run sweep:
          *
          * 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
-         *
          * 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
-         *
-         * 3. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
+         * 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
+         * the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
+         * 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
          * feature flag.
          */
-        this.shouldRunSweep = false; // disable while TEMPORARY measure hardcoding snapshotCacheExpiryMs is here
-        // this.shouldRunGC
-        // && this.sweepTimeoutMs !== undefined
-        // && (this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
-        this.trackGCState = this.mc.config.getBoolean(exports.trackGCStateKey) === true;
+        this.shouldRunSweep =
+            this.shouldRunGC
+                && this.sweepTimeoutMs !== undefined
+                && ((_e = this.mc.config.getBoolean(garbageCollectionConstants_1.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
+        this.trackGCState = this.mc.config.getBoolean(garbageCollectionConstants_1.trackGCStateKey) === true;
         // Override inactive timeout if test config or gc options to override it is set.
-        this.inactiveTimeoutMs = (_f = (_e = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _e !== void 0 ? _e : this.gcOptions.inactiveTimeoutMs) !== null && _f !== void 0 ? _f : exports.defaultInactiveTimeoutMs;
+        this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : garbageCollectionConstants_1.defaultInactiveTimeoutMs;
         // Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
         if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
             throw new container_utils_1.UsageError("inactive timeout should not be greater than the sweep timeout");
         }
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
-        this.testMode = (_g = this.mc.config.getBoolean(exports.gcTestModeKey)) !== null && _g !== void 0 ? _g : this.gcOptions.runGCInTestMode === true;
-        // GC state is written into root of the summary tree by default. Can be overridden via feature flag for now.
-        this._writeDataAtRoot = (_h = this.mc.config.getBoolean(writeAtRootKey)) !== null && _h !== void 0 ? _h : true;
-        if (this._writeDataAtRoot) {
-            // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
-            // contain GC tree and GC is enabled.
-            const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[exports.gcTreeKey]) !== undefined;
-            this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
-        }
-        // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
-        // this once since it involves fetching blobs from storage which is expensive.
-        const baseSummaryStateP = new common_utils_1.LazyPromise(async () => {
+        this.testMode = (_h = this.mc.config.getBoolean(garbageCollectionConstants_1.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
+        // Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
+        this.tombstoneMode = this.mc.config.getBoolean(garbageCollectionConstants_1.disableTombstoneKey) !== true;
+        // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
+        // contain GC tree and GC is enabled.
+        const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey]) !== undefined;
+        this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
+        // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
+        // it involves fetching blobs from storage which is expensive.
+        this.baseSnapshotDataP = new common_utils_1.LazyPromise(async () => {
             var _a;
             if (baseSnapshot === undefined) {
                 return undefined;
             }
             try {
                 // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
-                const gcSnapshotTree = baseSnapshot.trees[exports.gcTreeKey];
+                const gcSnapshotTree = baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey];
                 if (gcSnapshotTree !== undefined) {
-                    // If the GC tree is written at root, we should also do the same.
-                    this._writeDataAtRoot = true;
-                    const baseGCState = await getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);
-                    if (this.trackGCState) {
-                        this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(baseGCState));
-                    }
-                    return baseGCState;
+                    return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
                 }
                 // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
                 // consolidate into IGarbageCollectionState format.
@@ -362,7 +337,7 @@ class GarbageCollector {
                 }
                 // If there is only one node (root node just added above), either GC is disabled or we are loading from
                 // the first summary generated by detached container. In both cases, GC was not run - return undefined.
-                return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
+                return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
             }
             catch (error) {
                 const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
@@ -371,11 +346,11 @@ class GarbageCollector {
             }
         });
         /**
-         * Set up the initializer which initializes the base GC state from the base snapshot. Note that the reference
-         * timestamp maybe from old ops which were not summarized and stored in the file. So, the unreferenced state
-         * may be out of date. This is fine because the state is updated every time GC runs based on the time then.
+         * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
+         * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
+         * GC state and updates their inactive or sweep ready state.
          */
-        this.initializeBaseStateP = new common_utils_1.LazyPromise(async () => {
+        this.initializeGCStateFromBaseSnapshotP = new common_utils_1.LazyPromise(async () => {
             const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
             /**
              * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
@@ -393,34 +368,41 @@ class GarbageCollector {
                 });
                 return;
             }
-            const baseState = await baseSummaryStateP;
+            const baseSnapshotData = await this.baseSnapshotDataP;
             /**
-             * The base state will not be present if the container is loaded from:
+             * The base snapshot data will not be present if the container is loaded from:
              * 1. The first summary created by the detached container.
              * 2. A summary that was generated with GC disabled.
              * 3. A summary that was generated before GC even existed.
              */
-            if (baseState === undefined) {
+            if (baseSnapshotData === undefined) {
                 return;
             }
             const gcNodes = {};
-            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
                 if (nodeData.unreferencedTimestampMs !== undefined) {
                     this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
                 }
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
             this.previousGCDataFromLastRun = { gcNodes };
+            // If tracking state across summaries, update latest summary data from the base snapshot's GC data.
+            if (this.trackGCState) {
+                this.latestSummaryData = {
+                    serializedGCState: JSON.stringify(generateSortedGCState(baseSnapshotData.gcState)),
+                    serializedTombstones: JSON.stringify(baseSnapshotData.tombstones),
+                };
+            }
         });
         // Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
         // which the caller uses to initialize each node's GC state.
         this.baseGCDetailsP = new common_utils_1.LazyPromise(async () => {
-            const baseState = await baseSummaryStateP;
-            if (baseState === undefined) {
+            const baseSnapshotData = await this.baseSnapshotDataP;
+            if (baseSnapshotData === undefined) {
                 return new Map();
             }
             const gcNodes = {};
-            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
             // Run GC on the nodes in the base summary to get the routes used in each node in the container.
@@ -430,7 +412,7 @@ class GarbageCollector {
             const baseGCDetailsMap = (0, garbage_collector_1.unpackChildNodesGCDetails)({ gcData: { gcNodes }, usedRoutes });
             // Currently, the nodes may write the GC data. So, we need to update its base GC details with the
             // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
-            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
                 if (nodeData.unreferencedTimestampMs !== undefined) {
                     const dataStoreGCDetails = baseGCDetailsMap.get(nodeId.slice(1));
                     if (dataStoreGCDetails !== undefined) {
@@ -469,12 +451,29 @@ class GarbageCollector {
         return this.initialStateNeedsReset ||
             (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion);
     }
-    get writeDataAtRoot() {
-        return this._writeDataAtRoot;
-    }
     /** Returns a list of all the configurations for garbage collection. */
     get configs() {
-        return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, writeAtRoot: this._writeDataAtRoot, testMode: this.testMode, sessionExpiry: this.sessionExpiryTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
+        return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
+    }
+    /**
+     * Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
+     * before they are loaded or used. This is important to get accurate information of whether tombstoned object are
+     * in use or not.
+     */
+    async initializeBaseState() {
+        const baseSnapshotData = await this.baseSnapshotDataP;
+        /**
+         * The base snapshot data or tombstone state will not be present if the container is loaded from:
+         * 1. The first summary created by the detached container.
+         * 2. A summary that was generated with GC disabled.
+         * 3. A summary that was generated before GC even existed.
+         * 4. A summary that was generated with tombstone feature disabled.
+         */
+        if (!this.tombstoneMode || (baseSnapshotData === null || baseSnapshotData === void 0 ? void 0 : baseSnapshotData.tombstones) === undefined) {
+            return;
+        }
+        this.tombstones = baseSnapshotData.tombstones;
+        this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
     }
     /**
      * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
@@ -484,16 +483,20 @@ class GarbageCollector {
      */
     setConnectionState(connected, clientId) {
         /**
-         * For non-summarizer clients, initialize the base state when the container becomes active, i.e., it transitions
+         * For all clients, initialize the base state when the container becomes active, i.e., it transitions
          * to "write" mode. This will ensure that the container's own join op is processed and there is a recent
          * reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
          * could affect the GC state will have been processed.
          *
+         * If GC is up-to-date for the client and the summarizing client, there will be an doubling of both
+         * InactiveObject_Loaded and SweepReady_Loaded errors, as there will be one from the sending client and one from
+         * the receiving summarizer client.
+         *
          * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
          * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
          */
-        if (this.activeConnection() && !this.isSummarizerClient && this.shouldRunGC) {
-            this.initializeBaseStateP.catch((error) => { });
+        if (this.activeConnection() && this.shouldRunGC) {
+            this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
         }
     }
     /**
@@ -529,14 +532,14 @@ class GarbageCollector {
             const gcData = await this.runtime.getGCData(fullGC);
             const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcData.gcNodes, ["/"]);
             const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
-            event.end(Object.assign({}, gcStats));
+            event.end(Object.assign(Object.assign({}, gcStats), { timestamp: currentReferenceTimestampMs }));
             this.completedRuns++;
             return gcStats;
         }, { end: true, cancel: "error" });
     }
     async runPreGCSteps() {
-        // Ensure that base state has been initialized.
-        await this.initializeBaseStateP;
+        // Ensure that state has been initialized from the base snapshot data.
+        await this.initializeGCStateFromBaseSnapshotP;
         // Let the runtime update its pending state before GC runs.
         await this.runtime.updateStateBeforeGC();
     }
@@ -549,14 +552,20 @@ class GarbageCollector {
         this.updateStateSinceLastRun(gcData, logger);
         // Update the current state and update the runtime of all routes or ids that used as per the GC run.
         this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
-        this.runtime.updateUsedRoutes(gcResult.referencedNodeIds, currentReferenceTimestampMs);
+        this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
         // Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
         // delete these objects here instead.
         this.logSweepEvents(logger, currentReferenceTimestampMs);
         // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
         // involving access to deleted data.
         if (this.testMode) {
-            this.runtime.deleteUnusedRoutes(gcResult.deletedNodeIds);
+            this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds, false /* tombstone */);
+        }
+        else if (this.tombstoneMode) {
+            // If we are running in GC tombstone mode, tombstone objects for unused routes. This enables testing
+            // scenarios involving access to "deleted" data without actually deleting the data from summaries.
+            // Note: we will not tombstone in test mode
+            this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
         }
         // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
         // updates its state so that we don't send false positives based on intermediate state. For example, we may get
@@ -581,31 +590,69 @@ class GarbageCollector {
                 unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
             };
         }
-        const newSerializedSummaryState = JSON.stringify(generateSortedGCState(gcState));
+        const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
+        const serializedTombstones = this.tombstoneMode
+            ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
+            : undefined;
         /**
-         * As an optimization if the GC tree hasn't changed and we're tracking the gc state, return a tree handle
-         * instead of returning the whole GC tree. If there are changes, then we want to return the whole tree.
+         * Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
+         * summary, send summary handles for them. Otherwise, send the data in summary blobs.
          */
         if (this.trackGCState) {
-            this.pendingSerializedSummaryState = newSerializedSummaryState;
-            if (this.latestSerializedSummaryState !== undefined &&
-                this.latestSerializedSummaryState === newSerializedSummaryState &&
-                !fullTree &&
-                trackState) {
-                const stats = (0, runtime_utils_1.mergeStats)();
-                stats.handleNodeCount++;
-                return {
-                    summary: {
-                        type: protocol_definitions_1.SummaryType.Handle,
-                        handle: `/${exports.gcTreeKey}`,
-                        handleType: protocol_definitions_1.SummaryType.Tree,
-                    },
-                    stats,
-                };
+            this.pendingSummaryData = { serializedGCState, serializedTombstones };
+            if (trackState && !fullTree && this.latestSummaryData !== undefined) {
+                // If neither GC state or tombstone state changed, send a summary handle for the entire GC data.
+                if (this.latestSummaryData.serializedGCState === serializedGCState
+                    && this.latestSummaryData.serializedTombstones === serializedTombstones) {
+                    const stats = (0, runtime_utils_1.mergeStats)();
+                    stats.handleNodeCount++;
+                    return {
+                        summary: {
+                            type: protocol_definitions_1.SummaryType.Handle,
+                            handle: `/${garbageCollectionConstants_1.gcTreeKey}`,
+                            handleType: protocol_definitions_1.SummaryType.Tree,
+                        },
+                        stats,
+                    };
+                }
+                // If either or both of GC state or tombstone state changed, build a GC summary tree.
+                return this.buildGCSummaryTree(serializedGCState, serializedTombstones, true /* trackState */);
             }
         }
+        // If not tracking GC state, build a GC summary tree without any summary handles.
+        return this.buildGCSummaryTree(serializedGCState, serializedTombstones, false /* trackState */);
+    }
+    /**
+     * Builds the GC summary tree which contains GC state and tombstone state.
+     * If trackState is false, both GC state and tombstone state are written as summary blobs.
+     * If trackState is true, summary blob is written for GC state or tombstone state if they changed.
+     * @param serializedGCState - The GC state serialized as string.
+     * @param serializedTombstones - THe tombstone state serialized as string.
+     * @param trackState - Whether we are tracking GC state across summaries.
+     * @returns the GC summary tree.
+     */
+    buildGCSummaryTree(serializedGCState, serializedTombstones, trackState) {
+        var _a, _b;
+        const gcStateBlobKey = `${garbageCollectionConstants_1.gcBlobPrefix}_root`;
         const builder = new runtime_utils_1.SummaryTreeBuilder();
-        builder.addBlob(`${exports.gcBlobPrefix}_root`, newSerializedSummaryState);
+        // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
+        if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
+            builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${gcStateBlobKey}`);
+        }
+        else {
+            builder.addBlob(gcStateBlobKey, serializedGCState);
+        }
+        // If there is no tombstone data, return only the GC state.
+        if (serializedTombstones === undefined) {
+            return builder.getSummaryTree();
+        }
+        // If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
+        if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones && trackState) {
+            builder.addHandle(garbageCollectionConstants_1.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${garbageCollectionConstants_1.gcTombstoneBlobKey}`);
+        }
+        else {
+            builder.addBlob(garbageCollectionConstants_1.gcTombstoneBlobKey, serializedTombstones);
+        }
         return builder.getSummaryTree();
     }
     getMetadata() {
@@ -617,6 +664,7 @@ class GarbageCollector {
             gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
             sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
             sweepEnabled: this.sweepEnabled,
+            sweepTimeoutMs: this.sweepTimeoutMs,
         };
     }
     /**
@@ -640,8 +688,8 @@ class GarbageCollector {
             this.latestSummaryGCVersion = this.currentGCVersion;
             this.initialStateNeedsReset = false;
             if (this.trackGCState) {
-                this.latestSerializedSummaryState = this.pendingSerializedSummaryState;
-                this.pendingSerializedSummaryState = undefined;
+                this.latestSummaryData = this.pendingSummaryData;
+                this.pendingSummaryData = undefined;
             }
             return;
         }
@@ -653,15 +701,18 @@ class GarbageCollector {
             const metadata = await readAndParseBlob(metadataBlobId);
             this.latestSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
         }
-        const gcSnapshotTree = snapshot.trees[exports.gcTreeKey];
+        const gcSnapshotTree = snapshot.trees[garbageCollectionConstants_1.gcTreeKey];
         if (gcSnapshotTree !== undefined && this.trackGCState) {
-            const latestGCState = await getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);
-            this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(latestGCState));
+            const latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
+            this.latestSummaryData = {
+                serializedGCState: JSON.stringify(generateSortedGCState(latestGCData.gcState)),
+                serializedTombstones: JSON.stringify(latestGCData.tombstones),
+            };
         }
         else {
-            this.latestSerializedSummaryState = undefined;
+            this.latestSummaryData = undefined;
         }
-        this.pendingSerializedSummaryState = undefined;
+        this.pendingSummaryData = undefined;
     }
     /**
      * Called when a node with the given id is updated. If the node is inactive, log an error.
@@ -716,6 +767,7 @@ class GarbageCollector {
      */
     updateCurrentState(gcData, gcResult, currentReferenceTimestampMs) {
         this.previousGCDataFromLastRun = (0, garbage_collector_1.cloneGCData)(gcData);
+        this.tombstones = [];
         this.newReferencesSinceLastRun.clear();
         // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
         for (const nodeId of gcResult.referencedNodeIds) {
@@ -739,13 +791,24 @@ class GarbageCollector {
             }
             else {
                 nodeStateTracker.updateTracking(currentReferenceTimestampMs);
+                if (this.tombstoneMode && nodeStateTracker.state === exports.UnreferencedState.SweepReady) {
+                    const nodeType = this.runtime.getNodeType(nodeId);
+                    if (nodeType === exports.GCNodeType.DataStore || nodeType === exports.GCNodeType.Blob) {
+                        this.tombstones.push(nodeId);
+                    }
+                }
             }
         }
     }
     /**
      * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
-     * time. It's possible that nodes transition from `unreferenced -> referenced -> unreferenced` between two runs. The
-     * unreferenced timestamp of such nodes needs to be reset as they may have been accessed when they were referenced.
+     * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
+     * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
+     * these objects while there can be in-memory referenced to it:
+     * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
+     * added, the object may have been accessed and in-memory reference to it added.
+     * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
+     * unreferenced, they could have been accessed and in-memory reference to them added.
      *
      * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
      * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
@@ -757,7 +820,7 @@ class GarbageCollector {
         }
         // Find any references that haven't been identified correctly.
         const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, this.previousGCDataFromLastRun, this.newReferencesSinceLastRun);
-        if (this.writeDataAtRoot && missingExplicitReferences.length > 0) {
+        if (missingExplicitReferences.length > 0) {
             missingExplicitReferences.forEach((missingExplicitReference) => {
                 const event = {
                     eventName: "gcUnknownOutboundReferences",
@@ -778,21 +841,18 @@ class GarbageCollector {
          * run, and then add the references since last run.
          *
          * Note on why we need to combine the data from previous run, current run and all references in between -
-         *
          * 1. We need data from last run because some of its references may have been deleted since then. If those
-         * references added new outbound references before getting deleted, we need to detect them.
+         * references added new outbound references before they were deleted, we need to detect them.
          *
          * 2. We need new outbound references since last run because some of them may have been deleted later. If those
-         * references added new outbound references before getting deleted, we need to detect them.
+         * references added new outbound references before they were deleted, we need to detect them.
          *
          * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
-         *
-         * - We don't require DDSes handles to be stored in a referenced DDS. For this, we need GC at DDS level
-         * which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
-         *
+         * - We don't require DDSes handles to be stored in a referenced DDS.
          * - A new data store may have "root" DDSes already created and we don't detect them today.
          */
         const gcDataSuperSet = (0, garbage_collector_1.concatGarbageCollectionData)(this.previousGCDataFromLastRun, currentGCData);
+        const newOutboundRoutesSinceLastRun = [];
         this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
             if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
                 gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
@@ -800,19 +860,22 @@ class GarbageCollector {
             else {
                 gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
             }
+            newOutboundRoutesSinceLastRun.push(...outboundRoutes);
         });
         /**
-         * Run GC on the above reference graph to find all nodes that are referenced. For each one, if they are
+         * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
+         * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
          * unreferenced, stop tracking them and remove from unreferenced list.
-         * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
+         * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
+         * unreferenced and add unreferenced state.
          */
-        const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcDataSuperSet.gcNodes, ["/"]);
+        const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcDataSuperSet.gcNodes, ["/", ...newOutboundRoutesSinceLastRun]);
         for (const nodeId of gcResult.referencedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker !== undefined) {
                 // Stop tracking so as to clear out any running timers.
                 nodeStateTracker.stopTracking();
-                // Delete the node as we don't need to track it any more.
+                // Delete the unreferenced state as we don't need to track it any more.
                 this.unreferencedNodesState.delete(nodeId);
             }
         }
@@ -918,7 +981,7 @@ class GarbageCollector {
      * this will give us a view into how much deleted content a container has.
      */
     logSweepEvents(logger, currentReferenceTimestampMs) {
-        if (this.mc.config.getBoolean(exports.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
+        if (this.mc.config.getBoolean(garbageCollectionConstants_1.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
             return;
         }
         this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
@@ -968,31 +1031,24 @@ class GarbageCollector {
             return;
         }
         this.loggedUnreferencedEvents.add(uniqueEventId);
-        const propsToLog = {
-            id: nodeId,
-            type: nodeType,
-            unrefTime: nodeStateTracker.unreferencedTimestampMs,
-            age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
-            timeout: nodeStateTracker.state === exports.UnreferencedState.Inactive
+        const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === exports.UnreferencedState.Inactive
                 ? this.inactiveTimeoutMs
-                : this.sweepTimeoutMs,
-            completedGCRuns: this.completedRuns,
-            lastSummaryTime: this.getLastSummaryTimestampMs(),
-            externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.externalRequest],
-            viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle],
-            fromId: fromNodeId,
-        };
+                : this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.externalRequest], viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle], fromId: fromNodeId });
         // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
         // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
         // but it's a good signal nonetheless and we can consume it with a grain of salt.
+        // Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
+        // SweepReady errors are usages of Objects that will be deleted by GC Sweep!
         if (this.isSummarizerClient) {
             this.pendingEventsQueue.push(Object.assign(Object.assign({}, propsToLog), { usageType, state }));
         }
         else {
             // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
             // summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
+            // Events generated:
+            // InactiveObject_Loaded, SweepReadyObject_Loaded
             if (usageType === "Loaded") {
-                this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: packagePath ? { value: packagePath.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined, stack: (0, telemetry_utils_1.generateStack)() }));
+                this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(packagePath), stack: (0, telemetry_utils_1.generateStack)() }));
             }
             // If SweepReady Usage Detection is enabed, the handler may close the interactive container.
             // Once Sweep is fully implemented, this will be removed since the objects will be gone
@@ -1003,6 +1059,11 @@ class GarbageCollector {
         }
     }
     async logUnreferencedEvents(logger) {
+        // Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
+        // summary time they are then logged.
+        // Events generated:
+        // InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
+        // SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
         for (const eventProps of this.pendingEventsQueue) {
             const { usageType, state } = eventProps, propsToLog = __rest(eventProps, ["usageType", "state"]);
             /**
@@ -1024,14 +1085,19 @@ class GarbageCollector {
 }
 exports.GarbageCollector = GarbageCollector;
 /**
- * Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
- * Merge the GC state from all such blobs and return the merged GC state.
+ * Gets the base garbage collection state from the given snapshot tree. It contains GC state and tombstone state.
+ * The GC state may be written into multiple blobs. Merge the GC state from all such blobs into one.
  */
-async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
+async function getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob) {
     let rootGCState = { gcNodes: {} };
+    let tombstones;
     for (const key of Object.keys(gcSnapshotTree.blobs)) {
+        if (key === garbageCollectionConstants_1.gcTombstoneBlobKey) {
+            tombstones = await readAndParseBlob(gcSnapshotTree.blobs[key]);
+            continue;
+        }
         // Skip blobs that do not start with the GC prefix.
-        if (!key.startsWith(exports.gcBlobPrefix)) {
+        if (!key.startsWith(garbageCollectionConstants_1.gcBlobPrefix)) {
             continue;
         }
         const blobId = gcSnapshotTree.blobs[key];
@@ -1043,7 +1109,7 @@ async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
         // Merge the GC state of this blob into the root GC state.
         rootGCState = (0, garbage_collector_1.concatGarbageCollectionStates)(rootGCState, gcState);
     }
-    return rootGCState;
+    return { gcState: rootGCState, tombstones };
 }
 function generateSortedGCState(gcState) {
     const sortableArray = Object.entries(gcState.gcNodes);