@fluidframework/container-runtime 2.0.0-internal.2.0.4 → 2.0.0-internal.2.1.1

package/src/dataStoreContext.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IDisposable, ITelemetryLogger, ITelemetryProperties } from "@fluidframework/common-definitions";
 import {
     FluidObject,
     IRequest,
@@ -65,7 +65,11 @@ import {
     TelemetryDataTag,
     ThresholdCounter,
 } from "@fluidframework/telemetry-utils";
-import { DataProcessingError } from "@fluidframework/container-utils";
+import {
+    DataCorruptionError,
+    DataProcessingError,
+    extractSafePropertiesFromMessage,
+} from "@fluidframework/container-utils";
 
 import { ContainerRuntime } from "./containerRuntime";
 import {
@@ -190,6 +194,13 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     private _disposed = false;
     public get disposed() { return this._disposed; }
 
+    /**
+     * Tombstone is a temporary feature that prevents a data store from sending / receiving ops, signals and from
+     * loading.
+     */
+    private _tombstoned = false;
+    public get tombstoned() { return this._tombstoned; }
+
     public get attachState(): AttachState {
         return this._attachState;
     }
@@ -305,6 +316,14 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         }
     }
 
+    public setTombstone(tombstone: boolean) {
+        if (this.tombstoned === tombstone) {
+            return;
+        }
+
+        this._tombstoned = tombstone;
+    }
+
     private rejectDeferredRealize(reason: string, packageName?: string): never {
         throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.CodeArtifact } });
     }
@@ -381,7 +400,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
      * its new client ID when we are connecting or connected.
      */
     public setConnectionState(connected: boolean, clientId?: string) {
-        this.verifyNotClosed();
+        // ConnectionState should not fail in tombstone mode as this is internally run
+        this.verifyNotClosed("setConnectionState", false /* checkTombstone */);
 
         // Connection events are ignored if the store is not yet loaded
         if (!this.loaded) {
@@ -395,7 +415,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public process(messageArg: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("process", true, extractSafePropertiesFromMessage(messageArg));
 
         const innerContents = messageArg.contents as FluidDataStoreMessage;
         const message = {
@@ -417,7 +437,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public processSignal(message: IInboundSignalMessage, local: boolean): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("processSignal");
 
         // Signals are ignored if the store is not yet loaded
         if (!this.loaded) {
@@ -582,7 +602,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public submitMessage(type: string, content: any, localOpMetadata: unknown): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("submitMessage");
         assert(!!this.channel, 0x146 /* "Channel must exist when submitting message" */);
         const fluidDataStoreContent: FluidDataStoreMessage = {
             content,
@@ -604,7 +624,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
      *
      */
     public setChannelDirty(address: string): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("setChannelDirty");
 
         // Get the latest sequence number.
         const latestSequenceNumber = this.deltaManager.lastSequenceNumber;
@@ -619,7 +639,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public submitSignal(type: string, content: any) {
-        this.verifyNotClosed();
+        this.verifyNotClosed("submitSignal");
+
         assert(!!this.channel, 0x147 /* "Channel must exist on submitting signal" */);
         return this._containerRuntime.submitDataStoreSignal(this.id, type, content);
     }
@@ -732,9 +753,17 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         return this.channel.applyStashedOp(innerContents.content);
     }
 
-    private verifyNotClosed() {
+    private verifyNotClosed(callSite: string, checkTombstone = true, safeTelemetryProps: ITelemetryProperties = {}) {
         if (this._disposed) {
-            throw new Error("Context is closed");
+            throw new Error(`Context is closed! Call site [${callSite}]`);
+        }
+
+        if (checkTombstone && this.tombstoned) {
+            const messageString = `Context is tombstoned! Call site [${callSite}]`;
+            throw new DataCorruptionError(messageString, {
+                errorMessage: messageString,
+                ...safeTelemetryProps,
+            });
         }
     }
 
@@ -991,6 +1020,15 @@ export class LocalDetachedFluidDataStoreContext
 
         super.bindRuntime(dataStoreChannel);
 
+        // Load the handle to the data store's entryPoint to make sure that for a detached data store, the entryPoint
+        // initialization function is called before the data store gets attached and potentially connected to the
+        // delta stream, so it gets a chance to do things while the data store is still "purely local".
+        // This preserves the behavior from before we introduced entryPoints, where the instantiateDataStore method
+        // of data store factories tends to construct the data object (at least kick off an async method that returns
+        // it); that code moved to the entryPoint initialization function, so we want to ensure it still executes
+        // before the data store is attached.
+        await dataStoreChannel.entryPoint?.get();
+
         if (await this.isRoot()) {
             dataStoreChannel.makeVisibleAndAttachGraph();
         }

package/src/dataStoreContexts.ts

@@ -87,7 +87,7 @@ import { FluidDataStoreContext, LocalFluidDataStoreContext } from "./dataStoreCo
             return undefined;
         }
 
-        return this._contexts.get(id) as LocalFluidDataStoreContext;
+        return context as LocalFluidDataStoreContext;
     }
 
     /**

package/src/dataStores.ts

@@ -32,6 +32,7 @@ import {
     convertSnapshotTreeToSummaryTree,
     convertToSummaryTree,
     create404Response,
+    createResponseError,
     responseToException,
     SummaryTreeBuilder,
 } from "@fluidframework/runtime-utils";
@@ -419,14 +420,25 @@ export class DataStores implements IDisposable {
         );
     }
 
-    public async getDataStore(id: string, wait: boolean): Promise<FluidDataStoreContext> {
+    public async getDataStore(id: string, wait: boolean, viaHandle: boolean): Promise<FluidDataStoreContext> {
         const context = await this.contexts.getBoundOrRemoted(id, wait);
+        const request = { url: id };
         if (context === undefined) {
             // The requested data store does not exits. Throw a 404 response exception.
-            const request = { url: id };
             throw responseToException(create404Response(request), request);
         }
 
+        if (context.tombstoned) {
+            // Note: if a user writes a request to look like it's viaHandle, we will also send this telemetry event
+            this.logger.sendErrorEvent({
+                eventName: "TombstonedDataStoreRequested",
+                url: request.url,
+                viaHandle,
+            });
+            // The requested data store is removed by gc. Throw a 404 gc response exception.
+            throw responseToException(createResponseError(404, "Datastore removed by gc", request), request);
+        }
+
         return context;
     }
 
@@ -597,6 +609,11 @@ export class DataStores implements IDisposable {
         // Verify that the used routes are correct.
         for (const [id] of usedDataStoreRoutes) {
             assert(this.contexts.has(id), 0x167 /* "Used route does not belong to any known data store" */);
+
+            // Revive datastores regardless of whether or not tombstone the tombstone flag is flipped
+            const dataStore = this.contexts.get(id);
+            assert(dataStore !== undefined, 0x46e /* No data store retrieved with specified id */);
+            dataStore.setTombstone(false /* tombstone */);
         }
 
         // Update the used routes in each data store. Used routes is empty for unused data stores.
@@ -609,8 +626,9 @@ export class DataStores implements IDisposable {
      * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
      * scenarios with accessing deleted content.
      * @param unusedRoutes - The routes that are unused in all data stores in this Container.
+     * @param tombstone - set the objects corresponding to routes as tombstones.
      */
-    public deleteUnusedRoutes(unusedRoutes: string[]) {
+    public deleteUnusedRoutes(unusedRoutes: string[], tombstone: boolean = false) {
         for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             // Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data
@@ -620,6 +638,19 @@ export class DataStores implements IDisposable {
             }
             const dataStoreId = pathParts[1];
             assert(this.contexts.has(dataStoreId), 0x2d7 /* No data store with specified id */);
+
+            /**
+             * When running GC in tombstone mode, datastore contexts are tombstoned. Tombstoned datastore contexts
+             * enable testing scenarios with accessing deleted content without actually deleting content from
+             * summaries.
+             */
+            if (tombstone) {
+                const dataStore = this.contexts.get(dataStoreId);
+                assert(dataStore !== undefined, 0x442 /* No data store retrieved with specified id */);
+                dataStore.setTombstone(true /* tombstone */);
+                continue;
+            }
+
             // Delete the contexts of unused data stores.
             this.contexts.delete(dataStoreId);
             // Delete the summarizer node of the unused data stores.

package/src/garbageCollection.ts

@@ -25,6 +25,7 @@ import {
     ISummarizeResult,
     ITelemetryContext,
     IGarbageCollectionNodeData,
+    ISummaryTreeWithStats,
 } from "@fluidframework/runtime-definitions";
 import {
     mergeStats,
@@ -61,6 +62,8 @@ const GCVersion = 1;
 export const gcTreeKey = "gc";
 // They prefix for GC blobs in the GC tree in summary.
 export const gcBlobPrefix = "__gc";
+// The key for tombstone blob in the GC tree in summary.
+export const gcTombstoneBlobKey = "__tombstones";
 
 // Feature gate key to turn GC on / off.
 export const runGCKey = "Fluid.GarbageCollection.RunGC";
@@ -74,6 +77,8 @@ export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
 export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
 // Feature gate key to turn GC sweep log off.
 export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
+// Feature gate key to tombstone datastores.
+export const testTombstoneKey = "Fluid.GarbageCollection.Test.Tombstone";
 
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
@@ -214,6 +219,14 @@ interface IUnreferencedEventProps {
     viaHandle?: boolean;
 }
 
+/**
+ * The GC data that is tracked for a summary that is submitted.
+ */
+interface IGCSummaryTrackingData {
+    serializedGCState: string | undefined;
+    serializedTombstones: string | undefined;
+}
+
 /**
  * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
  * be deleted by the sweep phase.
@@ -370,6 +383,7 @@ export class GarbageCollector implements IGarbageCollector {
     public readonly trackGCState: boolean;
 
     private readonly testMode: boolean;
+    private readonly tombstoneMode: boolean;
     private readonly mc: MonitoringContext;
 
     /**
@@ -393,17 +407,19 @@ export class GarbageCollector implements IGarbageCollector {
 
     // Keeps track of the GC state from the last run.
     private previousGCDataFromLastRun: IGarbageCollectionData | undefined;
+    // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
+    // outbound routes from that node.
+    private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
+    private tombstones: string[] = [];
+
     /**
-     * Keeps track of the serialized GC blob from the latest summary successfully submitted to the server.
+     * Keeps track of the GC data from the latest summary successfully submitted to and acked from the server.
      */
-    private latestSerializedSummaryState: string | undefined;
+    private latestSummaryData: IGCSummaryTrackingData | undefined;
     /**
-     * Keeps track of the serialized GC blob from the last GC run of the client.
+     * Keeps track of the GC data from the last summary submitted to the server but not yet acked.
      */
-    private pendingSerializedSummaryState: string | undefined;
-    // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
-    // outbound routes from that node.
-    private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
+    private pendingSummaryData: IGCSummaryTrackingData | undefined;
 
     // Promise when resolved initializes the base state of the nodes from the base summary state.
     private readonly initializeBaseStateP: Promise<void>;
@@ -449,6 +465,7 @@ export class GarbageCollector implements IGarbageCollector {
             runGC: this.shouldRunGC,
             runSweep: this.shouldRunSweep,
             testMode: this.testMode,
+            tombstoneMode: this.tombstoneMode,
             sessionExpiry: this.sessionExpiryTimeoutMs,
             sweepTimeout: this.sweepTimeoutMs,
             inactiveTimeout: this.inactiveTimeoutMs,
@@ -606,6 +623,7 @@ export class GarbageCollector implements IGarbageCollector {
 
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
         this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
+        this.tombstoneMode = this.mc.config.getBoolean(testTombstoneKey) ?? false;
 
         // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
         // contain GC tree and GC is enabled.
@@ -623,14 +641,20 @@ export class GarbageCollector implements IGarbageCollector {
                 // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
                 const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
                 if (gcSnapshotTree !== undefined) {
-                    const baseGCState = await getGCStateFromSnapshot(
+                    const baseGCData = await getGCDataFromSnapshot(
                         gcSnapshotTree,
                         readAndParseBlob,
                     );
+                    if (baseGCData.tombstones !== undefined && this.tombstoneMode) {
+                        this.tombstones = baseGCData.tombstones;
+                    }
                     if (this.trackGCState) {
-                        this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(baseGCState));
+                        this.latestSummaryData = {
+                            serializedGCState: JSON.stringify(generateSortedGCState(baseGCData.gcState)),
+                            serializedTombstones: JSON.stringify(baseGCData.tombstones),
+                        };
                     }
-                    return baseGCState;
+                    return baseGCData.gcState;
                 }
 
                 // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
@@ -787,15 +811,19 @@ export class GarbageCollector implements IGarbageCollector {
      */
     public setConnectionState(connected: boolean, clientId?: string | undefined): void {
         /**
-         * For non-summarizer clients, initialize the base state when the container becomes active, i.e., it transitions
+         * For all clients, initialize the base state when the container becomes active, i.e., it transitions
          * to "write" mode. This will ensure that the container's own join op is processed and there is a recent
          * reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
          * could affect the GC state will have been processed.
          *
+         * If GC is up-to-date for the client and the summarizing client, there will be an doubling of both
+         * InactiveObject_Loaded and SweepReady_Loaded errors, as there will be one from the sending client and one from
+         * the receiving summarizer client.
+         *
          * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
          * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
          */
-        if (this.activeConnection() && !this.isSummarizerClient && this.shouldRunGC) {
+        if (this.activeConnection() && this.shouldRunGC) {
             this.initializeBaseStateP.catch((error) => {});
         }
     }
@@ -884,6 +912,23 @@ export class GarbageCollector implements IGarbageCollector {
         // involving access to deleted data.
         if (this.testMode) {
             this.runtime.deleteUnusedRoutes(gcResult.deletedNodeIds);
+        } else {
+            // If we are running in GC tombstone mode, tombstone objects for unused routes. This enables testing
+            // scenarios involving access to "deleted" data without actually deleting the data from summaries.
+            // Note: we will not tombstone in test mode
+            if (this.tombstoneMode) {
+                const tombstoneRoutes: string[] = [];
+                // Currently only tombstone datastores
+                for (const [key, value] of this.unreferencedNodesState.entries()) {
+                    if (
+                        value.state === UnreferencedState.SweepReady &&
+                        this.runtime.getNodeType(key) === GCNodeType.DataStore
+                    ) {
+                        tombstoneRoutes.push(key);
+                    }
+                }
+                this.runtime.deleteUnusedRoutes(tombstoneRoutes);
+            }
         }
 
         // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
@@ -916,35 +961,74 @@ export class GarbageCollector implements IGarbageCollector {
             };
         }
 
-        const newSerializedSummaryState = JSON.stringify(generateSortedGCState(gcState));
+        const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
+        const serializedTombstones = this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined;
 
         /**
-         * As an optimization if the GC tree hasn't changed and we're tracking the gc state, return a tree handle
-         * instead of returning the whole GC tree. If there are changes, then we want to return the whole tree.
+         * Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
+         * summary, send summary handles for them. Otherwise, send the data in summary blobs.
          */
         if (this.trackGCState) {
-            this.pendingSerializedSummaryState = newSerializedSummaryState;
-            if (
-                this.latestSerializedSummaryState !== undefined &&
-                this.latestSerializedSummaryState === newSerializedSummaryState &&
-                !fullTree &&
-                trackState
-            ) {
-                const stats = mergeStats();
-                stats.handleNodeCount++;
-                return {
-                    summary: {
-                        type: SummaryType.Handle,
-                        handle: `/${gcTreeKey}`,
-                        handleType: SummaryType.Tree,
-                    },
-                    stats,
-                };
+            this.pendingSummaryData = { serializedGCState, serializedTombstones };
+            if (trackState && !fullTree && this.latestSummaryData !== undefined) {
+                // If neither GC state or tombstone state changed, send a summary handle for the entire GC data.
+                if (this.latestSummaryData.serializedGCState === serializedGCState
+                    && this.latestSummaryData.serializedTombstones === serializedTombstones) {
+                    const stats = mergeStats();
+                    stats.handleNodeCount++;
+                    return {
+                        summary: {
+                            type: SummaryType.Handle,
+                            handle: `/${gcTreeKey}`,
+                            handleType: SummaryType.Tree,
+                        },
+                        stats,
+                    };
+                }
+
+                // If either or both of GC state or tombstone state changed, build a GC summary tree.
+                return this.buildGCSummaryTree(serializedGCState, serializedTombstones, true /* trackState */);
             }
         }
+        // If not tracking GC state, build a GC summary tree without any summary handles.
+        return this.buildGCSummaryTree(serializedGCState, serializedTombstones, false /* trackState */);
+    }
 
+    /**
+     * Builds the GC summary tree which contains GC state and tombstone state.
+     * If trackState is false, both GC state and tombstone state are written as summary blobs.
+     * If trackState is true, summary blob is written for GC state or tombstone state if they changed.
+     * @param serializedGCState - The GC state serialized as string.
+     * @param serializedTombstones - THe tombstone state serialized as string.
+     * @param trackState - Whether we are tracking GC state across summaries.
+     * @returns the GC summary tree.
+     */
+    private buildGCSummaryTree(
+        serializedGCState: string,
+        serializedTombstones: string | undefined,
+        trackState: boolean,
+    ): ISummaryTreeWithStats {
+        const gcStateBlobKey = `${gcBlobPrefix}_root`;
         const builder = new SummaryTreeBuilder();
-        builder.addBlob(`${gcBlobPrefix}_root`, newSerializedSummaryState);
+
+        // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
+        if (this.latestSummaryData?.serializedGCState === serializedGCState && trackState) {
+            builder.addHandle(gcStateBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcStateBlobKey}`);
+        } else {
+            builder.addBlob(gcStateBlobKey, serializedGCState);
+        }
+
+        // If there is no tombstone data, return only the GC state.
+        if (serializedTombstones === undefined) {
+            return builder.getSummaryTree();
+        }
+
+        // If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
+        if (this.latestSummaryData?.serializedTombstones === serializedTombstones && trackState) {
+            builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
+        } else {
+            builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
+        }
         return builder.getSummaryTree();
     }
 
@@ -987,8 +1071,8 @@ export class GarbageCollector implements IGarbageCollector {
             this.latestSummaryGCVersion = this.currentGCVersion;
             this.initialStateNeedsReset = false;
             if (this.trackGCState) {
-                this.latestSerializedSummaryState = this.pendingSerializedSummaryState;
-                this.pendingSerializedSummaryState = undefined;
+                this.latestSummaryData = this.pendingSummaryData;
+                this.pendingSummaryData = undefined;
             }
             return;
         }
@@ -1003,15 +1087,18 @@ export class GarbageCollector implements IGarbageCollector {
 
         const gcSnapshotTree = snapshot.trees[gcTreeKey];
         if (gcSnapshotTree !== undefined && this.trackGCState) {
-            const latestGCState = await getGCStateFromSnapshot(
+            const latestGCData = await getGCDataFromSnapshot(
                 gcSnapshotTree,
                 readAndParseBlob,
             );
-            this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(latestGCState));
+            this.latestSummaryData = {
+                serializedGCState: JSON.stringify(generateSortedGCState(latestGCData.gcState)),
+                serializedTombstones: JSON.stringify(latestGCData.tombstones),
+            };
         } else {
-            this.latestSerializedSummaryState = undefined;
+            this.latestSummaryData = undefined;
         }
-        this.pendingSerializedSummaryState = undefined;
+        this.pendingSummaryData = undefined;
     }
 
     /**
@@ -1089,6 +1176,7 @@ export class GarbageCollector implements IGarbageCollector {
         currentReferenceTimestampMs: number,
     ) {
         this.previousGCDataFromLastRun = cloneGCData(gcData);
+        this.tombstones = [];
         this.newReferencesSinceLastRun.clear();
 
         // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
@@ -1121,6 +1209,12 @@ export class GarbageCollector implements IGarbageCollector {
                 );
             } else {
                 nodeStateTracker.updateTracking(currentReferenceTimestampMs);
+                if (this.tombstoneMode && nodeStateTracker.state === UnreferencedState.SweepReady) {
+                    const nodeType = this.runtime.getNodeType(nodeId);
+                    if (nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) {
+                        this.tombstones.push(nodeId);
+                    }
+                }
             }
         }
     }
@@ -1461,15 +1555,21 @@ export class GarbageCollector implements IGarbageCollector {
 }
 
 /**
- * Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
- * Merge the GC state from all such blobs and return the merged GC state.
+ * Gets the garbage collection data from the given snapshot tree. It contains GC state and tombstone state.
+ * The GC state may be written into multiple blobs. Merge the GC state from all such blobs into one.
  */
-async function getGCStateFromSnapshot(
+async function getGCDataFromSnapshot(
     gcSnapshotTree: ISnapshotTree,
     readAndParseBlob: ReadAndParseBlob,
-): Promise<IGarbageCollectionState> {
+) {
     let rootGCState: IGarbageCollectionState = { gcNodes: {} };
+    let tombstones: string[] | undefined;
     for (const key of Object.keys(gcSnapshotTree.blobs)) {
+        if (key === gcTombstoneBlobKey) {
+            tombstones = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]);
+            continue;
+        }
+
         // Skip blobs that do not start with the GC prefix.
         if (!key.startsWith(gcBlobPrefix)) {
             continue;
@@ -1484,7 +1584,7 @@ async function getGCStateFromSnapshot(
         // Merge the GC state of this blob into the root GC state.
         rootGCState = concatGarbageCollectionStates(rootGCState, gcState);
     }
-    return rootGCState;
+    return { gcState: rootGCState, tombstones };
 }
 
 function generateSortedGCState(gcState: IGarbageCollectionState): IGarbageCollectionState {

package/src/index.ts

@@ -28,8 +28,8 @@ export {
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
 export {
     gcBlobPrefix,
+    gcTombstoneBlobKey,
     gcTreeKey,
-    IGarbageCollectionRuntime,
     IGCStats,
 } from "./garbageCollection";
 export {

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.2.0.4";
+export const pkgVersion = "2.0.0-internal.2.1.1";

package/src/summarizerClientElection.ts

@@ -77,7 +77,7 @@ export class SummarizerClientElection
                 // Log and elect a new summarizer client.
                 const opsSinceLastReport = sequenceNumber - this.lastReportedSeq;
                 if (opsSinceLastReport > this.maxOpsSinceLastSummary) {
-                    this.logger.sendErrorEvent({
+                    this.logger.sendTelemetryEvent({
                         eventName: "ElectedClientNotSummarizing",
                         electedClientId,
                         lastSummaryAckSeqForClient: this.lastSummaryAckSeqForClient,

package/src/summaryGenerator.ts

@@ -231,13 +231,14 @@ export class SummaryGenerator {
             const category = cancellationToken.cancelled || error?.errorType === DriverErrorType.offlineError ?
                 "generic" : "error";
 
+            const message = getFailMessage(errorCode);
             summarizeEvent.cancel({
                  ...properties,
                  reason: errorCode,
                  category,
                  retryAfterSeconds,
-            }, error);
-            resultsBuilder.fail(getFailMessage(errorCode), error, nackSummaryResult, retryAfterSeconds);
+            }, error ?? message); // disconnect & summaryAckTimeout do not have proper error.
+            resultsBuilder.fail(message, error, nackSummaryResult, retryAfterSeconds);
         };
 
         // Wait to generate and send summary