@fluidframework/container-runtime 2.0.0-internal.2.3.0 → 2.0.0-internal.2.4.0

package/src/containerRuntime.ts

@@ -524,6 +524,30 @@ export enum RuntimeHeaders {
     viaHandle = "viaHandle",
 }
 
+/** True if a tombstoned object should be returned without erroring */
+export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
+
+/** Tombstone error responses will have this header set to true */
+export const TombstoneResponseHeaderKey = "isTombstoned"
+
+/**
+ * The full set of parsed header data that may be found on Runtime requests
+ */
+export interface RuntimeHeaderData {
+    wait?: boolean;
+    externalRequest?: boolean;
+    viaHandle?: boolean;
+    allowTombstone?: boolean;
+}
+
+/** Default values for Runtime Headers */
+export const defaultRuntimeHeaderData: Required<RuntimeHeaderData> = {
+    wait: true,
+    externalRequest: false,
+    viaHandle: false,
+    allowTombstone: false,
+}
+
 /**
  * Available compression algorithms for op compression.
  */
@@ -738,7 +762,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 if (loadSequenceNumberVerification === "log") {
                     logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
                 } else {
-                    // Call both close and dispose as close implementation will no longer dispose runtime in future (2.0.0-internal.3.0.0)
+                    // Call both close and dispose as closeFn implementation will no longer dispose runtime in future
                     context.closeFn(error);
                     context.disposeFn?.(error);
                 }
@@ -1174,6 +1198,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             (fromPath: string, toPath: string) => this.garbageCollector.addedOutboundReference(fromPath, toPath),
             this,
             pendingRuntimeState?.pendingAttachmentBlobs,
+            () => this.getCurrentReferenceTimestampMs(),
         );
 
         this.scheduleManager = new ScheduleManager(
@@ -1460,16 +1485,20 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
     }
 
     private async getDataStoreFromRequest(id: string, request: IRequest): Promise<IFluidRouter> {
-        const wait = typeof request.headers?.[RuntimeHeaders.wait] === "boolean"
-            ? request.headers?.[RuntimeHeaders.wait]
-            : true;
-        const viaHandle = typeof request.headers?.[RuntimeHeaders.viaHandle] === "boolean"
-            ? request.headers?.[RuntimeHeaders.viaHandle]
-            : false;
+        const headerData: RuntimeHeaderData = {};
+        if (typeof request.headers?.[RuntimeHeaders.wait] === "boolean") {
+            headerData.wait = request.headers[RuntimeHeaders.wait];
+        }
+        if (typeof request.headers?.[RuntimeHeaders.viaHandle] === "boolean") {
+            headerData.viaHandle = request.headers[RuntimeHeaders.viaHandle];
+        }
+        if (typeof request.headers?.[AllowTombstoneRequestHeaderKey] === "boolean") {
+            headerData.allowTombstone = request.headers[AllowTombstoneRequestHeaderKey];
+        }
 
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
-        const dataStoreContext = await this.dataStores.getDataStore(internalId, wait, viaHandle);
+        const dataStoreContext = await this.dataStores.getDataStore(internalId, headerData);
 
         /**
          * If GC should run and this an external app request with "externalRequest" header, we need to return
@@ -1860,7 +1889,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
     private async getRootDataStoreChannel(id: string, wait = true): Promise<IFluidDataStoreChannel> {
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
-        const context = await this.dataStores.getDataStore(internalId, wait, false /* viaHandle */);
+        const context = await this.dataStores.getDataStore(internalId, { wait });
         assert(await context.isRoot(), 0x12b /* "did not get root data store" */);
         return context.realize();
     }

package/src/dataStoreContext.ts

@@ -775,20 +775,19 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
 
         if (checkTombstone && this.tombstoned) {
             const messageString = `Context is tombstoned! Call site [${callSite}]`;
-            const error = new DataCorruptionError(messageString, {
-                errorMessage: messageString,
-                ...safeTelemetryProps,
-            });
-
-            // Always log an error when tombstoned data store is used. However, throw an error only if
-            // throwOnTombstoneUsage is set.
-            const event = {
-                eventName: "GC_Tombstone_DataStore_Changed",
-                callSite,
-            };
-            sendGCTombstoneEvent(this.mc, event, this.clientDetails.type === summarizerClientType, this.pkg, error);
-            // Always log an error when tombstoned data store is used. However, throw an error only if
-            // throwOnTombstoneUsage is set and the client is not a summarizer.
+            const error = new DataCorruptionError(messageString, safeTelemetryProps);
+
+            sendGCTombstoneEvent(
+                this.mc,
+                {
+                    eventName: "GC_Tombstone_DataStore_Changed",
+                    category: this.throwOnTombstoneUsage ? "error" : "generic",
+                    isSummarizerClient: this.clientDetails.type === summarizerClientType,
+                    callSite,
+                },
+                this.pkg,
+                error,
+            );
             if (this.throwOnTombstoneUsage) {
                 throw error;
             }

package/src/dataStores.ts

@@ -43,7 +43,7 @@ import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
 import { v4 as uuid } from "uuid";
 import { GCDataBuilder, unpackChildNodesGCDetails, unpackChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
 import { DataStoreContexts } from "./dataStoreContexts";
-import { ContainerRuntime } from "./containerRuntime";
+import { ContainerRuntime, defaultRuntimeHeaderData, RuntimeHeaderData, TombstoneResponseHeaderKey } from "./containerRuntime";
 import {
     FluidDataStoreContext,
     RemoteFluidDataStoreContext,
@@ -54,7 +54,7 @@ import {
 import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summaryFormat";
 import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
 import { GCNodeType } from "./garbageCollection";
-import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+import { throwOnTombstoneLoadKey } from "./garbageCollectionConstants";
 import { summarizerClientType } from "./summarizerClientElection";
 import { sendGCTombstoneEvent } from "./garbageCollectionTombstoneUtils";
 
@@ -85,7 +85,7 @@ export class DataStores implements IDisposable {
     // root data stores that are added.
     private dataStoresSinceLastGC: string[] = [];
     /** If true, throw an error when a tombstone data store is retrieved. */
-    private readonly throwOnTombstoneUsage: boolean;
+    private readonly throwOnTombstoneLoad: boolean;
     // The handle to the container runtime. This is used mainly for GC purposes to represent outbound reference from
     // the container runtime to other nodes.
     private readonly containerRuntimeHandle: IFluidHandle;
@@ -118,8 +118,8 @@ export class DataStores implements IDisposable {
             return baseGCDetails.get(dataStoreId);
         };
         // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
-        this.throwOnTombstoneUsage =
-            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+        this.throwOnTombstoneLoad =
+            this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
             this.runtime.clientDetails.type !== summarizerClientType;
 
         // Extract stores stored inside the snapshot
@@ -430,8 +430,10 @@ export class DataStores implements IDisposable {
         );
     }
 
-    public async getDataStore(id: string, wait: boolean, viaHandle: boolean): Promise<FluidDataStoreContext> {
-        const context = await this.contexts.getBoundOrRemoted(id, wait);
+    public async getDataStore(id: string, requestHeaderData: RuntimeHeaderData): Promise<FluidDataStoreContext> {
+        const headerData = { ...defaultRuntimeHeaderData, ...requestHeaderData };
+
+        const context = await this.contexts.getBoundOrRemoted(id, headerData.wait);
         const request = { url: id };
         if (context === undefined) {
             // The requested data store does not exits. Throw a 404 response exception.
@@ -439,24 +441,28 @@ export class DataStores implements IDisposable {
         }
 
         if (context.tombstoned) {
+            const shouldFail = this.throwOnTombstoneLoad && !headerData.allowTombstone;
+
             // The requested data store is removed by gc. Create a 404 gc response exception.
-            const error = responseToException(createResponseError(404, "Datastore removed by gc", request), request);
-            // Note: if a user writes a request to look like it's viaHandle, we will also send this telemetry event
-            const event = {
-                eventName: "GC_Tombstone_DataStore_Requested",
-                url: request.url,
-                viaHandle,
-            };
+            const error = responseToException(createResponseError(
+                404,
+                "DataStore was deleted",
+                request,
+                { [TombstoneResponseHeaderKey]: true },
+            ), request);
             sendGCTombstoneEvent(
                 this.mc,
-                event,
-                this.runtime.clientDetails.type === summarizerClientType,
+                {
+                    eventName: "GC_Tombstone_DataStore_Requested",
+                    category: shouldFail ? "error" : "generic",
+                    isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
+                    headers: JSON.stringify(requestHeaderData),
+                },
                 context.isLoaded ? context.packagePath : undefined,
                 error,
             );
-            // Always log an error when tombstoned data store is used. However, throw an error only if
-            // throwOnTombstoneUsage is set.
-            if (this.throwOnTombstoneUsage) {
+
+            if (shouldFail) {
                 throw error;
             }
         }

package/src/garbageCollection.ts

@@ -30,6 +30,7 @@ import {
     IGarbageCollectionNodeData,
     IGarbageCollectionSummaryDetailsLegacy,
     ISummaryTreeWithStats,
+    gcDeletedBlobKey,
 } from "@fluidframework/runtime-definitions";
 import {
     mergeStats,
@@ -62,9 +63,9 @@ import {
     runSessionExpiryKey,
     runSweepKey,
     stableGCVersion,
-    throwOnTombstoneUsageKey,
     trackGCStateKey
 } from "./garbageCollectionConstants";
+import { sendGCTombstoneEvent } from "./garbageCollectionTombstoneUtils";
 import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
 import {
     getGCVersion,
@@ -172,6 +173,8 @@ export interface IGarbageCollector {
     ): void;
     /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
     addedOutboundReference(fromNodePath: string, toNodePath: string): void;
+    /** Returns true if this node has been deleted by GC during sweep phase. */
+    isNodeDeleted(nodePath: string): boolean;
     setConnectionState(connected: boolean, clientId?: string): void;
     dispose(): void;
 }
@@ -226,6 +229,7 @@ interface IUnreferencedEventProps {
 interface IGCSummaryTrackingData {
     serializedGCState: string | undefined;
     serializedTombstones: string | undefined;
+    serializedDeletedNodes: string | undefined;
 }
 
 /**
@@ -419,7 +423,10 @@ export class GarbageCollector implements IGarbageCollector {
     // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
     // outbound routes from that node.
     private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
+    // A list of nodes that have been tombstoned.
     private tombstones: string[] = [];
+    // A list of nodes that have been deleted during sweep phase.
+    private deletedNodes: Set<string> = new Set();
 
     /**
      * Keeps track of the GC data from the latest summary successfully submitted to and acked from the server.
@@ -565,8 +572,7 @@ export class GarbageCollector implements IGarbageCollector {
             // flag in GC options to false.
             this.gcEnabled = this.gcOptions.gcAllowed !== false;
             // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
-            // ...unless we're using the TestOverride
-            this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
+            this.sweepEnabled = this.gcOptions.sweepAllowed === true;
 
             // Set the Session Expiry only if the flag is enabled and GC is enabled.
             if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
@@ -640,8 +646,9 @@ export class GarbageCollector implements IGarbageCollector {
 
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
         this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
-        // Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
-        this.tombstoneMode = this.mc.config.getBoolean(disableTombstoneKey) !== true;
+        // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
+        // via feature flags.
+        this.tombstoneMode = !this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
 
         // If GC ran in the container that generated the base snapshot, it will have a GC tree.
         this.wasGCRunInLatestSummary = baseSnapshot?.trees[gcTreeKey] !== undefined;
@@ -704,7 +711,9 @@ export class GarbageCollector implements IGarbageCollector {
                 }
                 // If there is only one node (root node just added above), either GC is disabled or we are loading from
                 // the first summary generated by detached container. In both cases, GC was not run - return undefined.
-                return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
+                return Object.keys(gcState.gcNodes).length === 1
+                    ? undefined
+                    : { gcState, tombstones: undefined, deletedNodes: undefined };
             } catch (error) {
                 const dpe = DataProcessingError.wrapIfUnrecognized(
                     error,
@@ -782,24 +791,33 @@ export class GarbageCollector implements IGarbageCollector {
     }
 
     /**
-     * Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
-     * before they are loaded or used. This is important to get accurate information of whether tombstoned object are
-     * in use or not.
+     * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
+     * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
      */
     public async initializeBaseState(): Promise<void> {
         const baseSnapshotData = await this.baseSnapshotDataP;
         /**
-         * The base snapshot data or tombstone state will not be present if the container is loaded from:
+         * The base snapshot data will not be present if the container is loaded from:
          * 1. The first summary created by the detached container.
          * 2. A summary that was generated with GC disabled.
          * 3. A summary that was generated before GC even existed.
-         * 4. A summary that was generated with tombstone feature disabled.
          */
-        if (!this.tombstoneMode || baseSnapshotData?.tombstones === undefined) {
+        if (baseSnapshotData === undefined) {
             return;
         }
-        this.tombstones = Array.from(baseSnapshotData.tombstones);
-        this.runtime.updateTombstonedRoutes(this.tombstones);
+
+        // Initialize the deleted nodes from the snapshot. This is done irrespective of whether sweep is enabled or not
+        // to identify deleted nodes' usage.
+        if (baseSnapshotData.deletedNodes !== undefined) {
+            this.deletedNodes = new Set(baseSnapshotData.deletedNodes);
+        }
+
+        // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
+        // tombstone routes.
+        if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
+            this.tombstones = Array.from(baseSnapshotData.tombstones);
+            this.runtime.updateTombstonedRoutes(this.tombstones);
+        }
     }
 
     /**
@@ -831,9 +849,29 @@ export class GarbageCollector implements IGarbageCollector {
         };
         this.unreferencedNodesState.clear();
 
-        // If tombstone mode is enabled, update tombstone information and also update all tombstoned nodes in the
-        // container as per the state in the snapshot data.
-        if (this.tombstoneMode) {
+        // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
+        // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
+        // tombstones.
+        // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
+        // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
+        if (this.shouldRunSweep) {
+            const snapshotDeletedNodes = snapshotData?.tombstones ? new Set(snapshotData.tombstones) : undefined;
+            // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
+            // delete them.
+            if (snapshotDeletedNodes !== undefined) {
+                const newDeletedNodes: string[] = [];
+                for (const nodeId of snapshotDeletedNodes) {
+                    if (!this.deletedNodes.has(nodeId)) {
+                        newDeletedNodes.push(nodeId);
+                    }
+                }
+                if (newDeletedNodes.length > 0) {
+                    // Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
+                }
+            }
+        } else if (this.tombstoneMode) {
+            // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
+            // notify the runtime to update its state as well.
             this.tombstones = snapshotData?.tombstones ? Array.from(snapshotData.tombstones) : [];
             this.runtime.updateTombstonedRoutes(this.tombstones);
         }
@@ -869,6 +907,7 @@ export class GarbageCollector implements IGarbageCollector {
             this.latestSummaryData = {
                 serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
                 serializedTombstones: JSON.stringify(snapshotData.tombstones),
+                serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
             };
         }
     }
@@ -1020,18 +1059,26 @@ export class GarbageCollector implements IGarbageCollector {
         }
 
         const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
+        // Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
+        // to identify deleted nodes' usage.
+        const serializedDeletedNodes = this.deletedNodes.size > 0
+            ? JSON.stringify(Array.from(this.deletedNodes).sort())
+            : undefined;
+        // If running in tombstone mode, serialize and write tombstones, if any.
         const serializedTombstones = this.tombstoneMode
             ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
             : undefined;
 
         /**
-         * Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
-         * summary, send summary handles for them. Otherwise, send the data in summary blobs.
+         * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
+         * write summary handle instead of summary tree for GC.
+         * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
+         * for each of these that did not change, write a summary handle.
          */
         if (this.trackGCState) {
-            this.pendingSummaryData = { serializedGCState, serializedTombstones };
+            this.pendingSummaryData = { serializedGCState, serializedTombstones, serializedDeletedNodes };
             if (trackState && !fullTree && this.latestSummaryData !== undefined) {
-                // If neither GC state or tombstone state changed, send a summary handle for the entire GC data.
+                // If nothing changed since last summary, send a summary handle for the entire GC data.
                 if (this.latestSummaryData.serializedGCState === serializedGCState
                     && this.latestSummaryData.serializedTombstones === serializedTombstones) {
                     const stats = mergeStats();
@@ -1046,26 +1093,30 @@ export class GarbageCollector implements IGarbageCollector {
                     };
                 }
 
-                // If either or both of GC state or tombstone state changed, build a GC summary tree.
-                return this.buildGCSummaryTree(serializedGCState, serializedTombstones, true /* trackState */);
+                // If some state changed, build a GC summary tree.
+                return this.buildGCSummaryTree(
+                    serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
             }
         }
         // If not tracking GC state, build a GC summary tree without any summary handles.
-        return this.buildGCSummaryTree(serializedGCState, serializedTombstones, false /* trackState */);
+        return this.buildGCSummaryTree(
+            serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
     }
 
     /**
-     * Builds the GC summary tree which contains GC state and tombstone state.
-     * If trackState is false, both GC state and tombstone state are written as summary blobs.
-     * If trackState is true, summary blob is written for GC state or tombstone state if they changed.
+     * Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
+     * If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
+     * If trackState is true, only states that changed are written. Rest are written as handles.
      * @param serializedGCState - The GC state serialized as string.
-     * @param serializedTombstones - THe tombstone state serialized as string.
+     * @param serializedTombstones - The tombstone state serialized as string.
+     * @param serializedDeletedNodes - Deleted nodes serialized as string.
      * @param trackState - Whether we are tracking GC state across summaries.
      * @returns the GC summary tree.
      */
     private buildGCSummaryTree(
         serializedGCState: string,
         serializedTombstones: string | undefined,
+        serializedDeletedNodes: string | undefined,
         trackState: boolean,
     ): ISummaryTreeWithStats {
         const gcStateBlobKey = `${gcBlobPrefix}_root`;
@@ -1078,16 +1129,26 @@ export class GarbageCollector implements IGarbageCollector {
             builder.addBlob(gcStateBlobKey, serializedGCState);
         }
 
-        // If there is no tombstone data, return only the GC state.
-        if (serializedTombstones === undefined) {
+        // If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
+        // summary blob.
+        if (serializedTombstones !== undefined) {
+            if (this.latestSummaryData?.serializedTombstones === serializedTombstones && trackState) {
+                builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
+            } else {
+                builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
+            }
+        }
+
+        // If there are no deleted nodes, return the summary tree.
+        if (serializedDeletedNodes === undefined) {
             return builder.getSummaryTree();
         }
 
-        // If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
-        if (this.latestSummaryData?.serializedTombstones === serializedTombstones && trackState) {
-            builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
+        // If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
+        if (this.latestSummaryData?.serializedDeletedNodes === serializedDeletedNodes && trackState) {
+            builder.addHandle(gcDeletedBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcDeletedBlobKey}`);
         } else {
-            builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
+            builder.addBlob(gcDeletedBlobKey, serializedDeletedNodes);
         }
         return builder.getSummaryTree();
     }
@@ -1243,16 +1304,28 @@ export class GarbageCollector implements IGarbageCollector {
                 eventName = "GC_Tombstone_Blob_Revived";
             }
 
-            this.mc.logger.sendTelemetryEvent({
-                eventName,
-                isSummarizerClient: this.isSummarizerClient,
-                url: trimLeadingSlashes(toNodePath),
-                nodeType,
-                throwOnTombstoneUsage: this.mc.config.getBoolean(throwOnTombstoneUsageKey) ?? false,
-            });
+            sendGCTombstoneEvent(
+                this.mc,
+                {
+                    eventName,
+                    category: "generic",
+                    isSummarizerClient: this.isSummarizerClient,
+                    url: trimLeadingSlashes(toNodePath),
+                    nodeType,
+                 },
+                undefined /* packagePath */,
+            );
         }
     }
 
+    /**
+     * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
+     * cases where objects are used after they are deleted and throw / log errors accordingly.
+     */
+    public isNodeDeleted(nodePath: string): boolean {
+        return this.deletedNodes.has(nodePath);
+    }
+
     public dispose(): void {
         this.sessionExpiryTimer?.clear();
         this.sessionExpiryTimer = undefined;
@@ -1636,7 +1709,7 @@ export class GarbageCollector implements IGarbageCollector {
                 }
             }
 
-            // If SweepReady Usage Detection is enabed, the handler may close the interactive container.
+            // If SweepReady Usage Detection is enabled, the handler may close the interactive container.
             // Once Sweep is fully implemented, this will be removed since the objects will be gone
             // and errors will arise elsewhere in the runtime
             if (state === UnreferencedState.SweepReady) {

package/src/garbageCollectionConstants.ts

@@ -24,7 +24,9 @@ export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
 export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
 // Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
 export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
-// Feature gate to enable throwing an error when tombstone object is used.
+// Feature gate to enable throwing an error when tombstone object is loaded (requested).
+export const throwOnTombstoneLoadKey = "Fluid.GarbageCollection.ThrowOnTombstoneLoad";
+// Feature gate to enable throwing an error when tombstone object is used (e.g. outgoing or incoming ops).
 export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
 // Feature gate to enable GC version upgrade.
 export const gcVersionUpgradeToV2Key = "Fluid.GarbageCollection.GCVersionUpgradeToV2";

package/src/garbageCollectionTombstoneUtils.ts

@@ -6,26 +6,23 @@
 import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
 import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
 import { MonitoringContext } from "@fluidframework/telemetry-utils";
-import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+import { disableTombstoneKey, throwOnTombstoneLoadKey, throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
 
 /**
- * Decides whether or not to send an error event or a generic event for gc tombstone scenarios
- *
- * Adds isSummarizerClient, packagePath, and error to telemetry properties.
+ * Consolidates info / logic for logging when we encounter a Tombstone
  */
 export function sendGCTombstoneEvent(
     mc: MonitoringContext,
-    event: ITelemetryGenericEvent,
-    isSummarizerClient: boolean,
+    event: ITelemetryGenericEvent & { category: "error" | "generic", isSummarizerClient: boolean },
     packagePath: readonly string[] | undefined,
-    error?: any,
+    error?: unknown,
 ) {
-    const throwOnTombstoneUsage = event.throwOnTombstoneUsage = mc.config.getBoolean(throwOnTombstoneUsageKey) ?? false;
     event.pkg = packagePathToTelemetryProperty(packagePath);
-    event.isSummarizerClient = isSummarizerClient;
-    if(throwOnTombstoneUsage) {
-        mc.logger.sendErrorEvent(event, error);
-    } else {
-        mc.logger.sendTelemetryEvent(event, error);
-    }
+    event.tombstoneFlags = JSON.stringify({
+        DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
+        ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
+        ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
+    });
+
+    mc.logger.sendTelemetryEvent(event, error);
 }

package/src/index.ts

@@ -19,6 +19,8 @@ export {
     agentSchedulerId,
     ContainerRuntime,
     RuntimeHeaders,
+    AllowTombstoneRequestHeaderKey,
+    TombstoneResponseHeaderKey,
     ISummaryConfiguration,
     DefaultSummaryConfiguration,
     ICompressionRuntimeOptions,

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.2.3.0";
+export const pkgVersion = "2.0.0-internal.2.4.0";

package/src/summarizer.ts

@@ -160,7 +160,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         // This will result in "summarizerClientDisconnected" stop reason recorded in telemetry,
         // unless stop() was called earlier
         this.dispose();
-        (this.runtime.disposeFn ?? this.runtime.closeFn)()
+        (this.runtime.disposeFn ?? this.runtime.closeFn)();
     }
 
     private async runCore(onBehalfOf: string): Promise<SummarizerStopReason> {