@fluidframework/container-runtime 2.0.0-internal.2.2.0 → 2.0.0-internal.2.3.0

package/src/containerRuntime.ts

@@ -69,6 +69,7 @@ import {
 } from "@fluidframework/protocol-definitions";
 import {
     FlushMode,
+    gcTreeKey,
     InboundAttachMessage,
     IFluidDataStoreContextDetached,
     IFluidDataStoreRegistry,
@@ -155,9 +156,6 @@ import {
     IGarbageCollector,
     IGCStats,
 } from "./garbageCollection";
-import {
-    gcTreeKey,
-} from "./garbageCollectionConstants";
 import {
     channelToDataStore,
     IDataStoreAliasMessage,
@@ -482,6 +480,25 @@ export interface IContainerRuntimeOptions {
      * @experimental This config should be driven by the connection with the service and will be moved in the future.
      */
     readonly maxBatchSizeInBytes?: number;
+    /**
+     * If the op payload needs to be chunked in order to work around the maximum size of the batch, this value represents
+     * how large the individual chunks will be. This is only supported when compression is enabled.
+     *
+     * If unspecified, if a batch exceeds `maxBatchSizeInBytes` after compression, the container will close with an instance
+     * of `GenericError` with the `BatchTooLarge` message.
+     *
+     * @experimental Not ready for use.
+     */
+    readonly chunkSizeInBytes?: number;
+    /**
+     * If enabled, the runtime will block all attempts to send an op with a different reference sequence number
+     * from the previous ops submitted in the same JS turn. This happens when ops are reentrant (an op is created as a
+     * response to another op, likely from an event handler).
+     *
+     * By default, the feature is disabled. If enabled from options, the `Fluid.ContainerRuntime.DisableOpReentryCheck`
+     * can be used to disable it at runtime.
+     */
+    readonly enableOpReentryCheck?: boolean;
 }
 
 /**
@@ -662,6 +679,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 compressionAlgorithm: CompressionAlgorithms.lz4
             },
             maxBatchSizeInBytes = defaultMaxBatchSizeInBytes,
+            chunkSizeInBytes = Number.POSITIVE_INFINITY,
+            enableOpReentryCheck = false,
         } = runtimeOptions;
 
         const pendingRuntimeState = context.pendingLocalState as IPendingRuntimeState | undefined;
@@ -719,7 +738,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 if (loadSequenceNumberVerification === "log") {
                     logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
                 } else {
+                    // Call both close and dispose as close implementation will no longer dispose runtime in future (2.0.0-internal.3.0.0)
                     context.closeFn(error);
+                    context.disposeFn?.(error);
                 }
             }
         }
@@ -739,6 +760,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 enableOfflineLoad,
                 compressionOptions,
                 maxBatchSizeInBytes,
+                chunkSizeInBytes,
+                enableOpReentryCheck,
             },
             containerScope,
             logger,
@@ -790,8 +813,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         return this.reSubmit;
     }
 
+    public get disposeFn(): (error?: ICriticalContainerError) => void {
+        // In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
+        return this.context.disposeFn ?? this.context.closeFn;
+    }
+
     public get closeFn(): (error?: ICriticalContainerError) => void {
-        return this.context.closeFn;
+        // Also call disposeFn to retain functionality of runtime being disposed on close
+        return (error?: ICriticalContainerError) => {
+            this.context.closeFn(error);
+            this.context.disposeFn?.(error);
+        };
     }
 
     public get flushMode(): FlushMode {
@@ -863,6 +895,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
     private dirtyContainer: boolean;
     private emitDirtyDocumentEvent = true;
+    private readonly enableOpReentryCheck: boolean;
 
     private readonly defaultTelemetrySignalSampleCount = 100;
     private _perfSignalData: IPerfSignalReport = {
@@ -1010,17 +1043,28 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         this.messageAtLastSummary = metadata?.message;
 
         this._connected = this.context.connected;
-        this.remoteMessageProcessor = new RemoteMessageProcessor(new OpSplitter(chunks), new OpDecompressor());
 
-        this.handleContext = new ContainerFluidHandleContext("", this);
+        this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
+
+        const opSplitter = new OpSplitter(
+            chunks,
+            this.context.submitBatchFn,
+            this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true ?
+                Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes,
+            runtimeOptions.maxBatchSizeInBytes,
+            this.mc.logger);
+        this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor());
 
-        this.mc = loggerToMonitoringContext(
-            ChildLogger.create(this.logger, "ContainerRuntime"));
+        this.handleContext = new ContainerFluidHandleContext("", this);
 
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
 
+        this.enableOpReentryCheck = runtimeOptions.enableOpReentryCheck === true
+            // Allow for a break-glass config to override the options
+            && this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
+
         this.summariesDisabled = this.isSummariesDisabled();
         this.heuristicsDisabled = this.isHeuristicsDisabled();
         this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
@@ -1079,6 +1123,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 // If GC should not run, let the summarizer node know so that it does not track GC state.
                 gcDisabled: !this.garbageCollector.shouldRunGC,
             },
+            // Function to get GC data if needed. This will always be called by the root summarizer node to get GC data.
+            async (fullGC?: boolean) => this.getGCDataInternal(fullGC),
+            // Function to get the GC details from the base snapshot we loaded from.
+            async () => this.garbageCollector.getBaseGCDetails(),
         );
 
         if (baseSnapshot) {
@@ -1092,7 +1140,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             (id: string, createParam: CreateChildSummarizerNodeParam) => (
                 summarizeInternal: SummarizeInternalFn,
                 getGCDataFn: (fullGC?: boolean) => Promise<IGarbageCollectionData>,
-                getBaseGCDetailsFn: () => Promise<IGarbageCollectionDetailsBase>,
+                getBaseGCDetailsFn?: () => Promise<IGarbageCollectionDetailsBase>,
             ) => this.summarizerNode.createChild(
                 summarizeInternal,
                 id,
@@ -1117,12 +1165,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             this.handleContext,
             blobManagerSnapshot,
             () => this.storage,
-            (blobId, localId) => {
+            (localId: string, blobId?: string) => {
                 if (!this.disposed) {
-                    this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId, localId });
+                    this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { localId, blobId });
                 }
             },
             (blobPath: string) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
+            (fromPath: string, toPath: string) => this.garbageCollector.addedOutboundReference(fromPath, toPath),
             this,
             pendingRuntimeState?.pendingAttachmentBlobs,
         );
@@ -1147,15 +1196,24 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             },
             pendingRuntimeState?.pending);
 
+        const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true ?
+        {
+            minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
+            compressionAlgorithm: CompressionAlgorithms.lz4
+        } : runtimeOptions.compressionOptions;
+
         this.outbox = new Outbox({
             shouldSend: () => this.canSendOps(),
             pendingStateManager: this.pendingStateManager,
             containerContext: this.context,
             compressor: new OpCompressor(this.mc.logger),
+            splitter: opSplitter,
             config: {
-                compressionOptions: runtimeOptions.compressionOptions,
+                compressionOptions,
                 maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
+                enableOpReentryCheck: this.enableOpReentryCheck,
             },
+            logger: this.mc.logger,
         });
 
         this.context.quorum.on("removeMember", (clientId: string) => {
@@ -1638,7 +1696,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             if (!this.shouldContinueReconnecting()) {
                 this.closeFn(
                     DataProcessingError.create(
-                        // eslint-disable-next-line max-len
                         "Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)",
                         "setConnectionState",
                         undefined,
@@ -1685,7 +1742,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
         try {
             let localOpMetadata: unknown;
-            if (local && runtimeMessage) {
+            if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
 
@@ -1820,9 +1877,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         assert(this.outbox.isEmpty, 0x3cf /* reentrancy */);
     }
 
-    public orderSequentially(callback: () => void): void {
+    public orderSequentially<T>(callback: () => T): T {
         let checkpoint: IBatchCheckpoint | undefined;
-
+        let result: T;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
             // Note: we are not touching this.pendingAttachBatch here, for two reasons:
             // 1. It would not help, as we flush attach ops as they become available.
@@ -1831,7 +1888,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         }
         try {
             this._orderSequentiallyCalls++;
-            callback();
+            result = callback();
         } catch (error) {
             if (checkpoint) {
                 // This will throw and close the container if rollback fails
@@ -1863,6 +1920,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         if (this.flushMode === FlushMode.Immediate && this._orderSequentiallyCalls === 0) {
             this.flush();
         }
+        return result;
     }
 
     public async createDataStore(pkg: string | string[]): Promise<IDataStore> {
@@ -2112,6 +2170,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         return this.dataStores.updateStateBeforeGC();
     }
 
+    private async getGCDataInternal(fullGC?: boolean): Promise<IGarbageCollectionData> {
+        return this.dataStores.getGCData(fullGC);
+    }
+
     /**
      * Implementation of IGarbageCollectionRuntime::getGCData.
      * Generates and returns the GC data for this container.
@@ -2119,7 +2181,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
      */
     public async getGCData(fullGC?: boolean): Promise<IGarbageCollectionData> {
         const builder = new GCDataBuilder();
-        const dsGCData = await this.dataStores.getGCData(fullGC);
+        const dsGCData = await this.summarizerNode.getGCData(fullGC);
         builder.addNodes(dsGCData.gcNodes);
 
         const blobsGCData = this.blobManager.getGCData(fullGC);
@@ -2138,40 +2200,28 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
 
-        const blobManagerUsedRoutes: string[] = [];
-        const dataStoreUsedRoutes: string[] = [];
-        for (const route of usedRoutes) {
-            if (this.isBlobPath(route)) {
-                blobManagerUsedRoutes.push(route);
-            } else {
-                dataStoreUsedRoutes.push(route);
-            }
-        }
-
-        this.blobManager.updateUsedRoutes(blobManagerUsedRoutes);
-        this.dataStores.updateUsedRoutes(dataStoreUsedRoutes);
+        const { dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(usedRoutes);
+        this.dataStores.updateUsedRoutes(dataStoreRoutes);
     }
 
     /**
-     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
-     * tombstones.
-     * @param unusedRoutes - The routes that are unused in all data stores and attachment blobs in this Container.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
+     * This is called to update objects whose routes are unused.
+     * @param unusedRoutes - Data store and attachment blob routes that are unused in this Container.
      */
-    public updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean) {
-        const blobManagerUnusedRoutes: string[] = [];
-        const dataStoreUnusedRoutes: string[] = [];
-        for (const route of unusedRoutes) {
-            if (this.isBlobPath(route)) {
-                blobManagerUnusedRoutes.push(route);
-            } else {
-                dataStoreUnusedRoutes.push(route);
-            }
-        }
+    public updateUnusedRoutes(unusedRoutes: string[]) {
+        const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
+        this.blobManager.updateUnusedRoutes(blobManagerRoutes);
+        this.dataStores.updateUnusedRoutes(dataStoreRoutes);
+    }
 
-        this.blobManager.updateUnusedRoutes(blobManagerUnusedRoutes, tombstone);
-        this.dataStores.updateUnusedRoutes(dataStoreUnusedRoutes, tombstone);
+    /**
+     * This is called to update objects that are tombstones.
+     * @param tombstonedRoutes - Data store and attachment blob routes that are tombstones in this Container.
+     */
+    public updateTombstonedRoutes(tombstonedRoutes: string[]) {
+        const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(tombstonedRoutes);
+        this.blobManager.updateTombstonedRoutes(blobManagerRoutes);
+        this.dataStores.updateTombstonedRoutes(dataStoreRoutes);
     }
 
     /**
@@ -2201,7 +2251,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
     public async getGCNodePackagePath(nodePath: string): Promise<readonly string[] | undefined> {
         switch (this.getNodeType(nodePath)) {
             case GCNodeType.Blob:
-                return ["_blobs"];
+                return [BlobManager.basePath];
             case GCNodeType.DataStore:
             case GCNodeType.SubDataStore:
                 return this.dataStores.getDataStorePackagePath(nodePath);
@@ -2221,6 +2271,25 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         return true;
     }
 
+    /**
+     * From a given list of routes, separate and return routes that belong to blob manager and data stores.
+     * @param routes - A list of routes that can belong to data stores or blob manager.
+     * @returns - Two route lists - One that contains routes for blob manager and another one that contains routes
+     * for data stores.
+     */
+    private getDataStoreAndBlobManagerRoutes(routes: string[]) {
+        const blobManagerRoutes: string[] = [];
+        const dataStoreRoutes: string[] = [];
+        for (const route of routes) {
+            if (this.isBlobPath(route)) {
+                blobManagerRoutes.push(route);
+            } else {
+                dataStoreRoutes.push(route);
+            }
+        }
+        return { blobManagerRoutes, dataStoreRoutes };
+    }
+
     /**
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
      * @returns the statistics of the garbage collection run; undefined if GC did not run.
@@ -2315,7 +2384,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 if (this.deltaManager.lastSequenceNumber !== summaryRefSeqNum) {
                     return {
                         continue: false,
-                        // eslint-disable-next-line max-len
                         error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
                     };
                 }
@@ -2325,7 +2393,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 if (lastAck !== this.summaryCollection.latestAck) {
                     return {
                         continue: false,
-                        // eslint-disable-next-line max-len
                         error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
                     };
                 }
@@ -2587,7 +2654,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             } else if (!this.flushMicroTaskExists) {
                 this.flushMicroTaskExists = true;
                 // Queue a microtask to detect the end of the turn and force a flush.
-                // eslint-disable-next-line @typescript-eslint/no-floating-promises
                 Promise.resolve().then(() => {
                     this.flushMicroTaskExists = false;
                     this.flush();
@@ -2708,17 +2774,42 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
         const snapshotTreeFetcher = async () => {
-            const fetchResult = await this.fetchSnapshotFromStorage(
-                null,
+            const fetchResult = await this.fetchLatestSnapshotFromStorage(
                 summaryLogger,
                 {
                     eventName: "RefreshLatestSummaryGetSnapshot",
                     ackHandle,
                     summaryRefSeq,
                     fetchLatest: true,
-                });
+                },
+            );
 
             const latestSnapshotRefSeq = await seqFromTree(fetchResult.snapshotTree, readAndParseBlob);
+            /**
+             * If the fetched snapshot is older than the one for which the ack was received, close the container.
+             * This should never happen because an ack should be sent after the latest summary is updated in the server.
+             * However, there are couple of scenarios where it's possible:
+             * 1. A file was modified externally resulting in modifying the snapshot's sequence number. This can lead to
+             * the document being unusable and we should not proceed.
+             * 2. The server DB failed after the ack was sent which may delete the corresponding snapshot. Ideally, in
+             * such cases, the file will be rolled back along with the ack and we will eventually reach a consistent
+             * state.
+             */
+            if (latestSnapshotRefSeq < summaryRefSeq) {
+                const error = DataProcessingError.create(
+                    "Fetched snapshot is older than the received ack",
+                    "RefreshLatestSummaryAck",
+                    undefined /* sequencedMessage */,
+                    {
+                        ackHandle,
+                        summaryRefSeq,
+                        latestSnapshotRefSeq,
+                    },
+                );
+                this.closeFn(error);
+                throw error;
+            }
+
             summaryLogger.sendTelemetryEvent(
                 {
                     eventName: "LatestSummaryRetrieved",
@@ -2744,7 +2835,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         );
 
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(
+            result,
+            proposalHandle,
+            summaryRefSeq,
+            readAndParseBlob,
+        );
     }
 
     /**
@@ -2756,11 +2852,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
     private async refreshLatestSummaryAckFromServer(
         summaryLogger: ITelemetryLogger,
     ): Promise<{ latestSnapshotRefSeq: number; latestSnapshotVersionId: string | undefined; }> {
-        const { snapshotTree, versionId } = await this.fetchSnapshotFromStorage(null, summaryLogger, {
-            eventName: "RefreshLatestSummaryGetSnapshot",
-            fetchLatest: true,
-        },
-            FetchSource.noCache,
+        const { snapshotTree, versionId } = await this.fetchLatestSnapshotFromStorage(
+            summaryLogger,
+            {
+                eventName: "RefreshLatestSummaryGetSnapshot",
+                fetchLatest: true,
+            },
         );
 
         const readAndParseBlob = async <T>(id: string) => readAndParse<T>(this.storage, id);
@@ -2775,16 +2872,19 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         );
 
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(
+            result,
+            undefined,
+            latestSnapshotRefSeq,
+            readAndParseBlob,
+        )
 
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
 
-    private async fetchSnapshotFromStorage(
-        versionId: string | null,
+    private async fetchLatestSnapshotFromStorage(
         logger: ITelemetryLogger,
         event: ITelemetryGenericEvent,
-        fetchSource?: FetchSource,
     ): Promise<{ snapshotTree: ISnapshotTree; versionId: string; }> {
         return PerformanceEvent.timedExecAsync(
             logger, event, async (perfEvent: {
@@ -2797,7 +2897,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             const trace = Trace.start();
 
             const versions = await this.storage.getVersions(
-                versionId, 1, "refreshLatestSummaryAckFromServer", fetchSource);
+                null, 1, "refreshLatestSummaryAckFromServer", FetchSource.noCache);
             assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
 
@@ -2960,6 +3060,7 @@ const waitForSeq = async (
 ): Promise<void> => new Promise<void>((resolve, reject) => {
     // TODO: remove cast to any when actual event is determined
     deltaManager.on("closed" as any, reject);
+    deltaManager.on("disposed" as any, reject);
 
     // If we already reached target sequence number, simply resolve the promise.
     if (deltaManager.lastSequenceNumber >= targetSeq) {

package/src/dataStoreContext.ts

@@ -61,7 +61,6 @@ import {
 import {
     addBlobToSummary,
     convertSummaryTreeToITree,
-    packagePathToTelemetryProperty,
 } from "@fluidframework/runtime-utils";
 import {
     ChildLogger,
@@ -88,6 +87,7 @@ import {
     getFluidDataStoreAttributes,
 } from "./summaryFormat";
 import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+import { sendGCTombstoneEvent } from "./garbageCollectionTombstoneUtils";
 import { summarizerClientType } from "./summarizerClientElection";
 
 function createAttributes(
@@ -782,11 +782,11 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
 
             // Always log an error when tombstoned data store is used. However, throw an error only if
             // throwOnTombstoneUsage is set.
-            this.mc.logger.sendErrorEvent({
+            const event = {
                 eventName: "GC_Tombstone_DataStore_Changed",
                 callSite,
-                pkg: packagePathToTelemetryProperty(this.pkg),
-            }, error);
+            };
+            sendGCTombstoneEvent(this.mc, event, this.clientDetails.type === summarizerClientType, this.pkg, error);
             // Always log an error when tombstoned data store is used. However, throw an error only if
             // throwOnTombstoneUsage is set and the client is not a summarizer.
             if (this.throwOnTombstoneUsage) {
@@ -799,7 +799,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         return (
             summarizeInternal: SummarizeInternalFn,
             getGCDataFn: (fullGC?: boolean) => Promise<IGarbageCollectionData>,
-            getBaseGCDetailsFn: () => Promise<IGarbageCollectionDetailsBase>,
+            getBaseGCDetailsFn?: () => Promise<IGarbageCollectionDetailsBase>,
         ) => this.summarizerNode.createChild(
             summarizeInternal,
             id,

package/src/dataStores.ts

@@ -34,7 +34,6 @@ import {
     create404Response,
     createResponseError,
     responseToException,
-    packagePathToTelemetryProperty,
     SummaryTreeBuilder,
 } from "@fluidframework/runtime-utils";
 import { ChildLogger, loggerToMonitoringContext, LoggingError, MonitoringContext, TelemetryDataTag } from "@fluidframework/telemetry-utils";
@@ -42,7 +41,7 @@ import { AttachState } from "@fluidframework/container-definitions";
 import { BlobCacheStorageService, buildSnapshotTree } from "@fluidframework/driver-utils";
 import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
 import { v4 as uuid } from "uuid";
-import { GCDataBuilder, unpackChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
+import { GCDataBuilder, unpackChildNodesGCDetails, unpackChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
 import { DataStoreContexts } from "./dataStoreContexts";
 import { ContainerRuntime } from "./containerRuntime";
 import {
@@ -57,6 +56,7 @@ import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
 import { GCNodeType } from "./garbageCollection";
 import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
 import { summarizerClientType } from "./summarizerClientElection";
+import { sendGCTombstoneEvent } from "./garbageCollectionTombstoneUtils";
 
 type PendingAliasResolve = (success: boolean) => void;
 
@@ -99,7 +99,7 @@ export class DataStores implements IDisposable {
             (id: string, createParam: CreateChildSummarizerNodeParam) => CreateChildSummarizerNodeFn,
         private readonly deleteChildSummarizerNodeFn: (id: string) => void,
         baseLogger: ITelemetryBaseLogger,
-        getBaseGCDetails: () => Promise<Map<string, IGarbageCollectionDetailsBase>>,
+        getBaseGCDetails: () => Promise<IGarbageCollectionDetailsBase>,
         private readonly gcNodeUpdated: (
             nodePath: string, timestampMs: number, packagePath?: readonly string[]) => void,
         private readonly aliasMap: Map<string, string>,
@@ -109,7 +109,8 @@ export class DataStores implements IDisposable {
         this.containerRuntimeHandle = new FluidObjectHandle(this.runtime, "/", this.runtime.IFluidHandleContext);
 
         const baseGCDetailsP = new LazyPromise(async () => {
-            return getBaseGCDetails();
+            const baseGCDetails = await getBaseGCDetails();
+            return unpackChildNodesGCDetails(baseGCDetails);
         });
         // Returns the base GC details for the data store with the given id.
         const dataStoreBaseGCDetails = async (dataStoreId: string) => {
@@ -441,12 +442,18 @@ export class DataStores implements IDisposable {
             // The requested data store is removed by gc. Create a 404 gc response exception.
             const error = responseToException(createResponseError(404, "Datastore removed by gc", request), request);
             // Note: if a user writes a request to look like it's viaHandle, we will also send this telemetry event
-            this.mc.logger.sendErrorEvent({
+            const event = {
                 eventName: "GC_Tombstone_DataStore_Requested",
                 url: request.url,
-                pkg: packagePathToTelemetryProperty(context.isLoaded ? context.packagePath : undefined),
                 viaHandle,
-            }, error);
+            };
+            sendGCTombstoneEvent(
+                this.mc,
+                event,
+                this.runtime.clientDetails.type === summarizerClientType,
+                context.isLoaded ? context.packagePath : undefined,
+                error,
+            );
             // Always log an error when tombstoned data store is used. However, throw an error only if
             // throwOnTombstoneUsage is set.
             if (this.throwOnTombstoneUsage) {
@@ -624,11 +631,6 @@ export class DataStores implements IDisposable {
         // Verify that the used routes are correct.
         for (const [id] of usedDataStoreRoutes) {
             assert(this.contexts.has(id), 0x167 /* "Used route does not belong to any known data store" */);
-
-            // Revive datastores regardless of whether or not tombstone the tombstone flag is flipped
-            const dataStore = this.contexts.get(id);
-            assert(dataStore !== undefined, 0x46e /* No data store retrieved with specified id */);
-            dataStore.setTombstone(false /* tombstone */);
         }
 
         // Update the used routes in each data store. Used routes is empty for unused data stores.
@@ -638,13 +640,10 @@ export class DataStores implements IDisposable {
     }
 
     /**
-     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
-     * tombstones.
+     * This is called to update objects whose routes are unused. The unused objects are deleted.
      * @param unusedRoutes - The routes that are unused in all data stores in this Container.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
      */
-    public updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean) {
+    public updateUnusedRoutes(unusedRoutes: string[]) {
         for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             // Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data
@@ -654,19 +653,6 @@ export class DataStores implements IDisposable {
             }
             const dataStoreId = pathParts[1];
             assert(this.contexts.has(dataStoreId), 0x2d7 /* No data store with specified id */);
-
-            /**
-             * When running GC in tombstone mode, datastore contexts are tombstoned. Tombstoned datastore contexts
-             * enable testing scenarios with accessing deleted content without actually deleting content from
-             * summaries.
-             */
-            if (tombstone) {
-                const dataStore = this.contexts.get(dataStoreId);
-                assert(dataStore !== undefined, 0x442 /* No data store retrieved with specified id */);
-                dataStore.setTombstone(true /* tombstone */);
-                continue;
-            }
-
             // Delete the contexts of unused data stores.
             this.contexts.delete(dataStoreId);
             // Delete the summarizer node of the unused data stores.
@@ -674,6 +660,30 @@ export class DataStores implements IDisposable {
         }
     }
 
+    /**
+     * This is called to update objects whose routes are tombstones. Tombstoned datastore contexts enable testing
+     * scenarios with accessing deleted content without actually deleting content from summaries.
+     * @param tombstonedRoutes - The routes that are tombstones in all data stores in this Container.
+     */
+    public updateTombstonedRoutes(tombstonedRoutes: string[]) {
+        const tombstonedDataStoresSet: Set<string> = new Set();
+        for (const route of tombstonedRoutes) {
+            const pathParts = route.split("/");
+            // Tombstone data store only if its route (/datastoreId) is directly in tombstoneRoutes.
+            if (pathParts.length > 2) {
+                continue;
+            }
+            const dataStoreId = pathParts[1];
+            assert(this.contexts.has(dataStoreId), 0x510 /* No data store with specified id */);
+            tombstonedDataStoresSet.add(dataStoreId);
+        }
+
+        // Update the used routes in each data store. Used routes is empty for unused data stores.
+        for (const [contextId, context] of this.contexts) {
+            context.setTombstone(tombstonedDataStoresSet.has(contextId));
+        }
+    }
+
     /**
      * Returns the outbound routes of this channel. Only root data stores are considered referenced and their paths are
      * part of outbound routes.