@fluidframework/container-runtime 2.0.0-dev.1.4.5.105745 → 2.0.0-dev.2.2.0.111723

package/src/dataStore.ts

@@ -7,7 +7,12 @@ import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, unreachableCase } from "@fluidframework/common-utils";
 import { AttachState } from "@fluidframework/container-definitions";
 import { UsageError } from "@fluidframework/container-utils";
-import { IRequest, IResponse } from "@fluidframework/core-interfaces";
+import {
+    FluidObject,
+    IFluidHandle,
+    IRequest,
+    IResponse,
+} from "@fluidframework/core-interfaces";
 import { AliasResult, IDataStore, IFluidDataStoreChannel } from "@fluidframework/runtime-definitions";
 import { TelemetryDataTag } from "@fluidframework/telemetry-utils";
 import { ContainerRuntime } from "./containerRuntime";
@@ -151,6 +156,13 @@ class DataStore implements IDataStore {
         return this.fluidDataStoreChannel.request(request);
     }
 
+    /**
+     * {@inheritDoc @fluidframework/runtime-definitions#IDataStore.entryPoint}
+     */
+    get entryPoint(): IFluidHandle<FluidObject> | undefined {
+        return this.fluidDataStoreChannel.entryPoint;
+    }
+
     constructor(
         private readonly fluidDataStoreChannel: IFluidDataStoreChannel,
         private readonly internalId: string,

package/src/dataStoreContext.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IDisposable, ITelemetryLogger, ITelemetryProperties } from "@fluidframework/common-definitions";
 import {
     FluidObject,
     IRequest,
@@ -42,7 +42,6 @@ import {
     CreateChildSummarizerNodeFn,
     CreateChildSummarizerNodeParam,
     FluidDataStoreRegistryEntry,
-    gcBlobKey,
     IAttachMessage,
     IFluidDataStoreChannel,
     IFluidDataStoreContext,
@@ -51,7 +50,6 @@ import {
     IFluidDataStoreRegistry,
     IGarbageCollectionData,
     IGarbageCollectionDetailsBase,
-    IGarbageCollectionSummaryDetails,
     IInboundSignalMessage,
     IProvideFluidDataStoreFactory,
     ISummarizeInternalResult,
@@ -60,14 +58,24 @@ import {
     SummarizeInternalFn,
     ITelemetryContext,
 } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, convertSummaryTreeToITree } from "@fluidframework/runtime-utils";
+import {
+    addBlobToSummary,
+    convertSummaryTreeToITree,
+    packagePathToTelemetryProperty,
+} from "@fluidframework/runtime-utils";
 import {
     ChildLogger,
+    loggerToMonitoringContext,
     LoggingError,
+    MonitoringContext,
     TelemetryDataTag,
     ThresholdCounter,
 } from "@fluidframework/telemetry-utils";
-import { DataProcessingError } from "@fluidframework/container-utils";
+import {
+    DataCorruptionError,
+    DataProcessingError,
+    extractSafePropertiesFromMessage,
+} from "@fluidframework/container-utils";
 
 import { ContainerRuntime } from "./containerRuntime";
 import {
@@ -79,6 +87,8 @@ import {
     getAttributesFormatVersion,
     getFluidDataStoreAttributes,
 } from "./summaryFormat";
+import { throwOnTombstoneUsageKey } from "./garbageCollection";
+import { summarizerClientType } from "./summarizerClientElection";
 
 function createAttributes(
     pkg: readonly string[],
@@ -117,7 +127,6 @@ export interface IFluidDataStoreContextProps {
     readonly storage: IDocumentStorageService;
     readonly scope: FluidObject;
     readonly createSummarizerNodeFn: CreateChildSummarizerNodeFn;
-    readonly writeGCDataAtRoot: boolean;
     readonly pkg?: Readonly<string[]>;
 }
 
@@ -135,7 +144,7 @@ export interface ILocalFluidDataStoreContextProps extends IFluidDataStoreContext
 
 /** Properties necessary for creating a remote FluidDataStoreContext */
 export interface IRemoteFluidDataStoreContextProps extends IFluidDataStoreContextProps {
-    readonly snapshotTree: ISnapshotTree | string | undefined;
+    readonly snapshotTree: ISnapshotTree | undefined;
     readonly getBaseGCDetails: () => Promise<IGarbageCollectionDetailsBase | undefined>;
 }
 
@@ -193,6 +202,15 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     private _disposed = false;
     public get disposed() { return this._disposed; }
 
+    /**
+     * Tombstone is a temporary feature that prevents a data store from sending / receiving ops, signals and from
+     * loading.
+     */
+    private _tombstoned = false;
+    public get tombstoned() { return this._tombstoned; }
+    /** If true, throw an error when a tombstone data store is used. */
+    private readonly throwOnTombstoneUsage: boolean;
+
     public get attachState(): AttachState {
         return this._attachState;
     }
@@ -235,19 +253,18 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     protected _attachState: AttachState;
     private _isInMemoryRoot: boolean = false;
     protected readonly summarizerNode: ISummarizerNodeWithGC;
-    private readonly subLogger: ITelemetryLogger;
+    private readonly mc: MonitoringContext;
     private readonly thresholdOpsCounter: ThresholdCounter;
     private static readonly pendingOpsCountThreshold = 1000;
 
-    // The used state of this node as per the last GC run. This is used to update the used state of the channel
+    // The used routes of this node as per the last GC run. This is used to update the used routes of the channel
     // if it realizes after GC is run.
-    private lastUsedState: { usedRoutes: string[]; gcTimestamp?: number; } | undefined;
+    private lastUsedRoutes: string[] | undefined;
 
     public readonly id: string;
     private readonly _containerRuntime: ContainerRuntime;
     public readonly storage: IDocumentStorageService;
     public readonly scope: FluidObject;
-    private readonly writeGCDataAtRoot: boolean;
     protected pkg?: readonly string[];
 
     constructor(
@@ -263,7 +280,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         this.id = props.id;
         this.storage = props.storage;
         this.scope = props.scope;
-        this.writeGCDataAtRoot = props.writeGCDataAtRoot;
         this.pkg = props.pkg;
 
         // URIs use slashes as delimiters. Handles use URIs.
@@ -291,8 +307,13 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
             async () => this.getBaseGCDetails(),
         );
 
-        this.subLogger = ChildLogger.create(this.logger, "FluidDataStoreContext");
-        this.thresholdOpsCounter = new ThresholdCounter(FluidDataStoreContext.pendingOpsCountThreshold, this.subLogger);
+        this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "FluidDataStoreContext"));
+        this.thresholdOpsCounter = new ThresholdCounter(FluidDataStoreContext.pendingOpsCountThreshold, this.mc.logger);
+
+        // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
+        this.throwOnTombstoneUsage =
+            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+            this.clientDetails.type !== summarizerClientType;
     }
 
     public dispose(): void {
@@ -310,6 +331,14 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         }
     }
 
+    public setTombstone(tombstone: boolean) {
+        if (this.tombstoned === tombstone) {
+            return;
+        }
+
+        this._tombstoned = tombstone;
+    }
+
     private rejectDeferredRealize(reason: string, packageName?: string): never {
         throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.CodeArtifact } });
     }
@@ -386,7 +415,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
      * its new client ID when we are connecting or connected.
      */
     public setConnectionState(connected: boolean, clientId?: string) {
-        this.verifyNotClosed();
+        // ConnectionState should not fail in tombstone mode as this is internally run
+        this.verifyNotClosed("setConnectionState", false /* checkTombstone */);
 
         // Connection events are ignored if the store is not yet loaded
         if (!this.loaded) {
@@ -400,7 +430,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public process(messageArg: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("process", true, extractSafePropertiesFromMessage(messageArg));
 
         const innerContents = messageArg.contents as FluidDataStoreMessage;
         const message = {
@@ -422,7 +452,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public processSignal(message: IInboundSignalMessage, local: boolean): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("processSignal");
 
         // Signals are ignored if the store is not yet loaded
         if (!this.loaded) {
@@ -474,11 +504,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         const attributes = createAttributes(pkg, isRoot);
         addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes));
 
-        // Add GC data to the summary if it's not written at the root.
-        if (!this.writeGCDataAtRoot) {
-            addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
-        }
-
         // If we are not referenced, mark the summary tree as unreferenced. Also, update unreferenced blob
         // size in the summary stats with the blobs size of this data store.
         if (!this.summarizerNode.isReferenced()) {
@@ -532,21 +557,17 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
      * 5. To update the timestamp when this data store or any children are marked as unreferenced.
      *
      * @param usedRoutes - The routes that are used in this data store.
-     * @param gcTimestamp - The time when GC was run that generated these used routes. If any node becomes unreferenced
-     * as part of this GC run, this should be used to update the time when it happens.
      */
-    public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
+    public updateUsedRoutes(usedRoutes: string[]) {
         // Update the used routes in this data store's summarizer node.
-        this.summarizerNode.updateUsedRoutes(usedRoutes, gcTimestamp);
+        this.summarizerNode.updateUsedRoutes(usedRoutes);
 
         /**
-         * If the data store has not been realized yet, we need this used state to update the used state of the channel
-         * when it realizes. It's safe to keep only the last used state because if something changes because of this GC
-         * run, the data store will be immediately realized as part of the summary that follows GC. For example, if a
-         * child's reference state changes, the gcTimestamp has to be used to update its unreferencedTimestamp. Since
-         * it will result in a change in this data store's used routes, it will be realized to regenerate its summary.
+         * Store the used routes to update the channel if the data store is not loaded yet. If the used routes changed
+         * since the previous run, the data store will be loaded during summarize since the used state changed. So, it's
+         * safe to only store the last used routes.
          */
-        this.lastUsedState = { usedRoutes, gcTimestamp };
+        this.lastUsedRoutes = usedRoutes;
 
         // If we are loaded, call the channel so it can update the used routes of the child contexts.
         // If we are not loaded, we will update this when we are realized.
@@ -575,16 +596,16 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         assert(this.loaded, 0x144 /* "Channel should be loaded when updating used routes" */);
         assert(this.channel !== undefined, 0x145 /* "Channel should be present when data store is loaded" */);
 
-        // If there is no lastUsedState, GC has not run up until this point.
-        if (this.lastUsedState === undefined) {
+        // If there is no lastUsedRoutes, GC has not run up until this point.
+        if (this.lastUsedRoutes === undefined) {
             return;
         }
 
         // Remove the route to this data store, if it exists.
-        const usedChannelRoutes = this.lastUsedState.usedRoutes.filter(
+        const usedChannelRoutes = this.lastUsedRoutes.filter(
             (id: string) => { return id !== "/" && id !== ""; },
         );
-        this.channel.updateUsedRoutes(usedChannelRoutes, this.lastUsedState.gcTimestamp);
+        this.channel.updateUsedRoutes(usedChannelRoutes);
     }
 
     /**
@@ -596,7 +617,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public submitMessage(type: string, content: any, localOpMetadata: unknown): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("submitMessage");
         assert(!!this.channel, 0x146 /* "Channel must exist when submitting message" */);
         const fluidDataStoreContent: FluidDataStoreMessage = {
             content,
@@ -618,7 +639,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
      *
      */
     public setChannelDirty(address: string): void {
-        this.verifyNotClosed();
+        this.verifyNotClosed("setChannelDirty");
 
         // Get the latest sequence number.
         const latestSequenceNumber = this.deltaManager.lastSequenceNumber;
@@ -633,7 +654,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     }
 
     public submitSignal(type: string, content: any) {
-        this.verifyNotClosed();
+        this.verifyNotClosed("submitSignal");
+
         assert(!!this.channel, 0x147 /* "Channel must exist on submitting signal" */);
         return this._containerRuntime.submitDataStoreSignal(this.id, type, content);
     }
@@ -718,11 +740,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         this._isInMemoryRoot = true;
     }
 
-    /**
-     * @deprecated Renamed to `{@link FluidDataStoreContext.getBaseGCDetails}()`.
-     */
-    public abstract getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails>;
-
     public abstract getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
 
     public reSubmit(contents: any, localOpMetadata: unknown) {
@@ -751,9 +768,30 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         return this.channel.applyStashedOp(innerContents.content);
     }
 
-    private verifyNotClosed() {
+    private verifyNotClosed(callSite: string, checkTombstone = true, safeTelemetryProps: ITelemetryProperties = {}) {
         if (this._disposed) {
-            throw new Error("Context is closed");
+            throw new Error(`Context is closed! Call site [${callSite}]`);
+        }
+
+        if (checkTombstone && this.tombstoned) {
+            const messageString = `Context is tombstoned! Call site [${callSite}]`;
+            const error = new DataCorruptionError(messageString, {
+                errorMessage: messageString,
+                ...safeTelemetryProps,
+            });
+
+            // Always log an error when tombstoned data store is used. However, throw an error only if
+            // throwOnTombstoneUsage is set.
+            this.mc.logger.sendErrorEvent({
+                eventName: "GC_Tombstone_DataStore_Changed",
+                callSite,
+                pkg: packagePathToTelemetryProperty(this.pkg),
+            }, error);
+            // Always log an error when tombstoned data store is used. However, throw an error only if
+            // throwOnTombstoneUsage is set and the client is not a summarizer.
+            if (this.throwOnTombstoneUsage) {
+                throw error;
+            }
         }
     }
 
@@ -779,7 +817,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
 }
 
 export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
-    private readonly initSnapshotValue: ISnapshotTree | string | undefined;
+    private readonly initSnapshotValue: ISnapshotTree | undefined;
     private readonly baseGCDetailsP: Promise<IGarbageCollectionDetailsBase>;
 
     constructor(props: IRemoteFluidDataStoreContextProps) {
@@ -797,28 +835,20 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
         this.baseGCDetailsP = new LazyPromise<IGarbageCollectionDetailsBase>(async () => {
             return (await props.getBaseGCDetails()) ?? {};
         });
+
+        if (props.snapshotTree !== undefined) {
+            this.summarizerNode.updateBaseSummaryState(props.snapshotTree);
+        }
     }
 
     private readonly initialSnapshotDetailsP = new LazyPromise<ISnapshotDetails>(async () => {
-        let tree: ISnapshotTree | undefined;
+        let tree = this.initSnapshotValue;
         let isRootDataStore = true;
 
-        if (typeof this.initSnapshotValue === "string") {
-            const commit = (await this.storage.getVersions(this.initSnapshotValue, 1))[0];
-            tree = await this.storage.getSnapshotTree(commit) ?? undefined;
-        } else {
-            tree = this.initSnapshotValue;
-        }
-
-        const localReadAndParse = async <T>(id: string) => readAndParse<T>(this.storage, id);
-        if (tree) {
-            tree = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
-        }
-
         if (!!tree && tree.blobs[dataStoreAttributesBlobName] !== undefined) {
             // Need to get through snapshot and use that to populate extraBlobs
             const attributes =
-                await localReadAndParse<ReadFluidDataStoreAttributes>(tree.blobs[dataStoreAttributesBlobName]);
+                await readAndParse<ReadFluidDataStoreAttributes>(this.storage, tree.blobs[dataStoreAttributesBlobName]);
 
             let pkgFromSnapshot: string[];
             // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot.
@@ -859,13 +889,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
         return this.initialSnapshotDetailsP;
     }
 
-    /**
-     * @deprecated Renamed to {@link RemoteFluidDataStoreContext.getBaseGCDetails}.
-     */
-    public async getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails> {
-        return this.getBaseGCDetails();
-    }
-
     public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
         return this.baseGCDetailsP;
     }
@@ -974,14 +997,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
         };
     }
 
-    /**
-     * @deprecated Renamed to {@link LocalFluidDataStoreContextBase.getBaseGCDetails}.
-     */
-    public async getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails> {
-        // Local data store does not have initial summary.
-        return {};
-    }
-
     public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
         // Local data store does not have initial summary.
         return {};
@@ -1033,6 +1048,15 @@ export class LocalDetachedFluidDataStoreContext
 
         super.bindRuntime(dataStoreChannel);
 
+        // Load the handle to the data store's entryPoint to make sure that for a detached data store, the entryPoint
+        // initialization function is called before the data store gets attached and potentially connected to the
+        // delta stream, so it gets a chance to do things while the data store is still "purely local".
+        // This preserves the behavior from before we introduced entryPoints, where the instantiateDataStore method
+        // of data store factories tends to construct the data object (at least kick off an async method that returns
+        // it); that code moved to the entryPoint initialization function, so we want to ensure it still executes
+        // before the data store is attached.
+        await dataStoreChannel.entryPoint?.get();
+
         if (await this.isRoot()) {
             dataStoreChannel.makeVisibleAndAttachGraph();
         }

package/src/dataStoreContexts.ts

@@ -87,7 +87,7 @@ import { FluidDataStoreContext, LocalFluidDataStoreContext } from "./dataStoreCo
             return undefined;
         }
 
-        return this._contexts.get(id) as LocalFluidDataStoreContext;
+        return context as LocalFluidDataStoreContext;
     }
 
     /**

package/src/dataStores.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger, ITelemetryBaseLogger, IDisposable } from "@fluidframework/common-definitions";
+import { ITelemetryBaseLogger, IDisposable } from "@fluidframework/common-definitions";
 import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
 import { IFluidHandle } from "@fluidframework/core-interfaces";
 import { FluidObjectHandle } from "@fluidframework/datastore";
@@ -32,10 +32,12 @@ import {
     convertSnapshotTreeToSummaryTree,
     convertToSummaryTree,
     create404Response,
+    createResponseError,
     responseToException,
+    packagePathToTelemetryProperty,
     SummaryTreeBuilder,
 } from "@fluidframework/runtime-utils";
-import { ChildLogger, LoggingError, TelemetryDataTag } from "@fluidframework/telemetry-utils";
+import { ChildLogger, loggerToMonitoringContext, LoggingError, MonitoringContext, TelemetryDataTag } from "@fluidframework/telemetry-utils";
 import { AttachState } from "@fluidframework/container-definitions";
 import { BlobCacheStorageService, buildSnapshotTree } from "@fluidframework/driver-utils";
 import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
@@ -52,7 +54,8 @@ import {
 } from "./dataStoreContext";
 import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summaryFormat";
 import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
-import { GCNodeType } from "./garbageCollection";
+import { throwOnTombstoneUsageKey, GCNodeType } from "./garbageCollection";
+import { summarizerClientType } from "./summarizerClientElection";
 
 type PendingAliasResolve = (success: boolean) => void;
 
@@ -66,7 +69,7 @@ export class DataStores implements IDisposable {
     // 0.24 back-compat attachingBeforeSummary
     public readonly attachOpFiredForDataStore = new Set<string>();
 
-    private readonly logger: ITelemetryLogger;
+    private readonly mc: MonitoringContext;
 
     private readonly disposeOnce = new Lazy<void>(() => this.contexts.dispose());
 
@@ -80,6 +83,8 @@ export class DataStores implements IDisposable {
     // Stores the ids of new data stores between two GC runs. This is used to notify the garbage collector of new
     // root data stores that are added.
     private dataStoresSinceLastGC: string[] = [];
+    /** If true, throw an error when a tombstone data store is retrieved. */
+    private readonly throwOnTombstoneUsage: boolean;
     // The handle to the container runtime. This is used mainly for GC purposes to represent outbound reference from
     // the container runtime to other nodes.
     private readonly containerRuntimeHandle: IFluidHandle;
@@ -97,10 +102,9 @@ export class DataStores implements IDisposable {
         private readonly gcNodeUpdated: (
             nodePath: string, timestampMs: number, packagePath?: readonly string[]) => void,
         private readonly aliasMap: Map<string, string>,
-        private readonly writeGCDataAtRoot: boolean,
         private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
     ) {
-        this.logger = ChildLogger.create(baseLogger);
+        this.mc = loggerToMonitoringContext(ChildLogger.create(baseLogger));
         this.containerRuntimeHandle = new FluidObjectHandle(this.runtime, "/", this.runtime.IFluidHandleContext);
 
         const baseGCDetailsP = new LazyPromise(async () => {
@@ -111,6 +115,10 @@ export class DataStores implements IDisposable {
             const baseGCDetails = await baseGCDetailsP;
             return baseGCDetails.get(dataStoreId);
         };
+        // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
+        this.throwOnTombstoneUsage =
+            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+            this.runtime.clientDetails.type !== summarizerClientType;
 
         // Extract stores stored inside the snapshot
         const fluidDataStores = new Map<string, ISnapshotTree>();
@@ -142,7 +150,6 @@ export class DataStores implements IDisposable {
                         key,
                         { type: CreateSummarizerNodeSource.FromSummary },
                     ),
-                    writeGCDataAtRoot: this.writeGCDataAtRoot,
                 });
             } else {
                 if (typeof value !== "object") {
@@ -162,7 +169,6 @@ export class DataStores implements IDisposable {
                     makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(key),
                     snapshotTree,
                     isRootDataStore: undefined,
-                    writeGCDataAtRoot: this.writeGCDataAtRoot,
                 });
             }
             this.contexts.addBoundOrRemoted(dataStoreContext);
@@ -247,7 +253,6 @@ export class DataStores implements IDisposable {
                     },
                 },
             ),
-            writeGCDataAtRoot: this.writeGCDataAtRoot,
             pkg,
         });
 
@@ -283,7 +288,7 @@ export class DataStores implements IDisposable {
 
         const context = this.contexts.get(aliasMessage.internalId);
         if (context === undefined) {
-            this.logger.sendErrorEvent({
+            this.mc.logger.sendErrorEvent({
                 eventName: "AliasFluidDataStoreNotFound",
                 fluidDataStoreId: aliasMessage.internalId,
             });
@@ -351,7 +356,6 @@ export class DataStores implements IDisposable {
             makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: isRoot,
-            writeGCDataAtRoot: this.writeGCDataAtRoot,
         });
         this.contexts.addUnbound(context);
         return context;
@@ -372,7 +376,6 @@ export class DataStores implements IDisposable {
             makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: false,
-            writeGCDataAtRoot: this.writeGCDataAtRoot,
             createProps: props,
         });
         this.contexts.addUnbound(context);
@@ -425,14 +428,31 @@ export class DataStores implements IDisposable {
         );
     }
 
-    public async getDataStore(id: string, wait: boolean): Promise<FluidDataStoreContext> {
+    public async getDataStore(id: string, wait: boolean, viaHandle: boolean): Promise<FluidDataStoreContext> {
         const context = await this.contexts.getBoundOrRemoted(id, wait);
+        const request = { url: id };
         if (context === undefined) {
             // The requested data store does not exits. Throw a 404 response exception.
-            const request = { url: id };
             throw responseToException(create404Response(request), request);
         }
 
+        if (context.tombstoned) {
+            // The requested data store is removed by gc. Create a 404 gc response exception.
+            const error = responseToException(createResponseError(404, "Datastore removed by gc", request), request);
+            // Note: if a user writes a request to look like it's viaHandle, we will also send this telemetry event
+            this.mc.logger.sendErrorEvent({
+                eventName: "GC_Tombstone_DataStore_Requested",
+                url: request.url,
+                pkg: packagePathToTelemetryProperty(context.isLoaded ? context.packagePath : undefined),
+                viaHandle,
+            }, error);
+            // Always log an error when tombstoned data store is used. However, throw an error only if
+            // throwOnTombstoneUsage is set.
+            if (this.throwOnTombstoneUsage) {
+                throw error;
+            }
+        }
+
         return context;
     }
 
@@ -441,7 +461,7 @@ export class DataStores implements IDisposable {
         if (!context) {
             // Attach message may not have been processed yet
             assert(!local, 0x163 /* "Missing datastore for local signal" */);
-            this.logger.sendTelemetryEvent({
+            this.mc.logger.sendTelemetryEvent({
                 eventName: "SignalFluidDataStoreNotFound",
                 fluidDataStoreId: {
                     value: address,
@@ -459,7 +479,7 @@ export class DataStores implements IDisposable {
             try {
                 context.setConnectionState(connected, clientId);
             } catch (error) {
-                this.logger.sendErrorEvent({
+                this.mc.logger.sendErrorEvent({
                     eventName: "SetConnectionStateError",
                     clientId,
                     fluidDataStore,
@@ -595,30 +615,35 @@ export class DataStores implements IDisposable {
     /**
      * After GC has run, called to notify this Container's data stores of routes that are used in it.
      * @param usedRoutes - The routes that are used in all data stores in this Container.
-     * @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
-     * unreferenced as part of this GC run, this should be used to update the time when it happens.
      */
-    public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
+    public updateUsedRoutes(usedRoutes: string[]) {
         // Get a map of data store ids to routes used in it.
         const usedDataStoreRoutes = unpackChildNodesUsedRoutes(usedRoutes);
 
         // Verify that the used routes are correct.
         for (const [id] of usedDataStoreRoutes) {
             assert(this.contexts.has(id), 0x167 /* "Used route does not belong to any known data store" */);
+
+            // Revive datastores regardless of whether or not tombstone the tombstone flag is flipped
+            const dataStore = this.contexts.get(id);
+            assert(dataStore !== undefined, 0x46e /* No data store retrieved with specified id */);
+            dataStore.setTombstone(false /* tombstone */);
         }
 
         // Update the used routes in each data store. Used routes is empty for unused data stores.
         for (const [contextId, context] of this.contexts) {
-            context.updateUsedRoutes(usedDataStoreRoutes.get(contextId) ?? [], gcTimestamp);
+            context.updateUsedRoutes(usedDataStoreRoutes.get(contextId) ?? []);
         }
     }
 
     /**
-     * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
-     * scenarios with accessing deleted content.
+     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
+     * tombstones.
      * @param unusedRoutes - The routes that are unused in all data stores in this Container.
+     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
+     * are deleted.
      */
-    public deleteUnusedRoutes(unusedRoutes: string[]) {
+    public updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean) {
         for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             // Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data
@@ -628,6 +653,19 @@ export class DataStores implements IDisposable {
             }
             const dataStoreId = pathParts[1];
             assert(this.contexts.has(dataStoreId), 0x2d7 /* No data store with specified id */);
+
+            /**
+             * When running GC in tombstone mode, datastore contexts are tombstoned. Tombstoned datastore contexts
+             * enable testing scenarios with accessing deleted content without actually deleting content from
+             * summaries.
+             */
+            if (tombstone) {
+                const dataStore = this.contexts.get(dataStoreId);
+                assert(dataStore !== undefined, 0x442 /* No data store retrieved with specified id */);
+                dataStore.setTombstone(true /* tombstone */);
+                continue;
+            }
+
             // Delete the contexts of unused data stores.
             this.contexts.delete(dataStoreId);
             // Delete the summarizer node of the unused data stores.