@fluidframework/container-runtime 2.0.0-internal.2.1.1 → 2.0.0-internal.2.2.0

package/src/dataStoreContext.ts

@@ -58,10 +58,16 @@ import {
     SummarizeInternalFn,
     ITelemetryContext,
 } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, convertSummaryTreeToITree } from "@fluidframework/runtime-utils";
+import {
+    addBlobToSummary,
+    convertSummaryTreeToITree,
+    packagePathToTelemetryProperty,
+} from "@fluidframework/runtime-utils";
 import {
     ChildLogger,
+    loggerToMonitoringContext,
     LoggingError,
+    MonitoringContext,
     TelemetryDataTag,
     ThresholdCounter,
 } from "@fluidframework/telemetry-utils";
@@ -81,6 +87,8 @@ import {
     getAttributesFormatVersion,
     getFluidDataStoreAttributes,
 } from "./summaryFormat";
+import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+import { summarizerClientType } from "./summarizerClientElection";
 
 function createAttributes(
     pkg: readonly string[],
@@ -200,6 +208,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
      */
     private _tombstoned = false;
     public get tombstoned() { return this._tombstoned; }
+    /** If true, throw an error when a tombstone data store is used. */
+    private readonly throwOnTombstoneUsage: boolean;
 
     public get attachState(): AttachState {
         return this._attachState;
@@ -243,7 +253,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
     protected _attachState: AttachState;
     private _isInMemoryRoot: boolean = false;
     protected readonly summarizerNode: ISummarizerNodeWithGC;
-    private readonly subLogger: ITelemetryLogger;
+    private readonly mc: MonitoringContext;
     private readonly thresholdOpsCounter: ThresholdCounter;
     private static readonly pendingOpsCountThreshold = 1000;
 
@@ -297,8 +307,13 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
             async () => this.getBaseGCDetails(),
         );
 
-        this.subLogger = ChildLogger.create(this.logger, "FluidDataStoreContext");
-        this.thresholdOpsCounter = new ThresholdCounter(FluidDataStoreContext.pendingOpsCountThreshold, this.subLogger);
+        this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "FluidDataStoreContext"));
+        this.thresholdOpsCounter = new ThresholdCounter(FluidDataStoreContext.pendingOpsCountThreshold, this.mc.logger);
+
+        // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
+        this.throwOnTombstoneUsage =
+            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+            this.clientDetails.type !== summarizerClientType;
     }
 
     public dispose(): void {
@@ -760,10 +775,23 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
 
         if (checkTombstone && this.tombstoned) {
             const messageString = `Context is tombstoned! Call site [${callSite}]`;
-            throw new DataCorruptionError(messageString, {
+            const error = new DataCorruptionError(messageString, {
                 errorMessage: messageString,
                 ...safeTelemetryProps,
             });
+
+            // Always log an error when tombstoned data store is used. However, throw an error only if
+            // throwOnTombstoneUsage is set.
+            this.mc.logger.sendErrorEvent({
+                eventName: "GC_Tombstone_DataStore_Changed",
+                callSite,
+                pkg: packagePathToTelemetryProperty(this.pkg),
+            }, error);
+            // Always log an error when tombstoned data store is used. However, throw an error only if
+            // throwOnTombstoneUsage is set and the client is not a summarizer.
+            if (this.throwOnTombstoneUsage) {
+                throw error;
+            }
         }
     }
 

package/src/dataStores.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger, ITelemetryBaseLogger, IDisposable } from "@fluidframework/common-definitions";
+import { ITelemetryBaseLogger, IDisposable } from "@fluidframework/common-definitions";
 import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
 import { IFluidHandle } from "@fluidframework/core-interfaces";
 import { FluidObjectHandle } from "@fluidframework/datastore";
@@ -34,9 +34,10 @@ import {
     create404Response,
     createResponseError,
     responseToException,
+    packagePathToTelemetryProperty,
     SummaryTreeBuilder,
 } from "@fluidframework/runtime-utils";
-import { ChildLogger, LoggingError, TelemetryDataTag } from "@fluidframework/telemetry-utils";
+import { ChildLogger, loggerToMonitoringContext, LoggingError, MonitoringContext, TelemetryDataTag } from "@fluidframework/telemetry-utils";
 import { AttachState } from "@fluidframework/container-definitions";
 import { BlobCacheStorageService, buildSnapshotTree } from "@fluidframework/driver-utils";
 import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
@@ -54,6 +55,8 @@ import {
 import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summaryFormat";
 import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
 import { GCNodeType } from "./garbageCollection";
+import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+import { summarizerClientType } from "./summarizerClientElection";
 
 type PendingAliasResolve = (success: boolean) => void;
 
@@ -67,7 +70,7 @@ export class DataStores implements IDisposable {
     // 0.24 back-compat attachingBeforeSummary
     public readonly attachOpFiredForDataStore = new Set<string>();
 
-    private readonly logger: ITelemetryLogger;
+    private readonly mc: MonitoringContext;
 
     private readonly disposeOnce = new Lazy<void>(() => this.contexts.dispose());
 
@@ -81,6 +84,8 @@ export class DataStores implements IDisposable {
     // Stores the ids of new data stores between two GC runs. This is used to notify the garbage collector of new
     // root data stores that are added.
     private dataStoresSinceLastGC: string[] = [];
+    /** If true, throw an error when a tombstone data store is retrieved. */
+    private readonly throwOnTombstoneUsage: boolean;
     // The handle to the container runtime. This is used mainly for GC purposes to represent outbound reference from
     // the container runtime to other nodes.
     private readonly containerRuntimeHandle: IFluidHandle;
@@ -100,7 +105,7 @@ export class DataStores implements IDisposable {
         private readonly aliasMap: Map<string, string>,
         private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
     ) {
-        this.logger = ChildLogger.create(baseLogger);
+        this.mc = loggerToMonitoringContext(ChildLogger.create(baseLogger));
         this.containerRuntimeHandle = new FluidObjectHandle(this.runtime, "/", this.runtime.IFluidHandleContext);
 
         const baseGCDetailsP = new LazyPromise(async () => {
@@ -111,6 +116,10 @@ export class DataStores implements IDisposable {
             const baseGCDetails = await baseGCDetailsP;
             return baseGCDetails.get(dataStoreId);
         };
+        // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
+        this.throwOnTombstoneUsage =
+            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+            this.runtime.clientDetails.type !== summarizerClientType;
 
         // Extract stores stored inside the snapshot
         const fluidDataStores = new Map<string, ISnapshotTree>();
@@ -280,7 +289,7 @@ export class DataStores implements IDisposable {
 
         const context = this.contexts.get(aliasMessage.internalId);
         if (context === undefined) {
-            this.logger.sendErrorEvent({
+            this.mc.logger.sendErrorEvent({
                 eventName: "AliasFluidDataStoreNotFound",
                 fluidDataStoreId: aliasMessage.internalId,
             });
@@ -429,14 +438,20 @@ export class DataStores implements IDisposable {
         }
 
         if (context.tombstoned) {
+            // The requested data store is removed by gc. Create a 404 gc response exception.
+            const error = responseToException(createResponseError(404, "Datastore removed by gc", request), request);
             // Note: if a user writes a request to look like it's viaHandle, we will also send this telemetry event
-            this.logger.sendErrorEvent({
-                eventName: "TombstonedDataStoreRequested",
+            this.mc.logger.sendErrorEvent({
+                eventName: "GC_Tombstone_DataStore_Requested",
                 url: request.url,
+                pkg: packagePathToTelemetryProperty(context.isLoaded ? context.packagePath : undefined),
                 viaHandle,
-            });
-            // The requested data store is removed by gc. Throw a 404 gc response exception.
-            throw responseToException(createResponseError(404, "Datastore removed by gc", request), request);
+            }, error);
+            // Always log an error when tombstoned data store is used. However, throw an error only if
+            // throwOnTombstoneUsage is set.
+            if (this.throwOnTombstoneUsage) {
+                throw error;
+            }
         }
 
         return context;
@@ -447,7 +462,7 @@ export class DataStores implements IDisposable {
         if (!context) {
             // Attach message may not have been processed yet
             assert(!local, 0x163 /* "Missing datastore for local signal" */);
-            this.logger.sendTelemetryEvent({
+            this.mc.logger.sendTelemetryEvent({
                 eventName: "SignalFluidDataStoreNotFound",
                 fluidDataStoreId: {
                     value: address,
@@ -465,7 +480,7 @@ export class DataStores implements IDisposable {
             try {
                 context.setConnectionState(connected, clientId);
             } catch (error) {
-                this.logger.sendErrorEvent({
+                this.mc.logger.sendErrorEvent({
                     eventName: "SetConnectionStateError",
                     clientId,
                     fluidDataStore,
@@ -623,12 +638,13 @@ export class DataStores implements IDisposable {
     }
 
     /**
-     * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
-     * scenarios with accessing deleted content.
+     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
+     * tombstones.
      * @param unusedRoutes - The routes that are unused in all data stores in this Container.
-     * @param tombstone - set the objects corresponding to routes as tombstones.
+     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
+     * are deleted.
      */
-    public deleteUnusedRoutes(unusedRoutes: string[], tombstone: boolean = false) {
+    public updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean) {
         for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             // Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data

package/src/garbageCollection.ts

@@ -29,6 +29,7 @@ import {
 } from "@fluidframework/runtime-definitions";
 import {
     mergeStats,
+    packagePathToTelemetryProperty,
     ReadAndParseBlob,
     RefreshSummaryResult,
     SummaryTreeBuilder,
@@ -44,6 +45,21 @@ import {
 
 import { IGCRuntimeOptions, RuntimeHeaders } from "./containerRuntime";
 import { getSummaryForDatastores } from "./dataStores";
+import {
+    defaultInactiveTimeoutMs,
+    defaultSessionExpiryDurationMs,
+    disableSweepLogKey,
+    disableTombstoneKey,
+    gcBlobPrefix,
+    gcTestModeKey,
+    gcTombstoneBlobKey,
+    gcTreeKey,
+    oneDayMs,
+    runGCKey,
+    runSessionExpiryKey,
+    runSweepKey,
+    trackGCStateKey
+} from "./garbageCollectionConstants";
 import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
 import {
     getGCVersion,
@@ -53,39 +69,12 @@ import {
     ReadFluidDataStoreAttributes,
     dataStoreAttributesBlobName,
     IGCMetadata,
+    ICreateContainerMetadata,
 } from "./summaryFormat";
 
 /** This is the current version of garbage collection. */
 const GCVersion = 1;
 
-// The key for the GC tree in summary.
-export const gcTreeKey = "gc";
-// They prefix for GC blobs in the GC tree in summary.
-export const gcBlobPrefix = "__gc";
-// The key for tombstone blob in the GC tree in summary.
-export const gcTombstoneBlobKey = "__tombstones";
-
-// Feature gate key to turn GC on / off.
-export const runGCKey = "Fluid.GarbageCollection.RunGC";
-// Feature gate key to turn GC sweep on / off.
-export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
-// Feature gate key to turn GC test mode on / off.
-export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
-// Feature gate key to expire a session after a set period of time.
-export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
-// Feature gate key to write the gc blob as a handle if the data is the same.
-export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
-// Feature gate key to turn GC sweep log off.
-export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
-// Feature gate key to tombstone datastores.
-export const testTombstoneKey = "Fluid.GarbageCollection.Test.Tombstone";
-
-// One day in milliseconds.
-export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
-
-export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
-export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days
-
 /** The statistics of the system state after a garbage collection run. */
 export interface IGCStats {
     /** The number of nodes in the container. */
@@ -129,8 +118,8 @@ export interface IGarbageCollectionRuntime {
     getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
     /** After GC has run, called to notify the runtime of routes that are used in it. */
     updateUsedRoutes(usedRoutes: string[]): void;
-    /** After GC has run, called to delete objects in the runtime whose routes are unused. */
-    deleteUnusedRoutes(unusedRoutes: string[]): void;
+    /** After GC has run, called to notify the runtime of routes that are unused in it. */
+    updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean): void;
     /** Returns a referenced timestamp to be used to track unreferenced nodes. */
     getCurrentReferenceTimestampMs(): number | undefined;
     /** Returns the type of the GC node. */
@@ -146,6 +135,8 @@ export interface IGarbageCollector {
     /** Tells whether the GC state in summary needs to be reset in the next summary. */
     readonly summaryStateNeedsReset: boolean;
     readonly trackGCState: boolean;
+    /** Initialize the state from the base snapshot after its creation. */
+    initializeBaseState(): Promise<void>;
     /** Run garbage collection and update the reference / used state of the system. */
     collectGarbage(
         options: { logger?: ITelemetryLogger; runSweep?: boolean; fullGC?: boolean; },
@@ -183,6 +174,7 @@ export interface IGarbageCollectorCreateParams {
     readonly baseLogger: ITelemetryLogger;
     readonly existing: boolean;
     readonly metadata: IContainerRuntimeMetadata | undefined;
+    readonly createContainerMetadata: ICreateContainerMetadata;
     readonly baseSnapshot: ISnapshotTree | undefined;
     readonly isSummarizerClient: boolean;
     readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
@@ -227,6 +219,14 @@ interface IGCSummaryTrackingData {
     serializedTombstones: string | undefined;
 }
 
+/**
+ * The GC data that is read from a snapshot. It contains the GC state and tombstone state.
+ */
+interface IGCSnapshotData {
+    gcState: IGarbageCollectionState;
+    tombstones: string[] | undefined;
+}
+
 /**
  * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
  * be deleted by the sweep phase.
@@ -421,8 +421,10 @@ export class GarbageCollector implements IGarbageCollector {
      */
     private pendingSummaryData: IGCSummaryTrackingData | undefined;
 
-    // Promise when resolved initializes the base state of the nodes from the base summary state.
-    private readonly initializeBaseStateP: Promise<void>;
+    // Promise when resolved returns the GC data data in the base snapshot.
+    private readonly baseSnapshotDataP: Promise<IGCSnapshotData | undefined>;
+    // Promise when resolved initializes the GC state from the data in the base snapshot.
+    private readonly initializeGCStateFromBaseSnapshotP: Promise<void>;
     // The map of data store ids to their GC details in the base summary returned in getDataStoreGCDetails().
     private readonly baseGCDetailsP: Promise<Map<string, IGarbageCollectionDetailsBase>>;
     // Map of node ids to their unreferenced state tracker.
@@ -440,6 +442,7 @@ export class GarbageCollector implements IGarbageCollector {
     private completedRuns = 0;
 
     private readonly runtime: IGarbageCollectionRuntime;
+    private readonly createContainerMetadata: ICreateContainerMetadata;
     private readonly gcOptions: IGCRuntimeOptions;
     private readonly isSummarizerClient: boolean;
 
@@ -481,6 +484,7 @@ export class GarbageCollector implements IGarbageCollector {
         this.runtime = createParams.runtime;
         this.isSummarizerClient = createParams.isSummarizerClient;
         this.gcOptions = createParams.gcOptions;
+        this.createContainerMetadata = createParams.createContainerMetadata;
         this.getNodePackagePath = createParams.getNodePackagePath;
         this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
         this.activeConnection = createParams.activeConnection;
@@ -623,16 +627,17 @@ export class GarbageCollector implements IGarbageCollector {
 
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
         this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
-        this.tombstoneMode = this.mc.config.getBoolean(testTombstoneKey) ?? false;
+        // Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
+        this.tombstoneMode = this.mc.config.getBoolean(disableTombstoneKey) !== true;
 
         // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
         // contain GC tree and GC is enabled.
         const gcTreePresent = baseSnapshot?.trees[gcTreeKey] !== undefined;
         this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
 
-        // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
-        // this once since it involves fetching blobs from storage which is expensive.
-        const baseSummaryStateP = new LazyPromise<IGarbageCollectionState | undefined>(async () => {
+        // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
+        // it involves fetching blobs from storage which is expensive.
+        this.baseSnapshotDataP = new LazyPromise<IGCSnapshotData | undefined>(async () => {
             if (baseSnapshot === undefined) {
                 return undefined;
             }
@@ -641,20 +646,10 @@ export class GarbageCollector implements IGarbageCollector {
                 // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
                 const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
                 if (gcSnapshotTree !== undefined) {
-                    const baseGCData = await getGCDataFromSnapshot(
+                    return getGCDataFromSnapshot(
                         gcSnapshotTree,
                         readAndParseBlob,
                     );
-                    if (baseGCData.tombstones !== undefined && this.tombstoneMode) {
-                        this.tombstones = baseGCData.tombstones;
-                    }
-                    if (this.trackGCState) {
-                        this.latestSummaryData = {
-                            serializedGCState: JSON.stringify(generateSortedGCState(baseGCData.gcState)),
-                            serializedTombstones: JSON.stringify(baseGCData.tombstones),
-                        };
-                    }
-                    return baseGCData.gcState;
                 }
 
                 // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
@@ -698,7 +693,7 @@ export class GarbageCollector implements IGarbageCollector {
                 }
                 // If there is only one node (root node just added above), either GC is disabled or we are loading from
                 // the first summary generated by detached container. In both cases, GC was not run - return undefined.
-                return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
+                return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
             } catch (error) {
                 const dpe = DataProcessingError.wrapIfUnrecognized(
                     error,
@@ -710,11 +705,11 @@ export class GarbageCollector implements IGarbageCollector {
         });
 
         /**
-         * Set up the initializer which initializes the base GC state from the base snapshot. Note that the reference
-         * timestamp maybe from old ops which were not summarized and stored in the file. So, the unreferenced state
-         * may be out of date. This is fine because the state is updated every time GC runs based on the time then.
+         * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
+         * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
+         * GC state and updates their inactive or sweep ready state.
          */
-        this.initializeBaseStateP = new LazyPromise<void>(async () => {
+        this.initializeGCStateFromBaseSnapshotP = new LazyPromise<void>(async () => {
             const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
             /**
              * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
@@ -733,19 +728,19 @@ export class GarbageCollector implements IGarbageCollector {
                 return;
             }
 
-            const baseState = await baseSummaryStateP;
+            const baseSnapshotData = await this.baseSnapshotDataP;
             /**
-             * The base state will not be present if the container is loaded from:
+             * The base snapshot data will not be present if the container is loaded from:
              * 1. The first summary created by the detached container.
              * 2. A summary that was generated with GC disabled.
              * 3. A summary that was generated before GC even existed.
              */
-            if (baseState === undefined) {
+            if (baseSnapshotData === undefined) {
                 return;
             }
 
             const gcNodes: { [id: string]: string[]; } = {};
-            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
                 if (nodeData.unreferencedTimestampMs !== undefined) {
                     this.unreferencedNodesState.set(
                         nodeId,
@@ -760,18 +755,26 @@ export class GarbageCollector implements IGarbageCollector {
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
             this.previousGCDataFromLastRun = { gcNodes };
+
+            // If tracking state across summaries, update latest summary data from the base snapshot's GC data.
+            if (this.trackGCState) {
+                this.latestSummaryData = {
+                    serializedGCState: JSON.stringify(generateSortedGCState(baseSnapshotData.gcState)),
+                    serializedTombstones: JSON.stringify(baseSnapshotData.tombstones),
+                };
+            }
         });
 
         // Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
         // which the caller uses to initialize each node's GC state.
         this.baseGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionDetailsBase>>(async () => {
-            const baseState = await baseSummaryStateP;
-            if (baseState === undefined) {
+            const baseSnapshotData = await this.baseSnapshotDataP;
+            if (baseSnapshotData === undefined) {
                 return new Map();
             }
 
             const gcNodes: { [id: string]: string[]; } = {};
-            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
             // Run GC on the nodes in the base summary to get the routes used in each node in the container.
@@ -782,7 +785,7 @@ export class GarbageCollector implements IGarbageCollector {
             const baseGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
             // Currently, the nodes may write the GC data. So, we need to update its base GC details with the
             // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
-            for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
+            for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
                 if (nodeData.unreferencedTimestampMs !== undefined) {
                     const dataStoreGCDetails = baseGCDetailsMap.get(nodeId.slice(1));
                     if (dataStoreGCDetails !== undefined) {
@@ -803,6 +806,27 @@ export class GarbageCollector implements IGarbageCollector {
         }
     }
 
+    /**
+     * Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
+     * before they are loaded or used. This is important to get accurate information of whether tombstoned object are
+     * in use or not.
+     */
+    public async initializeBaseState(): Promise<void> {
+        const baseSnapshotData = await this.baseSnapshotDataP;
+        /**
+         * The base snapshot data or tombstone state will not be present if the container is loaded from:
+         * 1. The first summary created by the detached container.
+         * 2. A summary that was generated with GC disabled.
+         * 3. A summary that was generated before GC even existed.
+         * 4. A summary that was generated with tombstone feature disabled.
+         */
+        if (!this.tombstoneMode || baseSnapshotData?.tombstones === undefined) {
+            return;
+        }
+        this.tombstones = baseSnapshotData.tombstones;
+        this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
+    }
+
     /**
      * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
      * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
@@ -824,7 +848,7 @@ export class GarbageCollector implements IGarbageCollector {
          * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
          */
         if (this.activeConnection() && this.shouldRunGC) {
-            this.initializeBaseStateP.catch((error) => {});
+            this.initializeGCStateFromBaseSnapshotP.catch((error) => {});
         }
     }
 
@@ -880,8 +904,8 @@ export class GarbageCollector implements IGarbageCollector {
     }
 
     private async runPreGCSteps() {
-        // Ensure that base state has been initialized.
-        await this.initializeBaseStateP;
+        // Ensure that state has been initialized from the base snapshot data.
+        await this.initializeGCStateFromBaseSnapshotP;
         // Let the runtime update its pending state before GC runs.
         await this.runtime.updateStateBeforeGC();
     }
@@ -911,24 +935,12 @@ export class GarbageCollector implements IGarbageCollector {
         // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
         // involving access to deleted data.
         if (this.testMode) {
-            this.runtime.deleteUnusedRoutes(gcResult.deletedNodeIds);
-        } else {
+            this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds, false /* tombstone */);
+        } else if (this.tombstoneMode) {
             // If we are running in GC tombstone mode, tombstone objects for unused routes. This enables testing
             // scenarios involving access to "deleted" data without actually deleting the data from summaries.
             // Note: we will not tombstone in test mode
-            if (this.tombstoneMode) {
-                const tombstoneRoutes: string[] = [];
-                // Currently only tombstone datastores
-                for (const [key, value] of this.unreferencedNodesState.entries()) {
-                    if (
-                        value.state === UnreferencedState.SweepReady &&
-                        this.runtime.getNodeType(key) === GCNodeType.DataStore
-                    ) {
-                        tombstoneRoutes.push(key);
-                    }
-                }
-                this.runtime.deleteUnusedRoutes(tombstoneRoutes);
-            }
+            this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
         }
 
         // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
@@ -962,7 +974,9 @@ export class GarbageCollector implements IGarbageCollector {
         }
 
         const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
-        const serializedTombstones = this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined;
+        const serializedTombstones = this.tombstoneMode
+            ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
+            : undefined;
 
         /**
          * Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
@@ -1497,6 +1511,7 @@ export class GarbageCollector implements IGarbageCollector {
                 : this.sweepTimeoutMs,
             completedGCRuns: this.completedRuns,
             lastSummaryTime: this.getLastSummaryTimestampMs(),
+            ...this.createContainerMetadata,
             externalRequest: requestHeaders?.[RuntimeHeaders.externalRequest],
             viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
             fromId: fromNodeId,
@@ -1505,16 +1520,20 @@ export class GarbageCollector implements IGarbageCollector {
         // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
         // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
         // but it's a good signal nonetheless and we can consume it with a grain of salt.
+        // Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
+        // SweepReady errors are usages of Objects that will be deleted by GC Sweep!
         if (this.isSummarizerClient) {
             this.pendingEventsQueue.push({ ...propsToLog, usageType, state });
         } else {
             // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
             // summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
+            // Events generated:
+            // InactiveObject_Loaded, SweepReadyObject_Loaded
             if (usageType === "Loaded") {
                 this.mc.logger.sendErrorEvent({
                     ...propsToLog,
                     eventName: `${state}Object_${usageType}`,
-                    pkg: packagePath ? { value: packagePath.join("/"), tag: TelemetryDataTag.CodeArtifact } : undefined,
+                    pkg: packagePathToTelemetryProperty(packagePath),
                     stack: generateStack(),
                 });
             }
@@ -1529,6 +1548,11 @@ export class GarbageCollector implements IGarbageCollector {
     }
 
     private async logUnreferencedEvents(logger: ITelemetryLogger) {
+        // Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
+        // summary time they are then logged.
+        // Events generated:
+        // InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
+        // SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
         for (const eventProps of this.pendingEventsQueue) {
             const { usageType, state, ...propsToLog } = eventProps;
             /**
@@ -1555,13 +1579,13 @@ export class GarbageCollector implements IGarbageCollector {
 }
 
 /**
- * Gets the garbage collection data from the given snapshot tree. It contains GC state and tombstone state.
+ * Gets the base garbage collection state from the given snapshot tree. It contains GC state and tombstone state.
  * The GC state may be written into multiple blobs. Merge the GC state from all such blobs into one.
  */
 async function getGCDataFromSnapshot(
     gcSnapshotTree: ISnapshotTree,
     readAndParseBlob: ReadAndParseBlob,
-) {
+): Promise<IGCSnapshotData> {
     let rootGCState: IGarbageCollectionState = { gcNodes: {} };
     let tombstones: string[] | undefined;
     for (const key of Object.keys(gcSnapshotTree.blobs)) {