@fluidframework/container-runtime 0.58.2002 → 0.59.1000

package/src/connectionTelemetry.ts

@@ -9,6 +9,7 @@ import { IDeltaManager } from "@fluidframework/container-definitions";
 import {
     IDocumentMessage,
     ISequencedDocumentMessage,
+    MessageType,
 } from "@fluidframework/protocol-definitions";
 import { assert, performance } from "@fluidframework/common-utils";
 
@@ -17,17 +18,51 @@ import { assert, performance } from "@fluidframework/common-utils";
  */
 export const latencyThreshold = 5000;
 
+// Phases in OpPerfTelemetry:
+// 1.	Op sits in a buffer in DeltaManager (DM) queue, then in outbound queue for some time.
+// 	 - Note: We do not differentiate these two today in telemetry, but first one is due to batches,
+//           second one might happen due to outbound queue being paused.
+// 2.	Op is sent to service and back.
+// 3.	Op sits in inbound queue.
+// 4.	Op is processed.
+interface IOpPerfTelemetryProperties {
+    /** Measure time between (1) and (2) - Measure time outbound op is sitting in queue due to active batch */
+    durationOutboundQueue: number;
+    /** Measure time between (2) and (3) - Track how long op is sitting in inbound queue until it is processed */
+    durationInboundQueue: number;
+    /** Measure time between (3) and (4) - Time between DM's inbound "push" event until DM's "op" event */
+    durationInboundToProcessing: number;
+    /** Length of the DeltaManager's inbound queue at the time of the DM's inbound "push" event (3) */
+    lenghtInboundQueue: number;
+}
+
+/**
+ * Timings collected at various moments during the op processing.
+ */
+ interface IOpPerfTimings {
+     /** Starting time for (1) */
+    opStartTimeForLatencyStatistics: number;
+     /** Starting time for (2) */
+     opStartTimeSittingInboundQueue: number;
+     /** Starting time for (3) */
+     opStartTimeInboundPushEvent: number;
+}
+
 class OpPerfTelemetry {
     private pongCount: number = 0;
     private pingLatency: number | undefined;
 
     // Collab window tracking. This is timestamp of %1000 message.
-    private opSendTimeForLatencyStatisticsForMsnStatistics: number | undefined;
-
-    // To track round trip time for every %1000 client message.
-    private opSendTimeForLatencyStatistics: number | undefined;
+    private sequenceNumberForMsnTracking: number | undefined;
+    private msnTrackingTimestamp: number = 0;
+    // To track round trip time for every %500 client message.
     private clientSequenceNumberForLatencyStatistics: number | undefined;
 
+    private opProcessingTimes: Partial<IOpPerfTimings> = {};
+
+    // Performance Data to be reported for ops round trips and processing.
+    private opPerfData: Partial<IOpPerfTelemetryProperties> = {};
+
     private firstConnection = true;
     private connectionOpSeqNumber: number | undefined;
     private readonly bootTime = performance.now();
@@ -61,12 +96,49 @@ class OpPerfTelemetry {
             }
         });
         this.deltaManager.on("disconnect", () => {
-            this.opSendTimeForLatencyStatisticsForMsnStatistics = undefined;
+            this.sequenceNumberForMsnTracking = undefined;
             this.clientSequenceNumberForLatencyStatistics = undefined;
+            this.opProcessingTimes = {};
+            this.opPerfData = {};
             this.connectionOpSeqNumber = undefined;
             this.firstConnection = false;
         });
 
+        this.deltaManager.outbound.on("push", (messages) => {
+            for (const msg of messages) {
+                if (msg.type === MessageType.Operation &&
+                    this.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber) {
+                    assert(this.opProcessingTimes.opStartTimeSittingInboundQueue === undefined,
+                        0x2c8 /* "opStartTimeSittingInboundQueue should be undefined" */);
+                    assert(this.opPerfData.durationInboundQueue === undefined,
+                        0x2c9 /* "durationInboundQueue should be undefined" */);
+                    this.opProcessingTimes.opStartTimeSittingInboundQueue = Date.now();
+
+                    assert(this.opPerfData.durationOutboundQueue === undefined,
+                        0x2ca /* "durationOutboundQueue should be undefined" */);
+
+                    assert(this.opProcessingTimes.opStartTimeForLatencyStatistics !== undefined,
+                        0x2cb /* "opStartTimeForLatencyStatistics should be undefined" */);
+
+                    this.opPerfData.durationOutboundQueue = this.opProcessingTimes.opStartTimeSittingInboundQueue
+                        - this.opProcessingTimes.opStartTimeForLatencyStatistics;
+                }
+            }
+        });
+
+        this.deltaManager.inbound.on("push", (message: ISequencedDocumentMessage) => {
+            if (this.clientId === message.clientId &&
+                message.type === MessageType.Operation &&
+                this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber &&
+                this.opProcessingTimes.opStartTimeSittingInboundQueue !== undefined) {
+                this.opProcessingTimes.opStartTimeInboundPushEvent = Date.now();
+                this.opPerfData.durationInboundQueue = this.opProcessingTimes.opStartTimeInboundPushEvent
+                                     - this.opProcessingTimes.opStartTimeSittingInboundQueue;
+                this.opProcessingTimes.opStartTimeSittingInboundQueue = undefined;
+                this.opPerfData.lenghtInboundQueue = this.deltaManager.inbound.length;
+            }
+        });
+
         this.deltaManager.inbound.on("idle", (count: number, duration: number) => {
             // Do not want to log zero for sure.
             // We are more interested in aggregates, so logging only if we are processing some number of ops
@@ -113,8 +185,13 @@ class OpPerfTelemetry {
 
     private beforeOpSubmit(message: IDocumentMessage) {
         // start with first client op and measure latency every 500 client ops
-        if (this.clientSequenceNumberForLatencyStatistics === undefined && message.clientSequenceNumber % 500 === 1) {
-            this.opSendTimeForLatencyStatistics = Date.now();
+        if (this.clientSequenceNumberForLatencyStatistics === undefined &&
+            message.clientSequenceNumber % 500 === 1) {
+            assert(this.opProcessingTimes.opStartTimeSittingInboundQueue === undefined,
+                0x2cc /* "OpTimeSittingInboundQueue should be undefined" */);
+            assert(this.opPerfData.durationInboundQueue === undefined,
+                0x2cd /* "durationInboundQueue should be undefined" */);
+            this.opProcessingTimes.opStartTimeForLatencyStatistics = Date.now();
             this.clientSequenceNumberForLatencyStatistics = message.clientSequenceNumber;
         }
     }
@@ -127,24 +204,35 @@ class OpPerfTelemetry {
         }
 
         // Record collab window max size after every 1000th op.
-        if (sequenceNumber % 1000 === 0) {
-            if (this.opSendTimeForLatencyStatisticsForMsnStatistics !== undefined) {
-                this.logger.sendPerformanceEvent({
-                    eventName: "MsnStatistics",
-                    sequenceNumber,
-                    msnDistance: this.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,
-                    duration: message.timestamp - this.opSendTimeForLatencyStatisticsForMsnStatistics,
-                });
-            }
-            this.opSendTimeForLatencyStatisticsForMsnStatistics = message.timestamp;
+        if (this.sequenceNumberForMsnTracking === undefined && sequenceNumber % 1000 === 0) {
+            this.sequenceNumberForMsnTracking = sequenceNumber;
+            this.msnTrackingTimestamp = message.timestamp;
+        }
+        if (this.sequenceNumberForMsnTracking !== undefined &&
+                message.minimumSequenceNumber >= this.sequenceNumberForMsnTracking) {
+            assert(this.msnTrackingTimestamp !== undefined,
+                0x2ce /* "msnTrackingTimestamp should not be undefined" */);
+            this.logger.sendPerformanceEvent({
+                eventName: "MsnStatistics",
+                sequenceNumber,
+                msnDistance: sequenceNumber - this.sequenceNumberForMsnTracking,
+                duration: message.timestamp - this.msnTrackingTimestamp,
+            });
+            this.sequenceNumberForMsnTracking = undefined;
         }
 
         if (this.clientId === message.clientId &&
             this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber) {
-            assert(this.opSendTimeForLatencyStatistics !== undefined,
+            assert(this.opProcessingTimes.opStartTimeForLatencyStatistics !== undefined,
                 0x120 /* "Undefined latency statistics (op send time)" */);
+            const currentTime = Date.now();
+
+            if (this.opProcessingTimes.opStartTimeInboundPushEvent !== undefined) {
+                this.opPerfData.durationInboundToProcessing = currentTime
+                - this.opProcessingTimes.opStartTimeInboundPushEvent;
+            }
 
-            const duration = Date.now() - this.opSendTimeForLatencyStatistics;
+            const duration = currentTime - this.opProcessingTimes.opStartTimeForLatencyStatistics;
 
             // One of the core expectations for Fluid service is to be fast.
             // When it's not the case, we want to learn about it and be able to investigate, so
@@ -162,8 +250,11 @@ class OpPerfTelemetry {
                 duration,
                 category,
                 pingLatency: this.pingLatency,
+                msnDistance: this.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,
+                ...this.opPerfData,
             });
             this.clientSequenceNumberForLatencyStatistics = undefined;
+            this.opPerfData = {};
         }
     }
 }

package/src/containerRuntime.ts

@@ -10,7 +10,6 @@ import {
     FluidObject,
     IFluidHandle,
     IFluidHandleContext,
-    IFluidObject,
     IFluidRouter,
     IRequest,
     IResponse,
@@ -91,7 +90,6 @@ import {
 import {
     addBlobToSummary,
     addTreeToSummary,
-    convertToSummaryTree,
     createRootSummarizerNodeWithGC,
     IRootSummarizerNodeWithGC,
     RequestParser,
@@ -100,7 +98,9 @@ import {
     requestFluidObject,
     responseToException,
     seqFromTree,
+    calculateStats,
 } from "@fluidframework/runtime-utils";
+import { GCDataBuilder } from "@fluidframework/garbage-collector";
 import { v4 as uuid } from "uuid";
 import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
@@ -141,6 +141,7 @@ import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
 import {
     GarbageCollector,
+    GCNodeType,
     gcTreeKey,
     IGarbageCollectionRuntime,
     IGarbageCollector,
@@ -152,6 +153,7 @@ import {
     isDataStoreAliasMessage,
 } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
+import { OpTracker } from "./opTelemetry";
 
 export enum ContainerMessageType {
     // An op to be delivered to store
@@ -277,8 +279,8 @@ export interface ISummaryRuntimeOptions {
  * Options for container runtime.
  */
 export interface IContainerRuntimeOptions {
-    summaryOptions?: ISummaryRuntimeOptions;
-    gcOptions?: IGCRuntimeOptions;
+    readonly summaryOptions?: ISummaryRuntimeOptions;
+    readonly gcOptions?: IGCRuntimeOptions;
     /**
      * Affects the behavior while loading the runtime when the data verification check which
      * compares the DeltaManager sequence number (obtained from protocol in summary) to the
@@ -287,13 +289,20 @@ export interface IContainerRuntimeOptions {
      * 2. "log" will log an error event to telemetry, but still continue to load.
      * 3. "bypass" will skip the check entirely. This is not recommended.
      */
-    loadSequenceNumberVerification?: "close" | "log" | "bypass";
+    readonly loadSequenceNumberVerification?: "close" | "log" | "bypass";
     /**
      * Should the runtime use data store aliasing for creating root datastores.
      * In case of aliasing conflicts, the runtime will raise an exception which does
      * not effect the status of the container.
      */
-    useDataStoreAliasing?: boolean;
+    readonly useDataStoreAliasing?: boolean;
+    /**
+     * Sets the flush mode for the runtime. In Immediate flush mode the runtime will immediately
+     * send all operations to the driver layer, while in TurnBased the operations will be buffered
+     * and then sent them as a single batch at the end of the turn.
+     * By default, flush mode is TurnBased.
+     */
+    readonly flushMode?: FlushMode;
 }
 
 type IRuntimeMessageMetadata = undefined | {
@@ -347,6 +356,13 @@ const maxOpSizeInBytesKey = "Fluid.ContainerRuntime.MaxOpSizeInBytes";
 // to not reach the 1MB limits in socket.io and Kafka.
 const defaultMaxOpSizeInBytes = 768000;
 
+// By default, the size of the contents for the incoming ops is tracked.
+// However, in certain situations, this may incur a performance hit.
+// The feature-gate below can be used to disable this feature.
+const disableOpTrackingKey = "Fluid.ContainerRuntime.DisableOpTracking";
+
+const defaultFlushMode = FlushMode.TurnBased;
+
 export enum RuntimeMessage {
     FluidDataStoreOp = "component",
     Attach = "attach",
@@ -394,6 +410,7 @@ class ScheduleManagerCore {
     private currentBatchClientId: string | undefined;
     private localPaused = false;
     private timePaused = 0;
+    private batchCount = 0;
 
     constructor(
         private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
@@ -482,14 +499,30 @@ class ScheduleManagerCore {
         this.deltaManager.inbound.pause();
     }
 
-    private resumeQueue(startBatch: number, endBatch: number) {
+    private resumeQueue(startBatch: number, messageEndBatch: ISequencedDocumentMessage) {
+        const endBatch = messageEndBatch.sequenceNumber;
+        const duration = performance.now() - this.timePaused;
+
+        this.batchCount++;
+        if (this.batchCount % 1000 === 1) {
+            this.logger.sendTelemetryEvent({
+                eventName: "BatchStats",
+                sequenceNumber: endBatch,
+                length: endBatch - startBatch + 1,
+                msnDistance: endBatch - messageEndBatch.minimumSequenceNumber,
+                duration,
+                batchCount: this.batchCount,
+                interrupted: this.localPaused,
+            });
+        }
+
         // Return early if no change in value
         if (!this.localPaused) {
             return;
         }
 
         this.localPaused = false;
-        const duration = performance.now() - this.timePaused;
+
         // Random round number - we want to know when batch waiting paused op processing.
         if (duration > latencyThreshold) {
             this.logger.sendErrorEvent({
@@ -564,7 +597,7 @@ class ScheduleManagerCore {
         } else if (batchMetadata === false) {
             assert(this.pauseSequenceNumber !== undefined, 0x2a0 /* "batch presence was validated above" */);
             // Batch is complete, we can process it!
-            this.resumeQueue(this.pauseSequenceNumber, message.sequenceNumber);
+            this.resumeQueue(this.pauseSequenceNumber, message);
             this.pauseSequenceNumber = undefined;
             this.currentBatchClientId = undefined;
         } else {
@@ -605,7 +638,7 @@ export class ScheduleManager {
 
             // This could be the beginning of a new batch or an individual message.
             this.emitter.emit("batchBegin", message);
-            this.deltaScheduler.batchBegin();
+            this.deltaScheduler.batchBegin(message);
 
             const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
             if (batch) {
@@ -626,7 +659,7 @@ export class ScheduleManager {
             this.hitError = true;
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", error, message);
-            this.deltaScheduler.batchEnd();
+            this.deltaScheduler.batchEnd(message);
             return;
         }
 
@@ -636,7 +669,7 @@ export class ScheduleManager {
         if (this.batchClientId === undefined || batch === false) {
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", undefined, message);
-            this.deltaScheduler.batchEnd();
+            this.deltaScheduler.batchEnd(message);
             return;
         }
     }
@@ -710,6 +743,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             gcOptions = {},
             loadSequenceNumberVerification = "close",
             useDataStoreAliasing = false,
+            flushMode = defaultFlushMode,
         } = runtimeOptions;
 
         // We pack at data store level only. If isolated channels are disabled,
@@ -805,6 +839,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 gcOptions,
                 loadSequenceNumberVerification,
                 useDataStoreAliasing,
+                flushMode,
             },
             containerScope,
             logger,
@@ -873,7 +908,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         return this._flushMode;
     }
 
-    public get scope(): IFluidObject & FluidObject {
+    public get scope(): FluidObject {
         return this.containerScope;
     }
 
@@ -909,7 +944,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
     private readonly defaultMaxConsecutiveReconnects = 15;
 
     private _orderSequentiallyCalls: number = 0;
-    private _flushMode: FlushMode = FlushMode.TurnBased;
+    private _flushMode: FlushMode;
     private needsFlush = false;
     private flushTrigger = false;
 
@@ -983,6 +1018,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
     private readonly createContainerMetadata: ICreateContainerMetadata;
     private summaryCount: number | undefined;
+    private readonly opTracker: OpTracker;
 
     private constructor(
         private readonly context: IContainerContext,
@@ -1037,16 +1073,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         this.maxConsecutiveReconnects =
             this.mc.config.getNumber(maxConsecutiveReconnectsKey) ?? this.defaultMaxConsecutiveReconnects;
 
+        this._flushMode = runtimeOptions.flushMode;
         this.garbageCollector = GarbageCollector.create(
             this,
             this.runtimeOptions.gcOptions,
-            (unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
-            (nodePath: string) => this.dataStores.getNodePackagePath(nodePath),
-            /**
-             * Returns the timestamp of the last message seen by this client. This is used by garbage collector as
-             * the current reference timestamp for tracking unreferenced objects.
-             */
-            () => this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp,
+            (nodePath: string) => this.getGCNodePackagePath(nodePath),
             () => this.messageAtLastSummary?.timestamp,
             context.baseSnapshot,
             async <T>(id: string) => readAndParse<T>(this.storage, id),
@@ -1098,7 +1129,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 ),
             (id: string) => this.summarizerNode.deleteChild(id),
             this.mc.logger,
-            async () => this.garbageCollector.getDataStoreBaseGCDetails(),
+            async () => this.garbageCollector.getBaseGCDetails(),
             (path: string, timestampMs: number, packagePath?: readonly string[]) => this.garbageCollector.nodeUpdated(
                 path,
                 "Changed",
@@ -1277,6 +1308,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
         ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
         BindBatchTracker(this, this.logger);
+        this.opTracker = new OpTracker(this.deltaManager, this.mc.config.getBoolean(disableOpTrackingKey) === true);
     }
 
     public dispose(error?: Error): void {
@@ -1446,13 +1478,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
             addBlobToSummary(summaryTree, electedSummarizerBlobName, electedSummarizerContent);
         }
-        const snapshot = this.blobManager.snapshot();
 
+        const summary = this.blobManager.summarize();
         // Some storage (like git) doesn't allow empty tree, so we can omit it.
         // and the blob manager can handle the tree not existing when loading
-        if (snapshot.entries.length !== 0) {
-            const blobsTree = convertToSummaryTree(snapshot, false);
-            addTreeToSummary(summaryTree, blobsTreeName, blobsTree);
+        if (Object.keys(summary.summary.tree).length > 0) {
+            addTreeToSummary(summaryTree, blobsTreeName, summary);
         }
 
         if (this.garbageCollector.writeDataAtRoot) {
@@ -1700,6 +1731,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             return;
         }
 
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "FlushMode Updated",
+            old: this._flushMode,
+            new: mode,
+        });
+
         // Flush any pending batches if switching to immediate
         if (mode === FlushMode.Immediate) {
             this.flush();
@@ -1791,7 +1828,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
      */
     private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
         const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
-        fluidDataStore.bindToContext();
+        // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
+        // older versions, we still have to call bindToContext.
+        if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
+            fluidDataStore.makeVisibleAndAttachGraph();
+        } else {
+            fluidDataStore.bindToContext();
+        }
         return fluidDataStore;
     }
 
@@ -1863,7 +1906,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
             Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
         if (isRoot) {
-            fluidDataStore.bindToContext();
+            // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
+            // For older versions, we still have to call bindToContext.
+            if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
+                fluidDataStore.makeVisibleAndAttachGraph();
+            } else {
+                fluidDataStore.bindToContext();
+            }
+            this.logger.sendTelemetryEvent({
+                eventName: "Root datastore with props",
+                hasProps: props !== undefined,
+            });
         }
         return channelToDataStore(fluidDataStore, id, this, this.dataStores, this.mc.logger);
     }
@@ -2041,11 +2094,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
         }
 
-        const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
-        assert(summarizeResult.summary.type === SummaryType.Tree,
+        const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState);
+
+        assert(summary.type === SummaryType.Tree,
             0x12f /* "Container Runtime's summarize should always return a tree" */);
 
-        return { ...summarizeResult, gcStats } as IRootSummaryTreeWithStats;
+        return { stats, summary, gcStats };
     }
 
     /**
@@ -2064,7 +2118,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
      * @param fullGC - true to bypass optimizations and force full generation of GC data.
      */
     public async getGCData(fullGC?: boolean): Promise<IGarbageCollectionData> {
-        return this.dataStores.getGCData(fullGC);
+        const builder = new GCDataBuilder();
+        const dsGCData = await this.dataStores.getGCData(fullGC);
+        builder.addNodes(dsGCData.gcNodes);
+
+        const blobsGCData = this.blobManager.getGCData(fullGC);
+        builder.addNodes(blobsGCData.gcNodes);
+        return builder.getGCData();
     }
 
     /**
@@ -2080,7 +2140,83 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
 
-        return this.dataStores.updateUsedRoutes(usedRoutes, gcTimestamp);
+        const dataStoreUsedRoutes: string[] = [];
+        for (const route of usedRoutes) {
+            if (route.split("/")[1] !== BlobManager.basePath) {
+                dataStoreUsedRoutes.push(route);
+            }
+        }
+
+        return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes, gcTimestamp);
+    }
+
+    /**
+     * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
+     * scenarios with accessing deleted content.
+     * @param unusedRoutes - The routes that are unused in all data stores in this Container.
+     */
+    public deleteUnusedRoutes(unusedRoutes: string[]) {
+        const blobManagerUnusedRoutes: string[] = [];
+        const dataStoreUnusedRoutes: string[] = [];
+        for (const route of unusedRoutes) {
+            if (this.isBlobPath(route)) {
+                blobManagerUnusedRoutes.push(route);
+            } else {
+                dataStoreUnusedRoutes.push(route);
+            }
+        }
+
+        this.blobManager.deleteUnusedRoutes(blobManagerUnusedRoutes);
+        this.dataStores.deleteUnusedRoutes(dataStoreUnusedRoutes);
+    }
+
+    /**
+     * Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
+     */
+    public getCurrentReferenceTimestampMs(): number | undefined {
+        // Use the timestamp of the last message seen by this client as that is server generated. If no messages have
+        // been processed, use the timestamp of the message from the last summary.
+        return this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp;
+    }
+
+    /**
+     * Returns the type of the GC node. Currently, there are nodes that belong to data store and nodes that belong
+     * to the blob manager.
+     */
+    public getNodeType(nodePath: string): GCNodeType {
+        if (this.isBlobPath(nodePath)) {
+            return GCNodeType.Blob;
+        }
+        if (this.dataStores.isDataStoreNode(nodePath)) {
+            return GCNodeType.DataStore;
+        }
+        // Root node ("/") and DDS nodes belong to "Other" node types.
+        return GCNodeType.Other;
+    }
+
+    /**
+     * Called by GC to retrieve the package path of the node with the given path. The node should belong to a
+     * data store or an attachment blob.
+     */
+    public getGCNodePackagePath(nodePath: string): readonly string[] | undefined {
+        // If the node is a blob, return "_blobs" as the package path.
+        if (this.isBlobPath(nodePath)) {
+            return ["_blobs"];
+        }
+        const dataStorePkgPath = this.dataStores.getDataStorePackagePath(nodePath);
+        assert(dataStorePkgPath !== undefined, 0x2d6 /* "Package path requested for unknown node type." */);
+        return dataStorePkgPath;
+    }
+
+    /**
+     * Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
+     */
+    private isBlobPath(path: string): boolean {
+        const pathParts = path.split("/");
+        if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
+            return false;
+        }
+        return true;
     }
 
     /**
@@ -2144,6 +2280,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             await this.deltaManager.inbound.pause();
 
             const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
+            const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
 
             // We should be here is we haven't processed be here. If we are of if the last message's sequence number
@@ -2188,7 +2325,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
 
             let continueResult = checkContinue();
             if (!continueResult.continue) {
-                return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error: continueResult.error };
+                return {
+                    stage: "base",
+                    referenceSequenceNumber: summaryRefSeqNum,
+                    minimumSequenceNumber,
+                    error: continueResult.error,
+                };
             }
 
             // increment summary count
@@ -2211,7 +2353,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                     runGC: this.garbageCollector.shouldRunGC,
                 });
             } catch (error) {
-                return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error };
+                return {
+                    stage: "base",
+                    referenceSequenceNumber: summaryRefSeqNum,
+                    minimumSequenceNumber,
+                    error,
+                };
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
 
@@ -2226,15 +2373,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter(
                 (value) => value.type === SummaryType.Handle).length;
+            const gcSummaryTreeStats = summaryTree.tree[gcTreeKey]
+                ? calculateStats((summaryTree.tree[gcTreeKey] as ISummaryTree))
+                : undefined;
 
             const summaryStats: IGeneratedSummaryStats = {
                 dataStoreCount: this.dataStores.size,
                 summarizedDataStoreCount: this.dataStores.size - handleCount,
                 gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
+                gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
+                gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
+                opsSizesSinceLastSummary: this.opTracker.opsSizeAccumulator,
+                nonSystemOpsSinceLastSummary: this.opTracker.nonSystemOpCount,
                 ...partialStats,
             };
             const generateSummaryData = {
                 referenceSequenceNumber: summaryRefSeqNum,
+                minimumSequenceNumber,
                 summaryTree,
                 summaryStats,
                 generateDuration: trace.trace().duration,
@@ -2301,7 +2456,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             } as const;
 
             this.summarizerNode.completeSummary(handle);
-
+            this.opTracker.reset();
             return submitData;
         } finally {
             // Cleanup wip summary in case of failure