@fluidframework/container-runtime 0.58.3000-61081 → 0.59.2000-61729

package/src/containerRuntime.ts

@@ -10,7 +10,6 @@ import {
     FluidObject,
     IFluidHandle,
     IFluidHandleContext,
-    IFluidObject,
     IFluidRouter,
     IRequest,
     IResponse,
@@ -101,6 +100,7 @@ import {
     seqFromTree,
     calculateStats,
 } from "@fluidframework/runtime-utils";
+import { GCDataBuilder } from "@fluidframework/garbage-collector";
 import { v4 as uuid } from "uuid";
 import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
@@ -141,6 +141,7 @@ import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
 import {
     GarbageCollector,
+    GCNodeType,
     gcTreeKey,
     IGarbageCollectionRuntime,
     IGarbageCollector,
@@ -637,7 +638,7 @@ export class ScheduleManager {
 
             // This could be the beginning of a new batch or an individual message.
             this.emitter.emit("batchBegin", message);
-            this.deltaScheduler.batchBegin();
+            this.deltaScheduler.batchBegin(message);
 
             const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
             if (batch) {
@@ -658,7 +659,7 @@ export class ScheduleManager {
             this.hitError = true;
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", error, message);
-            this.deltaScheduler.batchEnd();
+            this.deltaScheduler.batchEnd(message);
             return;
         }
 
@@ -668,7 +669,7 @@ export class ScheduleManager {
         if (this.batchClientId === undefined || batch === false) {
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", undefined, message);
-            this.deltaScheduler.batchEnd();
+            this.deltaScheduler.batchEnd(message);
             return;
         }
     }
@@ -907,7 +908,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         return this._flushMode;
     }
 
-    public get scope(): IFluidObject & FluidObject {
+    public get scope(): FluidObject {
         return this.containerScope;
     }
 
@@ -1076,13 +1077,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         this.garbageCollector = GarbageCollector.create(
             this,
             this.runtimeOptions.gcOptions,
-            (unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
-            (nodePath: string) => this.dataStores.getNodePackagePath(nodePath),
-            /**
-             * Returns the timestamp of the last message seen by this client. This is used by garbage collector as
-             * the current reference timestamp for tracking unreferenced objects.
-             */
-            () => this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp,
+            (nodePath: string) => this.getGCNodePackagePath(nodePath),
             () => this.messageAtLastSummary?.timestamp,
             context.baseSnapshot,
             async <T>(id: string) => readAndParse<T>(this.storage, id),
@@ -1134,7 +1129,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 ),
             (id: string) => this.summarizerNode.deleteChild(id),
             this.mc.logger,
-            async () => this.garbageCollector.getDataStoreBaseGCDetails(),
+            async () => this.garbageCollector.getBaseGCDetails(),
             (path: string, timestampMs: number, packagePath?: readonly string[]) => this.garbageCollector.nodeUpdated(
                 path,
                 "Changed",
@@ -1833,7 +1828,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
      */
     private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
         const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
-        fluidDataStore.bindToContext();
+        // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
+        // older versions, we still have to call bindToContext.
+        if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
+            fluidDataStore.makeVisibleAndAttachGraph();
+        } else {
+            fluidDataStore.bindToContext();
+        }
         return fluidDataStore;
     }
 
@@ -1905,7 +1906,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
             Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
         if (isRoot) {
-            fluidDataStore.bindToContext();
+            // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
+            // For older versions, we still have to call bindToContext.
+            if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
+                fluidDataStore.makeVisibleAndAttachGraph();
+            } else {
+                fluidDataStore.bindToContext();
+            }
             this.logger.sendTelemetryEvent({
                 eventName: "Root datastore with props",
                 hasProps: props !== undefined,
@@ -2111,7 +2118,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
      * @param fullGC - true to bypass optimizations and force full generation of GC data.
      */
     public async getGCData(fullGC?: boolean): Promise<IGarbageCollectionData> {
-        return this.dataStores.getGCData(fullGC);
+        const builder = new GCDataBuilder();
+        const dsGCData = await this.dataStores.getGCData(fullGC);
+        builder.addNodes(dsGCData.gcNodes);
+
+        const blobsGCData = this.blobManager.getGCData(fullGC);
+        builder.addNodes(blobsGCData.gcNodes);
+        return builder.getGCData();
     }
 
     /**
@@ -2127,7 +2140,83 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
 
-        return this.dataStores.updateUsedRoutes(usedRoutes, gcTimestamp);
+        const dataStoreUsedRoutes: string[] = [];
+        for (const route of usedRoutes) {
+            if (route.split("/")[1] !== BlobManager.basePath) {
+                dataStoreUsedRoutes.push(route);
+            }
+        }
+
+        return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes, gcTimestamp);
+    }
+
+    /**
+     * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
+     * scenarios with accessing deleted content.
+     * @param unusedRoutes - The routes that are unused in all data stores in this Container.
+     */
+    public deleteUnusedRoutes(unusedRoutes: string[]) {
+        const blobManagerUnusedRoutes: string[] = [];
+        const dataStoreUnusedRoutes: string[] = [];
+        for (const route of unusedRoutes) {
+            if (this.isBlobPath(route)) {
+                blobManagerUnusedRoutes.push(route);
+            } else {
+                dataStoreUnusedRoutes.push(route);
+            }
+        }
+
+        this.blobManager.deleteUnusedRoutes(blobManagerUnusedRoutes);
+        this.dataStores.deleteUnusedRoutes(dataStoreUnusedRoutes);
+    }
+
+    /**
+     * Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
+     */
+    public getCurrentReferenceTimestampMs(): number | undefined {
+        // Use the timestamp of the last message seen by this client as that is server generated. If no messages have
+        // been processed, use the timestamp of the message from the last summary.
+        return this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp;
+    }
+
+    /**
+     * Returns the type of the GC node. Currently, there are nodes that belong to data store and nodes that belong
+     * to the blob manager.
+     */
+    public getNodeType(nodePath: string): GCNodeType {
+        if (this.isBlobPath(nodePath)) {
+            return GCNodeType.Blob;
+        }
+        if (this.dataStores.isDataStoreNode(nodePath)) {
+            return GCNodeType.DataStore;
+        }
+        // Root node ("/") and DDS nodes belong to "Other" node types.
+        return GCNodeType.Other;
+    }
+
+    /**
+     * Called by GC to retrieve the package path of the node with the given path. The node should belong to a
+     * data store or an attachment blob.
+     */
+    public getGCNodePackagePath(nodePath: string): readonly string[] | undefined {
+        // If the node is a blob, return "_blobs" as the package path.
+        if (this.isBlobPath(nodePath)) {
+            return ["_blobs"];
+        }
+        const dataStorePkgPath = this.dataStores.getDataStorePackagePath(nodePath);
+        assert(dataStorePkgPath !== undefined, 0x2d6 /* "Package path requested for unknown node type." */);
+        return dataStorePkgPath;
+    }
+
+    /**
+     * Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
+     */
+    private isBlobPath(path: string): boolean {
+        const pathParts = path.split("/");
+        if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
+            return false;
+        }
+        return true;
     }
 
     /**

package/src/dataStore.ts

@@ -75,7 +75,13 @@ class DataStore implements IDataStore {
             alias,
         };
 
-        this.fluidDataStoreChannel.bindToContext();
+        // back-compat 0.58.2000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
+        // older versions, we still have to call bindToContext.
+        if (this.fluidDataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
+            this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
+        } else {
+            this.fluidDataStoreChannel.bindToContext();
+        }
 
         if (this.runtime.attachState === AttachState.Detached) {
             const localResult = this.datastores.processAliasMessageCore(message);

package/src/dataStoreContext.ts

@@ -132,7 +132,7 @@ export interface ILocalFluidDataStoreContextProps extends IFluidDataStoreContext
     readonly pkg: Readonly<string[]> | undefined;
     readonly snapshotTree: ISnapshotTree | undefined;
     readonly isRootDataStore: boolean | undefined;
-    readonly bindChannelFn: (channel: IFluidDataStoreChannel) => void;
+    readonly makeLocallyVisibleFn: () => void;
     /**
      * @deprecated 0.16 Issue #1635, #3631
      */
@@ -261,7 +261,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         private readonly existing: boolean,
         private bindState: BindState,
         public readonly isLocalDataStore: boolean,
-        bindChannelFn: (channel: IFluidDataStoreChannel) => void,
+        private readonly makeLocallyVisibleFn: () => void,
     ) {
         super();
 
@@ -284,7 +284,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
             assert(this.bindState === BindState.NotBound, 0x13b /* "datastore context is already in bound state" */);
             this.bindState = BindState.Binding;
             assert(this.channel !== undefined, 0x13c /* "undefined channel on datastore context" */);
-            bindChannelFn(this.channel);
+            this.makeLocallyVisible();
             this.bindState = BindState.Bound;
         };
 
@@ -627,6 +627,15 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
         return this._containerRuntime.submitDataStoreSignal(this.id, type, content);
     }
 
+    /**
+     * This is called by the data store channel when it becomes locally visible indicating that it is ready to become
+     * globally visible now.
+     */
+    public makeLocallyVisible() {
+        assert(this.channel !== undefined, 0x2cf /* "undefined channel on datastore context" */);
+        this.makeLocallyVisibleFn();
+    }
+
     protected bindRuntime(channel: IFluidDataStoreChannel) {
         if (this.channel) {
             throw new Error("Runtime already bound");
@@ -864,7 +873,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
             props.snapshotTree !== undefined ? true : false /* existing */,
             props.snapshotTree ? BindState.Bound : BindState.NotBound,
             true /* isLocalDataStore */,
-            props.bindChannelFn,
+            props.makeLocallyVisibleFn,
         );
 
         this.snapshotTree = props.snapshotTree;
@@ -993,7 +1002,7 @@ export class LocalDetachedFluidDataStoreContext
 
     public async attachRuntime(
         registry: IProvideFluidDataStoreFactory,
-        dataStoreRuntime: IFluidDataStoreChannel)
+        dataStoreChannel: IFluidDataStoreChannel)
     {
         assert(this.detachedRuntimeCreation, 0x154 /* "runtime creation is already attached" */);
         assert(this.channelDeferred === undefined, 0x155 /* "channel deferral is already set" */);
@@ -1009,10 +1018,16 @@ export class LocalDetachedFluidDataStoreContext
         this.detachedRuntimeCreation = false;
         this.channelDeferred = new Deferred<IFluidDataStoreChannel>();
 
-        super.bindRuntime(dataStoreRuntime);
+        super.bindRuntime(dataStoreChannel);
 
         if (await this.isRoot()) {
-            dataStoreRuntime.bindToContext();
+            // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
+            // For older versions, we still have to call bindToContext.
+            if (dataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
+                dataStoreChannel.makeVisibleAndAttachGraph();
+            } else {
+                dataStoreChannel.bindToContext();
+            }
         }
     }
 

package/src/dataStores.ts

@@ -18,7 +18,6 @@ import {
     CreateSummarizerNodeSource,
     IAttachMessage,
     IEnvelope,
-    IFluidDataStoreChannel,
     IFluidDataStoreContextDetached,
     IGarbageCollectionData,
     IGarbageCollectionDetailsBase,
@@ -157,7 +156,7 @@ export class DataStores implements IDisposable {
                         key,
                         { type: CreateSummarizerNodeSource.FromSummary },
                     ),
-                    bindChannelFn: (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr),
+                    makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(key),
                     snapshotTree,
                     isRootDataStore: undefined,
                     writeGCDataAtRoot: this.writeGCDataAtRoot,
@@ -291,14 +290,20 @@ export class DataStores implements IDisposable {
         return this.aliasMap.get(id) !== undefined || this.contexts.get(id) !== undefined;
     }
 
-    public bindFluidDataStore(fluidDataStoreRuntime: IFluidDataStoreChannel): void {
-        const id = fluidDataStoreRuntime.id;
+    /**
+     * Make the data stores locally visible in the container graph by moving the data store context from unbound to
+     * bound list. This data store can now be reached from the root.
+     * @param id - The id of the data store context to make visible.
+     */
+    private makeDataStoreLocallyVisible(id: string): void {
         const localContext = this.contexts.getUnbound(id);
         assert(!!localContext, 0x15f /* "Could not find unbound context to bind" */);
 
-        // If the container is detached, we don't need to send OP or add to pending attach because
-        // we will summarize it while uploading the create new summary and make it known to other
-        // clients.
+        /**
+         * If the container is not detached, it is globally visible to all clients. This data store should also be
+         * globally visible. Move it to attaching state and send an "attach" op for it.
+         * If the container is detached, this data store will be part of the summary that makes the container attached.
+         */
         if (this.runtime.attachState !== AttachState.Detached) {
             localContext.emit("attaching");
             const message = localContext.generateAttachMessage();
@@ -308,7 +313,7 @@ export class DataStores implements IDisposable {
             this.attachOpFiredForDataStore.add(id);
         }
 
-        this.contexts.bind(fluidDataStoreRuntime.id);
+        this.contexts.bind(id);
     }
 
     public createDetachedDataStoreCore(
@@ -326,7 +331,7 @@ export class DataStores implements IDisposable {
                 id,
                 { type: CreateSummarizerNodeSource.Local },
             ),
-            bindChannelFn: (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr),
+            makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: isRoot,
             writeGCDataAtRoot: this.writeGCDataAtRoot,
@@ -347,7 +352,7 @@ export class DataStores implements IDisposable {
                 id,
                 { type: CreateSummarizerNodeSource.Local },
             ),
-            bindChannelFn: (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr),
+            makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: isRoot,
             writeGCDataAtRoot: this.writeGCDataAtRoot,
@@ -587,7 +592,14 @@ export class DataStores implements IDisposable {
      */
     public deleteUnusedRoutes(unusedRoutes: string[]) {
         for (const route of unusedRoutes) {
-            const dataStoreId = route.split("/")[1];
+            const pathParts = route.split("/");
+            // Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data
+            // store based on its DDS being unused.
+            if (pathParts.length > 2) {
+                continue;
+            }
+            const dataStoreId = pathParts[1];
+            assert(this.contexts.has(dataStoreId), 0x2d7 /* `${dataStoreId} is not a data store` */);
             // Delete the contexts of unused data stores.
             this.contexts.delete(dataStoreId);
             // Delete the summarizer node of the unused data stores.
@@ -611,15 +623,24 @@ export class DataStores implements IDisposable {
     }
 
     /**
-     * Returns the package path of the node with the given path. This is used by GC to log when an inactive / deleted
-     * node is used.
+     * Called during GC to retrieve the package path of a data store node with the given path.
+     */
+    public getDataStorePackagePath(nodePath: string): readonly string[] | undefined {
+        // If the node belongs to a data store, return its package path if the data store is loaded. For DDSs, we return
+        // the package path of the data store that contains it.
+        const context = this.contexts.get(nodePath.split("/")[1]);
+        return context?.isLoaded ? context.packagePath : undefined;
+    }
+
+    /**
+     * Called by GC to know if a node is a data store or not. Data store ids are of the format "/dataStoreId".
      */
-    public getNodePackagePath(nodePath: string): readonly string[] | undefined {
-        // Currently, only return the data store package path for the node since GC is only interested in data stores.
-        const dataStoreId = nodePath.split("/")[1];
-        const context = this.contexts.get(dataStoreId);
-        assert(context !== undefined, 0x2b9 /* "Data store with given id does not exist" */);
-        return context.isLoaded ? context.packagePath : undefined;
+    public isDataStoreNode(nodePath: string): boolean {
+        const pathParts = nodePath.split("/");
+        if (pathParts.length === 2 && this.contexts.has(pathParts[1])) {
+            return true;
+        }
+        return false;
     }
 }
 

package/src/deltaScheduler.ts

@@ -11,6 +11,9 @@ import {
     ISequencedDocumentMessage,
 } from "@fluidframework/protocol-definitions";
 
+import {
+    TelemetryLogger,
+} from "@fluidframework/telemetry-utils";
 /**
  * DeltaScheduler is responsible for the scheduling of inbound delta queue in cases where there
  * is more than one op a particular run of the queue. It does not schedule if there is just one
@@ -25,18 +28,13 @@ import {
 export class DeltaScheduler {
     private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>;
     // The time for processing ops in a single turn.
-    public static readonly processingTime = 20;
+    public static readonly processingTime = 50;
 
     // The increase in time for processing ops after each turn.
     private readonly processingTimeIncrement = 10;
 
     private processingStartTime: number | undefined;
-    private totalProcessingTime: number = DeltaScheduler.processingTime;
-
-    // This keeps track of whether the delta scheduler is scheduling a particular run of the
-    // the inbound delta queue. Basically, every time the delta queue starts processing with
-    // more than one op, this will be set to true until the run completes.
-    private isScheduling: boolean = false;
+    private currentAllowedProcessingTimeForTurn: number = DeltaScheduler.processingTime;
 
     // This keeps track of the number of times inbound queue has been scheduled. After a particular
     // count, we log telemetry for the number of ops processed, the time and number of turns it took
@@ -44,9 +42,13 @@ export class DeltaScheduler {
     private schedulingCount: number = 0;
 
     private schedulingLog: {
-        numberOfOps: number;
+        opsRemainingToProcess: number;
         totalProcessingTime: number;
         numberOfTurns: number;
+        numberOfBatchesProcessed: number;
+        lastSequenceNumber: number;
+        firstSequenceNumber: number;
+        startTime: number;
     } | undefined;
 
     constructor(
@@ -57,50 +59,72 @@ export class DeltaScheduler {
         this.deltaManager.inbound.on("idle", () => { this.inboundQueueIdle(); });
     }
 
-    public batchBegin() {
+    public batchBegin(message: ISequencedDocumentMessage) {
         if (!this.processingStartTime) {
             this.processingStartTime = performance.now();
         }
+        if (this.schedulingLog === undefined && this.schedulingCount % 500 === 0) {
+            // Every 500th time we are scheduling the inbound queue, we log telemetry for the
+            // number of ops processed, the time and number of turns it took to process the ops.
+            this.schedulingLog = {
+                opsRemainingToProcess: 0,
+                numberOfTurns: 1,
+                totalProcessingTime: 0,
+                numberOfBatchesProcessed: 0,
+                firstSequenceNumber: message.sequenceNumber,
+                lastSequenceNumber: message.sequenceNumber,
+                startTime: performance.now(),
+            };
+        }
     }
 
-    public batchEnd() {
-        if (this.shouldRunScheduler()) {
-            if (!this.isScheduling) {
-                this.isScheduling = true;
-                // Every 2000th time we are scheduling the inbound queue, we log telemetry for the
-                // number of ops processed, the time and number of turns it took to process the ops.
-                if (this.schedulingCount % 2000 === 0) {
-                    this.schedulingLog = {
-                        numberOfOps: this.deltaManager.inbound.length,
-                        numberOfTurns: 1,
-                        totalProcessingTime: 0,
-                    };
-                }
-            }
+    public batchEnd(message: ISequencedDocumentMessage) {
+        if (this.schedulingLog) {
+            this.schedulingLog.numberOfBatchesProcessed++;
+            this.schedulingLog.lastSequenceNumber = message.sequenceNumber;
+            this.schedulingLog.opsRemainingToProcess = this.deltaManager.inbound.length;
+        }
 
+        if (this.shouldRunScheduler()) {
+            const currentTime = performance.now();
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            const elapsedTime = performance.now() - this.processingStartTime!;
-            if (elapsedTime > this.totalProcessingTime) {
+            const elapsedTime = currentTime - this.processingStartTime!;
+            if (elapsedTime > this.currentAllowedProcessingTimeForTurn) {
                 // We have processed ops for more than the total processing time. So, pause the
                 // queue, yield the thread and schedule a resume.
 
                 // eslint-disable-next-line @typescript-eslint/no-floating-promises
                 this.deltaManager.inbound.pause();
-                setTimeout(() => {
-                    this.deltaManager.inbound.resume();
-                });
 
-                this.processingStartTime = undefined;
-                // Increase the total processing time. Keep doing this after each turn until all the ops have
+                 // Increase the total processing time. Keep doing this after each turn until all the ops have
                 // been processed. This way we keep the responsiveness at the beginning while also making sure
                 // that all the ops process fairly quickly.
-                this.totalProcessingTime += this.processingTimeIncrement;
+                this.currentAllowedProcessingTimeForTurn += this.processingTimeIncrement;
 
                 // If we are logging the telemetry this time, update the telemetry log object.
                 if (this.schedulingLog) {
                     this.schedulingLog.numberOfTurns++;
                     this.schedulingLog.totalProcessingTime += elapsedTime;
                 }
+
+                setTimeout(() => {
+                    if (this.schedulingLog) {
+                        this.logger.sendTelemetryEvent({
+                            eventName: "InboundOpsPartialProcessingTime",
+                            duration: TelemetryLogger.formatTick(elapsedTime),
+                            opsProcessed: this.schedulingLog.lastSequenceNumber -
+                                this.schedulingLog.firstSequenceNumber + 1,
+                            opsRemainingToProcess: this.deltaManager.inbound.length,
+                            processingTime: TelemetryLogger.formatTick(this.schedulingLog.totalProcessingTime),
+                            numberOfTurns: this.schedulingLog.numberOfTurns,
+                            batchesProcessed: this.schedulingLog.numberOfBatchesProcessed,
+                            timeToResume: TelemetryLogger.formatTick(performance.now() - currentTime),
+                        });
+                    }
+                    this.deltaManager.inbound.resume();
+                });
+
+                this.processingStartTime = undefined;
             }
         }
     }
@@ -109,14 +133,19 @@ export class DeltaScheduler {
         if (this.schedulingLog) {
             // Add the time taken for processing the final ops to the total processing time in the
             // telemetry log object.
+            const currentTime = performance.now();
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            this.schedulingLog.totalProcessingTime += performance.now() - this.processingStartTime!;
+            this.schedulingLog.totalProcessingTime += currentTime - this.processingStartTime!;
 
             this.logger.sendTelemetryEvent({
                 eventName: "InboundOpsProcessingTime",
-                numberOfOps: this.schedulingLog.numberOfOps,
+                opsRemainingToProcess: this.schedulingLog.opsRemainingToProcess,
                 numberOfTurns: this.schedulingLog.numberOfTurns,
-                processingTime: this.schedulingLog.totalProcessingTime,
+                processingTime: TelemetryLogger.formatTick(this.schedulingLog.totalProcessingTime),
+                opsProcessed: this.schedulingLog.lastSequenceNumber - this.schedulingLog.firstSequenceNumber + 1,
+                batchesProcessed: this.schedulingLog.numberOfBatchesProcessed,
+                duration: TelemetryLogger.formatTick(currentTime - this.schedulingLog.startTime),
+                schedulingCount: this.schedulingCount,
             });
 
             this.schedulingLog = undefined;
@@ -124,14 +153,11 @@ export class DeltaScheduler {
 
         // If we scheduled this batch of the inbound queue, increment the counter that tracks the
         // number of times we have done this.
-        if (this.isScheduling) {
-            this.isScheduling = false;
-            this.schedulingCount++;
-        }
+        this.schedulingCount++;
 
         // Reset the processing times.
         this.processingStartTime = undefined;
-        this.totalProcessingTime = DeltaScheduler.processingTime;
+        this.currentAllowedProcessingTimeForTurn = DeltaScheduler.processingTime;
     }
 
     /**