@fluidframework/container-runtime 0.51.3 → 0.53.0-46105

package/lib/containerRuntime.js

@@ -3,21 +3,20 @@
  * Licensed under the MIT License.
  */
 import { AttachState, } from "@fluidframework/container-definitions";
-import { assert, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
+import { assert, Trace, TypedEventEmitter, unreachableCase, performance, } from "@fluidframework/common-utils";
 import { ChildLogger, raiseConnectedEvent, PerformanceEvent, normalizeError, TaggedLoggerAdapter, } from "@fluidframework/telemetry-utils";
 import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
-import { DataCorruptionError, GenericError } from "@fluidframework/container-utils";
-import { BlobTreeEntry, TreeTreeEntry, } from "@fluidframework/protocol-base";
-import { MessageType, } from "@fluidframework/protocol-definitions";
+import { DataCorruptionError, GenericError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
+import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
 import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, addTreeToSummary, convertToSummaryTree, createRootSummarizerNodeWithGC, FluidSerializer, RequestParser, create404Response, exceptionToResponse, responseToException, seqFromTree, } from "@fluidframework/runtime-utils";
+import { addBlobToSummary, addTreeToSummary, convertToSummaryTree, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, responseToException, seqFromTree, convertSummaryTreeToITree, } from "@fluidframework/runtime-utils";
 import { v4 as uuid } from "uuid";
 import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
 import { Summarizer } from "./summarizer";
 import { formRequestSummarizerFn, SummaryManager } from "./summaryManager";
 import { DeltaScheduler } from "./deltaScheduler";
-import { ReportOpPerfTelemetry } from "./connectionTelemetry";
+import { ReportOpPerfTelemetry, latencyThreshold } from "./connectionTelemetry";
 import { PendingStateManager } from "./pendingStateManager";
 import { pkgVersion } from "./packageVersion";
 import { BlobManager } from "./blobManager";
@@ -29,7 +28,7 @@ import { OrderedClientCollection, OrderedClientElection } from "./orderedClientE
 import { SummarizerClientElection, summarizerClientType } from "./summarizerClientElection";
 import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
-import { GarbageCollector, } from "./garbageCollection";
+import { GarbageCollector, gcTreeKey, } from "./garbageCollection";
 export var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -92,13 +91,16 @@ export function unpackRuntimeMessage(message) {
     }
     return message;
 }
-export class ScheduleManager {
-    constructor(deltaManager, emitter, logger) {
+/**
+ * This class controls pausing and resuming of inbound queue to ensure that we never
+ * start processing ops in a batch IF we do not have all ops in the batch.
+ */
+class ScheduleManagerCore {
+    constructor(deltaManager, logger) {
         this.deltaManager = deltaManager;
-        this.emitter = emitter;
         this.logger = logger;
         this.localPaused = false;
-        this.deltaScheduler = new DeltaScheduler(this.deltaManager, ChildLogger.create(this.logger, "DeltaScheduler"));
+        this.timePaused = 0;
         // Listen for delta manager sends and add batch metadata to messages
         this.deltaManager.on("prepareSend", (messages) => {
             if (messages.length === 0) {
@@ -106,7 +108,7 @@ export class ScheduleManager {
             }
             // First message will have the batch flag set to true if doing a batched send
             const firstMessageMetadata = messages[0].metadata;
-            if (!firstMessageMetadata || !firstMessageMetadata.batch) {
+            if (!(firstMessageMetadata === null || firstMessageMetadata === void 0 ? void 0 : firstMessageMetadata.batch)) {
                 return;
             }
             // If the batch contains only a single op, clear the batch flag.
@@ -121,29 +123,143 @@ export class ScheduleManager {
         // Listen for updates and peek at the inbound
         this.deltaManager.inbound.on("push", (message) => {
             this.trackPending(message);
-            this.updatePauseState(message.sequenceNumber);
         });
+        // Start with baseline - empty inbound queue.
+        assert(!this.localPaused, 0x293 /* "initial state" */);
         const allPending = this.deltaManager.inbound.toArray();
         for (const pending of allPending) {
             this.trackPending(pending);
         }
-        // Based on track pending update the pause state
-        this.updatePauseState(this.deltaManager.lastSequenceNumber);
     }
-    beginOperation(message) {
+    /**
+     * The only public function in this class - called when we processed an op,
+     * to make decision if op processing should be paused or not afer that.
+     */
+    afterOpProcessing(sequenceNumber) {
+        assert(!this.localPaused, 0x294 /* "can't have op processing paused if we are processing an op" */);
+        // If the inbound queue is ever empty, nothing to do!
+        if (this.deltaManager.inbound.length === 0) {
+            assert(this.pauseSequenceNumber === undefined, 0x295 /* "there should be no pending batch if we have no ops" */);
+            return;
+        }
+        // The queue is
+        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
+        //    - here (processing ops until reaching start of incomplete batch)
+        //    - in trackPending(), when queue was empty and start of batch showed up.
+        // 2. resumed when batch end comes in (in trackPending())
+        // do we have incomplete batch to worry about?
+        if (this.pauseSequenceNumber !== undefined) {
+            assert(sequenceNumber < this.pauseSequenceNumber, 0x296 /* "we should never start processing incomplete batch!" */);
+            // If the next op is the start of incomplete batch, then we can't process it until it's fully in - pause!
+            if (sequenceNumber + 1 === this.pauseSequenceNumber) {
+                this.pauseQueue();
+            }
+        }
+    }
+    pauseQueue() {
+        assert(!this.localPaused, 0x297 /* "always called from resumed state" */);
+        this.localPaused = true;
+        this.timePaused = performance.now();
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        this.deltaManager.inbound.pause();
+    }
+    resumeQueue(startBatch, endBatch) {
+        // Return early if no change in value
+        if (!this.localPaused) {
+            return;
+        }
+        this.localPaused = false;
+        const duration = performance.now() - this.timePaused;
+        // Random round number - we want to know when batch waiting paused op processing.
+        if (duration > latencyThreshold) {
+            this.logger.sendErrorEvent({
+                eventName: "MaxBatchWaitTimeExceeded",
+                duration,
+                sequenceNumber: endBatch,
+                length: endBatch - startBatch,
+            });
+        }
+        this.deltaManager.inbound.resume();
+    }
+    /**
+     * Called for each incoming op (i.e. inbound "push" notification)
+     */
+    trackPending(message) {
+        assert(this.deltaManager.inbound.length !== 0, 0x298 /* "we have something in the queue that generates this event" */);
+        assert((this.currentBatchClientId === undefined) === (this.pauseSequenceNumber === undefined), 0x299 /* "non-synchronized state" */);
+        const metadata = message.metadata;
+        const batchMetadata = metadata === null || metadata === void 0 ? void 0 : metadata.batch;
+        // Protocol messages are never part of a runtime batch of messages
+        if (!isRuntimeMessage(message)) {
+            // Protocol messages should never show up in the middle of the batch!
+            assert(this.currentBatchClientId === undefined, 0x29a /* "System message in the middle of batch!" */);
+            assert(batchMetadata === undefined, 0x29b /* "system op in a batch?" */);
+            assert(!this.localPaused, 0x29c /* "we should be processing ops when there is no active batch" */);
+            return;
+        }
+        if (this.currentBatchClientId === undefined && batchMetadata === undefined) {
+            assert(!this.localPaused, 0x29d /* "we should be processing ops when there is no active batch" */);
+            return;
+        }
+        // If the client ID changes then we can move the pause point. If it stayed the same then we need to check.
+        // If batchMetadata is not undefined then if it's true we've begun a new batch - if false we've ended
+        // the previous one
+        if (this.currentBatchClientId !== undefined || batchMetadata === false) {
+            if (this.currentBatchClientId !== message.clientId) {
+                // "Batch not closed, yet message from another client!"
+                throw new DataCorruptionError("OpBatchIncomplete", Object.assign({ batchClientId: this.currentBatchClientId }, extractSafePropertiesFromMessage(message)));
+            }
+        }
+        // The queue is
+        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
+        //    - in afterOpProcessing() - processing ops until reaching start of incomplete batch
+        //    - here (batchMetadata == false below), when queue was empty and start of batch showed up.
+        // 2. resumed when batch end comes in (batchMetadata === true case below)
+        if (batchMetadata) {
+            assert(this.currentBatchClientId === undefined, 0x29e /* "there can't be active batch" */);
+            assert(!this.localPaused, 0x29f /* "we should be processing ops when there is no active batch" */);
+            this.pauseSequenceNumber = message.sequenceNumber;
+            this.currentBatchClientId = message.clientId;
+            // Start of the batch
+            // Only pause processing if queue has no other ops!
+            // If there are any other ops in the queue, processing will be stopped when they are processed!
+            if (this.deltaManager.inbound.length === 1) {
+                this.pauseQueue();
+            }
+        }
+        else if (batchMetadata === false) {
+            assert(this.pauseSequenceNumber !== undefined, 0x2a0 /* "batch presence was validated above" */);
+            // Batch is complete, we can process it!
+            this.resumeQueue(this.pauseSequenceNumber, message.sequenceNumber);
+            this.pauseSequenceNumber = undefined;
+            this.currentBatchClientId = undefined;
+        }
+        else {
+            // Continuation of current batch. Do nothing
+            assert(this.currentBatchClientId !== undefined, 0x2a1 /* "logic error" */);
+        }
+    }
+}
+/**
+ * This class has the following responsibilities:
+ * 1. It tracks batches as we process ops and raises "batchBegin" and "batchEnd" events.
+ *    As part of it, it validates batch correctness (i.e. no system ops in the middle of batch)
+ * 2. It creates instance of ScheduleManagerCore that ensures we never start processing ops from batch
+ *    unless all ops of the batch are in.
+ */
+export class ScheduleManager {
+    constructor(deltaManager, emitter, logger) {
+        this.deltaManager = deltaManager;
+        this.emitter = emitter;
+        this.logger = logger;
+        this.hitError = false;
+        this.deltaScheduler = new DeltaScheduler(this.deltaManager, ChildLogger.create(this.logger, "DeltaScheduler"));
+        this.scheduler = new ScheduleManagerCore(deltaManager, logger);
+    }
+    beforeOpProcessing(message) {
         var _a;
         if (this.batchClientId !== message.clientId) {
-            // As a back stop for any bugs marking the end of a batch - if the client ID flipped, we
-            // consider the previous batch over.
-            if (this.batchClientId) {
-                this.emitter.emit("batchEnd", "Did not receive real batchEnd message", undefined);
-                this.deltaScheduler.batchEnd();
-                this.logger.sendTelemetryEvent({
-                    eventName: "BatchEndNotReceived",
-                    clientId: this.batchClientId,
-                    sequenceNumber: message.sequenceNumber,
-                });
-            }
+            assert(this.batchClientId === undefined, 0x2a2 /* "Batch is interrupted by other client op. Should be caught by trackPending()" */);
             // This could be the beginning of a new batch or an individual message.
             this.emitter.emit("batchBegin", message);
             this.deltaScheduler.batchBegin();
@@ -156,79 +272,32 @@ export class ScheduleManager {
             }
         }
     }
-    endOperation(error, message) {
+    afterOpProcessing(error, message) {
         var _a;
+        // If this is no longer true, we need to revisit what we do where we set this.hitError.
+        assert(!this.hitError, 0x2a3 /* "container should be closed on any error" */);
+        // Let the scheduler know how far we progressed, to decide if op processing
+        // should be paused or not.
+        this.scheduler.afterOpProcessing(message.sequenceNumber);
         if (error) {
+            // We assume here that loader will close container and stop processing all future ops.
+            // This is implicit dependency. If this flow changes, this code might no longer be correct.
+            this.hitError = true;
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", error, message);
             this.deltaScheduler.batchEnd();
             return;
         }
-        this.updatePauseState(message.sequenceNumber);
         const batch = (_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.batch;
         // If no batchClientId has been set then we're in an individual batch. Else, if we get
         // batch end metadata, this is end of the current batch.
-        if (!this.batchClientId || batch === false) {
+        if (this.batchClientId === undefined || batch === false) {
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", undefined, message);
             this.deltaScheduler.batchEnd();
             return;
         }
     }
-    setPaused(localPaused) {
-        // Return early if no change in value
-        if (this.localPaused === localPaused) {
-            return;
-        }
-        this.localPaused = localPaused;
-        if (localPaused) {
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-            this.deltaManager.inbound.pause();
-        }
-        else {
-            this.deltaManager.inbound.resume();
-        }
-    }
-    updatePauseState(sequenceNumber) {
-        // If the inbound queue is ever empty we pause it and wait for new events
-        if (this.deltaManager.inbound.length === 0) {
-            this.setPaused(true);
-            return;
-        }
-        // If no message has caused the pause flag to be set, or the next message up is not the one we need to pause at
-        // then we simply continue processing
-        if (!this.pauseSequenceNumber || sequenceNumber + 1 < this.pauseSequenceNumber) {
-            this.setPaused(false);
-        }
-        else {
-            // Otherwise the next message requires us to pause
-            this.setPaused(true);
-        }
-    }
-    trackPending(message) {
-        const metadata = message.metadata;
-        // Protocol messages are never part of a runtime batch of messages
-        if (!isRuntimeMessage(message)) {
-            this.pauseSequenceNumber = undefined;
-            this.pauseClientId = undefined;
-            return;
-        }
-        const batchMetadata = metadata ? metadata.batch : undefined;
-        // If the client ID changes then we can move the pause point. If it stayed the same then we need to check.
-        if (this.pauseClientId === message.clientId) {
-            if (batchMetadata !== undefined) {
-                // If batchMetadata is not undefined then if it's true we've begun a new batch - if false we've ended
-                // the previous one
-                this.pauseSequenceNumber = batchMetadata ? message.sequenceNumber : undefined;
-                this.pauseClientId = batchMetadata ? this.pauseClientId : undefined;
-            }
-        }
-        else {
-            // We check the batch flag for the new clientID - if true we pause otherwise we reset the tracking data
-            this.pauseSequenceNumber = batchMetadata ? message.sequenceNumber : undefined;
-            this.pauseClientId = batchMetadata ? message.clientId : undefined;
-        }
-    }
 }
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
@@ -236,6 +305,20 @@ export class ScheduleManager {
  * ContainerRuntime's perspective.
  */
 export const agentSchedulerId = "_scheduler";
+// safely check navigator and get the hardware spec value
+export function getDeviceSpec() {
+    try {
+        if (typeof navigator === "object" && navigator !== null) {
+            return {
+                deviceMemory: navigator.deviceMemory,
+                hardwareConcurrency: navigator.hardwareConcurrency,
+            };
+        }
+    }
+    catch (_a) {
+    }
+    return {};
+}
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
@@ -299,9 +382,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             else {
                 // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // generateSummaries is turned off. We are throwing instead of returning a failure here,
+                // disableSummaries is turned on. We are throwing instead of returning a failure here,
                 // because it is a misuse of the API rather than an expected failure.
-                throw new Error(`Can't summarize, generateSummaries: ${this.runtimeOptions.summaryOptions.generateSummaries}`);
+                throw new Error(`Can't summarize, disableSummaries: ${this.summariesDisabled()}`);
             }
         };
         this.enqueueSummarize = (...args) => {
@@ -315,18 +398,43 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // If we're not the summarizer, and we don't have a summaryManager, we expect that
                 // generateSummaries is turned off. We are throwing instead of returning a failure here,
                 // because it is a misuse of the API rather than an expected failure.
-                throw new Error(`Can't summarize, generateSummaries: ${this.runtimeOptions.summaryOptions.generateSummaries}`);
+                throw new Error(`Can't summarize, disableSummaries: ${this.summariesDisabled()}`);
             }
         };
         this.baseSummaryMessage = metadata === null || metadata === void 0 ? void 0 : metadata.message;
+        // If this is an existing container, we get values from metadata.
+        // otherwise, we initialize them.
+        if (existing) {
+            this.createContainerMetadata = {
+                createContainerRuntimeVersion: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerRuntimeVersion,
+                createContainerTimestamp: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerTimestamp,
+            };
+            this.summaryCount = metadata === null || metadata === void 0 ? void 0 : metadata.summaryCount;
+        }
+        else {
+            this.createContainerMetadata = {
+                createContainerRuntimeVersion: pkgVersion,
+                createContainerTimestamp: Date.now(),
+            };
+        }
         // Default to false (enabled).
         this.disableIsolatedChannels = (_a = this.runtimeOptions.summaryOptions.disableIsolatedChannels) !== null && _a !== void 0 ? _a : false;
         this._connected = this.context.connected;
         this.chunkMap = new Map(chunks);
         this.IFluidHandleContext = new ContainerFluidHandleContext("", this);
-        this.IFluidSerializer = new FluidSerializer(this.IFluidHandleContext);
         this._logger = ChildLogger.create(this.logger, "ContainerRuntime");
-        this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), this._logger, existing, metadata);
+        /**
+         * Function that return the current server timestamp. This is used by the garbage collector to set the
+         * time when a node becomes unreferenced.
+         * For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
+         * we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
+         * of this client's connection - https://github.com/microsoft/FluidFramework/issues/8375.
+         */
+        const getCurrentTimestamp = () => {
+            var _a, _b;
+            return (_b = (_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp) !== null && _b !== void 0 ? _b : Date.now();
+        };
+        this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), getCurrentTimestamp, context.baseSnapshot, async (id) => readAndParse(this.storage, id), this._logger, existing, metadata);
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"), 
         // Summarize function to call when summarize is called. Summarizer node always tracks summary state.
@@ -343,13 +451,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             throwOnFailure: true,
             // If GC should not run, let the summarizer node know so that it does not track GC state.
             gcDisabled: !this.garbageCollector.shouldRunGC,
-            // The max duration for which objects can be unreferenced before they are eligible for deletion.
-            maxUnreferencedDurationMs: this.runtimeOptions.gcOptions.maxUnreferencedDurationMs,
         });
         if (this.context.baseSnapshot) {
             this.summarizerNode.loadBaseSummaryWithoutDifferential(this.context.baseSnapshot);
         }
-        this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getInitialGCSummaryDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getInitialGCSummaryDetailsFn), (id) => this.summarizerNode.deleteChild(id), this._logger);
+        this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getInitialGCSummaryDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getInitialGCSummaryDetailsFn), (id) => this.summarizerNode.deleteChild(id), this._logger, async () => this.garbageCollector.getDataStoreBaseGCDetails(), (id) => this.garbageCollector.nodeChanged(id));
         this.blobManager = new BlobManager(this.IFluidHandleContext, blobManagerSnapshot, () => this.storage, (blobId) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }), this, this.logger);
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, ChildLogger.create(this.logger, "ScheduleManager"));
         this.deltaSender = this.deltaManager;
@@ -357,14 +463,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.context.quorum.on("removeMember", (clientId) => {
             this.clearPartialChunks(clientId);
         });
-        this.context.quorum.on("addProposal", (proposal) => {
-            if (proposal.key === "code" || proposal.key === "code2") {
-                this.emit("codeDetailsProposed", proposal.value, proposal);
-            }
-        });
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
         // Only create a SummaryManager if summaries are enabled and we are not the summarizer client
+        // Map the deprecated generateSummaries flag to disableSummaries.
         if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
+            this.runtimeOptions.summaryOptions.disableSummaries = true;
+        }
+        if (this.summariesDisabled()) {
             this._logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
         }
         else {
@@ -414,7 +519,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.deltaManager.on("readonly", (readonly) => {
             // we accumulate ops while being in read-only state.
             // once user gets write permissions and we have active connection, flush all pending ops.
-            assert(readonly === this.deltaManager.readonly, 0x124 /* "inconsistent readonly property/event state" */);
+            // eslint-disable-next-line max-len
+            assert(readonly === this.deltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
             // We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
             // when we either never send an op, or attempted to send it but we know for sure it was not
             // sequenced by server and will never be sequenced (i.e. was lost)
@@ -433,6 +539,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (context.pendingLocalState !== undefined) {
             this.deltaManager.on("op", this.onOp);
         }
+        // logging hardware telemetry
+        logger.sendTelemetryEvent(Object.assign({ eventName: "DeviceSpec" }, getDeviceSpec()));
+        // logging container load stats
+        this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryCount: this.summaryCount, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature }));
         ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
     }
     get IContainerRuntime() { return this; }
@@ -454,7 +564,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 runtimeVersion: pkgVersion,
             },
         });
-        const { summaryOptions = { generateSummaries: true }, gcOptions = {}, loadSequenceNumberVerification = "close", } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", } = runtimeOptions;
         // We pack at data store level only. If isolated channels are disabled,
         // then there are no .channel layers, we pack at level 1, otherwise we pack at level 2
         const packingLevel = summaryOptions.disableIsolatedChannels ? 1 : 2;
@@ -505,7 +615,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" && runtimeSequenceNumber !== protocolSequenceNumber) {
-                const error = new DataCorruptionError("Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber", { runtimeSequenceNumber, protocolSequenceNumber });
+                // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
+                const error = new DataCorruptionError("SummaryMetadataMismatch", { runtimeSequenceNumber, protocolSequenceNumber });
                 if (loadSequenceNumberVerification === "log") {
                     logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
                 }
@@ -584,6 +695,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         return Object.assign(Object.assign(Object.assign({}, DefaultSummaryConfiguration), (_b = (_a = this.context) === null || _a === void 0 ? void 0 : _a.serviceConfiguration) === null || _b === void 0 ? void 0 : _b.summary), (_c = this.runtimeOptions.summaryOptions) === null || _c === void 0 ? void 0 : _c.summaryConfigOverrides);
     }
     get disposed() { return this._disposed; }
+    /**
+     * True, if GC data should be written at root of the summary tree.
+     * False, if data stores should write GC blobs in their summary tree.
+     */
+    get writeGCDataAtRoot() {
+        return this.garbageCollector.writeDataAtRoot;
+    }
     static get defaultFlushMode() {
         return getLocalStorageFeatureGate(turnBasedFlushModeKey) ? FlushMode.TurnBased : FlushMode.Immediate;
     }
@@ -704,14 +822,10 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     formMetadata() {
         var _a;
-        return {
-            summaryFormatVersion: 1,
-            disableIsolatedChannels: this.disableIsolatedChannels || undefined,
-            gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
+        return Object.assign(Object.assign({}, this.createContainerMetadata), { summaryCount: this.summaryCount, summaryFormatVersion: 1, disableIsolatedChannels: this.disableIsolatedChannels || undefined, gcFeature: this.garbageCollector.gcSummaryFeatureVersion, 
             // The last message processed at the time of summary. If there are no messages, nothing has changed from
             // the base summary we loaded from. So, use the message from its metadata blob.
-            message: (_a = extractSummaryMetadataMessage(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.baseSummaryMessage,
-        };
+            message: (_a = extractSummaryMetadataMessage(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.baseSummaryMessage });
     }
     /**
      * Retrieves the runtime for a data store if it's referenced as per the initially summary that it is loaded with.
@@ -738,22 +852,14 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @deprecated - Use summarize to get summary of the container runtime.
      */
     async snapshot() {
-        if (this.garbageCollector.shouldRunGC) {
-            await this.collectGarbage({ logger: this.logger, fullGC: true /* fullGC */ });
-        }
-        const root = { entries: [] };
-        const entries = await this.dataStores.snapshot();
-        if (this.disableIsolatedChannels) {
-            root.entries = root.entries.concat(entries);
-        }
-        else {
-            root.entries.push(new TreeTreeEntry(channelsTreeName, { entries }));
-        }
-        root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata())));
-        if (this.chunkMap.size > 0) {
-            root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap])));
-        }
-        return root;
+        const summaryResult = await this.summarize({
+            summaryLogger: this.logger,
+            fullTree: true,
+            trackState: false,
+            runGC: this.garbageCollector.shouldRunGC,
+            fullGC: true,
+        });
+        return convertSummaryTreeToITree(summaryResult.summary);
     }
     addContainerBlobsToSummary(summaryTree) {
         var _a;
@@ -773,6 +879,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             const blobsTree = convertToSummaryTree(snapshot, false);
             addTreeToSummary(summaryTree, blobsTreeName, blobsTree);
         }
+        if (this.writeGCDataAtRoot) {
+            const gcSummary = this.garbageCollector.summarize();
+            if (gcSummary !== undefined) {
+                addTreeToSummary(summaryTree, gcTreeKey, gcSummary);
+            }
+        }
     }
     replayPendingStates() {
         // We need to be able to send ops to replay states
@@ -842,11 +954,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         // We do not need to make deep copy, as each layer will just replace message.content itself,
         // but would not modify contents details
         let message = Object.assign({}, messageArg);
-        let error;
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
-        this.scheduleManager.beginOperation(message);
+        this.scheduleManager.beforeOpProcessing(message);
         try {
             message = unpackRuntimeMessage(message);
             // Chunk processing must come first given that we will transform the message to the unchunked version
@@ -874,14 +985,12 @@ export class ContainerRuntime extends TypedEventEmitter {
                 default:
             }
             this.emit("op", message);
+            this.scheduleManager.afterOpProcessing(undefined, message);
         }
         catch (e) {
-            error = e;
+            this.scheduleManager.afterOpProcessing(e, message);
             throw e;
         }
-        finally {
-            this.scheduleManager.endOperation(error, message);
-        }
     }
     processSignal(message, local) {
         const envelope = message.content;
@@ -1093,7 +1202,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
         }
         const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
-        assert(summarizeResult.summary.type === 1 /* Tree */, 0x12f /* "Container Runtime's summarize should always return a tree" */);
+        assert(summarizeResult.summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
         return summarizeResult;
     }
     /**
@@ -1108,19 +1217,16 @@ export class ContainerRuntime extends TypedEventEmitter {
      * Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
      * After GC has run, called to notify this container's nodes of routes that are used in it.
      * @param usedRoutes - The routes that are used in all nodes in this Container.
+     * @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
+     * unreferenced as part of this GC run, this should be used to update the time when it happens.
      * @returns the statistics of the used state of the data stores.
      */
-    updateUsedRoutes(usedRoutes) {
-        var _a;
+    updateUsedRoutes(usedRoutes, gcTimestamp) {
         // Update our summarizer node's used routes. Updating used routes in summarizer node before
         // summarizing is required and asserted by the the summarizer node. We are the root and are
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
-        return this.dataStores.updateUsedRoutes(usedRoutes, (_a = 
-        // For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
-        // we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
-        // of this client's connection - https://github.com/microsoft/FluidFramework/issues/7152.
-        this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp);
+        return this.dataStores.updateUsedRoutes(usedRoutes, gcTimestamp);
     }
     /**
      * Runs garbage collection and udpates the reference / used state of the nodes in the container.
@@ -1188,6 +1294,13 @@ export class ContainerRuntime extends TypedEventEmitter {
             if (!continueResult.continue) {
                 return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error: continueResult.error };
             }
+            // increment summary count
+            if (this.summaryCount !== undefined) {
+                this.summaryCount++;
+            }
+            else {
+                this.summaryCount = 1;
+            }
             const trace = Trace.start();
             let summarizeResult;
             try {
@@ -1209,8 +1322,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
             const dataStoreTree = this.disableIsolatedChannels ? summaryTree : summaryTree.tree[channelsTreeName];
-            assert(dataStoreTree.type === 1 /* Tree */, 0x1fc /* "summary is not a tree" */);
-            const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === 3 /* Handle */).length;
+            assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
+            const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === SummaryType.Handle).length;
             const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount }, partialStats);
             const generateSummaryData = {
                 referenceSequenceNumber: summaryRefSeqNum,
@@ -1487,6 +1600,14 @@ export class ContainerRuntime extends TypedEventEmitter {
     getPendingLocalState() {
         return this.pendingStateManager.getLocalState();
     }
+    /**
+     * @returns true if summaries are explicitly disabled for this ContainerRuntime, false otherwise
+     */
+    summariesDisabled() {
+        var _a;
+        return this.runtimeOptions.summaryOptions.disableSummaries === true ||
+            ((_a = this.runtimeOptions.summaryOptions.summaryConfigOverrides) === null || _a === void 0 ? void 0 : _a.disableSummaries) === true;
+    }
 }
 /**
  * Wait for a specific sequence number. Promise should resolve when we reach that number,