@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/lib/containerRuntime.js

@@ -1,35 +1,28 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, Trace, TypedEventEmitter, unreachableCase } from "@fluidframework/common-utils";
+import { assert, LazyPromise, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
 import { ChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, loggerToMonitoringContext, wrapError, } from "@fluidframework/telemetry-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
-import { FlushMode, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
-import { GCDataBuilder, trimLeadingAndTrailingSlashes } from "@fluidframework/garbage-collector";
+import { FlushMode, FlushModeExperimental, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
+import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, RequestParser, create404Response, exceptionToResponse, GCDataBuilder, requestFluidObject, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
 import { v4 as uuid } from "uuid";
 import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
-import { Summarizer } from "./summarizer";
-import { SummaryManager } from "./summaryManager";
 import { ReportOpPerfTelemetry } from "./connectionTelemetry";
 import { PendingStateManager } from "./pendingStateManager";
 import { pkgVersion } from "./packageVersion";
 import { BlobManager } from "./blobManager";
 import { DataStores, getSummaryForDatastores } from "./dataStores";
-import { aliasBlobName, blobsTreeName, chunksBlobName, electedSummarizerBlobName, extractSummaryMetadataMessage, metadataBlobName, wrapSummaryInChannelsTree, } from "./summaryFormat";
-import { SummaryCollection } from "./summaryCollection";
-import { OrderedClientCollection, OrderedClientElection, } from "./orderedClientElection";
-import { SummarizerClientElection, summarizerClientType } from "./summarizerClientElection";
+import { aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, metadataBlobName, Summarizer, SummaryManager, wrapSummaryInChannelsTree, SummaryCollection, OrderedClientCollection, OrderedClientElection, SummarizerClientElection, summarizerClientType, RunWhileConnectedCoordinator, } from "./summary";
 import { formExponentialFn, Throttler } from "./throttler";
-import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
-import { GarbageCollector, GCNodeType, } from "./garbageCollection";
+import { GarbageCollector, GCNodeType, gcTombstoneGenerationOptionName, shouldAllowGcTombstoneEnforcement, trimLeadingAndTrailingSlashes, } from "./gc";
 import { channelToDataStore, isDataStoreAliasMessage } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
-import { SerializedSnapshotStorage, } from "./serializedSnapshotStorage";
 import { ScheduleManager } from "./scheduleManager";
-import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, } from "./opLifecycle";
+import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, OpGroupingManager, } from "./opLifecycle";
+import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy";
 export var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -52,7 +45,7 @@ export const DefaultSummaryConfiguration = {
     maxTime: 60 * 1000,
     maxOps: 100,
     minOpsForLastSummaryAttempt: 10,
-    maxAckWaitTime: 10 * 60 * 1000,
+    maxAckWaitTime: 3 * 60 * 1000,
     maxOpsSinceLastSummary: 7000,
     initialSummarizerDelayMs: 5 * 1000,
     nonRuntimeOpWeight: 0.1,
@@ -66,11 +59,6 @@ export var RuntimeHeaders;
 (function (RuntimeHeaders) {
     /** True to wait for a data store to be created and loaded before returning it. */
     RuntimeHeaders["wait"] = "wait";
-    /**
-     * True if the request is from an external app. Used for GC to handle scenarios where a data store
-     * is deleted and requested via an external app.
-     */
-    RuntimeHeaders["externalRequest"] = "externalRequest";
     /** True if the request is coming from an IFluidHandle. */
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders || (RuntimeHeaders = {}));
@@ -81,7 +69,6 @@ export const TombstoneResponseHeaderKey = "isTombstoned";
 /** Default values for Runtime Headers */
 export const defaultRuntimeHeaderData = {
     wait: true,
-    externalRequest: false,
     viaHandle: false,
     allowTombstone: false,
 };
@@ -98,7 +85,13 @@ const defaultFlushMode = FlushMode.TurnBased;
 // We can't estimate it fully, as we
 // - do not know what properties relay service will add
 // - we do not stringify final op, thus we do not know how much escaping will be added.
-const defaultMaxBatchSizeInBytes = 950 * 1024;
+const defaultMaxBatchSizeInBytes = 700 * 1024;
+const defaultCompressionConfig = {
+    // Batches with content size exceeding this value will be compressed
+    minimumBatchSizeInBytes: 614400,
+    compressionAlgorithm: CompressionAlgorithms.lz4,
+};
+const defaultChunkSizeInBytes = 204800;
 /**
  * @deprecated - use ContainerRuntimeMessage instead
  */
@@ -145,8 +138,8 @@ export class ContainerRuntime extends TypedEventEmitter {
     /**
      * @internal
      */
-    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
-        var _a, _b, _c, _d, _e, _f;
+    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration, initializeEntryPoint) {
+        var _a, _b, _c, _d, _e, _f, _g, _h;
         if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
         super();
         this.context = context;
@@ -159,8 +152,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.summaryConfiguration = summaryConfiguration;
         this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
-        this.flushMicroTaskExists = false;
-        this.savedOps = [];
+        this.flushTaskExists = false;
         this.consecutiveReconnects = 0;
         this.ensureNoDataModelChangesCalls = 0;
         /**
@@ -207,6 +199,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                 throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
             }
         };
+        this.innerDeltaManager = context.deltaManager;
+        this.deltaManager = new DeltaManagerSummarizerProxy(context.deltaManager);
         let loadSummaryNumber;
         // Get the container creation metadata. For new container, we initialize these. For existing containers,
         // get the values from the metadata blob.
@@ -229,29 +223,47 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.nextSummaryNumber = loadSummaryNumber + 1;
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         this._connected = this.context.connected;
+        this.gcTombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement((_c = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix) === null || _c === void 0 ? void 0 : _c.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName] /* current */);
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
-        const opSplitter = new OpSplitter(chunks, this.context.submitBatchFn, this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true
-            ? Number.POSITIVE_INFINITY
-            : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
-        this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor());
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "GCFeatureMatrix",
+            metadataValue: JSON.stringify(metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix),
+            inputs: JSON.stringify({
+                gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName],
+            }),
+        });
+        this.telemetryDocumentId = (_d = metadata === null || metadata === void 0 ? void 0 : metadata.telemetryDocumentId) !== null && _d !== void 0 ? _d : uuid();
+        this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
+        const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionChunkingDisabled");
+        const opGroupingManager = new OpGroupingManager(this.groupedBatchingEnabled);
+        const opSplitter = new OpSplitter(chunks, this.context.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
+        this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor(this.mc.logger), opGroupingManager);
         this.handleContext = new ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
+        const disableOpReentryCheck = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck");
         this.enableOpReentryCheck =
             runtimeOptions.enableOpReentryCheck === true &&
                 // Allow for a break-glass config to override the options
-                this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
+                disableOpReentryCheck !== true;
         this.summariesDisabled = this.isSummariesDisabled();
         this.heuristicsDisabled = this.isHeuristicsDisabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
         this.maxConsecutiveReconnects =
-            (_c = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _c !== void 0 ? _c : this.defaultMaxConsecutiveReconnects;
-        this._flushMode = runtimeOptions.flushMode;
+            (_e = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _e !== void 0 ? _e : this.defaultMaxConsecutiveReconnects;
+        if (runtimeOptions.flushMode === FlushModeExperimental.Async &&
+            ((_f = context.supportedFeatures) === null || _f === void 0 ? void 0 : _f.get("referenceSequenceNumbers")) !== true) {
+            // The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
+            this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
+            this._flushMode = FlushMode.TurnBased;
+        }
+        else {
+            this._flushMode = runtimeOptions.flushMode;
+        }
         const pendingRuntimeState = context.pendingLocalState;
-        const baseSnapshot = (_d = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _d !== void 0 ? _d : context.baseSnapshot;
-        const maxSnapshotCacheDurationMs = (_f = (_e = this._storage) === null || _e === void 0 ? void 0 : _e.policies) === null || _f === void 0 ? void 0 : _f.maximumCacheDurationMs;
+        const maxSnapshotCacheDurationMs = (_h = (_g = this._storage) === null || _g === void 0 ? void 0 : _g.policies) === null || _h === void 0 ? void 0 : _h.maximumCacheDurationMs;
         if (maxSnapshotCacheDurationMs !== undefined &&
             maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
@@ -262,7 +274,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.garbageCollector = GarbageCollector.create({
             runtime: this,
             gcOptions: this.runtimeOptions.gcOptions,
-            baseSnapshot,
+            baseSnapshot: context.baseSnapshot,
             baseLogger: this.mc.logger,
             existing,
             metadata,
@@ -272,7 +284,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             getLastSummaryTimestampMs: () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; },
             readAndParseBlob: async (id) => readAndParse(this.storage, id),
             getContainerDiagnosticId: () => this.context.id,
-            activeConnection: () => this.deltaManager.active,
+            // GC runs in summarizer client and needs access to the real (non-proxy) active information. The proxy
+            // delta manager would always return false for summarizer client.
+            activeConnection: () => this.innerDeltaManager.active,
         });
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"), 
@@ -281,7 +295,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // Latest change sequence number, no changes since summary applied yet
         loadedFromSequenceNumber, 
         // Summary reference sequence number, undefined if no summary yet
-        baseSnapshot ? loadedFromSequenceNumber : undefined, {
+        context.baseSnapshot ? loadedFromSequenceNumber : undefined, {
             // Must set to false to prevent sending summary handle which would be pointing to
             // a summary with an older protocol state.
             canReuseHandle: false,
@@ -295,18 +309,24 @@ export class ContainerRuntime extends TypedEventEmitter {
         async (fullGC) => this.getGCDataInternal(fullGC), 
         // Function to get the GC details from the base snapshot we loaded from.
         async () => this.garbageCollector.getBaseGCDetails());
-        if (baseSnapshot) {
-            this.summarizerNode.updateBaseSummaryState(baseSnapshot);
+        if (context.baseSnapshot) {
+            this.summarizerNode.updateBaseSummaryState(context.baseSnapshot);
         }
-        this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap));
+        this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap));
         this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
             if (!this.disposed) {
-                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
-                    localId,
-                    blobId,
+                // eslint-disable-next-line @typescript-eslint/no-floating-promises
+                Promise.resolve().then(() => {
+                    // Blob attaches need to be in their own batch (grouped batching would hide metadata)
+                    this.flush();
+                    this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
+                        localId,
+                        blobId,
+                    });
+                    this.flush();
                 });
             }
-        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (fromPath, toPath) => this.garbageCollector.addedOutboundReference(fromPath, toPath), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs, () => this.getCurrentReferenceTimestampMs());
+        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs, (error) => this.closeFn(error));
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, () => this.clientId, ChildLogger.create(this.logger, "ScheduleManager"));
         this.pendingStateManager = new PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
@@ -317,12 +337,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             rollback: this.rollback.bind(this),
             orderSequentially: this.orderSequentially.bind(this),
         }, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
-        const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true
+        const disableCompression = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
+        const compressionOptions = disableCompression === true
             ? {
                 minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
                 compressionAlgorithm: CompressionAlgorithms.lz4,
             }
             : runtimeOptions.compressionOptions;
+        const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
         this.outbox = new Outbox({
             shouldSend: () => this.canSendOps(),
             pendingStateManager: this.pendingStateManager,
@@ -332,9 +354,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             config: {
                 compressionOptions,
                 maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
-                enableOpReentryCheck: this.enableOpReentryCheck,
+                disablePartialFlush: disablePartialFlush === true,
             },
             logger: this.mc.logger,
+            groupingManager: opGroupingManager,
         });
         this.context.quorum.on("removeMember", (clientId) => {
             this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
@@ -353,7 +376,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
             this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
             if (this.context.clientDetails.type === summarizerClientType) {
-                this._summarizer = new Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime));
+                this._summarizer = new Summarizer(this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime, 
+                // Summarization runs in summarizer client and needs access to the real (non-proxy) active
+                // information. The proxy delta manager would always return false for summarizer client.
+                () => this.innerDeltaManager.active));
             }
             else if (SummarizerClientElection.clientDetailsPermitElection(this.context.clientDetails)) {
                 // Only create a SummaryManager and SummarizerClientElection
@@ -388,7 +414,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.deltaManager.on("readonly", (readonly) => {
             // we accumulate ops while being in read-only state.
             // once user gets write permissions and we have active connection, flush all pending ops.
-            assert(readonly === this.deltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
+            // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
+            assert(readonly === this.innerDeltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
             // We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
             // when we either never send an op, or attempted to send it but we know for sure it was not
             // sequenced by server and will never be sequenced (i.e. was lost)
@@ -406,9 +433,22 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         // logging hardware telemetry
         logger.sendTelemetryEvent(Object.assign({ eventName: "DeviceSpec" }, getDeviceSpec()));
-        this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryNumber: loadSummaryNumber, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature }));
+        this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryNumber: loadSummaryNumber, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature, options: JSON.stringify(runtimeOptions), featureGates: JSON.stringify({
+                disableCompression,
+                disableOpReentryCheck,
+                disableChunking,
+                disableAttachReorder: this.disableAttachReorder,
+                disablePartialFlush,
+            }), telemetryDocumentId: this.telemetryDocumentId, groupedBatchingEnabled: this.groupedBatchingEnabled }));
         ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
         BindBatchTracker(this, this.logger);
+        this.entryPoint = new LazyPromise(async () => {
+            if (this.context.clientDetails.type === summarizerClientType) {
+                assert(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
+                return this._summarizer;
+            }
+            return initializeEntryPoint === null || initializeEntryPoint === void 0 ? void 0 : initializeEntryPoint(this);
+        });
     }
     get IContainerRuntime() {
         return this;
@@ -446,17 +486,20 @@ export class ContainerRuntime extends TypedEventEmitter {
      * Load the stores from a snapshot and returns the runtime.
      * @param params - An object housing the runtime properties:
      * - context - Context of the container.
-     * - registryEntries - Mapping to the stores.
-     * - existing - When loading from an existing snapshot
-     * - requestHandler - Request handlers for the container runtime
+     * - registryEntries - Mapping from data store types to their corresponding factories.
+     * - existing - Pass 'true' if loading from an existing snapshot.
+     * - requestHandler - (optional) Request handler for the request() method of the container runtime.
+     * Only relevant for back-compat while we remove the request() method and move fully to entryPoint as the main pattern.
      * - runtimeOptions - Additional options to be passed to the runtime
      * - containerScope - runtime services provided with context
      * - containerRuntimeCtor - Constructor to use to create the ContainerRuntime instance.
      * This allows mixin classes to leverage this method to define their own async initializer.
+     * - initializeEntryPoint - Promise that resolves to an object which will act as entryPoint for the Container.
+     * This object should provide all the functionality that the Container is expected to provide to the loader layer.
      */
     static async loadRuntime(params) {
         var _a, _b, _c, _d;
-        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
+        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, initializeEntryPoint, } = params;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
@@ -466,25 +509,16 @@ export class ContainerRuntime extends TypedEventEmitter {
                 runtimeVersion: pkgVersion,
             },
         });
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = {
-            minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
-            compressionAlgorithm: CompressionAlgorithms.lz4,
-        }, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, chunkSizeInBytes = Number.POSITIVE_INFINITY, enableOpReentryCheck = false, } = runtimeOptions;
-        const pendingRuntimeState = context.pendingLocalState;
-        const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
-        const storage = !pendingRuntimeState
-            ? context.storage
-            : new SerializedSnapshotStorage(() => {
-                return context.storage;
-            }, pendingRuntimeState.snapshotBlobs);
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, } = runtimeOptions;
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
-            const blobId = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.blobs[blobName];
-            if (baseSnapshot && blobId) {
+            var _a;
+            const blobId = (_a = context.baseSnapshot) === null || _a === void 0 ? void 0 : _a.blobs[blobName];
+            if (context.baseSnapshot && blobId) {
                 // IContainerContext storage api return type still has undefined in 0.39 package version.
                 // So once we release 0.40 container-defn package we can remove this check.
-                assert(storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
-                return readAndParse(storage, blobId);
+                assert(context.storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
+                return readAndParse(context.storage, blobId);
             }
         };
         const [chunks, metadata, electedSummarizerData, aliases] = await Promise.all([
@@ -495,16 +529,16 @@ export class ContainerRuntime extends TypedEventEmitter {
         ]);
         const loadExisting = existing === true || context.existing === true;
         // read snapshot blobs needed for BlobManager to load
-        const blobManagerSnapshot = await BlobManager.load(baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[blobsTreeName], async (id) => {
+        const blobManagerSnapshot = await BlobManager.load((_b = context.baseSnapshot) === null || _b === void 0 ? void 0 : _b.trees[blobsTreeName], async (id) => {
             // IContainerContext storage api return type still has undefined in 0.39 package version.
             // So once we release 0.40 container-defn package we can remove this check.
-            assert(storage !== undefined, 0x256 /* "storage undefined in attached container" */);
-            return readAndParse(storage, id);
+            assert(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
+            return readAndParse(context.storage, id);
         });
         // Verify summary runtime sequence number matches protocol sequence number.
         const runtimeSequenceNumber = (_c = metadata === null || metadata === void 0 ? void 0 : metadata.message) === null || _c === void 0 ? void 0 : _c.sequenceNumber;
         // When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
-        if (!pendingRuntimeState && runtimeSequenceNumber !== undefined) {
+        if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
             const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" &&
@@ -528,17 +562,16 @@ export class ContainerRuntime extends TypedEventEmitter {
             gcOptions,
             loadSequenceNumberVerification,
             flushMode,
-            enableOfflineLoad,
             compressionOptions,
             maxBatchSizeInBytes,
             chunkSizeInBytes,
             enableOpReentryCheck,
-        }, containerScope, logger, loadExisting, blobManagerSnapshot, storage, requestHandler);
-        if (pendingRuntimeState) {
-            await runtime.processSavedOps(pendingRuntimeState);
-            // delete these once runtime has seen them to save space
-            pendingRuntimeState.savedOps = [];
-        }
+            enableGroupedBatching,
+        }, containerScope, logger, loadExisting, blobManagerSnapshot, context.storage, requestHandler, undefined, // summaryConfiguration
+        initializeEntryPoint);
+        // It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
+        // at 1, so we won't see a replayed saved op with a sequence number of 0.
+        await runtime.pendingStateManager.applyStashedOpsAt(0);
         // Initialize the base state of the runtime before it's returned.
         await runtime.initializeBaseState();
         return runtime;
@@ -552,9 +585,6 @@ export class ContainerRuntime extends TypedEventEmitter {
     get clientDetails() {
         return this.context.clientDetails;
     }
-    get deltaManager() {
-        return this.context.deltaManager;
-    }
     get storage() {
         return this._storage;
     }
@@ -647,7 +677,6 @@ export class ContainerRuntime extends TypedEventEmitter {
      * Initializes the state from the base snapshot this container runtime loaded from.
      */
     async initializeBaseState() {
-        await this.initializeBaseSnapshotBlobs();
         await this.garbageCollector.initializeBaseState();
     }
     dispose(error) {
@@ -672,16 +701,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.emit("dispose");
         this.removeAllListeners();
     }
-    get IFluidTokenProvider() {
-        var _a;
-        if ((_a = this.options) === null || _a === void 0 ? void 0 : _a.intelligence) {
-            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-            return {
-                intelligence: this.options.intelligence,
-            };
-        }
-        return undefined;
-    }
     /**
      * Notifies this object about the request made to the container.
      * @param request - Request made to the handler.
@@ -744,12 +763,18 @@ export class ContainerRuntime extends TypedEventEmitter {
             return exceptionToResponse(error);
         }
     }
+    /**
+     * {@inheritDoc @fluidframework/container-definitions#IRuntime.getEntryPoint}
+     */
+    async getEntryPoint() {
+        return this.entryPoint;
+    }
     internalId(maybeAlias) {
         var _a;
         return (_a = this.dataStores.aliases.get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
     }
     async getDataStoreFromRequest(id, request) {
-        var _a, _b, _c, _d;
+        var _a, _b, _c;
         const headerData = {};
         if (typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a[RuntimeHeaders.wait]) === "boolean") {
             headerData.wait = request.headers[RuntimeHeaders.wait];
@@ -763,23 +788,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
         const dataStoreContext = await this.dataStores.getDataStore(internalId, headerData);
-        /**
-         * If GC should run and this an external app request with "externalRequest" header, we need to return
-         * an error if the data store being requested is marked as unreferenced as per the data store's base
-         * GC data.
-         *
-         * This is a workaround to handle scenarios where a data store shared with an external app is deleted
-         * and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
-         */
-        if (((_d = request.headers) === null || _d === void 0 ? void 0 : _d[RuntimeHeaders.externalRequest]) &&
-            this.garbageCollector.shouldRunGC) {
-            // The data store is referenced if used routes in the base summary has a route to self.
-            // Older documents may not have used routes in the summary. They are considered referenced.
-            const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
-            if (!(usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/"))) {
-                throw responseToException(create404Response(request), request);
-            }
-        }
         const dataStoreChannel = await dataStoreContext.realize();
         // Remove query params, leading and trailing slashes from the url. This is done to make sure the format is
         // the same as GC nodes id.
@@ -795,7 +803,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             summaryNumber: this.nextSummaryNumber++, summaryFormatVersion: 1 }), this.garbageCollector.getMetadata()), { 
             // The last message processed at the time of summary. If there are no new messages, use the message from the
             // last summary.
-            message: (_a = extractSummaryMetadataMessage(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary });
+            message: (_a = extractSummaryMetadataMessage(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary, telemetryDocumentId: this.telemetryDocumentId });
         addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
     }
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
@@ -911,7 +919,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         // If attachment blobs were added while disconnected, we need to delay
         // propagation of the "connected" event until we have uploaded them to
         // ensure we don't submit ops referencing a blob that has not been uploaded
-        const connecting = connected && !this._connected && !this.deltaManager.readOnlyInfo.readonly;
+        // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
+        const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
         if (connecting && this.blobManager.hasPendingOfflineUploads) {
             assert(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
             assert(!!clientId, 0x393 /* Must have clientId when connecting */);
@@ -961,19 +970,23 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.garbageCollector.setConnectionState(connected, clientId);
         raiseConnectedEvent(this.mc.logger, this, connected, clientId);
     }
+    async notifyOpReplay(message) {
+        await this.pendingStateManager.applyStashedOpsAt(message.sequenceNumber);
+    }
     process(messageArg, local) {
-        var _a;
         this.verifyNotClosed();
-        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
-            this.savedOps.push(messageArg);
-        }
         // Whether or not the message is actually a runtime message.
         // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
         // or something different, like a system message.
         const runtimeMessage = messageArg.type === MessageType.Operation;
         // Do shallow copy of message, as the processing flow will modify it.
         const messageCopy = Object.assign({}, messageArg);
-        const message = this.remoteMessageProcessor.process(messageCopy);
+        for (const message of this.remoteMessageProcessor.process(messageCopy)) {
+            this.processCore(message, local, runtimeMessage);
+        }
+    }
+    processCore(message, local, runtimeMessage) {
+        var _a;
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
@@ -1006,10 +1019,21 @@ export class ContainerRuntime extends TypedEventEmitter {
                 case ContainerMessageType.Rejoin:
                     break;
                 default:
-                    assert(!runtimeMessage, 0x3ce /* Runtime message of unknown type */);
+                    if (runtimeMessage) {
+                        const error = DataProcessingError.create(
+                        // Former assert 0x3ce
+                        "Runtime message of unknown type", "OpProcessing", message, {
+                            local,
+                            type: message.type,
+                            contentType: typeof message.contents,
+                            batch: (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.batch,
+                            compression: message.compression,
+                        });
+                        this.closeFn(error);
+                        throw error;
+                    }
             }
-            // For back-compat, notify only about runtime messages for now.
-            if (runtimeMessage) {
+            if (runtimeMessage || this.groupedBatchingEnabled) {
                 this.emit("op", message, runtimeMessage);
             }
             this.scheduleManager.afterOpProcessing(undefined, message);
@@ -1067,7 +1091,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             else if (envelope.clientSignalSequenceNumber ===
                 this._perfSignalData.trackingSignalSequenceNumber) {
-                this.sendSignalTelemetryEvent(envelope.clientSignalSequenceNumber);
+                // only logging for the first connection and the trackingSignalSequenceNUmber.
+                if (this.consecutiveReconnects === 0) {
+                    this.sendSignalTelemetryEvent(envelope.clientSignalSequenceNumber);
+                }
                 this._perfSignalData.trackingSignalSequenceNumber = undefined;
             }
         }
@@ -1164,13 +1191,15 @@ export class ContainerRuntime extends TypedEventEmitter {
             .realize();
     }
     canSendOps() {
-        return this.connected && !this.deltaManager.readOnlyInfo.readonly;
+        // Note that the real (non-proxy) delta manager is needed here to get the readonly info. This is because
+        // container runtime's ability to send ops depend on the actual readonly state of the delta manager.
+        return this.connected && !this.innerDeltaManager.readOnlyInfo.readonly;
     }
     /**
      * Are we in the middle of batching ops together?
      */
     currentlyBatching() {
-        return this.flushMode === FlushMode.TurnBased || this._orderSequentiallyCalls !== 0;
+        return this.flushMode !== FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
     }
     getQuorum() {
         return this.context.quorum;
@@ -1286,11 +1315,19 @@ export class ContainerRuntime extends TypedEventEmitter {
     async summarize(options) {
         this.verifyNotClosed();
         const { fullTree = false, trackState = true, summaryLogger = this.mc.logger, runGC = this.garbageCollector.shouldRunGC, runSweep, fullGC, } = options;
+        const telemetryContext = new TelemetryContext();
+        // Add the options that are used to generate this summary to the telemetry context.
+        telemetryContext.setMultiple("fluid_Summarize", "Options", {
+            fullTree,
+            trackState,
+            runGC,
+            fullGC,
+            runSweep,
+        });
         let gcStats;
         if (runGC) {
-            gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
+            gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
         }
-        const telemetryContext = new TelemetryContext();
         const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
         this.logger.sendTelemetryEvent({
             eventName: "SummarizeTelemetry",
@@ -1300,10 +1337,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         return { stats, summary, gcStats };
     }
     /**
-     * Implementation of IGarbageCollectionRuntime::updateStateBeforeGC.
      * Before GC runs, called by the garbage collector to update any pending GC state. This is mainly used to notify
      * the garbage collector of references detected since the last GC run. Most references are notified immediately
      * but there can be some for which async operation is required (such as detecting new root data stores).
+     * @see IGarbageCollectionRuntime.updateStateBeforeGC
      */
     async updateStateBeforeGC() {
         return this.dataStores.updateStateBeforeGC();
@@ -1312,9 +1349,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.dataStores.getGCData(fullGC);
     }
     /**
-     * Implementation of IGarbageCollectionRuntime::getGCData.
      * Generates and returns the GC data for this container.
      * @param fullGC - true to bypass optimizations and force full generation of GC data.
+     * @see IGarbageCollectionRuntime.getGCData
      */
     async getGCData(fullGC) {
         const builder = new GCDataBuilder();
@@ -1325,9 +1362,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         return builder.getGCData();
     }
     /**
-     * Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
      * After GC has run, called to notify this container's nodes of routes that are used in it.
      * @param usedRoutes - The routes that are used in all nodes in this Container.
+     * @see IGarbageCollectionRuntime.updateUsedRoutes
      */
     updateUsedRoutes(usedRoutes) {
         // Update our summarizer node's used routes. Updating used routes in summarizer node before
@@ -1346,6 +1383,22 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.blobManager.updateUnusedRoutes(blobManagerRoutes);
         this.dataStores.updateUnusedRoutes(dataStoreRoutes);
     }
+    /**
+     * @deprecated - Replaced by deleteSweepReadyNodes.
+     */
+    deleteUnusedNodes(unusedRoutes) {
+        throw new Error("deleteUnusedRoutes should not be called");
+    }
+    /**
+     * After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
+     * @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
+     * @returns - The routes of nodes that were deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyRoutes) {
+        const { dataStoreRoutes, blobManagerRoutes } = this.getDataStoreAndBlobManagerRoutes(sweepReadyRoutes);
+        const deletedRoutes = this.dataStores.deleteSweepReadyNodes(dataStoreRoutes);
+        return deletedRoutes.concat(this.blobManager.deleteSweepReadyNodes(blobManagerRoutes));
+    }
     /**
      * This is called to update objects that are tombstones.
      * @param tombstonedRoutes - Data store and attachment blob routes that are tombstones in this Container.
@@ -1423,8 +1476,8 @@ export class ContainerRuntime extends TypedEventEmitter {
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
      * @returns the statistics of the garbage collection run; undefined if GC did not run.
      */
-    async collectGarbage(options) {
-        return this.garbageCollector.collectGarbage(options);
+    async collectGarbage(options, telemetryContext) {
+        return this.garbageCollector.collectGarbage(options, telemetryContext);
     }
     /**
      * Called when a new outbound reference is added to another node. This is used by garbage collection to identify
@@ -1445,7 +1498,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     async submitSummary(options) {
         var _a, _b;
-        const { fullTree, refreshLatestAck, summaryLogger } = options;
+        const { fullTree = false, refreshLatestAck, summaryLogger } = options;
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
@@ -1461,8 +1514,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             // We might need to catch up to the latest summary's reference sequence number before pausing.
             await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
         }
+        const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         try {
             await this.deltaManager.inbound.pause();
+            if (shouldPauseInboundSignal) {
+                await this.deltaManager.inboundSignal.pause();
+            }
             const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
             const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
@@ -1518,7 +1575,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
             try {
                 summarizeResult = await this.summarize({
-                    fullTree: fullTree !== null && fullTree !== void 0 ? fullTree : forcedFullTree,
+                    fullTree: fullTree || forcedFullTree,
                     trackState: true,
                     summaryLogger: summaryNumberLogger,
                     runGC: this.garbageCollector.shouldRunGC,
@@ -1607,7 +1664,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             let clientSequenceNumber;
             try {
-                clientSequenceNumber = this.submitSummaryMessage(summaryMessage);
+                clientSequenceNumber = this.submitSummaryMessage(summaryMessage, summaryRefSeqNum);
             }
             catch (error) {
                 return Object.assign(Object.assign({ stage: "upload" }, uploadData), { error });
@@ -1621,6 +1678,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.summarizerNode.clearSummary();
             // Restart the delta manager
             this.deltaManager.inbound.resume();
+            if (shouldPauseInboundSignal) {
+                this.deltaManager.inboundSignal.resume();
+            }
         }
     }
     hasPendingMessages() {
@@ -1666,9 +1726,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
         assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
-        const deserializedContent = { type, contents };
-        const serializedContent = JSON.stringify(deserializedContent);
-        if (this.deltaManager.readOnlyInfo.readonly) {
+        const serializedContent = JSON.stringify({ type, contents });
+        // Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
+        // container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
+        if (this.innerDeltaManager.readOnlyInfo.readonly) {
             this.logger.sendTelemetryEvent({
                 eventName: "SubmitOpInReadonly",
                 connected: this.connected,
@@ -1676,7 +1737,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         const message = {
             contents: serializedContent,
-            deserializedContent,
+            deserializedContent: JSON.parse(serializedContent),
             metadata,
             localOpMetadata,
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
@@ -1704,7 +1765,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // optimization no longer makes sense (for example, batch compression may make it less appealing).
             if (this.currentlyBatching() &&
                 type === ContainerMessageType.Attach &&
-                this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
+                this.disableAttachReorder !== true) {
                 this.outbox.submitAttach(message);
             }
             else {
@@ -1713,17 +1774,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             if (!this.currentlyBatching()) {
                 this.flush();
             }
-            else if (!this.flushMicroTaskExists) {
-                this.flushMicroTaskExists = true;
-                // Queue a microtask to detect the end of the turn and force a flush.
-                Promise.resolve()
-                    .then(() => {
-                    this.flushMicroTaskExists = false;
-                    this.flush();
-                })
-                    .catch((error) => {
-                    this.closeFn(error);
-                });
+            else {
+                this.scheduleFlush();
             }
         }
         catch (error) {
@@ -1734,14 +1786,47 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.updateDocumentDirtyState(true);
         }
     }
-    submitSummaryMessage(contents) {
+    scheduleFlush() {
+        if (this.flushTaskExists) {
+            return;
+        }
+        this.flushTaskExists = true;
+        const flush = () => {
+            this.flushTaskExists = false;
+            try {
+                this.flush();
+            }
+            catch (error) {
+                this.closeFn(error);
+            }
+        };
+        switch (this.flushMode) {
+            case FlushMode.TurnBased:
+                // When in TurnBased flush mode the runtime will buffer operations in the current turn and send them as a single
+                // batch at the end of the turn
+                // eslint-disable-next-line @typescript-eslint/no-floating-promises
+                Promise.resolve().then(flush);
+                break;
+            // FlushModeExperimental is experimental and not exposed directly in the runtime APIs
+            case FlushModeExperimental.Async:
+                // When in Async flush mode, the runtime will accumulate all operations across JS turns and send them as a single
+                // batch when all micro-tasks are complete.
+                // Compared to TurnBased, this flush mode will capture more ops into the same batch.
+                setTimeout(flush, 0);
+                break;
+            default:
+                assert(this._orderSequentiallyCalls > 0, 0x587 /* Unreachable unless running under orderSequentially */);
+                break;
+        }
+    }
+    submitSummaryMessage(contents, referenceSequenceNumber) {
         this.verifyNotClosed();
         assert(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
         assert(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
         // back-compat: ADO #1385: Make this call unconditional in the future
         return this.context.submitSummaryFn !== undefined
-            ? this.context.submitSummaryFn(contents)
+            ? this.context.submitSummaryFn(contents, referenceSequenceNumber)
             : this.context.submitFn(MessageType.Summarize, contents, false);
     }
     /**
@@ -1837,14 +1922,12 @@ export class ContainerRuntime extends TypedEventEmitter {
         // The call to fetch the snapshot is very expensive and not always needed.
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
-        const snapshotTreeFetcher = async () => {
-            const fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
-                eventName: "RefreshLatestSummaryGetSnapshot",
+        const fetchLatestSnapshot = async () => {
+            let fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+                eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
-                summaryRefSeq,
-                fetchLatest: true,
-            });
-            const latestSnapshotRefSeq = await seqFromTree(fetchResult.snapshotTree, readAndParseBlob);
+                targetSequenceNumber: summaryRefSeq,
+            }, readAndParseBlob);
             /**
              * If the fetched snapshot is older than the one for which the ack was received, close the container.
              * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -1855,29 +1938,34 @@ export class ContainerRuntime extends TypedEventEmitter {
              * such cases, the file will be rolled back along with the ack and we will eventually reach a consistent
              * state.
              */
-            if (latestSnapshotRefSeq < summaryRefSeq) {
-                const error = DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+            if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
+                /* before failing, let's try to retrieve the latest snapshot for that specific ackHandle */
+                fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
+                    eventName: "RefreshLatestSummaryAckFetch",
                     ackHandle,
-                    summaryRefSeq,
-                    latestSnapshotRefSeq,
-                });
-                this.closeFn(error);
-                throw error;
+                    targetSequenceNumber: summaryRefSeq,
+                }, readAndParseBlob, ackHandle);
+                if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
+                    const error = DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+                        ackHandle,
+                        summaryRefSeq,
+                        fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
             }
-            summaryLogger.sendTelemetryEvent({
-                eventName: "LatestSummaryRetrieved",
-                ackHandle,
-                lastSequenceNumber: latestSnapshotRefSeq,
-                targetSequenceNumber: summaryRefSeq,
-            });
             // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
             // wait for the delta manager to catch up before refreshing the latest Summary.
-            await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryLogger);
-            return fetchResult.snapshotTree;
+            await this.waitForDeltaManagerToCatchup(fetchResult.latestSnapshotRefSeq, summaryLogger);
+            return {
+                snapshotTree: fetchResult.snapshotTree,
+                snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
+            };
         };
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, snapshotTreeFetcher, readAndParseBlob, summaryLogger);
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(result, proposalHandle, summaryRefSeq, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -1886,53 +1974,54 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @returns downloaded snapshot's reference sequence number
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
-        const { snapshotTree, versionId } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
-            eventName: "RefreshLatestSummaryGetSnapshot",
-            fetchLatest: true,
-        });
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        const latestSnapshotRefSeq = await seqFromTree(snapshotTree, readAndParseBlob);
-        const result = await this.summarizerNode.refreshLatestSummary(undefined, latestSnapshotRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
+        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+            eventName: "RefreshLatestSummaryFromServerFetch",
+        }, readAndParseBlob);
+        const fetchLatestSnapshot = {
+            snapshotTree,
+            snapshotRefSeq: latestSnapshotRefSeq,
+        };
+        const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(result, undefined, latestSnapshotRefSeq, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchLatestSnapshotFromStorage(logger, event) {
+    async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
+        return this.fetchSnapshotFromStorage(logger, event, readAndParseBlob, null /* latest */);
+    }
+    async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
+        var _a;
+        const recoveryMethod = this.mc.config.getString("Fluid.ContainerRuntime.Test.SummarizationRecoveryMethod");
+        if (recoveryMethod === "restart") {
+            const error = new GenericError("Restarting summarizer instead of refreshing");
+            this.mc.logger.sendTelemetryEvent(Object.assign(Object.assign({}, event), { eventName: "ClosingSummarizerOnSummaryStale", codePath: event.eventName, message: "Stopping fetch from storage", versionId: versionId != null ? versionId : undefined }), error);
+            (_a = this._summarizer) === null || _a === void 0 ? void 0 : _a.stop("latestSummaryStateStale");
+            this.closeFn();
+            throw error;
+        }
         return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
-            const versions = await this.storage.getVersions(null, 1, "refreshLatestSummaryAckFromServer", FetchSource.noCache);
+            const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? FetchSource.noCache : undefined);
             assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
             assert(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
             stats.getSnapshotDuration = trace.trace().duration;
+            const latestSnapshotRefSeq = await seqFromTree(maybeSnapshot, readAndParseBlob);
+            stats.snapshotRefSeq = latestSnapshotRefSeq;
+            stats.snapshotVersion = versions[0].id;
             perfEvent.end(stats);
-            return { snapshotTree: maybeSnapshot, versionId: versions[0].id };
+            return {
+                snapshotTree: maybeSnapshot,
+                versionId: versions[0].id,
+                latestSnapshotRefSeq,
+            };
         });
     }
-    notifyAttaching(snapshot) {
-        var _a;
-        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
-            this.baseSnapshotBlobs =
-                SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
-        }
-    }
-    async initializeBaseSnapshotBlobs() {
-        var _a;
-        if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) ||
-            this.attachState !== AttachState.Attached ||
-            this.context.pendingLocalState) {
-            return;
-        }
-        assert(!!this.context.baseSnapshot, 0x2e5 /* "Must have a base snapshot" */);
-        this.baseSnapshotBlobs = await SerializedSnapshotStorage.serializeTree(this.context.baseSnapshot, this.storage);
-    }
+    notifyAttaching() { } // do nothing (deprecated method)
     getPendingLocalState() {
-        var _a;
-        if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad)) {
-            throw new UsageError("can't get state when offline load disabled");
-        }
         if (this._orderSequentiallyCalls !== 0) {
             throw new UsageError("can't get state during orderSequentially");
         }
@@ -1940,24 +2029,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
         // to close current batch.
         this.flush();
-        const previousPendingState = this.context.pendingLocalState;
-        if (previousPendingState) {
-            return {
-                pending: this.pendingStateManager.getLocalState(),
-                pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
-                snapshotBlobs: previousPendingState.snapshotBlobs,
-                baseSnapshot: previousPendingState.baseSnapshot,
-                savedOps: this.savedOps,
-            };
-        }
-        assert(!!this.context.baseSnapshot, 0x2e6 /* "Must have a base snapshot" */);
-        assert(!!this.baseSnapshotBlobs, 0x2e7 /* "Must serialize base snapshot blobs before getting runtime state" */);
         return {
             pending: this.pendingStateManager.getLocalState(),
             pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
-            snapshotBlobs: this.baseSnapshotBlobs,
-            baseSnapshot: this.context.baseSnapshot,
-            savedOps: this.savedOps,
         };
     }
     /**
@@ -1986,20 +2060,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             return summarizer;
         };
     }
-    async processSavedOps(state) {
-        for (const op of state.savedOps) {
-            this.process(op, false);
-            await this.pendingStateManager.applyStashedOpsAt(op.sequenceNumber);
-        }
-        // we may not have seen every sequence number (because of system ops) so apply everything once we
-        // don't have any more saved ops
-        await this.pendingStateManager.applyStashedOpsAt();
-        // If it's not the case, we should take it into account when calculating dirty state.
-        assert(this.context.attachState === AttachState.Attached, 0x3d5 /* this function is called for attached containers only */);
-        if (!this.hasPendingMessages()) {
-            this.updateDocumentDirtyState(false);
-        }
-    }
     validateSummaryHeuristicConfiguration(configuration) {
         // eslint-disable-next-line no-restricted-syntax
         for (const prop in configuration) {
@@ -2011,6 +2071,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw new UsageError(`"minIdleTime" [${configuration.minIdleTime}] cannot be greater than "maxIdleTime" [${configuration.maxIdleTime}]`);
         }
     }
+    get groupedBatchingEnabled() {
+        const killSwitch = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
+        return killSwitch !== true && this.runtimeOptions.enableGroupedBatching;
+    }
 }
 /**
  * Wait for a specific sequence number. Promise should resolve when we reach that number,