@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.3.1.0.125672

package/lib/containerRuntime.js

@@ -1,11 +1,11 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
+import { assert, Trace, TypedEventEmitter, unreachableCase } from "@fluidframework/common-utils";
 import { ChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, loggerToMonitoringContext, wrapError, } from "@fluidframework/telemetry-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
-import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
+import { FlushMode, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
 import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
 import { GCDataBuilder, trimLeadingAndTrailingSlashes } from "@fluidframework/garbage-collector";
 import { v4 as uuid } from "uuid";
@@ -13,24 +13,23 @@ import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
 import { Summarizer } from "./summarizer";
 import { SummaryManager } from "./summaryManager";
-import { ReportOpPerfTelemetry, } from "./connectionTelemetry";
-import { PendingStateManager, } from "./pendingStateManager";
-import { BatchManager } from "./batchManager";
+import { ReportOpPerfTelemetry } from "./connectionTelemetry";
+import { PendingStateManager } from "./pendingStateManager";
 import { pkgVersion } from "./packageVersion";
 import { BlobManager } from "./blobManager";
 import { DataStores, getSummaryForDatastores } from "./dataStores";
 import { aliasBlobName, blobsTreeName, chunksBlobName, electedSummarizerBlobName, extractSummaryMetadataMessage, metadataBlobName, wrapSummaryInChannelsTree, } from "./summaryFormat";
 import { SummaryCollection } from "./summaryCollection";
-import { OrderedClientCollection, OrderedClientElection } from "./orderedClientElection";
+import { OrderedClientCollection, OrderedClientElection, } from "./orderedClientElection";
 import { SummarizerClientElection, summarizerClientType } from "./summarizerClientElection";
 import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
-import { GarbageCollector, GCNodeType, gcTreeKey, } from "./garbageCollection";
-import { channelToDataStore, isDataStoreAliasMessage, } from "./dataStore";
+import { GarbageCollector, GCNodeType, } from "./garbageCollection";
+import { channelToDataStore, isDataStoreAliasMessage } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
-import { SerializedSnapshotStorage } from "./serializedSnapshotStorage";
+import { SerializedSnapshotStorage, } from "./serializedSnapshotStorage";
 import { ScheduleManager } from "./scheduleManager";
-import { OpDecompressor } from "./opDecompressor";
+import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, } from "./opLifecycle";
 export var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -56,7 +55,6 @@ export const DefaultSummaryConfiguration = {
     maxAckWaitTime: 10 * 60 * 1000,
     maxOpsSinceLastSummary: 7000,
     initialSummarizerDelayMs: 5 * 1000,
-    summarizerClientElection: false,
     nonRuntimeOpWeight: 0.1,
     runtimeOpWeight: 1.0,
     nonRuntimeHeuristicThreshold: 20,
@@ -76,6 +74,17 @@ export var RuntimeHeaders;
     /** True if the request is coming from an IFluidHandle. */
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders || (RuntimeHeaders = {}));
+/** True if a tombstoned object should be returned without erroring */
+export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
+/** Tombstone error responses will have this header set to true */
+export const TombstoneResponseHeaderKey = "isTombstoned";
+/** Default values for Runtime Headers */
+export const defaultRuntimeHeaderData = {
+    wait: true,
+    externalRequest: false,
+    viaHandle: false,
+    allowTombstone: false,
+};
 /**
  * Available compression algorithms for op compression.
  */
@@ -109,37 +118,6 @@ export var RuntimeMessage;
 export function isRuntimeMessage(message) {
     return Object.values(RuntimeMessage).includes(message.type);
 }
-/**
- * Unpacks runtime messages
- *
- * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
- * @param message - message (as it observed in storage / service)
- * @returns unpacked runtime message
- *
- * @internal
- */
-export function unpackRuntimeMessage(message) {
-    if (message.type === MessageType.Operation) {
-        // legacy op format?
-        if (message.contents.address !== undefined && message.contents.type === undefined) {
-            message.type = ContainerMessageType.FluidDataStoreOp;
-        }
-        else {
-            // new format
-            const innerContents = message.contents;
-            message.type = innerContents.type;
-            message.contents = innerContents.contents;
-        }
-        return true;
-    }
-    else {
-        // Legacy format, but it's already "unpacked",
-        // i.e. message.type is actually ContainerMessageType.
-        // Or it's non-runtime message.
-        // Nothing to do in such case.
-        return false;
-    }
-}
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
  * special-case for document dirty state.  Ultimately we should have no special-cases from the
@@ -156,8 +134,7 @@ export function getDeviceSpec() {
             };
         }
     }
-    catch (_a) {
-    }
+    catch (_a) { }
     return {};
 }
 /**
@@ -165,6 +142,9 @@ export function getDeviceSpec() {
  * It will define the store level mappings.
  */
 export class ContainerRuntime extends TypedEventEmitter {
+    /**
+     * @internal
+     */
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
         var _a, _b, _c, _d, _e, _f;
         if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
@@ -177,12 +157,19 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._storage = _storage;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
-        this.opDecompressor = new OpDecompressor();
         this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
         this.flushMicroTaskExists = false;
         this.savedOps = [];
         this.consecutiveReconnects = 0;
+        this.ensureNoDataModelChangesCalls = 0;
+        /**
+         * Tracks the number of detected reentrant ops to report,
+         * in order to self-throttle the telemetry events.
+         *
+         * This should be removed as part of ADO:2322
+         */
+        this.opReentryCallsToReport = 5;
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
         this.defaultTelemetrySignalSampleCount = 100;
@@ -242,38 +229,31 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.nextSummaryNumber = loadSummaryNumber + 1;
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         this._connected = this.context.connected;
-        this.chunkMap = new Map(chunks);
-        this.handleContext = new ContainerFluidHandleContext("", this);
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
+        const opSplitter = new OpSplitter(chunks, this.context.submitBatchFn, this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true
+            ? Number.POSITIVE_INFINITY
+            : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
+        this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor());
+        this.handleContext = new ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
+        this.enableOpReentryCheck =
+            runtimeOptions.enableOpReentryCheck === true &&
+                // Allow for a break-glass config to override the options
+                this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
         this.summariesDisabled = this.isSummariesDisabled();
         this.heuristicsDisabled = this.isHeuristicsDisabled();
-        this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
         this.maxConsecutiveReconnects =
             (_c = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _c !== void 0 ? _c : this.defaultMaxConsecutiveReconnects;
         this._flushMode = runtimeOptions.flushMode;
-        // Provide lower soft limit - we want to have some number of ops to get efficiency in compression
-        // & bandwidth usage, but at the same time we want to send these ops sooner, to reduce overall
-        // latency of processing a batch.
-        // So there is some ballance here, that depends on compression algorithm and its efficiency working with smaller
-        // payloads. That number represents final (compressed) bits (once compression is implemented).
-        this.pendingAttachBatch = new BatchManager(this.mc.logger, {
-            hardLimit: runtimeOptions.maxBatchSizeInBytes,
-            softLimit: 64 * 1024,
-            compressionOptions: runtimeOptions.compressionOptions
-        });
-        this.pendingBatch = new BatchManager(this.mc.logger, {
-            hardLimit: runtimeOptions.maxBatchSizeInBytes,
-            compressionOptions: runtimeOptions.compressionOptions
-        });
         const pendingRuntimeState = context.pendingLocalState;
         const baseSnapshot = (_d = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _d !== void 0 ? _d : context.baseSnapshot;
         const maxSnapshotCacheDurationMs = (_f = (_e = this._storage) === null || _e === void 0 ? void 0 : _e.policies) === null || _f === void 0 ? void 0 : _f.maximumCacheDurationMs;
-        if (maxSnapshotCacheDurationMs !== undefined && maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
+        if (maxSnapshotCacheDurationMs !== undefined &&
+            maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
             // which dictates the value is either undefined or exactly 5 days in ms.
             // As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
@@ -310,34 +290,59 @@ export class ContainerRuntime extends TypedEventEmitter {
             throwOnFailure: true,
             // If GC should not run, let the summarizer node know so that it does not track GC state.
             gcDisabled: !this.garbageCollector.shouldRunGC,
-        });
+        }, 
+        // Function to get GC data if needed. This will always be called by the root summarizer node to get GC data.
+        async (fullGC) => this.getGCDataInternal(fullGC), 
+        // Function to get the GC details from the base snapshot we loaded from.
+        async () => this.garbageCollector.getBaseGCDetails());
         if (baseSnapshot) {
             this.summarizerNode.updateBaseSummaryState(baseSnapshot);
         }
         this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap));
-        this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId, localId) => {
+        this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
             if (!this.disposed) {
-                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId, localId });
+                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
+                    localId,
+                    blobId,
+                });
             }
-        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
+        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (fromPath, toPath) => this.garbageCollector.addedOutboundReference(fromPath, toPath), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs, () => this.getCurrentReferenceTimestampMs());
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, () => this.clientId, ChildLogger.create(this.logger, "ScheduleManager"));
-        this.deltaSender = this.deltaManager;
         this.pendingStateManager = new PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
             clientId: () => this.clientId,
             close: this.closeFn,
             connected: () => this.connected,
-            flush: this.flush.bind(this),
             reSubmit: this.reSubmit.bind(this),
             rollback: this.rollback.bind(this),
             orderSequentially: this.orderSequentially.bind(this),
         }, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
+        const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true
+            ? {
+                minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
+                compressionAlgorithm: CompressionAlgorithms.lz4,
+            }
+            : runtimeOptions.compressionOptions;
+        this.outbox = new Outbox({
+            shouldSend: () => this.canSendOps(),
+            pendingStateManager: this.pendingStateManager,
+            containerContext: this.context,
+            compressor: new OpCompressor(this.mc.logger),
+            splitter: opSplitter,
+            config: {
+                compressionOptions,
+                maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
+                enableOpReentryCheck: this.enableOpReentryCheck,
+            },
+            logger: this.mc.logger,
+        });
         this.context.quorum.on("removeMember", (clientId) => {
-            this.clearPartialChunks(clientId);
+            this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
         });
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
-        this.dirtyContainer = this.context.attachState !== AttachState.Attached
-            || this.pendingStateManager.hasPendingMessages();
+        this.dirtyContainer =
+            this.context.attachState !== AttachState.Attached ||
+                this.pendingStateManager.hasPendingMessages();
         this.context.updateDirtyContainerState(this.dirtyContainer);
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
@@ -346,7 +351,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const orderedClientLogger = ChildLogger.create(this.logger, "OrderedClientElection");
             const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
             const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
-            this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary, this.summarizerClientElectionEnabled);
+            this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
             if (this.context.clientDetails.type === summarizerClientType) {
                 this._summarizer = new Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime));
             }
@@ -405,18 +410,53 @@ export class ContainerRuntime extends TypedEventEmitter {
         ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
         BindBatchTracker(this, this.logger);
     }
-    get IContainerRuntime() { return this; }
-    get IFluidRouter() { return this; }
+    get IContainerRuntime() {
+        return this;
+    }
+    get IFluidRouter() {
+        return this;
+    }
     /**
+     * @deprecated - use loadRuntime instead.
      * Load the stores from a snapshot and returns the runtime.
      * @param context - Context of the container.
      * @param registryEntries - Mapping to the stores.
      * @param requestHandler - Request handlers for the container runtime
      * @param runtimeOptions - Additional options to be passed to the runtime
      * @param existing - (optional) When loading from an existing snapshot. Precedes context.existing if provided
+     * @param containerRuntimeCtor - (optional) Constructor to use to create the ContainerRuntime instance. This
+     * allows mixin classes to leverage this method to define their own async initializer.
      */
-    static async load(context, registryEntries, requestHandler, runtimeOptions = {}, containerScope = context.scope, existing) {
-        var _a, _b, _c;
+    static async load(context, registryEntries, requestHandler, runtimeOptions = {}, containerScope = context.scope, existing, containerRuntimeCtor = ContainerRuntime) {
+        let existingFlag = true;
+        if (!existing) {
+            existingFlag = false;
+        }
+        return this.loadRuntime({
+            context,
+            registryEntries,
+            existing: existingFlag,
+            requestHandler,
+            runtimeOptions,
+            containerScope,
+            containerRuntimeCtor,
+        });
+    }
+    /**
+     * Load the stores from a snapshot and returns the runtime.
+     * @param params - An object housing the runtime properties:
+     * - context - Context of the container.
+     * - registryEntries - Mapping to the stores.
+     * - existing - When loading from an existing snapshot
+     * - requestHandler - Request handlers for the container runtime
+     * - runtimeOptions - Additional options to be passed to the runtime
+     * - containerScope - runtime services provided with context
+     * - containerRuntimeCtor - Constructor to use to create the ContainerRuntime instance.
+     * This allows mixin classes to leverage this method to define their own async initializer.
+     */
+    static async loadRuntime(params) {
+        var _a, _b, _c, _d;
+        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
@@ -426,13 +466,17 @@ export class ContainerRuntime extends TypedEventEmitter {
                 runtimeVersion: pkgVersion,
             },
         });
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = { minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
-            compressionAlgorithm: CompressionAlgorithms.lz4 }, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = {
+            minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
+            compressionAlgorithm: CompressionAlgorithms.lz4,
+        }, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, chunkSizeInBytes = Number.POSITIVE_INFINITY, enableOpReentryCheck = false, } = runtimeOptions;
         const pendingRuntimeState = context.pendingLocalState;
         const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
-        const storage = !pendingRuntimeState ?
-            context.storage :
-            new SerializedSnapshotStorage(() => { return context.storage; }, pendingRuntimeState.snapshotBlobs);
+        const storage = !pendingRuntimeState
+            ? context.storage
+            : new SerializedSnapshotStorage(() => {
+                return context.storage;
+            }, pendingRuntimeState.snapshotBlobs);
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.blobs[blobName];
@@ -463,7 +507,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (!pendingRuntimeState && runtimeSequenceNumber !== undefined) {
             const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
-            if (loadSequenceNumberVerification !== "bypass" && runtimeSequenceNumber !== protocolSequenceNumber) {
+            if (loadSequenceNumberVerification !== "bypass" &&
+                runtimeSequenceNumber !== protocolSequenceNumber) {
                 // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
                 const error = new DataCorruptionError(
                 // pre-0.58 error message: SummaryMetadataMismatch
@@ -472,11 +517,13 @@ export class ContainerRuntime extends TypedEventEmitter {
                     logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
                 }
                 else {
+                    // Call both close and dispose as closeFn implementation will no longer dispose runtime in future
                     context.closeFn(error);
+                    (_d = context.disposeFn) === null || _d === void 0 ? void 0 : _d.call(context, error);
                 }
             }
         }
-        const runtime = new ContainerRuntime(context, registry, metadata, electedSummarizerData, chunks !== null && chunks !== void 0 ? chunks : [], aliases !== null && aliases !== void 0 ? aliases : [], {
+        const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks !== null && chunks !== void 0 ? chunks : [], aliases !== null && aliases !== void 0 ? aliases : [], {
             summaryOptions,
             gcOptions,
             loadSequenceNumberVerification,
@@ -484,6 +531,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             enableOfflineLoad,
             compressionOptions,
             maxBatchSizeInBytes,
+            chunkSizeInBytes,
+            enableOpReentryCheck,
         }, containerScope, logger, loadExisting, blobManagerSnapshot, storage, requestHandler);
         if (pendingRuntimeState) {
             await runtime.processSavedOps(pendingRuntimeState);
@@ -513,8 +562,18 @@ export class ContainerRuntime extends TypedEventEmitter {
         // eslint-disable-next-line @typescript-eslint/unbound-method
         return this.reSubmit;
     }
+    get disposeFn() {
+        var _a;
+        // In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
+        return (_a = this.context.disposeFn) !== null && _a !== void 0 ? _a : this.context.closeFn;
+    }
     get closeFn() {
-        return this.context.closeFn;
+        // Also call disposeFn to retain functionality of runtime being disposed on close
+        return (error) => {
+            var _a, _b;
+            this.context.closeFn(error);
+            (_b = (_a = this.context).disposeFn) === null || _b === void 0 ? void 0 : _b.call(_a, error);
+        };
     }
     get flushMode() {
         return this._flushMode;
@@ -531,6 +590,23 @@ export class ContainerRuntime extends TypedEventEmitter {
     get IFluidHandleContext() {
         return this.handleContext;
     }
+    /**
+     * Invokes the given callback and expects that no ops are submitted
+     * until execution finishes. If an op is submitted, an error will be raised.
+     *
+     * Can be disabled by feature gate `Fluid.ContainerRuntime.DisableOpReentryCheck`
+     *
+     * @param callback - the callback to be invoked
+     */
+    ensureNoDataModelChanges(callback) {
+        this.ensureNoDataModelChangesCalls++;
+        try {
+            return callback();
+        }
+        finally {
+            this.ensureNoDataModelChangesCalls--;
+        }
+    }
     get connected() {
         return this._connected;
     }
@@ -539,51 +615,20 @@ export class ContainerRuntime extends TypedEventEmitter {
         var _a;
         return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
     }
-    get disposed() { return this._disposed; }
-    get emptyBatch() {
-        return this.pendingBatch.empty && this.pendingAttachBatch.empty;
+    get disposed() {
+        return this._disposed;
     }
     get summarizer() {
         assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
     }
     isSummariesDisabled() {
-        // back-compat: disableSummaries was moved from ISummaryRuntimeOptions
-        //   to ISummaryConfiguration in 0.60.
-        if (this.runtimeOptions.summaryOptions.disableSummaries === true) {
-            return true;
-        }
         return this.summaryConfiguration.state === "disabled";
     }
     isHeuristicsDisabled() {
-        var _a;
-        // back-compat: disableHeuristics was moved from ISummarizerOptions
-        //   to ISummaryConfiguration in 0.60.
-        if (((_a = this.runtimeOptions.summaryOptions.summarizerOptions) === null || _a === void 0 ? void 0 : _a.disableHeuristics) === true) {
-            return true;
-        }
         return this.summaryConfiguration.state === "disableHeuristics";
     }
-    isSummarizerClientElectionEnabled() {
-        var _a;
-        if (this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) {
-            return (_a = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _a !== void 0 ? _a : true;
-        }
-        // back-compat: summarizerClientElection was moved from ISummaryRuntimeOptions
-        //   to ISummaryConfiguration in 0.60.
-        if (this.runtimeOptions.summaryOptions.summarizerClientElection === true) {
-            return true;
-        }
-        return this.summaryConfiguration.state !== "disabled"
-            ? this.summaryConfiguration.summarizerClientElection === true
-            : false;
-    }
     getMaxOpsSinceLastSummary() {
-        // back-compat: maxOpsSinceLastSummary was moved from ISummaryRuntimeOptions
-        //   to ISummaryConfiguration in 0.60.
-        if (this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary !== undefined) {
-            return this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary;
-        }
         return this.summaryConfiguration.state !== "disabled"
             ? this.summaryConfiguration.maxOpsSinceLastSummary
             : 0;
@@ -682,7 +727,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                         status: 200,
                         mimeType: "fluid/object",
                         value: blob,
-                    } : create404Response(request);
+                    }
+                    : create404Response(request);
             }
             else if (requestParser.pathParts.length > 0) {
                 const dataStore = await this.getDataStoreFromRequest(id, request);
@@ -703,16 +749,20 @@ export class ContainerRuntime extends TypedEventEmitter {
         return (_a = this.dataStores.aliases.get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
     }
     async getDataStoreFromRequest(id, request) {
-        var _a, _b, _c, _d, _e;
-        const wait = typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a[RuntimeHeaders.wait]) === "boolean"
-            ? (_b = request.headers) === null || _b === void 0 ? void 0 : _b[RuntimeHeaders.wait]
-            : true;
-        const viaHandle = typeof ((_c = request.headers) === null || _c === void 0 ? void 0 : _c[RuntimeHeaders.viaHandle]) === "boolean"
-            ? (_d = request.headers) === null || _d === void 0 ? void 0 : _d[RuntimeHeaders.viaHandle]
-            : false;
+        var _a, _b, _c, _d;
+        const headerData = {};
+        if (typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a[RuntimeHeaders.wait]) === "boolean") {
+            headerData.wait = request.headers[RuntimeHeaders.wait];
+        }
+        if (typeof ((_b = request.headers) === null || _b === void 0 ? void 0 : _b[RuntimeHeaders.viaHandle]) === "boolean") {
+            headerData.viaHandle = request.headers[RuntimeHeaders.viaHandle];
+        }
+        if (typeof ((_c = request.headers) === null || _c === void 0 ? void 0 : _c[AllowTombstoneRequestHeaderKey]) === "boolean") {
+            headerData.allowTombstone = request.headers[AllowTombstoneRequestHeaderKey];
+        }
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
-        const dataStoreContext = await this.dataStores.getDataStore(internalId, wait, viaHandle);
+        const dataStoreContext = await this.dataStores.getDataStore(internalId, headerData);
         /**
          * If GC should run and this an external app request with "externalRequest" header, we need to return
          * an error if the data store being requested is marked as unreferenced as per the data store's base
@@ -721,7 +771,8 @@ export class ContainerRuntime extends TypedEventEmitter {
          * This is a workaround to handle scenarios where a data store shared with an external app is deleted
          * and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
          */
-        if (((_e = request.headers) === null || _e === void 0 ? void 0 : _e[RuntimeHeaders.externalRequest]) && this.garbageCollector.shouldRunGC) {
+        if (((_d = request.headers) === null || _d === void 0 ? void 0 : _d[RuntimeHeaders.externalRequest]) &&
+            this.garbageCollector.shouldRunGC) {
             // The data store is referenced if used routes in the base summary has a route to self.
             // Older documents may not have used routes in the summary. They are considered referenced.
             const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
@@ -750,8 +801,8 @@ export class ContainerRuntime extends TypedEventEmitter {
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         var _a;
         this.addMetadataToSummary(summaryTree);
-        if (this.chunkMap.size > 0) {
-            const content = JSON.stringify([...this.chunkMap]);
+        if (this.remoteMessageProcessor.partialMessages.size > 0) {
+            const content = JSON.stringify([...this.remoteMessageProcessor.partialMessages]);
             addBlobToSummary(summaryTree, chunksBlobName, content);
         }
         const dataStoreAliases = this.dataStores.aliases;
@@ -895,9 +946,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (reconnection) {
             this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(DataProcessingError.create(
-                // eslint-disable-next-line max-len
-                "Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
+                this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
                     pendingMessages: this.pendingStateManager.pendingMessagesCount,
@@ -915,34 +964,23 @@ export class ContainerRuntime extends TypedEventEmitter {
     process(messageArg, local) {
         var _a;
         this.verifyNotClosed();
-        // Do shallow copy of message, as methods below will modify it.
-        // There might be multiple container instances receiving same message
-        // We do not need to make deep copy, as each layer will just replace message.content itself,
-        // but would not modify contents details
-        let message = Object.assign({}, messageArg);
-        // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
-        // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
-        // Old ops may contain empty string (I assume noops).
-        if (typeof message.contents === "string" && message.contents !== "") {
-            message.contents = JSON.parse(message.contents);
-        }
-        message = this.opDecompressor.processMessage(message);
-        // Caveat: This will return false for runtime message in very old format, that are used in snapshot tests
-        // This format was not shipped to production workflows.
-        const runtimeMessage = unpackRuntimeMessage(message);
         if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
             this.savedOps.push(messageArg);
         }
+        // Whether or not the message is actually a runtime message.
+        // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
+        // or something different, like a system message.
+        const runtimeMessage = messageArg.type === MessageType.Operation;
+        // Do shallow copy of message, as the processing flow will modify it.
+        const messageCopy = Object.assign({}, messageArg);
+        const message = this.remoteMessageProcessor.process(messageCopy);
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
         this.scheduleManager.beforeOpProcessing(message);
         try {
-            // Chunk processing must come first given that we will transform the message to the unchunked version
-            // once all pieces are available
-            message = this.processRemoteChunkedMessage(message);
             let localOpMetadata;
-            if (local && runtimeMessage) {
+            if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1015,7 +1053,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (message.clientId === this.clientId && this.connected) {
             // Check to see if the signal was lost.
             if (this._perfSignalData.trackingSignalSequenceNumber !== undefined &&
-                envelope.clientSignalSequenceNumber > this._perfSignalData.trackingSignalSequenceNumber) {
+                envelope.clientSignalSequenceNumber >
+                    this._perfSignalData.trackingSignalSequenceNumber) {
                 this._perfSignalData.signalsLost++;
                 this._perfSignalData.trackingSignalSequenceNumber = undefined;
                 this.logger.sendErrorEvent({
@@ -1026,7 +1065,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                     clientSignalSequenceNumber: envelope.clientSignalSequenceNumber,
                 });
             }
-            else if (envelope.clientSignalSequenceNumber === this._perfSignalData.trackingSignalSequenceNumber) {
+            else if (envelope.clientSignalSequenceNumber ===
+                this._perfSignalData.trackingSignalSequenceNumber) {
                 this.sendSignalTelemetryEvent(envelope.clientSignalSequenceNumber);
                 this._perfSignalData.trackingSignalSequenceNumber = undefined;
             }
@@ -1044,7 +1084,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     async getRootDataStoreChannel(id, wait = true) {
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
-        const context = await this.dataStores.getDataStore(internalId, wait, false /* viaHandle */);
+        const context = await this.dataStores.getDataStore(internalId, { wait });
         assert(await context.isRoot(), 0x12b /* "did not get root data store" */);
         return context.realize();
     }
@@ -1054,74 +1094,21 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     flush() {
         assert(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
-        this.flushBatch(this.pendingAttachBatch.popBatch());
-        this.flushBatch(this.pendingBatch.popBatch());
-        assert(this.emptyBatch, 0x3cf /* reentrancy */);
-    }
-    flushBatch(batch) {
-        var _a;
-        const length = batch.length;
-        if (length > 1) {
-            batch[0].metadata = Object.assign(Object.assign({}, batch[0].metadata), { batch: true });
-            batch[length - 1].metadata = Object.assign(Object.assign({}, batch[length - 1].metadata), { batch: false });
-            // This assert fires for the following reason (there might be more cases like that):
-            // AgentScheduler will send ops in response to ConsensusRegisterCollection's "atomicChanged" event handler,
-            // i.e. in the middle of op processing!
-            // Sending ops while processing ops is not good idea - it's not defined when
-            // referenceSequenceNumber changes in op processing sequence (at the beginning or end of op processing),
-            // If we send ops in response to processing multiple ops, then we for sure hit this assert!
-            // Tracked via ADO #1834
-            // assert(batch[0].referenceSequenceNumber === batch[length - 1].referenceSequenceNumber,
-            //    "Batch should be generated synchronously, without processing ops in the middle!");
-        }
-        let clientSequenceNumber = -1;
-        // Did we disconnect in the middle of turn-based batch?
-        // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
-        if (this.canSendOps()) {
-            if (this.context.submitBatchFn !== undefined) {
-                const batchToSend = [];
-                for (const message of batch) {
-                    batchToSend.push({ contents: message.contents, metadata: message.metadata });
-                }
-                // returns clientSequenceNumber of last message in a batch
-                clientSequenceNumber = this.context.submitBatchFn(batchToSend);
-            }
-            else {
-                // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
-                // version that has support for batches (submitBatchFn)
-                for (const message of batch) {
-                    // Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
-                    if ((_a = message.metadata) === null || _a === void 0 ? void 0 : _a.compressed) {
-                        delete message.metadata.compressed;
-                    }
-                    clientSequenceNumber = this.context.submitFn(MessageType.Operation, message.deserializedContent, true, // batch
-                    message.metadata);
-                }
-                this.deltaSender.flush();
-            }
-            // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
-            clientSequenceNumber -= batch.length - 1;
-            assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
-        }
-        // Let the PendingStateManager know that a message was submitted.
-        // In future, need to shift toward keeping batch as a whole!
-        for (const message of batch) {
-            this.pendingStateManager.onSubmitMessage(message.deserializedContent.type, clientSequenceNumber, message.referenceSequenceNumber, message.deserializedContent.contents, message.localOpMetadata, message.metadata);
-            clientSequenceNumber++;
-        }
-        this.pendingStateManager.onFlush();
+        this.outbox.flush();
+        assert(this.outbox.isEmpty, 0x3cf /* reentrancy */);
     }
     orderSequentially(callback) {
         let checkpoint;
+        let result;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
             // Note: we are not touching this.pendingAttachBatch here, for two reasons:
             // 1. It would not help, as we flush attach ops as they become available.
             // 2. There is no way to undo process of data store creation.
-            checkpoint = this.pendingBatch.checkpoint();
+            checkpoint = this.outbox.checkpoint().mainBatch;
         }
         try {
             this._orderSequentiallyCalls++;
-            callback();
+            result = callback();
         }
         catch (error) {
             if (checkpoint) {
@@ -1146,9 +1133,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         finally {
             this._orderSequentiallyCalls--;
         }
-        if (this.flushMode === FlushMode.Immediate && this._orderSequentiallyCalls === 0) {
+        // We don't flush on TurnBased since we expect all messages in the same JS turn to be part of the same batch
+        if (this.flushMode !== FlushMode.TurnBased && this._orderSequentiallyCalls === 0) {
             this.flush();
         }
+        return result;
     }
     async createDataStore(pkg) {
         const internalId = uuid();
@@ -1164,7 +1153,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.dataStores.createDetachedDataStoreCore(pkg, false);
     }
     async _createDataStoreWithProps(pkg, props, id = uuid()) {
-        const fluidDataStore = await this.dataStores._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props).realize();
+        const fluidDataStore = await this.dataStores
+            ._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
+            .realize();
         return channelToDataStore(fluidDataStore, id, this, this.dataStores, this.mc.logger);
     }
     async _createDataStore(pkg, id = uuid(), props) {
@@ -1301,7 +1292,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         const telemetryContext = new TelemetryContext();
         const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
-        this.logger.sendTelemetryEvent({ eventName: "SummarizeTelemetry", details: telemetryContext.serialize() });
+        this.logger.sendTelemetryEvent({
+            eventName: "SummarizeTelemetry",
+            details: telemetryContext.serialize(),
+        });
         assert(summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
         return { stats, summary, gcStats };
     }
@@ -1314,6 +1308,9 @@ export class ContainerRuntime extends TypedEventEmitter {
     async updateStateBeforeGC() {
         return this.dataStores.updateStateBeforeGC();
     }
+    async getGCDataInternal(fullGC) {
+        return this.dataStores.getGCData(fullGC);
+    }
     /**
      * Implementation of IGarbageCollectionRuntime::getGCData.
      * Generates and returns the GC data for this container.
@@ -1321,7 +1318,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     async getGCData(fullGC) {
         const builder = new GCDataBuilder();
-        const dsGCData = await this.dataStores.getGCData(fullGC);
+        const dsGCData = await this.summarizerNode.getGCData(fullGC);
         builder.addNodes(dsGCData.gcNodes);
         const blobsGCData = this.blobManager.getGCData(fullGC);
         builder.addNodes(blobsGCData.gcNodes);
@@ -1337,37 +1334,26 @@ export class ContainerRuntime extends TypedEventEmitter {
         // summarizing is required and asserted by the the summarizer node. We are the root and are
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
-        const dataStoreUsedRoutes = [];
-        for (const route of usedRoutes) {
-            if (route.split("/")[1] !== BlobManager.basePath) {
-                dataStoreUsedRoutes.push(route);
-            }
-        }
-        return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes);
+        const { dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(usedRoutes);
+        this.dataStores.updateUsedRoutes(dataStoreRoutes);
     }
     /**
-     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
-     * tombstones.
-     * @param unusedRoutes - The routes that are unused in all data stores in this Container.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
+     * This is called to update objects whose routes are unused.
+     * @param unusedRoutes - Data store and attachment blob routes that are unused in this Container.
      */
-    updateUnusedRoutes(unusedRoutes, tombstone) {
-        const blobManagerUnusedRoutes = [];
-        const dataStoreUnusedRoutes = [];
-        for (const route of unusedRoutes) {
-            if (this.isBlobPath(route)) {
-                blobManagerUnusedRoutes.push(route);
-            }
-            else {
-                dataStoreUnusedRoutes.push(route);
-            }
-        }
-        // Todo: Add tombstone for attachment blobs. For now, we ignore attachment blobs that should be tombstoned.
-        if (!tombstone) {
-            this.blobManager.deleteUnusedRoutes(blobManagerUnusedRoutes);
-        }
-        this.dataStores.updateUnusedRoutes(dataStoreUnusedRoutes, tombstone);
+    updateUnusedRoutes(unusedRoutes) {
+        const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
+        this.blobManager.updateUnusedRoutes(blobManagerRoutes);
+        this.dataStores.updateUnusedRoutes(dataStoreRoutes);
+    }
+    /**
+     * This is called to update objects that are tombstones.
+     * @param tombstonedRoutes - Data store and attachment blob routes that are tombstones in this Container.
+     */
+    updateTombstonedRoutes(tombstonedRoutes) {
+        const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(tombstonedRoutes);
+        this.blobManager.updateTombstonedRoutes(blobManagerRoutes);
+        this.dataStores.updateTombstonedRoutes(dataStoreRoutes);
     }
     /**
      * Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
@@ -1396,7 +1382,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     async getGCNodePackagePath(nodePath) {
         switch (this.getNodeType(nodePath)) {
             case GCNodeType.Blob:
-                return ["_blobs"];
+                return [BlobManager.basePath];
             case GCNodeType.DataStore:
             case GCNodeType.SubDataStore:
                 return this.dataStores.getDataStorePackagePath(nodePath);
@@ -1414,6 +1400,25 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         return true;
     }
+    /**
+     * From a given list of routes, separate and return routes that belong to blob manager and data stores.
+     * @param routes - A list of routes that can belong to data stores or blob manager.
+     * @returns - Two route lists - One that contains routes for blob manager and another one that contains routes
+     * for data stores.
+     */
+    getDataStoreAndBlobManagerRoutes(routes) {
+        const blobManagerRoutes = [];
+        const dataStoreRoutes = [];
+        for (const route of routes) {
+            if (this.isBlobPath(route)) {
+                blobManagerRoutes.push(route);
+            }
+            else {
+                dataStoreRoutes.push(route);
+            }
+        }
+        return { blobManagerRoutes, dataStoreRoutes };
+    }
     /**
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
      * @returns the statistics of the garbage collection run; undefined if GC did not run.
@@ -1447,7 +1452,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         const summaryNumberLogger = ChildLogger.create(summaryLogger, undefined, {
             all: { summaryNumber },
         });
-        assert(this.emptyBatch, 0x3d1 /* Can't trigger summary in the middle of a batch */);
+        assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
         let latestSnapshotVersionId;
         if (refreshLatestAck) {
             const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(ChildLogger.create(summaryNumberLogger, undefined, { all: { safeSummary: true } }));
@@ -1485,7 +1490,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 if (this.deltaManager.lastSequenceNumber !== summaryRefSeqNum) {
                     return {
                         continue: false,
-                        // eslint-disable-next-line max-len
                         error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
                     };
                 }
@@ -1493,7 +1497,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 if (lastAck !== this.summaryCollection.latestAck) {
                     return {
                         continue: false,
-                        // eslint-disable-next-line max-len
                         error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
                     };
                 }
@@ -1560,8 +1563,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             // latestSnapshotVersionId from storage and it does not match with the lastAck ackHandle, then use
             // the one fetched from storage as parent as that is the latest.
             let summaryContext;
-            if ((lastAck === null || lastAck === void 0 ? void 0 : lastAck.summaryAck.contents.handle) !== latestSnapshotVersionId
-                && latestSnapshotVersionId !== undefined) {
+            if ((lastAck === null || lastAck === void 0 ? void 0 : lastAck.summaryAck.contents.handle) !== latestSnapshotVersionId &&
+                latestSnapshotVersionId !== undefined) {
                 summaryContext = {
                     proposalHandle: undefined,
                     ackHandle: latestSnapshotVersionId,
@@ -1620,40 +1623,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.deltaManager.inbound.resume();
         }
     }
-    processRemoteChunkedMessage(message) {
-        if (message.type !== ContainerMessageType.ChunkedOp) {
-            return message;
-        }
-        const clientId = message.clientId;
-        const chunkedContent = message.contents;
-        this.addChunk(clientId, chunkedContent);
-        if (chunkedContent.chunkId === chunkedContent.totalChunks) {
-            const newMessage = Object.assign({}, message);
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            const serializedContent = this.chunkMap.get(clientId).join("");
-            newMessage.contents = JSON.parse(serializedContent);
-            newMessage.type = chunkedContent.originalType;
-            this.clearPartialChunks(clientId);
-            return newMessage;
-        }
-        return message;
-    }
-    addChunk(clientId, chunkedContent) {
-        let map = this.chunkMap.get(clientId);
-        if (map === undefined) {
-            map = [];
-            this.chunkMap.set(clientId, map);
-        }
-        assert(chunkedContent.chunkId === map.length + 1, 0x131 /* "Mismatch between new chunkId and expected chunkMap" */); // 1-based indexing
-        map.push(chunkedContent.contents);
-    }
-    clearPartialChunks(clientId) {
-        if (this.chunkMap.has(clientId)) {
-            this.chunkMap.delete(clientId);
-        }
-    }
     hasPendingMessages() {
-        return this.pendingStateManager.hasPendingMessages() || !this.emptyBatch;
+        return this.pendingStateManager.hasPendingMessages() || !this.outbox.isEmpty;
     }
     updateDocumentDirtyState(dirty) {
         if (this.attachState !== AttachState.Attached) {
@@ -1691,14 +1662,17 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.blobManager.createBlob(blob);
     }
     submit(type, contents, localOpMetadata = undefined, metadata = undefined) {
-        var _a, _b, _c, _d;
         this.verifyNotClosed();
+        this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
         assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         const deserializedContent = { type, contents };
         const serializedContent = JSON.stringify(deserializedContent);
         if (this.deltaManager.readOnlyInfo.readonly) {
-            this.logger.sendErrorEvent({ eventName: "SubmitOpInReadonly" });
+            this.logger.sendTelemetryEvent({
+                eventName: "SubmitOpInReadonly",
+                connected: this.connected,
+            });
         }
         const message = {
             contents: serializedContent,
@@ -1728,44 +1702,28 @@ export class ContainerRuntime extends TypedEventEmitter {
             //    issue than sending.
             // Please note that this does not change file format, so it can be disabled in the future if this
             // optimization no longer makes sense (for example, batch compression may make it less appealing).
-            if (this.currentlyBatching() && type === ContainerMessageType.Attach &&
-                this.mc.config.getBoolean("Fluid.ContainerRuntime.enableAttachOpReorder") === true) {
-                if (!this.pendingAttachBatch.push(message)) {
-                    // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
-                    // when queue is not empty.
-                    // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
-                    this.flushBatch(this.pendingAttachBatch.popBatch());
-                    if (!this.pendingAttachBatch.push(message)) {
-                        throw new GenericError("BatchTooLarge", 
-                        /* error */ undefined, {
-                            opSize: (_b = ((_a = message.contents) === null || _a === void 0 ? void 0 : _a.length)) !== null && _b !== void 0 ? _b : 0,
-                            count: this.pendingAttachBatch.length,
-                            limit: this.pendingAttachBatch.options.hardLimit,
-                        });
-                    }
-                }
+            if (this.currentlyBatching() &&
+                type === ContainerMessageType.Attach &&
+                this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
+                this.outbox.submitAttach(message);
             }
             else {
-                if (!this.pendingBatch.push(message)) {
-                    throw new GenericError("BatchTooLarge", 
-                    /* error */ undefined, {
-                        opSize: (_d = ((_c = message.contents) === null || _c === void 0 ? void 0 : _c.length)) !== null && _d !== void 0 ? _d : 0,
-                        count: this.pendingBatch.length,
-                        limit: this.pendingBatch.options.hardLimit,
-                    });
-                }
-                if (!this.currentlyBatching()) {
+                this.outbox.submit(message);
+            }
+            if (!this.currentlyBatching()) {
+                this.flush();
+            }
+            else if (!this.flushMicroTaskExists) {
+                this.flushMicroTaskExists = true;
+                // Queue a microtask to detect the end of the turn and force a flush.
+                Promise.resolve()
+                    .then(() => {
+                    this.flushMicroTaskExists = false;
                     this.flush();
-                }
-                else if (!this.flushMicroTaskExists) {
-                    this.flushMicroTaskExists = true;
-                    // Queue a microtask to detect the end of the turn and force a flush.
-                    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-                    Promise.resolve().then(() => {
-                        this.flushMicroTaskExists = false;
-                        this.flush();
-                    });
-                }
+                })
+                    .catch((error) => {
+                    this.closeFn(error);
+                });
             }
         }
         catch (error) {
@@ -1780,7 +1738,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.verifyNotClosed();
         assert(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
-        assert(this.emptyBatch, 0x3d4 /* System op in the middle of a batch */);
+        assert(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
         // back-compat: ADO #1385: Make this call unconditional in the future
         return this.context.submitSummaryFn !== undefined
             ? this.context.submitSummaryFn(contents)
@@ -1795,6 +1753,32 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw new Error("Runtime is closed");
         }
     }
+    verifyCanSubmitOps() {
+        if (this.ensureNoDataModelChangesCalls > 0) {
+            const errorMessage = "Op was submitted from within a `ensureNoDataModelChanges` callback";
+            if (this.opReentryCallsToReport > 0) {
+                this.mc.logger.sendTelemetryEvent({ eventName: "OpReentry" }, 
+                // We need to capture the call stack in order to inspect the source of this usage pattern
+                new UsageError(errorMessage));
+                this.opReentryCallsToReport--;
+            }
+            // Creating ops while processing ops can lead
+            // to undefined behavior and events observed in the wrong order.
+            // For example, we have two callbacks registered for a DDS, A and B.
+            // Then if on change #1 callback A creates change #2, the invocation flow will be:
+            //
+            // A because of #1
+            // A because of #2
+            // B because of #2
+            // B because of #1
+            //
+            // The runtime must enforce op coherence by not allowing ops to be submitted
+            // while ops are being processed.
+            if (this.enableOpReentryCheck) {
+                throw new UsageError(errorMessage);
+            }
+        }
+    }
     /**
      * Finds the right store and asks it to resubmit the message. This typically happens when we
      * reconnect and there are pending messages.
@@ -1854,13 +1838,32 @@ export class ContainerRuntime extends TypedEventEmitter {
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
         const snapshotTreeFetcher = async () => {
-            const fetchResult = await this.fetchSnapshotFromStorage(null, summaryLogger, {
+            const fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
                 eventName: "RefreshLatestSummaryGetSnapshot",
                 ackHandle,
                 summaryRefSeq,
                 fetchLatest: true,
             });
             const latestSnapshotRefSeq = await seqFromTree(fetchResult.snapshotTree, readAndParseBlob);
+            /**
+             * If the fetched snapshot is older than the one for which the ack was received, close the container.
+             * This should never happen because an ack should be sent after the latest summary is updated in the server.
+             * However, there are couple of scenarios where it's possible:
+             * 1. A file was modified externally resulting in modifying the snapshot's sequence number. This can lead to
+             * the document being unusable and we should not proceed.
+             * 2. The server DB failed after the ack was sent which may delete the corresponding snapshot. Ideally, in
+             * such cases, the file will be rolled back along with the ack and we will eventually reach a consistent
+             * state.
+             */
+            if (latestSnapshotRefSeq < summaryRefSeq) {
+                const error = DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+                    ackHandle,
+                    summaryRefSeq,
+                    latestSnapshotRefSeq,
+                });
+                this.closeFn(error);
+                throw error;
+            }
             summaryLogger.sendTelemetryEvent({
                 eventName: "LatestSummaryRetrieved",
                 ackHandle,
@@ -1874,7 +1877,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         };
         const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, snapshotTreeFetcher, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result, proposalHandle, summaryRefSeq, readAndParseBlob);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -1883,22 +1886,22 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @returns downloaded snapshot's reference sequence number
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
-        const { snapshotTree, versionId } = await this.fetchSnapshotFromStorage(null, summaryLogger, {
+        const { snapshotTree, versionId } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
             eventName: "RefreshLatestSummaryGetSnapshot",
             fetchLatest: true,
-        }, FetchSource.noCache);
+        });
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
         const latestSnapshotRefSeq = await seqFromTree(snapshotTree, readAndParseBlob);
         const result = await this.summarizerNode.refreshLatestSummary(undefined, latestSnapshotRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result, undefined, latestSnapshotRefSeq, readAndParseBlob);
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchSnapshotFromStorage(versionId, logger, event, fetchSource) {
+    async fetchLatestSnapshotFromStorage(logger, event) {
         return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
-            const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", fetchSource);
+            const versions = await this.storage.getVersions(null, 1, "refreshLatestSummaryAckFromServer", FetchSource.noCache);
             assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
@@ -1911,13 +1914,15 @@ export class ContainerRuntime extends TypedEventEmitter {
     notifyAttaching(snapshot) {
         var _a;
         if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
-            this.baseSnapshotBlobs = SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
+            this.baseSnapshotBlobs =
+                SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
         }
     }
     async initializeBaseSnapshotBlobs() {
         var _a;
         if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) ||
-            this.attachState !== AttachState.Attached || this.context.pendingLocalState) {
+            this.attachState !== AttachState.Attached ||
+            this.context.pendingLocalState) {
             return;
         }
         assert(!!this.context.baseSnapshot, 0x2e5 /* "Must have a base snapshot" */);
@@ -1928,13 +1933,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad)) {
             throw new UsageError("can't get state when offline load disabled");
         }
+        if (this._orderSequentiallyCalls !== 0) {
+            throw new UsageError("can't get state during orderSequentially");
+        }
         // Flush pending batch.
         // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
         // to close current batch.
         this.flush();
-        if (this._orderSequentiallyCalls !== 0) {
-            throw new UsageError("can't get state during orderSequentially");
-        }
         const previousPendingState = this.context.pendingLocalState;
         if (previousPendingState) {
             return {
@@ -2014,6 +2019,7 @@ export class ContainerRuntime extends TypedEventEmitter {
 const waitForSeq = async (deltaManager, targetSeq) => new Promise((resolve, reject) => {
     // TODO: remove cast to any when actual event is determined
     deltaManager.on("closed", reject);
+    deltaManager.on("disposed", reject);
     // If we already reached target sequence number, simply resolve the promise.
     if (deltaManager.lastSequenceNumber >= targetSeq) {
         resolve();