@fluidframework/container-runtime 2.0.0-internal.2.1.2 → 2.0.0-internal.2.2.1

package/dist/containerRuntime.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.unpackRuntimeMessage = exports.isRuntimeMessage = exports.RuntimeMessage = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
+exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.CompressionAlgorithms = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
 const container_definitions_1 = require("@fluidframework/container-definitions");
 const common_utils_1 = require("@fluidframework/common-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
@@ -12,14 +12,12 @@ const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
 const runtime_utils_1 = require("@fluidframework/runtime-utils");
 const garbage_collector_1 = require("@fluidframework/garbage-collector");
 const uuid_1 = require("uuid");
-const lz4js_1 = require("lz4js");
 const containerHandleContext_1 = require("./containerHandleContext");
 const dataStoreRegistry_1 = require("./dataStoreRegistry");
 const summarizer_1 = require("./summarizer");
 const summaryManager_1 = require("./summaryManager");
 const connectionTelemetry_1 = require("./connectionTelemetry");
 const pendingStateManager_1 = require("./pendingStateManager");
-const batchManager_1 = require("./batchManager");
 const packageVersion_1 = require("./packageVersion");
 const blobManager_1 = require("./blobManager");
 const dataStores_1 = require("./dataStores");
@@ -30,10 +28,12 @@ const summarizerClientElection_1 = require("./summarizerClientElection");
 const throttler_1 = require("./throttler");
 const runWhileConnectedCoordinator_1 = require("./runWhileConnectedCoordinator");
 const garbageCollection_1 = require("./garbageCollection");
+const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
 const dataStore_1 = require("./dataStore");
 const batchTracker_1 = require("./batchTracker");
 const serializedSnapshotStorage_1 = require("./serializedSnapshotStorage");
 const scheduleManager_1 = require("./scheduleManager");
+const opLifecycle_1 = require("./opLifecycle");
 var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -79,6 +79,13 @@ var RuntimeHeaders;
     /** True if the request is coming from an IFluidHandle. */
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders = exports.RuntimeHeaders || (exports.RuntimeHeaders = {}));
+/**
+ * Available compression algorithms for op compression.
+ */
+var CompressionAlgorithms;
+(function (CompressionAlgorithms) {
+    CompressionAlgorithms["lz4"] = "lz4";
+})(CompressionAlgorithms = exports.CompressionAlgorithms || (exports.CompressionAlgorithms = {}));
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
 const defaultFlushMode = runtime_definitions_1.FlushMode.TurnBased;
 // The actual limit is 1Mb (socket.io and Kafka limits)
@@ -106,48 +113,6 @@ function isRuntimeMessage(message) {
     return Object.values(RuntimeMessage).includes(message.type);
 }
 exports.isRuntimeMessage = isRuntimeMessage;
-/**
- * Unpacks runtime messages
- *
- * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
- * @param message - message (as it observed in storage / service)
- * @returns unpacked runtime message
- *
- * @internal
- */
-function unpackRuntimeMessage(message) {
-    var _a;
-    if ((_a = message.metadata) === null || _a === void 0 ? void 0 : _a.compressed) {
-        const contents = common_utils_1.IsoBuffer.from(message.contents.contents, "base64");
-        const decompressedMessage = (0, lz4js_1.decompress)(contents);
-        const intoString = new TextDecoder().decode(decompressedMessage);
-        const asObj = JSON.parse(intoString);
-        message.contents.contents = asObj;
-        message.metadata.compressed = false;
-    }
-    if (message.type === protocol_definitions_1.MessageType.Operation) {
-        // legacy op format?
-        if (message.contents.address !== undefined && message.contents.type === undefined) {
-            message.type = ContainerMessageType.FluidDataStoreOp;
-        }
-        else {
-            // new format
-            const innerContents = message.contents;
-            (0, common_utils_1.assert)(innerContents.type !== undefined, 0x121 /* "Undefined inner contents type!" */);
-            message.type = innerContents.type;
-            message.contents = innerContents.contents;
-        }
-        return true;
-    }
-    else {
-        // Legacy format, but it's already "unpacked",
-        // i.e. message.type is actually ContainerMessageType.
-        // Or it's non-runtime message.
-        // Nothing to do in such case.
-        return false;
-    }
-}
-exports.unpackRuntimeMessage = unpackRuntimeMessage;
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
  * special-case for document dirty state.  Ultimately we should have no special-cases from the
@@ -174,6 +139,9 @@ exports.getDeviceSpec = getDeviceSpec;
  * It will define the store level mappings.
  */
 class ContainerRuntime extends common_utils_1.TypedEventEmitter {
+    /**
+     * @internal
+     */
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
         var _a, _b, _c, _d, _e, _f;
         if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, exports.DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
@@ -191,7 +159,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.flushMicroTaskExists = false;
         this.savedOps = [];
         this.consecutiveReconnects = 0;
-        this.compressedOpCount = 0;
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
         this.defaultTelemetrySignalSampleCount = 100;
@@ -229,9 +196,29 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 throw new container_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
             }
         };
+        let loadSummaryNumber;
+        // Get the container creation metadata. For new container, we initialize these. For existing containers,
+        // get the values from the metadata blob.
+        if (existing) {
+            this.createContainerMetadata = {
+                createContainerRuntimeVersion: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerRuntimeVersion,
+                createContainerTimestamp: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerTimestamp,
+            };
+            // summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
+            // the count is reset to 0.
+            loadSummaryNumber = (_b = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _b !== void 0 ? _b : 0;
+        }
+        else {
+            this.createContainerMetadata = {
+                createContainerRuntimeVersion: packageVersion_1.pkgVersion,
+                createContainerTimestamp: Date.now(),
+            };
+            loadSummaryNumber = 0;
+        }
+        this.nextSummaryNumber = loadSummaryNumber + 1;
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         this._connected = this.context.connected;
-        this.chunkMap = new Map(chunks);
+        this.remoteMessageProcessor = new opLifecycle_1.RemoteMessageProcessor(new opLifecycle_1.OpSplitter(chunks), new opLifecycle_1.OpDecompressor());
         this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.logger, "ContainerRuntime"));
         if (this.summaryConfiguration.state === "enabled") {
@@ -243,18 +230,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
         this.maxConsecutiveReconnects =
-            (_b = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _b !== void 0 ? _b : this.defaultMaxConsecutiveReconnects;
+            (_c = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _c !== void 0 ? _c : this.defaultMaxConsecutiveReconnects;
         this._flushMode = runtimeOptions.flushMode;
-        // Provide lower soft limit - we want to have some number of ops to get efficiency in compression
-        // & bandwidth usage, but at the same time we want to send these ops sooner, to reduce overall
-        // latency of processing a batch.
-        // So there is some ballance here, that depends on compression algorithm and its efficiency working with smaller
-        // payloads. That number represents final (compressed) bits (once compression is implemented).
-        this.pendingAttachBatch = new batchManager_1.BatchManager(runtimeOptions.maxBatchSizeInBytes, 64 * 1024);
-        this.pendingBatch = new batchManager_1.BatchManager(runtimeOptions.maxBatchSizeInBytes);
         const pendingRuntimeState = context.pendingLocalState;
-        const baseSnapshot = (_c = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _c !== void 0 ? _c : context.baseSnapshot;
-        const maxSnapshotCacheDurationMs = (_e = (_d = this._storage) === null || _d === void 0 ? void 0 : _d.policies) === null || _e === void 0 ? void 0 : _e.maximumCacheDurationMs;
+        const baseSnapshot = (_d = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _d !== void 0 ? _d : context.baseSnapshot;
+        const maxSnapshotCacheDurationMs = (_f = (_e = this._storage) === null || _e === void 0 ? void 0 : _e.policies) === null || _f === void 0 ? void 0 : _f.maximumCacheDurationMs;
         if (maxSnapshotCacheDurationMs !== undefined && maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
             // which dictates the value is either undefined or exactly 5 days in ms.
@@ -268,6 +248,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             baseLogger: this.mc.logger,
             existing,
             metadata,
+            createContainerMetadata: this.createContainerMetadata,
             isSummarizerClient: this.context.clientDetails.type === summarizerClientElection_1.summarizerClientType,
             getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
             getLastSummaryTimestampMs: () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; },
@@ -302,7 +283,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
         }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
         this.scheduleManager = new scheduleManager_1.ScheduleManager(context.deltaManager, this, () => this.clientId, telemetry_utils_1.ChildLogger.create(this.logger, "ScheduleManager"));
-        this.deltaSender = this.deltaManager;
         this.pendingStateManager = new pendingStateManager_1.PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
             clientId: () => this.clientId,
@@ -313,8 +293,18 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             rollback: this.rollback.bind(this),
             orderSequentially: this.orderSequentially.bind(this),
         }, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
+        this.outbox = new opLifecycle_1.Outbox({
+            shouldSend: () => this.canSendOps(),
+            pendingStateManager: this.pendingStateManager,
+            containerContext: this.context,
+            compressor: new opLifecycle_1.OpCompressor(this.mc.logger),
+            config: {
+                compressionOptions: runtimeOptions.compressionOptions,
+                maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
+            },
+        });
         this.context.quorum.on("removeMember", (clientId) => {
-            this.clearPartialChunks(clientId);
+            this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
         });
         this.summaryCollection = new summaryCollection_1.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer = this.context.attachState !== container_definitions_1.AttachState.Attached
@@ -382,26 +372,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         });
         // logging hardware telemetry
         logger.sendTelemetryEvent(Object.assign({ eventName: "DeviceSpec" }, getDeviceSpec()));
-        let loadSummaryNumber;
-        // Get the container creation metadata. For new container, we initialize these. For existing containers,
-        // get the values from the metadata blob.
-        if (existing) {
-            this.createContainerMetadata = {
-                createContainerRuntimeVersion: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerRuntimeVersion,
-                createContainerTimestamp: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerTimestamp,
-            };
-            // summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
-            // the count is reset to 0.
-            loadSummaryNumber = (_f = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _f !== void 0 ? _f : 0;
-        }
-        else {
-            this.createContainerMetadata = {
-                createContainerRuntimeVersion: packageVersion_1.pkgVersion,
-                createContainerTimestamp: Date.now(),
-            };
-            loadSummaryNumber = 0;
-        }
-        this.nextSummaryNumber = loadSummaryNumber + 1;
         this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryNumber: loadSummaryNumber, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature }));
         (0, connectionTelemetry_1.ReportOpPerfTelemetry)(this.context.clientId, this.deltaManager, this.logger);
         (0, batchTracker_1.BindBatchTracker)(this, this.logger);
@@ -415,8 +385,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * @param requestHandler - Request handlers for the container runtime
      * @param runtimeOptions - Additional options to be passed to the runtime
      * @param existing - (optional) When loading from an existing snapshot. Precedes context.existing if provided
+     * @param containerRuntimeCtor - (optional) Constructor to use to create the ContainerRuntime instance. This
+     * allows mixin classes to leverage this method to define their own async initializer.
      */
-    static async load(context, registryEntries, requestHandler, runtimeOptions = {}, containerScope = context.scope, existing) {
+    static async load(context, registryEntries, requestHandler, runtimeOptions = {}, containerScope = context.scope, existing, containerRuntimeCtor = ContainerRuntime) {
         var _a, _b, _c;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
@@ -427,7 +399,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 runtimeVersion: packageVersion_1.pkgVersion,
             },
         });
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = {}, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = {
+            minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
+            compressionAlgorithm: CompressionAlgorithms.lz4
+        }, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, } = runtimeOptions;
         const pendingRuntimeState = context.pendingLocalState;
         const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
         const storage = !pendingRuntimeState ?
@@ -476,7 +451,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 }
             }
         }
-        const runtime = new ContainerRuntime(context, registry, metadata, electedSummarizerData, chunks !== null && chunks !== void 0 ? chunks : [], aliases !== null && aliases !== void 0 ? aliases : [], {
+        const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks !== null && chunks !== void 0 ? chunks : [], aliases !== null && aliases !== void 0 ? aliases : [], {
             summaryOptions,
             gcOptions,
             loadSequenceNumberVerification,
@@ -490,7 +465,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // delete these once runtime has seen them to save space
             pendingRuntimeState.savedOps = [];
         }
-        await runtime.getSnapshotBlobs();
+        // Initialize the base state of the runtime before it's returned.
+        await runtime.initializeBaseState();
         return runtime;
     }
     get options() {
@@ -539,9 +515,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
     }
     get disposed() { return this._disposed; }
-    get emptyBatch() {
-        return this.pendingBatch.empty && this.pendingAttachBatch.empty;
-    }
     get summarizer() {
         (0, common_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
@@ -597,6 +570,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             ? this.summaryConfiguration.initialSummarizerDelayMs
             : 0;
     }
+    /**
+     * Initializes the state from the base snapshot this container runtime loaded from.
+     */
+    async initializeBaseState() {
+        await this.initializeBaseSnapshotBlobs();
+        await this.garbageCollector.initializeBaseState();
+    }
     dispose(error) {
         var _a;
         if (this._disposed) {
@@ -742,8 +722,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         var _a;
         this.addMetadataToSummary(summaryTree);
-        if (this.chunkMap.size > 0) {
-            const content = JSON.stringify([...this.chunkMap]);
+        if (this.remoteMessageProcessor.partialMessages.size > 0) {
+            const content = JSON.stringify([...this.remoteMessageProcessor.partialMessages]);
             (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.chunksBlobName, content);
         }
         const dataStoreAliases = this.dataStores.aliases;
@@ -762,7 +742,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
         if (gcSummary !== undefined) {
-            (0, runtime_utils_1.addSummarizeResultToSummary)(summaryTree, garbageCollection_1.gcTreeKey, gcSummary);
+            (0, runtime_utils_1.addSummarizeResultToSummary)(summaryTree, garbageCollectionConstants_1.gcTreeKey, gcSummary);
         }
     }
     // Track how many times the container tries to reconnect with pending messages.
@@ -907,31 +887,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     process(messageArg, local) {
         var _a;
         this.verifyNotClosed();
-        // Do shallow copy of message, as methods below will modify it.
-        // There might be multiple container instances receiving same message
-        // We do not need to make deep copy, as each layer will just replace message.content itself,
-        // but would not modify contents details
-        let message = Object.assign({}, messageArg);
-        // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
-        // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
-        // Old ops may contain empty string (I assume noops).
-        if (typeof message.contents === "string" && message.contents !== "") {
-            message.contents = JSON.parse(message.contents);
-        }
-        // Caveat: This will return false for runtime message in very old format, that are used in snapshot tests
-        // This format was not shipped to production workflows.
-        const runtimeMessage = unpackRuntimeMessage(message);
         if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
             this.savedOps.push(messageArg);
         }
+        // Whether or not the message is actually a runtime message.
+        // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
+        // or something different, like a system message.
+        const runtimeMessage = messageArg.type === protocol_definitions_1.MessageType.Operation;
+        // Do shallow copy of message, as the processing flow will modify it.
+        const messageCopy = Object.assign({}, messageArg);
+        const message = this.remoteMessageProcessor.process(messageCopy);
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
         this.scheduleManager.beforeOpProcessing(message);
         try {
-            // Chunk processing must come first given that we will transform the message to the unchunked version
-            // once all pieces are available
-            message = this.processRemoteChunkedMessage(message);
             let localOpMetadata;
             if (local && runtimeMessage) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
@@ -1045,83 +1015,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      */
     flush() {
         (0, common_utils_1.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
-        this.flushBatch(this.pendingAttachBatch.popBatch());
-        this.flushBatch(this.pendingBatch.popBatch());
-        (0, common_utils_1.assert)(this.emptyBatch, 0x3cf /* reentrancy */);
-    }
-    flushBatch(batch) {
-        const length = batch.length;
-        if (length > 1) {
-            batch[0].metadata = Object.assign(Object.assign({}, batch[0].metadata), { batch: true });
-            batch[length - 1].metadata = Object.assign(Object.assign({}, batch[length - 1].metadata), { batch: false });
-            // This assert fires for the following reason (there might be more cases like that):
-            // AgentScheduler will send ops in response to ConsensusRegisterCollection's "atomicChanged" event handler,
-            // i.e. in the middle of op processing!
-            // Sending ops while processing ops is not good idea - it's not defined when
-            // referenceSequenceNumber changes in op processing sequence (at the beginning or end of op processing),
-            // If we send ops in response to processing multiple ops, then we for sure hit this assert!
-            // Tracked via ADO #1834
-            // assert(batch[0].referenceSequenceNumber === batch[length - 1].referenceSequenceNumber,
-            //    "Batch should be generated synchronously, without processing ops in the middle!");
-        }
-        let clientSequenceNumber = -1;
-        // Did we disconnect in the middle of turn-based batch?
-        // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
-        if (this.canSendOps()) {
-            if (this.context.submitBatchFn !== undefined) {
-                const batchToSend = [];
-                for (const message of batch) {
-                    let contents = message.contents;
-                    let metadata = message.metadata;
-                    if (this.runtimeOptions.compressionOptions.minimumSize &&
-                        this.runtimeOptions.compressionOptions.minimumSize < message.contents.length) {
-                        this.compressedOpCount++;
-                        const copiedMessage = Object.assign({}, message.deserializedContent);
-                        const compressionStart = Date.now();
-                        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(copiedMessage.contents));
-                        const compressedContents = (0, lz4js_1.compress)(contentsAsBuffer);
-                        const compressedContent = common_utils_1.IsoBuffer.from(compressedContents).toString("base64");
-                        const duration = Date.now() - compressionStart;
-                        if (this.compressedOpCount % 100) {
-                            this.mc.logger.sendPerformanceEvent({
-                                eventName: "compressedOp",
-                                duration,
-                                sizeBeforeCompression: message.contents.length,
-                                sizeAfterCompression: compressedContent.length,
-                            });
-                        }
-                        copiedMessage.contents = compressedContent;
-                        const stringifiedContents = JSON.stringify(copiedMessage);
-                        if (stringifiedContents.length < message.contents.length) {
-                            contents = JSON.stringify(copiedMessage);
-                            metadata = Object.assign(Object.assign({}, message.metadata), { compressed: true });
-                        }
-                    }
-                    batchToSend.push({ contents, metadata });
-                }
-                // returns clientSequenceNumber of last message in a batch
-                clientSequenceNumber = this.context.submitBatchFn(batchToSend);
-            }
-            else {
-                // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
-                // version that has support for batches (submitBatchFn)
-                for (const message of batch) {
-                    clientSequenceNumber = this.context.submitFn(protocol_definitions_1.MessageType.Operation, message.deserializedContent, true, // batch
-                    message.metadata);
-                }
-                this.deltaSender.flush();
-            }
-            // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
-            clientSequenceNumber -= batch.length - 1;
-            (0, common_utils_1.assert)(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
-        }
-        // Let the PendingStateManager know that a message was submitted.
-        // In future, need to shift toward keeping batch as a whole!
-        for (const message of batch) {
-            this.pendingStateManager.onSubmitMessage(message.deserializedContent.type, clientSequenceNumber, message.referenceSequenceNumber, message.deserializedContent.contents, message.localOpMetadata, message.metadata);
-            clientSequenceNumber++;
-        }
-        this.pendingStateManager.onFlush();
+        this.outbox.flush();
+        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
     }
     orderSequentially(callback) {
         let checkpoint;
@@ -1129,7 +1024,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Note: we are not touching this.pendingAttachBatch here, for two reasons:
             // 1. It would not help, as we flush attach ops as they become available.
             // 2. There is no way to undo process of data store creation.
-            checkpoint = this.pendingBatch.checkpoint();
+            checkpoint = this.outbox.checkpoint().mainBatch;
         }
         try {
             this._orderSequentiallyCalls++;
@@ -1349,33 +1244,27 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // summarizing is required and asserted by the the summarizer node. We are the root and are
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
+        const blobManagerUsedRoutes = [];
         const dataStoreUsedRoutes = [];
         for (const route of usedRoutes) {
-            if (route.split("/")[1] !== blobManager_1.BlobManager.basePath) {
+            if (this.isBlobPath(route)) {
+                blobManagerUsedRoutes.push(route);
+            }
+            else {
                 dataStoreUsedRoutes.push(route);
             }
         }
-        return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes);
+        this.blobManager.updateUsedRoutes(blobManagerUsedRoutes);
+        this.dataStores.updateUsedRoutes(dataStoreUsedRoutes);
     }
     /**
-     * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
-     * scenarios with accessing deleted content.
-     * @param unusedRoutes - The routes that are unused in all data stores and blobs in this Container.
+     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
+     * tombstones.
+     * @param unusedRoutes - The routes that are unused in all data stores and attachment blobs in this Container.
+     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
+     * are deleted.
      */
-    deleteUnusedRoutes(unusedRoutes) {
-        var _a;
-        /**
-         * When running GC in tombstone mode, this is called to tombstone datastore routes that are unused. This
-         * enables testing scenarios without actually deleting content. The content acts as if it's deleted to the
-         * external user, but the internal runtime does not delete it in summarizes, etc.
-         */
-        const tombstone = (_a = this.mc.config.getBoolean(garbageCollection_1.testTombstoneKey)) !== null && _a !== void 0 ? _a : false;
-        // TODO: add blobs
-        if (tombstone) {
-            // If blob routes are passed in here, tombstone will fail and hit an assert
-            this.dataStores.deleteUnusedRoutes(unusedRoutes, tombstone);
-            return;
-        }
+    updateUnusedRoutes(unusedRoutes, tombstone) {
         const blobManagerUnusedRoutes = [];
         const dataStoreUnusedRoutes = [];
         for (const route of unusedRoutes) {
@@ -1386,8 +1275,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 dataStoreUnusedRoutes.push(route);
             }
         }
-        this.blobManager.deleteUnusedRoutes(blobManagerUnusedRoutes);
-        this.dataStores.deleteUnusedRoutes(dataStoreUnusedRoutes);
+        this.blobManager.updateUnusedRoutes(blobManagerUnusedRoutes, tombstone);
+        this.dataStores.updateUnusedRoutes(dataStoreUnusedRoutes, tombstone);
     }
     /**
      * Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
@@ -1467,7 +1356,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         const summaryNumberLogger = telemetry_utils_1.ChildLogger.create(summaryLogger, undefined, {
             all: { summaryNumber },
         });
-        (0, common_utils_1.assert)(this.emptyBatch, 0x3d1 /* Can't trigger summary in the middle of a batch */);
+        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
         let latestSnapshotVersionId;
         if (refreshLatestAck) {
             const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(telemetry_utils_1.ChildLogger.create(summaryNumberLogger, undefined, { all: { safeSummary: true } }));
@@ -1558,8 +1447,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             const dataStoreTree = summaryTree.tree[runtime_definitions_1.channelsTreeName];
             (0, common_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === protocol_definitions_1.SummaryType.Handle).length;
-            const gcSummaryTreeStats = summaryTree.tree[garbageCollection_1.gcTreeKey]
-                ? (0, runtime_utils_1.calculateStats)(summaryTree.tree[garbageCollection_1.gcTreeKey])
+            const gcSummaryTreeStats = summaryTree.tree[garbageCollectionConstants_1.gcTreeKey]
+                ? (0, runtime_utils_1.calculateStats)(summaryTree.tree[garbageCollectionConstants_1.gcTreeKey])
                 : undefined;
             const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount, gcStateUpdatedDataStoreCount: (_a = summarizeResult.gcStats) === null || _a === void 0 ? void 0 : _a.updatedDataStoreCount, gcBlobNodeCount: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.blobNodeCount, gcTotalBlobsSize: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.totalBlobSize, summaryNumber }, partialStats);
             const generateSummaryData = {
@@ -1640,40 +1529,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.deltaManager.inbound.resume();
         }
     }
-    processRemoteChunkedMessage(message) {
-        if (message.type !== ContainerMessageType.ChunkedOp) {
-            return message;
-        }
-        const clientId = message.clientId;
-        const chunkedContent = message.contents;
-        this.addChunk(clientId, chunkedContent);
-        if (chunkedContent.chunkId === chunkedContent.totalChunks) {
-            const newMessage = Object.assign({}, message);
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            const serializedContent = this.chunkMap.get(clientId).join("");
-            newMessage.contents = JSON.parse(serializedContent);
-            newMessage.type = chunkedContent.originalType;
-            this.clearPartialChunks(clientId);
-            return newMessage;
-        }
-        return message;
-    }
-    addChunk(clientId, chunkedContent) {
-        let map = this.chunkMap.get(clientId);
-        if (map === undefined) {
-            map = [];
-            this.chunkMap.set(clientId, map);
-        }
-        (0, common_utils_1.assert)(chunkedContent.chunkId === map.length + 1, 0x131 /* "Mismatch between new chunkId and expected chunkMap" */); // 1-based indexing
-        map.push(chunkedContent.contents);
-    }
-    clearPartialChunks(clientId) {
-        if (this.chunkMap.has(clientId)) {
-            this.chunkMap.delete(clientId);
-        }
-    }
     hasPendingMessages() {
-        return this.pendingStateManager.hasPendingMessages() || !this.emptyBatch;
+        return this.pendingStateManager.hasPendingMessages() || !this.outbox.isEmpty;
     }
     updateDocumentDirtyState(dirty) {
         if (this.attachState !== container_definitions_1.AttachState.Attached) {
@@ -1717,7 +1574,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         const deserializedContent = { type, contents };
         const serializedContent = JSON.stringify(deserializedContent);
         if (this.deltaManager.readOnlyInfo.readonly) {
-            this.logger.sendErrorEvent({ eventName: "SubmitOpInReadonly" });
+            this.logger.sendTelemetryEvent({ eventName: "SubmitOpInReadonly", connected: this.connected });
         }
         const message = {
             contents: serializedContent,
@@ -1748,43 +1605,23 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Please note that this does not change file format, so it can be disabled in the future if this
             // optimization no longer makes sense (for example, batch compression may make it less appealing).
             if (this.currentlyBatching() && type === ContainerMessageType.Attach &&
-                this.mc.config.getBoolean("Fluid.ContainerRuntime.enableAttachOpReorder") === true) {
-                if (!this.pendingAttachBatch.push(message)) {
-                    // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
-                    // when queue is not empty.
-                    // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
-                    this.flushBatch(this.pendingAttachBatch.popBatch());
-                    if (!this.pendingAttachBatch.push(message)) {
-                        throw new container_utils_1.GenericError("BatchTooLarge", 
-                        /* error */ undefined, {
-                            opSize: message.contents.length,
-                            count: this.pendingAttachBatch.length,
-                            limit: this.pendingAttachBatch.limit,
-                        });
-                    }
-                }
+                this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
+                this.outbox.submitAttach(message);
             }
             else {
-                if (!this.pendingBatch.push(message)) {
-                    throw new container_utils_1.GenericError("BatchTooLarge", 
-                    /* error */ undefined, {
-                        opSize: message.contents.length,
-                        count: this.pendingBatch.length,
-                        limit: this.pendingBatch.limit,
-                    });
-                }
-                if (!this.currentlyBatching()) {
+                this.outbox.submit(message);
+            }
+            if (!this.currentlyBatching()) {
+                this.flush();
+            }
+            else if (!this.flushMicroTaskExists) {
+                this.flushMicroTaskExists = true;
+                // Queue a microtask to detect the end of the turn and force a flush.
+                // eslint-disable-next-line @typescript-eslint/no-floating-promises
+                Promise.resolve().then(() => {
+                    this.flushMicroTaskExists = false;
                     this.flush();
-                }
-                else if (!this.flushMicroTaskExists) {
-                    this.flushMicroTaskExists = true;
-                    // Queue a microtask to detect the end of the turn and force a flush.
-                    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-                    Promise.resolve().then(() => {
-                        this.flushMicroTaskExists = false;
-                        this.flush();
-                    });
-                }
+                }).catch((error) => { this.closeFn(error); });
             }
         }
         catch (error) {
@@ -1799,7 +1636,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyNotClosed();
         (0, common_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
-        (0, common_utils_1.assert)(this.emptyBatch, 0x3d4 /* System op in the middle of a batch */);
+        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
         // back-compat: ADO #1385: Make this call unconditional in the future
         return this.context.submitSummaryFn !== undefined
             ? this.context.submitSummaryFn(contents)
@@ -1933,7 +1770,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.baseSnapshotBlobs = serializedSnapshotStorage_1.SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
         }
     }
-    async getSnapshotBlobs() {
+    async initializeBaseSnapshotBlobs() {
         var _a;
         if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) ||
             this.attachState !== container_definitions_1.AttachState.Attached || this.context.pendingLocalState) {