@fluidframework/container-runtime 1.3.0 → 2.0.0-dev.1.4.5.105745

package/dist/containerRuntime.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.ScheduleManager = exports.unpackRuntimeMessage = exports.isRuntimeMessage = exports.RuntimeMessage = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
+exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.unpackRuntimeMessage = exports.isRuntimeMessage = exports.RuntimeMessage = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
 const container_definitions_1 = require("@fluidframework/container-definitions");
 const common_utils_1 = require("@fluidframework/common-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
@@ -16,9 +16,9 @@ const containerHandleContext_1 = require("./containerHandleContext");
 const dataStoreRegistry_1 = require("./dataStoreRegistry");
 const summarizer_1 = require("./summarizer");
 const summaryManager_1 = require("./summaryManager");
-const deltaScheduler_1 = require("./deltaScheduler");
 const connectionTelemetry_1 = require("./connectionTelemetry");
 const pendingStateManager_1 = require("./pendingStateManager");
+const batchManager_1 = require("./batchManager");
 const packageVersion_1 = require("./packageVersion");
 const blobManager_1 = require("./blobManager");
 const dataStores_1 = require("./dataStores");
@@ -32,7 +32,7 @@ const garbageCollection_1 = require("./garbageCollection");
 const dataStore_1 = require("./dataStore");
 const batchTracker_1 = require("./batchTracker");
 const serializedSnapshotStorage_1 = require("./serializedSnapshotStorage");
-const opTelemetry_1 = require("./opTelemetry");
+const scheduleManager_1 = require("./scheduleManager");
 var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -50,14 +50,17 @@ var ContainerMessageType;
 })(ContainerMessageType = exports.ContainerMessageType || (exports.ContainerMessageType = {}));
 exports.DefaultSummaryConfiguration = {
     state: "enabled",
-    idleTime: 5000 * 3,
-    maxTime: 5000 * 12,
+    minIdleTime: 0,
+    maxIdleTime: 30 * 1000,
+    maxTime: 60 * 1000,
     maxOps: 100,
     minOpsForLastSummaryAttempt: 10,
-    maxAckWaitTime: 6 * 10 * 1000,
+    maxAckWaitTime: 10 * 60 * 1000,
     maxOpsSinceLastSummary: 7000,
-    initialSummarizerDelayMs: 5000,
+    initialSummarizerDelayMs: 5 * 1000,
     summarizerClientElection: false,
+    nonRuntimeOpWeight: 0.1,
+    runtimeOpWeight: 1.0,
 };
 /**
  * Accepted header keys for requests coming to the runtime.
@@ -74,21 +77,11 @@ var RuntimeHeaders;
     /** True if the request is coming from an IFluidHandle. */
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders = exports.RuntimeHeaders || (exports.RuntimeHeaders = {}));
-const useDataStoreAliasingKey = "Fluid.ContainerRuntime.UseDataStoreAliasing";
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
-// Feature gate for the max op size. If the value is negative, chunking is enabled
-// and all ops over 16k would be chunked. If the value is positive, all ops with
-// a size strictly larger will be rejected and the container closed with an error.
-const maxOpSizeInBytesKey = "Fluid.ContainerRuntime.MaxOpSizeInBytes";
-// By default, we should reject any op larger than 768KB,
-// in order to account for some extra overhead from serialization
-// to not reach the 1MB limits in socket.io and Kafka.
-const defaultMaxOpSizeInBytes = 768000;
-// By default, the size of the contents for the incoming ops is tracked.
-// However, in certain situations, this may incur a performance hit.
-// The feature-gate below can be used to disable this feature.
-const disableOpTrackingKey = "Fluid.ContainerRuntime.DisableOpTracking";
 const defaultFlushMode = runtime_definitions_1.FlushMode.TurnBased;
+/**
+ * @deprecated - use ContainerRuntimeMessage instead
+ */
 var RuntimeMessage;
 (function (RuntimeMessage) {
     RuntimeMessage["FluidDataStoreOp"] = "component";
@@ -99,6 +92,9 @@ var RuntimeMessage;
     RuntimeMessage["Alias"] = "alias";
     RuntimeMessage["Operation"] = "op";
 })(RuntimeMessage = exports.RuntimeMessage || (exports.RuntimeMessage = {}));
+/**
+ * @deprecated - please use version in driver-utils
+ */
 function isRuntimeMessage(message) {
     if (Object.values(RuntimeMessage).includes(message.type)) {
         return true;
@@ -106,6 +102,15 @@ function isRuntimeMessage(message) {
     return false;
 }
 exports.isRuntimeMessage = isRuntimeMessage;
+/**
+ * Unpacks runtime messages
+ *
+ * @remarks This API makes no promises regarding backward-compatability. This is internal API.
+ * @param message - message (as it observed in storage / service)
+ * @returns unpacked runtime message
+ *
+ * @internal
+ */
 function unpackRuntimeMessage(message) {
     if (message.type === protocol_definitions_1.MessageType.Operation) {
         // legacy op format?
@@ -119,240 +124,17 @@ function unpackRuntimeMessage(message) {
             message.type = innerContents.type;
             message.contents = innerContents.contents;
         }
-        (0, common_utils_1.assert)((0, driver_utils_1.isUnpackedRuntimeMessage)(message), 0x122 /* "Message to unpack is not proper runtime message" */);
+        return true;
     }
     else {
         // Legacy format, but it's already "unpacked",
         // i.e. message.type is actually ContainerMessageType.
+        // Or it's non-runtime message.
         // Nothing to do in such case.
+        return false;
     }
-    return message;
 }
 exports.unpackRuntimeMessage = unpackRuntimeMessage;
-/**
- * This class controls pausing and resuming of inbound queue to ensure that we never
- * start processing ops in a batch IF we do not have all ops in the batch.
- */
-class ScheduleManagerCore {
-    constructor(deltaManager, logger) {
-        this.deltaManager = deltaManager;
-        this.logger = logger;
-        this.localPaused = false;
-        this.timePaused = 0;
-        this.batchCount = 0;
-        // Listen for delta manager sends and add batch metadata to messages
-        this.deltaManager.on("prepareSend", (messages) => {
-            if (messages.length === 0) {
-                return;
-            }
-            // First message will have the batch flag set to true if doing a batched send
-            const firstMessageMetadata = messages[0].metadata;
-            if (!(firstMessageMetadata === null || firstMessageMetadata === void 0 ? void 0 : firstMessageMetadata.batch)) {
-                return;
-            }
-            // If the batch contains only a single op, clear the batch flag.
-            if (messages.length === 1) {
-                delete firstMessageMetadata.batch;
-                return;
-            }
-            // Set the batch flag to false on the last message to indicate the end of the send batch
-            const lastMessage = messages[messages.length - 1];
-            lastMessage.metadata = Object.assign(Object.assign({}, lastMessage.metadata), { batch: false });
-        });
-        // Listen for updates and peek at the inbound
-        this.deltaManager.inbound.on("push", (message) => {
-            this.trackPending(message);
-        });
-        // Start with baseline - empty inbound queue.
-        (0, common_utils_1.assert)(!this.localPaused, 0x293 /* "initial state" */);
-        const allPending = this.deltaManager.inbound.toArray();
-        for (const pending of allPending) {
-            this.trackPending(pending);
-        }
-        // We are intentionally directly listening to the "op" to inspect system ops as well.
-        // If we do not observe system ops, we are likely to hit 0x296 assert when system ops
-        // precedes start of incomplete batch.
-        this.deltaManager.on("op", (message) => this.afterOpProcessing(message.sequenceNumber));
-    }
-    /**
-     * The only public function in this class - called when we processed an op,
-     * to make decision if op processing should be paused or not afer that.
-     */
-    afterOpProcessing(sequenceNumber) {
-        (0, common_utils_1.assert)(!this.localPaused, 0x294 /* "can't have op processing paused if we are processing an op" */);
-        // If the inbound queue is ever empty, nothing to do!
-        if (this.deltaManager.inbound.length === 0) {
-            (0, common_utils_1.assert)(this.pauseSequenceNumber === undefined, 0x295 /* "there should be no pending batch if we have no ops" */);
-            return;
-        }
-        // The queue is
-        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
-        //    - here (processing ops until reaching start of incomplete batch)
-        //    - in trackPending(), when queue was empty and start of batch showed up.
-        // 2. resumed when batch end comes in (in trackPending())
-        // do we have incomplete batch to worry about?
-        if (this.pauseSequenceNumber !== undefined) {
-            (0, common_utils_1.assert)(sequenceNumber < this.pauseSequenceNumber, 0x296 /* "we should never start processing incomplete batch!" */);
-            // If the next op is the start of incomplete batch, then we can't process it until it's fully in - pause!
-            if (sequenceNumber + 1 === this.pauseSequenceNumber) {
-                this.pauseQueue();
-            }
-        }
-    }
-    pauseQueue() {
-        (0, common_utils_1.assert)(!this.localPaused, 0x297 /* "always called from resumed state" */);
-        this.localPaused = true;
-        this.timePaused = common_utils_1.performance.now();
-        // eslint-disable-next-line @typescript-eslint/no-floating-promises
-        this.deltaManager.inbound.pause();
-    }
-    resumeQueue(startBatch, messageEndBatch) {
-        const endBatch = messageEndBatch.sequenceNumber;
-        const duration = this.localPaused ? (common_utils_1.performance.now() - this.timePaused) : undefined;
-        this.batchCount++;
-        if (this.batchCount % 1000 === 1) {
-            this.logger.sendTelemetryEvent({
-                eventName: "BatchStats",
-                sequenceNumber: endBatch,
-                length: endBatch - startBatch + 1,
-                msnDistance: endBatch - messageEndBatch.minimumSequenceNumber,
-                duration,
-                batchCount: this.batchCount,
-                interrupted: this.localPaused,
-            });
-        }
-        // Return early if no change in value
-        if (!this.localPaused) {
-            return;
-        }
-        this.localPaused = false;
-        // Random round number - we want to know when batch waiting paused op processing.
-        if (duration !== undefined && duration > connectionTelemetry_1.latencyThreshold) {
-            this.logger.sendErrorEvent({
-                eventName: "MaxBatchWaitTimeExceeded",
-                duration,
-                sequenceNumber: endBatch,
-                length: endBatch - startBatch,
-            });
-        }
-        this.deltaManager.inbound.resume();
-    }
-    /**
-     * Called for each incoming op (i.e. inbound "push" notification)
-     */
-    trackPending(message) {
-        (0, common_utils_1.assert)(this.deltaManager.inbound.length !== 0, 0x298 /* "we have something in the queue that generates this event" */);
-        (0, common_utils_1.assert)((this.currentBatchClientId === undefined) === (this.pauseSequenceNumber === undefined), 0x299 /* "non-synchronized state" */);
-        const metadata = message.metadata;
-        const batchMetadata = metadata === null || metadata === void 0 ? void 0 : metadata.batch;
-        // Protocol messages are never part of a runtime batch of messages
-        if (!(0, driver_utils_1.isUnpackedRuntimeMessage)(message)) {
-            // Protocol messages should never show up in the middle of the batch!
-            (0, common_utils_1.assert)(this.currentBatchClientId === undefined, 0x29a /* "System message in the middle of batch!" */);
-            (0, common_utils_1.assert)(batchMetadata === undefined, 0x29b /* "system op in a batch?" */);
-            (0, common_utils_1.assert)(!this.localPaused, 0x29c /* "we should be processing ops when there is no active batch" */);
-            return;
-        }
-        if (this.currentBatchClientId === undefined && batchMetadata === undefined) {
-            (0, common_utils_1.assert)(!this.localPaused, 0x29d /* "we should be processing ops when there is no active batch" */);
-            return;
-        }
-        // If the client ID changes then we can move the pause point. If it stayed the same then we need to check.
-        // If batchMetadata is not undefined then if it's true we've begun a new batch - if false we've ended
-        // the previous one
-        if (this.currentBatchClientId !== undefined || batchMetadata === false) {
-            if (this.currentBatchClientId !== message.clientId) {
-                // "Batch not closed, yet message from another client!"
-                throw new container_utils_1.DataCorruptionError("OpBatchIncomplete", Object.assign({ runtimeVersion: packageVersion_1.pkgVersion, batchClientId: this.currentBatchClientId }, (0, container_utils_1.extractSafePropertiesFromMessage)(message)));
-            }
-        }
-        // The queue is
-        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
-        //    - in afterOpProcessing() - processing ops until reaching start of incomplete batch
-        //    - here (batchMetadata == false below), when queue was empty and start of batch showed up.
-        // 2. resumed when batch end comes in (batchMetadata === true case below)
-        if (batchMetadata) {
-            (0, common_utils_1.assert)(this.currentBatchClientId === undefined, 0x29e /* "there can't be active batch" */);
-            (0, common_utils_1.assert)(!this.localPaused, 0x29f /* "we should be processing ops when there is no active batch" */);
-            this.pauseSequenceNumber = message.sequenceNumber;
-            this.currentBatchClientId = message.clientId;
-            // Start of the batch
-            // Only pause processing if queue has no other ops!
-            // If there are any other ops in the queue, processing will be stopped when they are processed!
-            if (this.deltaManager.inbound.length === 1) {
-                this.pauseQueue();
-            }
-        }
-        else if (batchMetadata === false) {
-            (0, common_utils_1.assert)(this.pauseSequenceNumber !== undefined, 0x2a0 /* "batch presence was validated above" */);
-            // Batch is complete, we can process it!
-            this.resumeQueue(this.pauseSequenceNumber, message);
-            this.pauseSequenceNumber = undefined;
-            this.currentBatchClientId = undefined;
-        }
-        else {
-            // Continuation of current batch. Do nothing
-            (0, common_utils_1.assert)(this.currentBatchClientId !== undefined, 0x2a1 /* "logic error" */);
-        }
-    }
-}
-/**
- * This class has the following responsibilities:
- * 1. It tracks batches as we process ops and raises "batchBegin" and "batchEnd" events.
- *    As part of it, it validates batch correctness (i.e. no system ops in the middle of batch)
- * 2. It creates instance of ScheduleManagerCore that ensures we never start processing ops from batch
- *    unless all ops of the batch are in.
- */
-class ScheduleManager {
-    constructor(deltaManager, emitter, logger) {
-        this.deltaManager = deltaManager;
-        this.emitter = emitter;
-        this.logger = logger;
-        this.hitError = false;
-        this.deltaScheduler = new deltaScheduler_1.DeltaScheduler(this.deltaManager, telemetry_utils_1.ChildLogger.create(this.logger, "DeltaScheduler"));
-        void new ScheduleManagerCore(deltaManager, logger);
-    }
-    beforeOpProcessing(message) {
-        var _a;
-        if (this.batchClientId !== message.clientId) {
-            (0, common_utils_1.assert)(this.batchClientId === undefined, 0x2a2 /* "Batch is interrupted by other client op. Should be caught by trackPending()" */);
-            // This could be the beginning of a new batch or an individual message.
-            this.emitter.emit("batchBegin", message);
-            this.deltaScheduler.batchBegin(message);
-            const batch = (_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.batch;
-            if (batch) {
-                this.batchClientId = message.clientId;
-            }
-            else {
-                this.batchClientId = undefined;
-            }
-        }
-    }
-    afterOpProcessing(error, message) {
-        var _a;
-        // If this is no longer true, we need to revisit what we do where we set this.hitError.
-        (0, common_utils_1.assert)(!this.hitError, 0x2a3 /* "container should be closed on any error" */);
-        if (error) {
-            // We assume here that loader will close container and stop processing all future ops.
-            // This is implicit dependency. If this flow changes, this code might no longer be correct.
-            this.hitError = true;
-            this.batchClientId = undefined;
-            this.emitter.emit("batchEnd", error, message);
-            this.deltaScheduler.batchEnd(message);
-            return;
-        }
-        const batch = (_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.batch;
-        // If no batchClientId has been set then we're in an individual batch. Else, if we get
-        // batch end metadata, this is end of the current batch.
-        if (this.batchClientId === undefined || batch === false) {
-            this.batchClientId = undefined;
-            this.emitter.emit("batchEnd", undefined, message);
-            this.deltaScheduler.batchEnd(message);
-            return;
-        }
-    }
-}
-exports.ScheduleManager = ScheduleManager;
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
  * special-case for document dirty state.  Ultimately we should have no special-cases from the
@@ -380,7 +162,7 @@ exports.getDeviceSpec = getDeviceSpec;
  */
 class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+        var _a, _b, _c, _d;
         if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, exports.DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
         super();
         this.context = context;
@@ -391,9 +173,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this._storage = _storage;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
-        this.defaultMaxConsecutiveReconnects = 15;
+        this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
-        this.needsFlush = false;
         this.flushTrigger = false;
         this.savedOps = [];
         this.consecutiveReconnects = 0;
@@ -406,6 +187,12 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             signalTimestamp: 0,
             trackingSignalSequenceNumber: undefined,
         };
+        // Provide lower soft limit - we want to have some number of ops to get efficiency in compression & bandwidth usage,
+        // but at the same time we want to send these ops sooner, to reduce overall latency of processing a batch.
+        // So there is some ballance here, that depends on compression algorithm and its efficiency working with smaller
+        // payloads. That number represents final (compressed) bits (once compression is implemented).
+        this.pendingAttachBatch = new batchManager_1.BatchManager(64 * 1024);
+        this.pendingBatch = new batchManager_1.BatchManager();
         this.summarizeOnDemand = (...args) => {
             if (this.clientDetails.type === summarizerClientElection_1.summarizerClientType) {
                 return this.summarizer.summarizeOnDemand(...args);
@@ -435,26 +222,23 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
         };
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
-        // Default to false (enabled).
-        this.disableIsolatedChannels = (_b = this.runtimeOptions.summaryOptions.disableIsolatedChannels) !== null && _b !== void 0 ? _b : false;
         this._connected = this.context.connected;
         this.chunkMap = new Map(chunks);
         this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.logger, "ContainerRuntime"));
+        if (this.summaryConfiguration.state === "enabled") {
+            this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
+        }
         this.summariesDisabled = this.isSummariesDisabled();
         this.heuristicsDisabled = this.isHeuristicsDisabled();
         this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
-        this._aliasingEnabled =
-            ((_c = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _c !== void 0 ? _c : false) ||
-                ((_d = runtimeOptions.useDataStoreAliasing) !== null && _d !== void 0 ? _d : false);
-        this._maxOpSizeInBytes = ((_e = this.mc.config.getNumber(maxOpSizeInBytesKey)) !== null && _e !== void 0 ? _e : defaultMaxOpSizeInBytes);
         this.maxConsecutiveReconnects =
-            (_f = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _f !== void 0 ? _f : this.defaultMaxConsecutiveReconnects;
+            (_b = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _b !== void 0 ? _b : this.defaultMaxConsecutiveReconnects;
         this._flushMode = runtimeOptions.flushMode;
         const pendingRuntimeState = context.pendingLocalState;
-        const baseSnapshot = (_g = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _g !== void 0 ? _g : context.baseSnapshot;
+        const baseSnapshot = (_c = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _c !== void 0 ? _c : context.baseSnapshot;
         this.garbageCollector = garbageCollection_1.GarbageCollector.create({
             runtime: this,
             gcOptions: this.runtimeOptions.gcOptions,
@@ -466,6 +250,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
             getLastSummaryTimestampMs: () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; },
             readAndParseBlob: async (id) => (0, driver_utils_1.readAndParse)(this.storage, id),
+            getContainerDiagnosticId: () => this.context.id,
+            activeConnection: () => this.deltaManager.active,
         });
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = (0, runtime_utils_1.createRootSummarizerNodeWithGC)(telemetry_utils_1.ChildLogger.create(this.logger, "SummarizerNode"), 
@@ -488,8 +274,12 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.summarizerNode.loadBaseSummaryWithoutDifferential(baseSnapshot);
         }
         this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
-        this.blobManager = new blobManager_1.BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }), (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, this.logger);
-        this.scheduleManager = new ScheduleManager(context.deltaManager, this, telemetry_utils_1.ChildLogger.create(this.logger, "ScheduleManager"));
+        this.blobManager = new blobManager_1.BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId, localId) => {
+            if (!this.disposed) {
+                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId, localId });
+            }
+        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
+        this.scheduleManager = new scheduleManager_1.ScheduleManager(context.deltaManager, this, () => this.clientId, telemetry_utils_1.ChildLogger.create(this.logger, "ScheduleManager"));
         this.deltaSender = this.deltaManager;
         this.pendingStateManager = new pendingStateManager_1.PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
@@ -499,7 +289,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             flush: this.flush.bind(this),
             flushMode: () => this.flushMode,
             reSubmit: this.reSubmit.bind(this),
-            rollback: this.rollback.bind(this),
             setFlushMode: (mode) => this.setFlushMode(mode),
         }, this._flushMode, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
         this.context.quorum.on("removeMember", (clientId) => {
@@ -579,9 +368,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 createContainerRuntimeVersion: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerRuntimeVersion,
                 createContainerTimestamp: metadata === null || metadata === void 0 ? void 0 : metadata.createContainerTimestamp,
             };
-            // back-compat 0.59.3000 - Older document may either write summaryCount or not write it at all. If it does
-            // not write it, initialize summaryNumber to 0.
-            loadSummaryNumber = (_j = (_h = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _h !== void 0 ? _h : metadata === null || metadata === void 0 ? void 0 : metadata.summaryCount) !== null && _j !== void 0 ? _j : 0;
+            // summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
+            // the count is reset to 0.
+            loadSummaryNumber = (_d = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _d !== void 0 ? _d : 0;
         }
         else {
             this.createContainerMetadata = {
@@ -594,7 +383,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryNumber: loadSummaryNumber, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature }));
         (0, connectionTelemetry_1.ReportOpPerfTelemetry)(this.context.clientId, this.deltaManager, this.logger);
         (0, batchTracker_1.BindBatchTracker)(this, this.logger);
-        this.opTracker = new opTelemetry_1.OpTracker(this.deltaManager, this.mc.config.getBoolean(disableOpTrackingKey) === true);
     }
     get IContainerRuntime() { return this; }
     get IFluidRouter() { return this; }
@@ -617,7 +405,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 runtimeVersion: packageVersion_1.pkgVersion,
             },
         });
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", useDataStoreAliasing = false, flushMode = defaultFlushMode, enableOfflineLoad = false, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, } = runtimeOptions;
         const pendingRuntimeState = context.pendingLocalState;
         const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
         const storage = !pendingRuntimeState ?
@@ -670,7 +458,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             summaryOptions,
             gcOptions,
             loadSequenceNumberVerification,
-            useDataStoreAliasing,
             flushMode,
             enableOfflineLoad,
         }, containerScope, logger, loadExisting, blobManagerSnapshot, storage, requestHandler);
@@ -728,6 +515,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
     }
     get disposed() { return this._disposed; }
+    get emptyBatch() {
+        return this.pendingBatch.empty && this.pendingAttachBatch.empty;
+    }
     get summarizer() {
         (0, common_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
@@ -759,12 +549,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         if (this.runtimeOptions.summaryOptions.summarizerClientElection === true) {
             return true;
         }
-        if (this.summaryConfiguration.state !== "disabled") {
-            return this.summaryConfiguration.summarizerClientElection === true;
-        }
-        else {
-            return false;
-        }
+        return this.summaryConfiguration.state !== "disabled"
+            ? this.summaryConfiguration.summarizerClientElection === true
+            : false;
     }
     getMaxOpsSinceLastSummary() {
         // back-compat: maxOpsSinceLastSummary was moved from ISummaryRuntimeOptions
@@ -772,12 +559,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         if (this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary !== undefined) {
             return this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary;
         }
-        if (this.summaryConfiguration.state !== "disabled") {
-            return this.summaryConfiguration.maxOpsSinceLastSummary;
-        }
-        else {
-            return 0;
-        }
+        return this.summaryConfiguration.state !== "disabled"
+            ? this.summaryConfiguration.maxOpsSinceLastSummary
+            : 0;
     }
     getInitialSummarizerDelayMs() {
         // back-compat: initialSummarizerDelayMs was moved from ISummaryRuntimeOptions
@@ -785,12 +569,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         if (this.runtimeOptions.summaryOptions.initialSummarizerDelayMs !== undefined) {
             return this.runtimeOptions.summaryOptions.initialSummarizerDelayMs;
         }
-        if (this.summaryConfiguration.state !== "disabled") {
-            return this.summaryConfiguration.initialSummarizerDelayMs;
-        }
-        else {
-            return 0;
-        }
+        return this.summaryConfiguration.state !== "disabled"
+            ? this.summaryConfiguration.initialSummarizerDelayMs
+            : 0;
     }
     dispose(error) {
         var _a;
@@ -863,17 +644,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 return this.resolveHandle(requestParser.createSubRequest(1));
             }
             if (id === blobManager_1.BlobManager.basePath && requestParser.isLeaf(2)) {
-                const handle = await this.blobManager.getBlob(requestParser.pathParts[1]);
-                if (handle) {
-                    return {
+                const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
+                return blob
+                    ? {
                         status: 200,
                         mimeType: "fluid/object",
-                        value: handle.get(),
-                    };
-                }
-                else {
-                    return (0, runtime_utils_1.create404Response)(request);
-                }
+                        value: blob,
+                    } : (0, runtime_utils_1.create404Response)(request);
             }
             else if (requestParser.pathParts.length > 0) {
                 const dataStore = await this.getDataStoreFromRequest(id, request);
@@ -891,13 +668,14 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     internalId(maybeAlias) {
         var _a;
-        return (_a = this.dataStores.aliases().get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
+        return (_a = this.dataStores.aliases.get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
     }
     async getDataStoreFromRequest(id, request) {
         var _a, _b, _c;
         const wait = typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a[RuntimeHeaders.wait]) === "boolean"
             ? (_b = request.headers) === null || _b === void 0 ? void 0 : _b[RuntimeHeaders.wait]
             : true;
+        await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
         const dataStoreContext = await this.dataStores.getDataStore(internalId, wait);
         /**
@@ -927,10 +705,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     addMetadataToSummary(summaryTree) {
         var _a;
         const metadata = Object.assign(Object.assign(Object.assign(Object.assign({}, this.createContainerMetadata), { 
-            // back-compat 0.59.3000: This is renamed to summaryNumber. Can be removed when 0.59.3000 saturates.
-            summaryCount: this.nextSummaryNumber, 
             // Increment the summary number for the next summary that will be generated.
-            summaryNumber: this.nextSummaryNumber++, summaryFormatVersion: 1, disableIsolatedChannels: this.disableIsolatedChannels || undefined }), this.garbageCollector.getMetadata()), { 
+            summaryNumber: this.nextSummaryNumber++, summaryFormatVersion: 1 }), this.garbageCollector.getMetadata()), { 
             // The last message processed at the time of summary. If there are no new messages, use the message from the
             // last summary.
             message: (_a = (0, summaryFormat_1.extractSummaryMetadataMessage)(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary });
@@ -943,7 +719,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             const content = JSON.stringify([...this.chunkMap]);
             (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.chunksBlobName, content);
         }
-        const dataStoreAliases = this.dataStores.aliases();
+        const dataStoreAliases = this.dataStores.aliases;
         if (dataStoreAliases.size > 0) {
             (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.aliasBlobName, JSON.stringify([...dataStoreAliases]));
         }
@@ -974,7 +750,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Feature disabled, we never stop reconnecting
             return true;
         }
-        if (!this.pendingStateManager.hasPendingMessages()) {
+        if (!this.hasPendingMessages()) {
             // If there are no pending messages, we can always reconnect
             this.resetReconnectCount();
             return true;
@@ -1040,22 +816,55 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
     }
     setConnectionState(connected, clientId) {
+        if (connected === false && this.delayConnectClientId !== undefined) {
+            this.delayConnectClientId = undefined;
+            this.mc.logger.sendTelemetryEvent({
+                eventName: "UnsuccessfulConnectedTransition",
+            });
+            // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
+            return;
+        }
+        // If attachment blobs were added while disconnected, we need to delay
+        // propagation of the "connected" event until we have uploaded them to
+        // ensure we don't submit ops referencing a blob that has not been uploaded
+        const connecting = connected && !this._connected && !this.deltaManager.readOnlyInfo.readonly;
+        if (connecting && this.blobManager.hasPendingOfflineUploads) {
+            (0, common_utils_1.assert)(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
+            (0, common_utils_1.assert)(!!clientId, 0x393 /* Must have clientId when connecting */);
+            this.delayConnectClientId = clientId;
+            this.blobManager.onConnected().then(() => {
+                // make sure we didn't reconnect before the promise resolved
+                if (this.delayConnectClientId === clientId && !this.disposed) {
+                    this.delayConnectClientId = undefined;
+                    this.setConnectionStateCore(connected, clientId);
+                }
+            }, (error) => this.closeFn(error));
+            return;
+        }
+        this.setConnectionStateCore(connected, clientId);
+    }
+    setConnectionStateCore(connected, clientId) {
+        (0, common_utils_1.assert)(!this.delayConnectClientId, 0x394 /* connect event delay must be cleared before propagating connect event */);
         this.verifyNotClosed();
         // There might be no change of state due to Container calling this API after loading runtime.
         const changeOfState = this._connected !== connected;
-        const reconnection = changeOfState && connected;
+        const reconnection = changeOfState && !connected;
         this._connected = connected;
         if (!connected) {
             this._perfSignalData.signalsLost = 0;
             this._perfSignalData.signalTimestamp = 0;
             this._perfSignalData.trackingSignalSequenceNumber = undefined;
         }
+        else {
+            (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
+        }
+        // Fail while disconnected
         if (reconnection) {
             this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(
-                // pre-0.58 error message: MaxReconnectsWithNoProgress
-                container_utils_1.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops", "setConnectionState", undefined, {
+                this.closeFn(container_utils_1.DataProcessingError.create(
+                // eslint-disable-next-line max-len
+                "Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
                     pendingMessages: this.pendingStateManager.pendingMessagesCount,
@@ -1067,46 +876,48 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.replayPendingStates();
         }
         this.dataStores.setConnectionState(connected, clientId);
+        this.garbageCollector.setConnectionState(connected, clientId);
         (0, telemetry_utils_1.raiseConnectedEvent)(this.mc.logger, this, connected, clientId);
     }
     process(messageArg, local) {
-        var _a, _b;
+        var _a;
         this.verifyNotClosed();
-        // If it's not message for runtime, bail out right away.
-        if (!(0, driver_utils_1.isUnpackedRuntimeMessage)(messageArg)) {
-            return;
-        }
-        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
-            this.savedOps.push(messageArg);
-        }
         // Do shallow copy of message, as methods below will modify it.
         // There might be multiple container instances receiving same message
         // We do not need to make deep copy, as each layer will just replace message.content itself,
         // but would not modify contents details
         let message = Object.assign({}, messageArg);
+        // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
+        // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
+        // Old ops may contain empty string (I assume noops).
+        if (typeof message.contents === "string" && message.contents !== "") {
+            message.contents = JSON.parse(message.contents);
+        }
+        // Caveat: This will return false for runtime message in very old format, that are used in snapshot tests
+        // This format was not shipped to production workflows.
+        const runtimeMessage = unpackRuntimeMessage(message);
+        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
+            this.savedOps.push(messageArg);
+        }
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
         this.scheduleManager.beforeOpProcessing(message);
         try {
-            message = unpackRuntimeMessage(message);
             // Chunk processing must come first given that we will transform the message to the unchunked version
             // once all pieces are available
             message = this.processRemoteChunkedMessage(message);
             let localOpMetadata;
-            if (local) {
-                // Call the PendingStateManager to process local messages.
-                // Do not process local chunked ops until all pieces are available.
-                if (message.type !== ContainerMessageType.ChunkedOp) {
-                    localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
-                }
+            if (local && runtimeMessage) {
+                localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
             // the document is no longer dirty.
-            if (!this.pendingStateManager.hasPendingMessages()) {
+            if (!this.hasPendingMessages()) {
                 this.updateDocumentDirtyState(false);
             }
-            switch (message.type) {
+            const type = message.type;
+            switch (type) {
                 case ContainerMessageType.Attach:
                     this.dataStores.processAttachMessage(message, local);
                     break;
@@ -1117,12 +928,18 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
                     break;
                 case ContainerMessageType.BlobAttach:
-                    (0, common_utils_1.assert)((_b = message === null || message === void 0 ? void 0 : message.metadata) === null || _b === void 0 ? void 0 : _b.blobId, 0x12a /* "Missing blob id on metadata" */);
-                    this.blobManager.processBlobAttachOp(message.metadata.blobId, local);
+                    this.blobManager.processBlobAttachOp(message, local);
+                    break;
+                case ContainerMessageType.ChunkedOp:
+                case ContainerMessageType.Rejoin:
                     break;
                 default:
+                    (0, common_utils_1.assert)(!runtimeMessage, 0x3ce /* Runtime message of unknown type */);
+            }
+            // For back-compat, notify only about runtime messages for now.
+            if (runtimeMessage) {
+                this.emit("op", message, runtimeMessage);
             }
-            this.emit("op", message);
             this.scheduleManager.afterOpProcessing(undefined, message);
             if (local) {
                 // If we have processed a local op, this means that the container is
@@ -1188,6 +1005,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.dataStores.processSignal(envelope.address, transformed, local);
     }
     async getRootDataStore(id, wait = true) {
+        return this.getRootDataStoreChannel(id, wait);
+    }
+    async getRootDataStoreChannel(id, wait = true) {
+        await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
         const context = await this.dataStores.getDataStore(internalId, wait);
         (0, common_utils_1.assert)(await context.isRoot(), 0x12b /* "did not get root data store" */);
@@ -1212,25 +1033,57 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     flush() {
         (0, common_utils_1.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
-        if (!this.deltaSender) {
-            return;
+        this.flushBatch(this.pendingAttachBatch.popBatch());
+        this.flushBatch(this.pendingBatch.popBatch());
+        (0, common_utils_1.assert)(this.emptyBatch, 0x3cf /* reentrancy */);
+    }
+    flushBatch(batch) {
+        const length = batch.length;
+        if (length > 1) {
+            batch[0].metadata = Object.assign(Object.assign({}, batch[0].metadata), { batch: true });
+            batch[length - 1].metadata = Object.assign(Object.assign({}, batch[length - 1].metadata), { batch: false });
+            // This assert fires for the following reason (there might be more cases like that):
+            // AgentScheduler will send ops in response to ConsensusRegisterCollection's "atomicChanged" event handler,
+            // i.e. in the middle of op processing!
+            // Sending ops while processing ops is not good idea - it's not defined when
+            // referenceSequenceNumber changes in op processing sequence (at the beginning or end of op processing),
+            // If we send ops in response to processing multiple ops, then we for sure hit this assert!
+            // Tracked via ADO #1834
+            // assert(batch[0].referenceSequenceNumber === batch[length - 1].referenceSequenceNumber,
+            //    "Batch should be generated synchronously, without processing ops in the middle!");
         }
-        // Let the PendingStateManager know that there was an attempt to flush messages.
-        // Note that this should happen before the `this.needsFlush` check below because in the scenario where we are
-        // not connected, `this.needsFlush` will be false but the PendingStateManager might have pending messages and
-        // hence needs to track this.
-        this.pendingStateManager.onFlush();
-        // If flush has already been called then exit early
-        if (!this.needsFlush) {
-            return;
-        }
-        this.needsFlush = false;
+        let clientSequenceNumber = -1;
         // Did we disconnect in the middle of turn-based batch?
         // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
-        if (!this.canSendOps()) {
-            return;
+        if (this.canSendOps()) {
+            if (this.context.submitBatchFn !== undefined) {
+                const batchToSend = [];
+                for (const message of batch) {
+                    batchToSend.push({ contents: message.contents, metadata: message.metadata });
+                }
+                // returns clientSequenceNumber of last message in a batch
+                clientSequenceNumber = this.context.submitBatchFn(batchToSend);
+            }
+            else {
+                // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
+                // version that has support for batches (submitBatchFn)
+                for (const message of batch) {
+                    clientSequenceNumber = this.context.submitFn(protocol_definitions_1.MessageType.Operation, message.deserializedContent, true, // batch
+                    message.metadata);
+                }
+                this.deltaSender.flush();
+            }
+            // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
+            clientSequenceNumber -= batch.length - 1;
+            (0, common_utils_1.assert)(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
+        }
+        // Let the PendingStateManager know that a message was submitted.
+        // In future, need to shift toward keeping batch as a whole!
+        for (const message of batch) {
+            this.pendingStateManager.onSubmitMessage(message.deserializedContent.type, clientSequenceNumber, message.referenceSequenceNumber, message.deserializedContent.contents, message.localOpMetadata, message.metadata);
+            clientSequenceNumber++;
         }
-        return this.deltaSender.flush();
+        this.pendingStateManager.onFlush();
     }
     orderSequentially(callback) {
         // If flush mode is already TurnBased we are either
@@ -1255,7 +1108,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     trackOrderSequentiallyCalls(callback) {
         let checkpoint;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
-            checkpoint = this.pendingStateManager.checkpoint();
+            // Note: we are not touching this.pendingAttachBatch here, for two reasons:
+            // 1. It would not help, as we flush attach ops as they become available.
+            // 2. There is no way to undo process of data store creation.
+            checkpoint = this.pendingBatch.checkpoint();
         }
         try {
             this._orderSequentiallyCalls++;
@@ -1264,7 +1120,16 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         catch (error) {
             if (checkpoint) {
                 // This will throw and close the container if rollback fails
-                checkpoint.rollback();
+                try {
+                    checkpoint.rollback((message) => this.rollback(message.deserializedContent.type, message.deserializedContent.contents, message.localOpMetadata));
+                }
+                catch (err) {
+                    const error2 = (0, telemetry_utils_1.wrapError)(err, (message) => {
+                        return container_utils_1.DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
+                    });
+                    this.closeFn(error2);
+                    throw error2;
+                }
             }
             else {
                 // pre-0.58 error message: orderSequentiallyCallbackException
@@ -1278,67 +1143,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     async createDataStore(pkg) {
         const internalId = (0, uuid_1.v4)();
-        return (0, dataStore_1.channelToDataStore)(await this._createDataStore(pkg, false /* isRoot */, internalId), internalId, this, this.dataStores, this.mc.logger);
-    }
-    /**
-     * Creates a root datastore directly with a user generated id and attaches it to storage.
-     * It is vulnerable to name collisions and should not be used.
-     *
-     * This method will be removed. See #6465.
-     */
-    async createRootDataStoreLegacy(pkg, rootDataStoreId) {
-        const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
-        // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
-        // older versions, we still have to call bindToContext.
-        if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
-            fluidDataStore.makeVisibleAndAttachGraph();
-        }
-        else {
-            fluidDataStore.bindToContext();
-        }
-        return fluidDataStore;
-    }
-    /**
-     * @deprecated - will be removed in an upcoming release. See #9660.
-     */
-    async createRootDataStore(pkg, rootDataStoreId) {
-        if (rootDataStoreId.includes("/")) {
-            throw new container_utils_1.UsageError(`Id cannot contain slashes: '${rootDataStoreId}'`);
-        }
-        return this._aliasingEnabled === true ?
-            this.createAndAliasDataStore(pkg, rootDataStoreId) :
-            this.createRootDataStoreLegacy(pkg, rootDataStoreId);
-    }
-    /**
-     * Creates a data store then attempts to alias it.
-     * If aliasing fails, it will raise an exception.
-     *
-     * This method will be removed. See #6465.
-     *
-     * @param pkg - Package name of the data store
-     * @param alias - Alias to be assigned to the data store
-     * @param props - Properties for the data store
-     * @returns - An aliased data store which can can be found / loaded by alias.
-     */
-    async createAndAliasDataStore(pkg, alias, props) {
-        const internalId = (0, uuid_1.v4)();
-        const dataStore = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
-        const aliasedDataStore = (0, dataStore_1.channelToDataStore)(dataStore, internalId, this, this.dataStores, this.mc.logger);
-        const result = await aliasedDataStore.trySetAlias(alias);
-        if (result !== "Success") {
-            throw new container_utils_1.GenericError("dataStoreAliasFailure", undefined /* error */, {
-                alias: {
-                    value: alias,
-                    tag: telemetry_utils_1.TelemetryDataTag.UserData,
-                },
-                internalId: {
-                    value: internalId,
-                    tag: telemetry_utils_1.TelemetryDataTag.PackageData,
-                },
-                aliasResult: result,
-            });
-        }
-        return aliasedDataStore;
+        return (0, dataStore_1.channelToDataStore)(await this._createDataStore(pkg, internalId), internalId, this, this.dataStores, this.mc.logger);
     }
     createDetachedRootDataStore(pkg, rootDataStoreId) {
         if (rootDataStoreId.includes("/")) {
@@ -1349,38 +1154,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     createDetachedDataStore(pkg) {
         return this.dataStores.createDetachedDataStoreCore(pkg, false);
     }
-    /**
-     * Creates a possibly root datastore directly with a possibly user generated id and attaches it to storage.
-     * It is vulnerable to name collisions if both aforementioned conditions are true, and should not be used.
-     *
-     * This method will be removed. See #6465.
-     */
-    async _createDataStoreWithPropsLegacy(pkg, props, id = (0, uuid_1.v4)(), isRoot = false) {
-        const fluidDataStore = await this.dataStores._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
-        if (isRoot) {
-            // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
-            // For older versions, we still have to call bindToContext.
-            if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
-                fluidDataStore.makeVisibleAndAttachGraph();
-            }
-            else {
-                fluidDataStore.bindToContext();
-            }
-            this.logger.sendTelemetryEvent({
-                eventName: "Root datastore with props",
-                hasProps: props !== undefined,
-            });
-        }
+    async _createDataStoreWithProps(pkg, props, id = (0, uuid_1.v4)()) {
+        const fluidDataStore = await this.dataStores._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props).realize();
         return (0, dataStore_1.channelToDataStore)(fluidDataStore, id, this, this.dataStores, this.mc.logger);
     }
-    async _createDataStoreWithProps(pkg, props, id = (0, uuid_1.v4)(), isRoot = false) {
-        return this._aliasingEnabled === true && isRoot ?
-            this.createAndAliasDataStore(pkg, id, props) :
-            this._createDataStoreWithPropsLegacy(pkg, props, id, isRoot);
-    }
-    async _createDataStore(pkg, isRoot, id = (0, uuid_1.v4)(), props) {
+    async _createDataStore(pkg, id = (0, uuid_1.v4)(), props) {
         return this.dataStores
-            ._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props)
+            ._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
             .realize();
     }
     canSendOps() {
@@ -1454,7 +1234,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
             this.emit("attached");
         }
-        if (attachState === container_definitions_1.AttachState.Attached && !this.pendingStateManager.hasPendingMessages()) {
+        if (attachState === container_definitions_1.AttachState.Attached && !this.hasPendingMessages()) {
             this.updateDocumentDirtyState(false);
         }
         this.dataStores.setAttachState(attachState);
@@ -1472,10 +1252,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.blobManager.setRedirectTable(blobRedirectTable);
         }
         const summarizeResult = this.dataStores.createSummary(telemetryContext);
-        if (!this.disableIsolatedChannels) {
-            // Wrap data store summaries in .channels subtree.
-            (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
-        }
+        // Wrap data store summaries in .channels subtree.
+        (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
         this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
         return summarizeResult.summary;
     }
@@ -1490,12 +1268,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     async summarizeInternal(fullTree, trackState, telemetryContext) {
         const summarizeResult = await this.dataStores.summarize(fullTree, trackState, telemetryContext);
-        let pathPartsForChildren;
-        if (!this.disableIsolatedChannels) {
-            // Wrap data store summaries in .channels subtree.
-            (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
-            pathPartsForChildren = [runtime_definitions_1.channelsTreeName];
-        }
+        // Wrap data store summaries in .channels subtree.
+        (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
+        const pathPartsForChildren = [runtime_definitions_1.channelsTreeName];
         this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
         return Object.assign(Object.assign({}, summarizeResult), { id: "", pathPartsForChildren });
     }
@@ -1623,7 +1398,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     /**
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
-     * @returns the statistics of the garbage collection run.
+     * @returns the statistics of the garbage collection run; undefined if GC did not run.
      */
     async collectGarbage(options) {
         return this.garbageCollector.collectGarbage(options);
@@ -1646,7 +1421,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * @param options - options controlling how the summary is generated or submitted
      */
     async submitSummary(options) {
-        var _a, _b, _c;
+        var _a, _b;
         const { fullTree, refreshLatestAck, summaryLogger } = options;
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
@@ -1654,6 +1429,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         const summaryNumberLogger = telemetry_utils_1.ChildLogger.create(summaryLogger, undefined, {
             all: { summaryNumber },
         });
+        (0, common_utils_1.assert)(this.emptyBatch, 0x3d1 /* Can't trigger summary in the middle of a batch */);
         let latestSnapshotVersionId;
         if (refreshLatestAck) {
             const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(telemetry_utils_1.ChildLogger.create(summaryNumberLogger, undefined, { all: { safeSummary: true } }));
@@ -1674,17 +1450,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
             const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
-            // We should be here is we haven't processed be here. If we are of if the last message's sequence number
-            // doesn't match the last processed sequence number, log an error.
-            if (summaryRefSeqNum !== ((_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.sequenceNumber)) {
-                summaryNumberLogger.sendErrorEvent({
-                    eventName: "LastSequenceMismatch",
-                    error: message,
-                });
-            }
+            const lastAck = this.summaryCollection.latestAck;
             this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger);
             // Helper function to check whether we should still continue between each async step.
             const checkContinue = () => {
+                var _a;
                 // Do not check for loss of connectivity directly! Instead leave it up to
                 // RunWhileConnectedCoordinator to control policy in a single place.
                 // This will allow easier change of design if we chose to. For example, we may chose to allow
@@ -1708,6 +1478,14 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                         error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
                     };
                 }
+                (0, common_utils_1.assert)(summaryRefSeqNum === ((_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.sequenceNumber), 0x395 /* it's one and the same thing */);
+                if (lastAck !== this.summaryCollection.latestAck) {
+                    return {
+                        continue: false,
+                        // eslint-disable-next-line max-len
+                        error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
+                    };
+                }
                 return { continue: true };
             };
             let continueResult = checkContinue();
@@ -1726,7 +1504,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
             try {
                 summarizeResult = await this.summarize({
-                    fullTree: fullTree || forcedFullTree,
+                    fullTree: fullTree !== null && fullTree !== void 0 ? fullTree : forcedFullTree,
                     trackState: true,
                     summaryLogger: summaryNumberLogger,
                     runGC: this.garbageCollector.shouldRunGC,
@@ -1746,13 +1524,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Counting dataStores and handles
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
-            const dataStoreTree = this.disableIsolatedChannels ? summaryTree : summaryTree.tree[runtime_definitions_1.channelsTreeName];
+            const dataStoreTree = summaryTree.tree[runtime_definitions_1.channelsTreeName];
             (0, common_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === protocol_definitions_1.SummaryType.Handle).length;
             const gcSummaryTreeStats = summaryTree.tree[garbageCollection_1.gcTreeKey]
                 ? (0, runtime_utils_1.calculateStats)(summaryTree.tree[garbageCollection_1.gcTreeKey])
                 : undefined;
-            const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount, gcStateUpdatedDataStoreCount: (_b = summarizeResult.gcStats) === null || _b === void 0 ? void 0 : _b.updatedDataStoreCount, gcBlobNodeCount: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.blobNodeCount, gcTotalBlobsSize: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.totalBlobSize, opsSizesSinceLastSummary: this.opTracker.opsSizeAccumulator, nonSystemOpsSinceLastSummary: this.opTracker.nonSystemOpCount, summaryNumber }, partialStats);
+            const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount, gcStateUpdatedDataStoreCount: (_a = summarizeResult.gcStats) === null || _a === void 0 ? void 0 : _a.updatedDataStoreCount, gcBlobNodeCount: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.blobNodeCount, gcTotalBlobsSize: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.totalBlobSize, summaryNumber }, partialStats);
             const generateSummaryData = {
                 referenceSequenceNumber: summaryRefSeqNum,
                 minimumSequenceNumber,
@@ -1770,7 +1548,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // submitting the summaryOp then we can't rely on summaryAck. So in case we have
             // latestSnapshotVersionId from storage and it does not match with the lastAck ackHandle, then use
             // the one fetched from storage as parent as that is the latest.
-            const lastAck = this.summaryCollection.latestAck;
             let summaryContext;
             if ((lastAck === null || lastAck === void 0 ? void 0 : lastAck.summaryAck.contents.handle) !== latestSnapshotVersionId
                 && latestSnapshotVersionId !== undefined) {
@@ -1783,7 +1560,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             else if (lastAck === undefined) {
                 summaryContext = {
                     proposalHandle: undefined,
-                    ackHandle: (_c = this.context.getLoadedFromVersion()) === null || _c === void 0 ? void 0 : _c.id,
+                    ackHandle: (_b = this.context.getLoadedFromVersion()) === null || _b === void 0 ? void 0 : _b.id,
                     referenceSequenceNumber: summaryRefSeqNum,
                 };
             }
@@ -1816,14 +1593,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
             let clientSequenceNumber;
             try {
-                clientSequenceNumber = this.submitSystemMessage(protocol_definitions_1.MessageType.Summarize, summaryMessage);
+                clientSequenceNumber = this.submitSummaryMessage(summaryMessage);
             }
             catch (error) {
                 return Object.assign(Object.assign({ stage: "upload" }, uploadData), { error });
             }
             const submitData = Object.assign(Object.assign({ stage: "submit" }, uploadData), { clientSequenceNumber, submitOpDuration: trace.trace().duration });
             this.summarizerNode.completeSummary(handle);
-            this.opTracker.reset();
             return submitData;
         }
         finally {
@@ -1865,7 +1641,17 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.chunkMap.delete(clientId);
         }
     }
+    hasPendingMessages() {
+        return this.pendingStateManager.hasPendingMessages() || !this.emptyBatch;
+    }
     updateDocumentDirtyState(dirty) {
+        if (this.attachState !== container_definitions_1.AttachState.Attached) {
+            (0, common_utils_1.assert)(dirty, 0x3d2 /* Non-attached container is dirty */);
+        }
+        else {
+            // Other way is not true = see this.isContainerMessageDirtyable()
+            (0, common_utils_1.assert)(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
+        }
         if (this.dirtyContainer === dirty) {
             return;
         }
@@ -1893,99 +1679,100 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyNotClosed();
         return this.blobManager.createBlob(blob);
     }
-    submit(type, content, localOpMetadata = undefined, opMetadata = undefined) {
+    submit(type, contents, localOpMetadata = undefined, metadata = undefined) {
         this.verifyNotClosed();
         // There should be no ops in detached container state!
         (0, common_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
-        let clientSequenceNumber = -1;
-        let opMetadataInternal = opMetadata;
-        if (this.canSendOps()) {
-            const serializedContent = JSON.stringify(content);
-            const maxOpSize = this.context.deltaManager.maxMessageSize;
-            // If in TurnBased flush mode we will trigger a flush at the next turn break
-            if (this.flushMode === runtime_definitions_1.FlushMode.TurnBased && !this.needsFlush) {
-                opMetadataInternal = Object.assign(Object.assign({}, opMetadata), { batch: true });
-                this.needsFlush = true;
-                // Use Promise.resolve().then() to queue a microtask to detect the end of the turn and force a flush.
-                if (!this.flushTrigger) {
-                    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-                    Promise.resolve().then(() => {
-                        this.flushTrigger = false;
-                        this.flush();
+        const deserializedContent = { type, contents };
+        const serializedContent = JSON.stringify(deserializedContent);
+        if (this.deltaManager.readOnlyInfo.readonly) {
+            this.logger.sendErrorEvent({ eventName: "SubmitOpInReadonly" });
+        }
+        const message = {
+            contents: serializedContent,
+            deserializedContent,
+            metadata,
+            localOpMetadata,
+            referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+        };
+        try {
+            // If this is attach message for new data store, and we are in a batch, send this op out of order
+            // Is it safe:
+            //    Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
+            //    They become visible only when aliased, or handle to some sub-element of newly created datastore
+            //    is stored in some DDS, i.e. only after some other op.
+            // Why:
+            //    Attach ops are large, and expensive to process. Plus there are scenarios where a lot of new data
+            //    stores are created, causing issues like relay service throttling (too many ops) and catastrophic
+            //    failure (batch is too large). Pushing them earlier and outside of main batch should alleviate
+            //    these issues.
+            // Cons:
+            //    1. With large batches, relay service may throttle clients. Clients may disconnect while throttled.
+            //    This change creates new possibility of a lot of newly created data stores never being referenced
+            //    because client died before it had a change to submit the rest of the ops. This will create more
+            //    garbage that needs to be collected leveraging GC (Garbage Collection) feature.
+            //    2. Sending ops out of order means they are excluded from rollback functionality. This is not an issue
+            //    today as rollback can't undo creation of data store. To some extent not sending them is a bigger
+            //    issue than sending.
+            // Please note that this does not change file format, so it can be disabled in the future if this
+            // optimization no longer makes sense (for example, batch compression may make it less appealing).
+            if (type === ContainerMessageType.Attach &&
+                this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
+                if (!this.pendingAttachBatch.push(message)) {
+                    // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
+                    // when queue is not empty.
+                    // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
+                    this.flushBatch(this.pendingAttachBatch.popBatch());
+                    if (!this.pendingAttachBatch.push(message)) {
+                        throw new container_utils_1.GenericError("BatchTooLarge", 
+                        /* error */ undefined, {
+                            opSize: message.contents.length,
+                            count: this.pendingAttachBatch.length,
+                            limit: this.pendingAttachBatch.limit,
+                        });
+                    }
+                }
+            }
+            else {
+                if (!this.pendingBatch.push(message)) {
+                    throw new container_utils_1.GenericError("BatchTooLarge", 
+                    /* error */ undefined, {
+                        opSize: message.contents.length,
+                        count: this.pendingBatch.length,
+                        limit: this.pendingBatch.limit,
                     });
                 }
             }
-            clientSequenceNumber = this.submitMaybeChunkedMessages(type, content, serializedContent, maxOpSize, this._flushMode === runtime_definitions_1.FlushMode.TurnBased, opMetadataInternal);
+            if (this._flushMode !== runtime_definitions_1.FlushMode.TurnBased) {
+                this.flush();
+            }
+            else if (!this.flushTrigger) {
+                this.flushTrigger = true;
+                // Queue a microtask to detect the end of the turn and force a flush.
+                // eslint-disable-next-line @typescript-eslint/no-floating-promises
+                Promise.resolve().then(() => {
+                    this.flushTrigger = false;
+                    this.flush();
+                });
+            }
         }
-        // Let the PendingStateManager know that a message was submitted.
-        this.pendingStateManager.onSubmitMessage(type, clientSequenceNumber, this.deltaManager.lastSequenceNumber, content, localOpMetadata, opMetadataInternal);
-        if (this.isContainerMessageDirtyable(type, content)) {
-            this.updateDocumentDirtyState(true);
+        catch (error) {
+            this.closeFn(error);
+            throw error;
         }
-    }
-    submitMaybeChunkedMessages(type, content, serializedContent, serverMaxOpSize, batch, opMetadataInternal = undefined) {
-        if (this._maxOpSizeInBytes >= 0) {
-            // Chunking disabled
-            if (!serializedContent || serializedContent.length <= this._maxOpSizeInBytes) {
-                return this.submitRuntimeMessage(type, content, batch, opMetadataInternal);
-            }
-            // When chunking is disabled, we ignore the server max message size
-            // and if the content length is larger than the client configured message size
-            // instead of splitting the content, we will fail by explicitly close the container
-            this.closeFn(new container_utils_1.GenericError("OpTooLarge", 
-            /* error */ undefined, {
-                length: {
-                    value: serializedContent.length,
-                    tag: telemetry_utils_1.TelemetryDataTag.PackageData,
-                },
-                limit: {
-                    value: this._maxOpSizeInBytes,
-                    tag: telemetry_utils_1.TelemetryDataTag.PackageData,
-                },
-            }));
-            return -1;
-        }
-        // Chunking enabled, fallback on the server's max message size
-        // and split the content accordingly
-        if (!serializedContent || serializedContent.length <= serverMaxOpSize) {
-            return this.submitRuntimeMessage(type, content, batch, opMetadataInternal);
-        }
-        return this.submitChunkedMessage(type, serializedContent, serverMaxOpSize);
-    }
-    submitChunkedMessage(type, content, maxOpSize) {
-        const contentLength = content.length;
-        const chunkN = Math.floor((contentLength - 1) / maxOpSize) + 1;
-        let offset = 0;
-        let clientSequenceNumber = 0;
-        for (let i = 1; i <= chunkN; i = i + 1) {
-            const chunkedOp = {
-                chunkId: i,
-                contents: content.substr(offset, maxOpSize),
-                originalType: type,
-                totalChunks: chunkN,
-            };
-            offset += maxOpSize;
-            clientSequenceNumber = this.submitRuntimeMessage(ContainerMessageType.ChunkedOp, chunkedOp, false);
+        if (this.isContainerMessageDirtyable(type, contents)) {
+            this.updateDocumentDirtyState(true);
         }
-        return clientSequenceNumber;
     }
-    submitSystemMessage(type, contents) {
+    submitSummaryMessage(contents) {
         this.verifyNotClosed();
         (0, common_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
-        // That said, we can preserve existing behavior by not flushing existing buffer.
-        // That might be not what caller hopes to get, but we can look deeper if telemetry tells us it's a problem.
-        const middleOfBatch = this.flushMode === runtime_definitions_1.FlushMode.TurnBased && this.needsFlush;
-        if (middleOfBatch) {
-            this.mc.logger.sendErrorEvent({ eventName: "submitSystemMessageError", type });
-        }
-        return this.context.submitFn(type, contents, middleOfBatch);
-    }
-    submitRuntimeMessage(type, contents, batch, appData) {
-        this.verifyNotClosed();
-        (0, common_utils_1.assert)(this.connected, 0x259 /* "Container disconnected when trying to submit system message" */);
-        const payload = { type, contents };
-        return this.context.submitFn(protocol_definitions_1.MessageType.Operation, payload, batch, appData);
+        (0, common_utils_1.assert)(this.emptyBatch, 0x3d4 /* System op in the middle of a batch */);
+        // back-compat: ADO #1385: Make this call unconditional in the future
+        return this.context.submitSummaryFn !== undefined
+            ? this.context.submitSummaryFn(contents)
+            : this.context.submitFn(protocol_definitions_1.MessageType.Summarize, contents, false);
     }
     /**
      * Throw an error if the runtime is closed.  Methods that are expected to potentially
@@ -2016,7 +1803,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             case ContainerMessageType.ChunkedOp:
                 throw new Error(`chunkedOp not expected here`);
             case ContainerMessageType.BlobAttach:
-                this.submit(type, content, localOpMetadata, opMetadata);
+                this.blobManager.reSubmit(opMetadata);
                 break;
             case ContainerMessageType.Rejoin:
                 this.submit(type, content);
@@ -2039,13 +1826,18 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(proposalHandle, ackHandle, summaryRefSeq, summaryLogger) {
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
-        const { snapshotTree } = await this.fetchSnapshotFromStorage(ackHandle, summaryLogger, {
-            eventName: "RefreshLatestSummaryGetSnapshot",
-            ackHandle,
-            summaryRefSeq,
-            fetchLatest: false,
-        });
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
+        // The call to fetch the snapshot is very expensive and not always needed.
+        // It should only be done by the summarizerNode, if required.
+        const snapshotTreeFetcher = async () => {
+            const fetchResult = await this.fetchSnapshotFromStorage(ackHandle, summaryLogger, {
+                eventName: "RefreshLatestSummaryGetSnapshot",
+                ackHandle,
+                summaryRefSeq,
+                fetchLatest: false,
+            });
+            return fetchResult.snapshotTree;
+        };
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, snapshotTreeFetcher, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
         await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
     }
@@ -2059,7 +1851,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         const { snapshotTree, versionId } = await this.fetchSnapshotFromStorage(null, summaryLogger, {
             eventName: "RefreshLatestSummaryGetSnapshot",
             fetchLatest: true,
-        });
+        }, driver_definitions_1.FetchSource.noCache);
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
         const latestSnapshotRefSeq = await (0, runtime_utils_1.seqFromTree)(snapshotTree, readAndParseBlob);
         const result = await this.summarizerNode.refreshLatestSummary(undefined, latestSnapshotRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
@@ -2067,11 +1859,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchSnapshotFromStorage(versionId, logger, event) {
+    async fetchSnapshotFromStorage(versionId, logger, event, fetchSource) {
         return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = common_utils_1.Trace.start();
-            const versions = await this.storage.getVersions(versionId, 1);
+            const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", fetchSource);
             (0, common_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
@@ -2101,10 +1893,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad)) {
             throw new container_utils_1.UsageError("can't get state when offline load disabled");
         }
+        // Flush pending batch.
+        // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
+        // to close current batch.
+        this.flush();
         const previousPendingState = this.context.pendingLocalState;
         if (previousPendingState) {
             return {
                 pending: this.pendingStateManager.getLocalState(),
+                pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
                 snapshotBlobs: previousPendingState.snapshotBlobs,
                 baseSnapshot: previousPendingState.baseSnapshot,
                 savedOps: this.savedOps,
@@ -2114,6 +1911,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         (0, common_utils_1.assert)(!!this.baseSnapshotBlobs, 0x2e7 /* "Must serialize base snapshot blobs before getting runtime state" */);
         return {
             pending: this.pendingStateManager.getLocalState(),
+            pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
             snapshotBlobs: this.baseSnapshotBlobs,
             baseSnapshot: this.context.baseSnapshot,
             savedOps: this.savedOps,
@@ -2153,6 +1951,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // we may not have seen every sequence number (because of system ops) so apply everything once we
         // don't have any more saved ops
         await this.pendingStateManager.applyStashedOpsAt();
+        // If it's not the case, we should take it into account when calculating dirty state.
+        (0, common_utils_1.assert)(this.context.attachState === container_definitions_1.AttachState.Attached, 0x3d5 /* this function is called for attached containers only */);
+        if (!this.hasPendingMessages()) {
+            this.updateDocumentDirtyState(false);
+        }
+    }
+    validateSummaryHeuristicConfiguration(configuration) {
+        // eslint-disable-next-line no-restricted-syntax
+        for (const prop in configuration) {
+            if (typeof configuration[prop] === "number" && configuration[prop] < 0) {
+                throw new container_utils_1.UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
+            }
+        }
     }
 }
 exports.ContainerRuntime = ContainerRuntime;