@fluidframework/container-runtime 2.0.0-internal.1.1.2 → 2.0.0-internal.1.2.0

package/lib/containerRuntime.js

@@ -1,8 +1,8 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
 import { assert, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
-import { ChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, loggerToMonitoringContext, } from "@fluidframework/telemetry-utils";
+import { ChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, loggerToMonitoringContext, wrapError, } from "@fluidframework/telemetry-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
-import { readAndParse, isUnpackedRuntimeMessage } from "@fluidframework/driver-utils";
+import { readAndParse } from "@fluidframework/driver-utils";
 import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
 import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
@@ -14,7 +14,8 @@ import { FluidDataStoreRegistry } from "./dataStoreRegistry";
 import { Summarizer } from "./summarizer";
 import { SummaryManager } from "./summaryManager";
 import { ReportOpPerfTelemetry, } from "./connectionTelemetry";
-import { PendingStateManager } from "./pendingStateManager";
+import { PendingStateManager, } from "./pendingStateManager";
+import { BatchManager } from "./batchManager";
 import { pkgVersion } from "./packageVersion";
 import { BlobManager } from "./blobManager";
 import { DataStores, getSummaryForDatastores } from "./dataStores";
@@ -75,11 +76,10 @@ export var RuntimeHeaders;
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders || (RuntimeHeaders = {}));
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
-// By default, we should reject any op larger than 768KB,
-// in order to account for some extra overhead from serialization
-// to not reach the 1MB limits in socket.io and Kafka.
-const defaultMaxOpSizeInBytes = 768000;
 const defaultFlushMode = FlushMode.TurnBased;
+/**
+ * @deprecated - use ContainerRuntimeMessage instead
+ */
 export var RuntimeMessage;
 (function (RuntimeMessage) {
     RuntimeMessage["FluidDataStoreOp"] = "component";
@@ -90,12 +90,24 @@ export var RuntimeMessage;
     RuntimeMessage["Alias"] = "alias";
     RuntimeMessage["Operation"] = "op";
 })(RuntimeMessage || (RuntimeMessage = {}));
+/**
+ * @deprecated - please use version in driver-utils
+ */
 export function isRuntimeMessage(message) {
     if (Object.values(RuntimeMessage).includes(message.type)) {
         return true;
     }
     return false;
 }
+/**
+ * Unpacks runtime messages
+ *
+ * @remarks This API makes no promises regarding backward-compatability. This is internal API.
+ * @param message - message (as it observed in storage / service)
+ * @returns unpacked runtime message
+ *
+ * @internal
+ */
 export function unpackRuntimeMessage(message) {
     if (message.type === MessageType.Operation) {
         // legacy op format?
@@ -109,14 +121,15 @@ export function unpackRuntimeMessage(message) {
             message.type = innerContents.type;
             message.contents = innerContents.contents;
         }
-        assert(isUnpackedRuntimeMessage(message), 0x122 /* "Message to unpack is not proper runtime message" */);
+        return true;
     }
     else {
         // Legacy format, but it's already "unpacked",
         // i.e. message.type is actually ContainerMessageType.
+        // Or it's non-runtime message.
         // Nothing to do in such case.
+        return false;
     }
-    return message;
 }
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
@@ -157,7 +170,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.summaryConfiguration = summaryConfiguration;
         this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
-        this.needsFlush = false;
         this.flushTrigger = false;
         this.savedOps = [];
         this.consecutiveReconnects = 0;
@@ -170,6 +182,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             signalTimestamp: 0,
             trackingSignalSequenceNumber: undefined,
         };
+        // Provide lower soft limit - we want to have some number of ops to get efficiency in compression & bandwidth usage,
+        // but at the same time we want to send these ops sooner, to reduce overall latency of processing a batch.
+        // So there is some ballance here, that depends on compression algorithm and its efficiency working with smaller
+        // payloads. That number represents final (compressed) bits (once compression is implemented).
+        this.pendingAttachBatch = new BatchManager(64 * 1024);
+        this.pendingBatch = new BatchManager();
         this.summarizeOnDemand = (...args) => {
             if (this.clientDetails.type === summarizerClientType) {
                 return this.summarizer.summarizeOnDemand(...args);
@@ -227,6 +245,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
             getLastSummaryTimestampMs: () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; },
             readAndParseBlob: async (id) => readAndParse(this.storage, id),
+            getContainerDiagnosticId: () => this.context.id,
+            activeConnection: () => this.deltaManager.active,
         });
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"), 
@@ -254,7 +274,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId, localId });
             }
         }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
-        this.scheduleManager = new ScheduleManager(context.deltaManager, this, ChildLogger.create(this.logger, "ScheduleManager"));
+        this.scheduleManager = new ScheduleManager(context.deltaManager, this, () => this.clientId, ChildLogger.create(this.logger, "ScheduleManager"));
         this.deltaSender = this.deltaManager;
         this.pendingStateManager = new PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
@@ -264,7 +284,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             flush: this.flush.bind(this),
             flushMode: () => this.flushMode,
             reSubmit: this.reSubmit.bind(this),
-            rollback: this.rollback.bind(this),
             setFlushMode: (mode) => this.setFlushMode(mode),
         }, this._flushMode, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
         this.context.quorum.on("removeMember", (clientId) => {
@@ -491,6 +510,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
     }
     get disposed() { return this._disposed; }
+    get emptyBatch() {
+        return this.pendingBatch.empty && this.pendingAttachBatch.empty;
+    }
     get summarizer() {
         assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
@@ -522,12 +544,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (this.runtimeOptions.summaryOptions.summarizerClientElection === true) {
             return true;
         }
-        if (this.summaryConfiguration.state !== "disabled") {
-            return this.summaryConfiguration.summarizerClientElection === true;
-        }
-        else {
-            return false;
-        }
+        return this.summaryConfiguration.state !== "disabled"
+            ? this.summaryConfiguration.summarizerClientElection === true
+            : false;
     }
     getMaxOpsSinceLastSummary() {
         // back-compat: maxOpsSinceLastSummary was moved from ISummaryRuntimeOptions
@@ -535,12 +554,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary !== undefined) {
             return this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary;
         }
-        if (this.summaryConfiguration.state !== "disabled") {
-            return this.summaryConfiguration.maxOpsSinceLastSummary;
-        }
-        else {
-            return 0;
-        }
+        return this.summaryConfiguration.state !== "disabled"
+            ? this.summaryConfiguration.maxOpsSinceLastSummary
+            : 0;
     }
     getInitialSummarizerDelayMs() {
         // back-compat: initialSummarizerDelayMs was moved from ISummaryRuntimeOptions
@@ -548,12 +564,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (this.runtimeOptions.summaryOptions.initialSummarizerDelayMs !== undefined) {
             return this.runtimeOptions.summaryOptions.initialSummarizerDelayMs;
         }
-        if (this.summaryConfiguration.state !== "disabled") {
-            return this.summaryConfiguration.initialSummarizerDelayMs;
-        }
-        else {
-            return 0;
-        }
+        return this.summaryConfiguration.state !== "disabled"
+            ? this.summaryConfiguration.initialSummarizerDelayMs
+            : 0;
     }
     dispose(error) {
         var _a;
@@ -627,16 +640,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             if (id === BlobManager.basePath && requestParser.isLeaf(2)) {
                 const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
-                if (blob) {
-                    return {
+                return blob
+                    ? {
                         status: 200,
                         mimeType: "fluid/object",
                         value: blob,
-                    };
-                }
-                else {
-                    return create404Response(request);
-                }
+                    } : create404Response(request);
             }
             else if (requestParser.pathParts.length > 0) {
                 const dataStore = await this.getDataStoreFromRequest(id, request);
@@ -736,7 +745,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Feature disabled, we never stop reconnecting
             return true;
         }
-        if (!this.pendingStateManager.hasPendingMessages()) {
+        if (!this.hasPendingMessages()) {
             // If there are no pending messages, we can always reconnect
             this.resetReconnectCount();
             return true;
@@ -841,13 +850,16 @@ export class ContainerRuntime extends TypedEventEmitter {
             this._perfSignalData.signalTimestamp = 0;
             this._perfSignalData.trackingSignalSequenceNumber = undefined;
         }
+        else {
+            assert(this.attachState === AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
+        }
         // Fail while disconnected
         if (reconnection) {
             this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(
-                // pre-0.58 error message: MaxReconnectsWithNoProgress
-                DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops", "setConnectionState", undefined, {
+                this.closeFn(DataProcessingError.create(
+                // eslint-disable-next-line max-len
+                "Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
                     pendingMessages: this.pendingStateManager.pendingMessagesCount,
@@ -859,46 +871,48 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.replayPendingStates();
         }
         this.dataStores.setConnectionState(connected, clientId);
+        this.garbageCollector.setConnectionState(connected, clientId);
         raiseConnectedEvent(this.mc.logger, this, connected, clientId);
     }
     process(messageArg, local) {
         var _a;
         this.verifyNotClosed();
-        // If it's not message for runtime, bail out right away.
-        if (!isUnpackedRuntimeMessage(messageArg)) {
-            return;
-        }
-        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
-            this.savedOps.push(messageArg);
-        }
         // Do shallow copy of message, as methods below will modify it.
         // There might be multiple container instances receiving same message
         // We do not need to make deep copy, as each layer will just replace message.content itself,
         // but would not modify contents details
         let message = Object.assign({}, messageArg);
+        // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
+        // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
+        // Old ops may contain empty string (I assume noops).
+        if (typeof message.contents === "string" && message.contents !== "") {
+            message.contents = JSON.parse(message.contents);
+        }
+        // Caveat: This will return false for runtime message in very old format, that are used in snapshot tests
+        // This format was not shipped to production workflows.
+        const runtimeMessage = unpackRuntimeMessage(message);
+        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
+            this.savedOps.push(messageArg);
+        }
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
         this.scheduleManager.beforeOpProcessing(message);
         try {
-            message = unpackRuntimeMessage(message);
             // Chunk processing must come first given that we will transform the message to the unchunked version
             // once all pieces are available
             message = this.processRemoteChunkedMessage(message);
             let localOpMetadata;
-            if (local) {
-                // Call the PendingStateManager to process local messages.
-                // Do not process local chunked ops until all pieces are available.
-                if (message.type !== ContainerMessageType.ChunkedOp) {
-                    localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
-                }
+            if (local && runtimeMessage) {
+                localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
             // the document is no longer dirty.
-            if (!this.pendingStateManager.hasPendingMessages()) {
+            if (!this.hasPendingMessages()) {
                 this.updateDocumentDirtyState(false);
             }
-            switch (message.type) {
+            const type = message.type;
+            switch (type) {
                 case ContainerMessageType.Attach:
                     this.dataStores.processAttachMessage(message, local);
                     break;
@@ -911,9 +925,16 @@ export class ContainerRuntime extends TypedEventEmitter {
                 case ContainerMessageType.BlobAttach:
                     this.blobManager.processBlobAttachOp(message, local);
                     break;
+                case ContainerMessageType.ChunkedOp:
+                case ContainerMessageType.Rejoin:
+                    break;
                 default:
+                    assert(!runtimeMessage, 0x3ce /* Runtime message of unknown type */);
+            }
+            // For back-compat, notify only about runtime messages for now.
+            if (runtimeMessage) {
+                this.emit("op", message, runtimeMessage);
             }
-            this.emit("op", message);
             this.scheduleManager.afterOpProcessing(undefined, message);
             if (local) {
                 // If we have processed a local op, this means that the container is
@@ -1007,25 +1028,57 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     flush() {
         assert(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
-        if (!this.deltaSender) {
-            return;
+        this.flushBatch(this.pendingAttachBatch.popBatch());
+        this.flushBatch(this.pendingBatch.popBatch());
+        assert(this.emptyBatch, 0x3cf /* reentrancy */);
+    }
+    flushBatch(batch) {
+        const length = batch.length;
+        if (length > 1) {
+            batch[0].metadata = Object.assign(Object.assign({}, batch[0].metadata), { batch: true });
+            batch[length - 1].metadata = Object.assign(Object.assign({}, batch[length - 1].metadata), { batch: false });
+            // This assert fires for the following reason (there might be more cases like that):
+            // AgentScheduler will send ops in response to ConsensusRegisterCollection's "atomicChanged" event handler,
+            // i.e. in the middle of op processing!
+            // Sending ops while processing ops is not good idea - it's not defined when
+            // referenceSequenceNumber changes in op processing sequence (at the beginning or end of op processing),
+            // If we send ops in response to processing multiple ops, then we for sure hit this assert!
+            // Tracked via ADO #1834
+            // assert(batch[0].referenceSequenceNumber === batch[length - 1].referenceSequenceNumber,
+            //    "Batch should be generated synchronously, without processing ops in the middle!");
         }
-        // Let the PendingStateManager know that there was an attempt to flush messages.
-        // Note that this should happen before the `this.needsFlush` check below because in the scenario where we are
-        // not connected, `this.needsFlush` will be false but the PendingStateManager might have pending messages and
-        // hence needs to track this.
-        this.pendingStateManager.onFlush();
-        // If flush has already been called then exit early
-        if (!this.needsFlush) {
-            return;
-        }
-        this.needsFlush = false;
+        let clientSequenceNumber = -1;
         // Did we disconnect in the middle of turn-based batch?
         // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
-        if (!this.canSendOps()) {
-            return;
+        if (this.canSendOps()) {
+            if (this.context.submitBatchFn !== undefined) {
+                const batchToSend = [];
+                for (const message of batch) {
+                    batchToSend.push({ contents: message.contents, metadata: message.metadata });
+                }
+                // returns clientSequenceNumber of last message in a batch
+                clientSequenceNumber = this.context.submitBatchFn(batchToSend);
+            }
+            else {
+                // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
+                // version that has support for batches (submitBatchFn)
+                for (const message of batch) {
+                    clientSequenceNumber = this.context.submitFn(MessageType.Operation, message.deserializedContent, true, // batch
+                    message.metadata);
+                }
+                this.deltaSender.flush();
+            }
+            // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
+            clientSequenceNumber -= batch.length - 1;
+            assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
+        }
+        // Let the PendingStateManager know that a message was submitted.
+        // In future, need to shift toward keeping batch as a whole!
+        for (const message of batch) {
+            this.pendingStateManager.onSubmitMessage(message.deserializedContent.type, clientSequenceNumber, message.referenceSequenceNumber, message.deserializedContent.contents, message.localOpMetadata, message.metadata);
+            clientSequenceNumber++;
         }
-        return this.deltaSender.flush();
+        this.pendingStateManager.onFlush();
     }
     orderSequentially(callback) {
         // If flush mode is already TurnBased we are either
@@ -1050,7 +1103,10 @@ export class ContainerRuntime extends TypedEventEmitter {
     trackOrderSequentiallyCalls(callback) {
         let checkpoint;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
-            checkpoint = this.pendingStateManager.checkpoint();
+            // Note: we are not touching this.pendingAttachBatch here, for two reasons:
+            // 1. It would not help, as we flush attach ops as they become available.
+            // 2. There is no way to undo process of data store creation.
+            checkpoint = this.pendingBatch.checkpoint();
         }
         try {
             this._orderSequentiallyCalls++;
@@ -1059,7 +1115,16 @@ export class ContainerRuntime extends TypedEventEmitter {
         catch (error) {
             if (checkpoint) {
                 // This will throw and close the container if rollback fails
-                checkpoint.rollback();
+                try {
+                    checkpoint.rollback((message) => this.rollback(message.deserializedContent.type, message.deserializedContent.contents, message.localOpMetadata));
+                }
+                catch (err) {
+                    const error2 = wrapError(err, (message) => {
+                        return DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
+                    });
+                    this.closeFn(error2);
+                    throw error2;
+                }
             }
             else {
                 // pre-0.58 error message: orderSequentiallyCallbackException
@@ -1164,7 +1229,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             assert(this.attachState === AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
             this.emit("attached");
         }
-        if (attachState === AttachState.Attached && !this.pendingStateManager.hasPendingMessages()) {
+        if (attachState === AttachState.Attached && !this.hasPendingMessages()) {
             this.updateDocumentDirtyState(false);
         }
         this.dataStores.setAttachState(attachState);
@@ -1328,7 +1393,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     /**
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
-     * @returns the statistics of the garbage collection run.
+     * @returns the statistics of the garbage collection run; undefined if GC did not run.
      */
     async collectGarbage(options) {
         return this.garbageCollector.collectGarbage(options);
@@ -1359,6 +1424,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         const summaryNumberLogger = ChildLogger.create(summaryLogger, undefined, {
             all: { summaryNumber },
         });
+        assert(this.emptyBatch, 0x3d1 /* Can't trigger summary in the middle of a batch */);
         let latestSnapshotVersionId;
         if (refreshLatestAck) {
             const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(ChildLogger.create(summaryNumberLogger, undefined, { all: { safeSummary: true } }));
@@ -1433,7 +1499,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
             try {
                 summarizeResult = await this.summarize({
-                    fullTree: fullTree || forcedFullTree,
+                    fullTree: fullTree !== null && fullTree !== void 0 ? fullTree : forcedFullTree,
                     trackState: true,
                     summaryLogger: summaryNumberLogger,
                     runGC: this.garbageCollector.shouldRunGC,
@@ -1522,7 +1588,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             let clientSequenceNumber;
             try {
-                clientSequenceNumber = this.submitSystemMessage(MessageType.Summarize, summaryMessage);
+                clientSequenceNumber = this.submitSummaryMessage(summaryMessage);
             }
             catch (error) {
                 return Object.assign(Object.assign({ stage: "upload" }, uploadData), { error });
@@ -1570,7 +1636,17 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.chunkMap.delete(clientId);
         }
     }
+    hasPendingMessages() {
+        return this.pendingStateManager.hasPendingMessages() || !this.emptyBatch;
+    }
     updateDocumentDirtyState(dirty) {
+        if (this.attachState !== AttachState.Attached) {
+            assert(dirty, 0x3d2 /* Non-attached container is dirty */);
+        }
+        else {
+            // Other way is not true = see this.isContainerMessageDirtyable()
+            assert(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
+        }
         if (this.dirtyContainer === dirty) {
             return;
         }
@@ -1598,64 +1674,100 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.verifyNotClosed();
         return this.blobManager.createBlob(blob);
     }
-    submit(type, content, localOpMetadata = undefined, opMetadata = undefined) {
+    submit(type, contents, localOpMetadata = undefined, metadata = undefined) {
         this.verifyNotClosed();
         // There should be no ops in detached container state!
         assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
-        let clientSequenceNumber = -1;
-        let opMetadataInternal = opMetadata;
-        if (this.canSendOps()) {
-            const serializedContent = JSON.stringify(content);
-            // If in TurnBased flush mode we will trigger a flush at the next turn break
-            if (this.flushMode === FlushMode.TurnBased && !this.needsFlush) {
-                opMetadataInternal = Object.assign(Object.assign({}, opMetadata), { batch: true });
-                this.needsFlush = true;
-                // Use Promise.resolve().then() to queue a microtask to detect the end of the turn and force a flush.
-                if (!this.flushTrigger) {
-                    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-                    Promise.resolve().then(() => {
-                        this.flushTrigger = false;
-                        this.flush();
+        const deserializedContent = { type, contents };
+        const serializedContent = JSON.stringify(deserializedContent);
+        if (this.deltaManager.readOnlyInfo.readonly) {
+            this.logger.sendErrorEvent({ eventName: "SubmitOpInReadonly" });
+        }
+        const message = {
+            contents: serializedContent,
+            deserializedContent,
+            metadata,
+            localOpMetadata,
+            referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+        };
+        try {
+            // If this is attach message for new data store, and we are in a batch, send this op out of order
+            // Is it safe:
+            //    Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
+            //    They become visible only when aliased, or handle to some sub-element of newly created datastore
+            //    is stored in some DDS, i.e. only after some other op.
+            // Why:
+            //    Attach ops are large, and expensive to process. Plus there are scenarios where a lot of new data
+            //    stores are created, causing issues like relay service throttling (too many ops) and catastrophic
+            //    failure (batch is too large). Pushing them earlier and outside of main batch should alleviate
+            //    these issues.
+            // Cons:
+            //    1. With large batches, relay service may throttle clients. Clients may disconnect while throttled.
+            //    This change creates new possibility of a lot of newly created data stores never being referenced
+            //    because client died before it had a change to submit the rest of the ops. This will create more
+            //    garbage that needs to be collected leveraging GC (Garbage Collection) feature.
+            //    2. Sending ops out of order means they are excluded from rollback functionality. This is not an issue
+            //    today as rollback can't undo creation of data store. To some extent not sending them is a bigger
+            //    issue than sending.
+            // Please note that this does not change file format, so it can be disabled in the future if this
+            // optimization no longer makes sense (for example, batch compression may make it less appealing).
+            if (type === ContainerMessageType.Attach &&
+                this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
+                if (!this.pendingAttachBatch.push(message)) {
+                    // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
+                    // when queue is not empty.
+                    // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
+                    this.flushBatch(this.pendingAttachBatch.popBatch());
+                    if (!this.pendingAttachBatch.push(message)) {
+                        throw new GenericError("BatchTooLarge", 
+                        /* error */ undefined, {
+                            opSize: message.contents.length,
+                            count: this.pendingAttachBatch.length,
+                            limit: this.pendingAttachBatch.limit,
+                        });
+                    }
+                }
+            }
+            else {
+                if (!this.pendingBatch.push(message)) {
+                    throw new GenericError("BatchTooLarge", 
+                    /* error */ undefined, {
+                        opSize: message.contents.length,
+                        count: this.pendingBatch.length,
+                        limit: this.pendingBatch.limit,
                     });
                 }
             }
-            if (!serializedContent || serializedContent.length <= defaultMaxOpSizeInBytes) {
-                clientSequenceNumber = this.submitRuntimeMessage(type, content, this._flushMode === FlushMode.TurnBased /* batch */, opMetadataInternal);
+            if (this._flushMode !== FlushMode.TurnBased) {
+                this.flush();
             }
-            else {
-                // If the content length is larger than the client configured message size
-                // instead of splitting the content, we will fail by explicitly closing the container
-                this.closeFn(new GenericError("OpTooLarge", 
-                /* error */ undefined, {
-                    length: serializedContent.length,
-                    limit: defaultMaxOpSizeInBytes,
-                }));
-                clientSequenceNumber = -1;
+            else if (!this.flushTrigger) {
+                this.flushTrigger = true;
+                // Queue a microtask to detect the end of the turn and force a flush.
+                // eslint-disable-next-line @typescript-eslint/no-floating-promises
+                Promise.resolve().then(() => {
+                    this.flushTrigger = false;
+                    this.flush();
+                });
             }
         }
-        // Let the PendingStateManager know that a message was submitted.
-        this.pendingStateManager.onSubmitMessage(type, clientSequenceNumber, this.deltaManager.lastSequenceNumber, content, localOpMetadata, opMetadataInternal);
-        if (this.isContainerMessageDirtyable(type, content)) {
+        catch (error) {
+            this.closeFn(error);
+            throw error;
+        }
+        if (this.isContainerMessageDirtyable(type, contents)) {
             this.updateDocumentDirtyState(true);
         }
     }
-    submitSystemMessage(type, contents) {
+    submitSummaryMessage(contents) {
         this.verifyNotClosed();
         assert(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
-        // That said, we can preserve existing behavior by not flushing existing buffer.
-        // That might be not what caller hopes to get, but we can look deeper if telemetry tells us it's a problem.
-        const middleOfBatch = this.flushMode === FlushMode.TurnBased && this.needsFlush;
-        if (middleOfBatch) {
-            this.mc.logger.sendErrorEvent({ eventName: "submitSystemMessageError", type });
-        }
-        return this.context.submitFn(type, contents, middleOfBatch);
-    }
-    submitRuntimeMessage(type, contents, batch, appData) {
-        this.verifyNotClosed();
-        assert(this.connected, 0x259 /* "Container disconnected when trying to submit system message" */);
-        const payload = { type, contents };
-        return this.context.submitFn(MessageType.Operation, payload, batch, appData);
+        assert(this.emptyBatch, 0x3d4 /* System op in the middle of a batch */);
+        // back-compat: ADO #1385: Make this call unconditional in the future
+        return this.context.submitSummaryFn !== undefined
+            ? this.context.submitSummaryFn(contents)
+            : this.context.submitFn(MessageType.Summarize, contents, false);
     }
     /**
      * Throw an error if the runtime is closed.  Methods that are expected to potentially
@@ -1709,13 +1821,18 @@ export class ContainerRuntime extends TypedEventEmitter {
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(proposalHandle, ackHandle, summaryRefSeq, summaryLogger) {
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        const { snapshotTree } = await this.fetchSnapshotFromStorage(ackHandle, summaryLogger, {
-            eventName: "RefreshLatestSummaryGetSnapshot",
-            ackHandle,
-            summaryRefSeq,
-            fetchLatest: false,
-        });
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
+        // The call to fetch the snapshot is very expensive and not always needed.
+        // It should only be done by the summarizerNode, if required.
+        const snapshotTreeFetcher = async () => {
+            const fetchResult = await this.fetchSnapshotFromStorage(ackHandle, summaryLogger, {
+                eventName: "RefreshLatestSummaryGetSnapshot",
+                ackHandle,
+                summaryRefSeq,
+                fetchLatest: false,
+            });
+            return fetchResult.snapshotTree;
+        };
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, snapshotTreeFetcher, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
         await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
     }
@@ -1771,6 +1888,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad)) {
             throw new UsageError("can't get state when offline load disabled");
         }
+        // Flush pending batch.
+        // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
+        // to close current batch.
+        this.flush();
         const previousPendingState = this.context.pendingLocalState;
         if (previousPendingState) {
             return {
@@ -1825,6 +1946,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         // we may not have seen every sequence number (because of system ops) so apply everything once we
         // don't have any more saved ops
         await this.pendingStateManager.applyStashedOpsAt();
+        // If it's not the case, we should take it into account when calculating dirty state.
+        assert(this.context.attachState === AttachState.Attached, 0x3d5 /* this function is called for attached containers only */);
+        if (!this.hasPendingMessages()) {
+            this.updateDocumentDirtyState(false);
+        }
     }
     validateSummaryHeuristicConfiguration(configuration) {
         // eslint-disable-next-line no-restricted-syntax