@fluidframework/container-runtime 2.5.0-302463 → 2.10.0-304831

package/lib/containerRuntime.js

@@ -89,6 +89,12 @@ export const TombstoneResponseHeaderKey = "isTombstoned";
  * to this was experimental and is no longer supported.
  */
 export const InactiveResponseHeaderKey = "isInactive";
+/** Default values for Runtime Headers */
+export const defaultRuntimeHeaderData = {
+    wait: true,
+    viaHandle: false,
+    allowTombstone: false,
+};
 /**
  * Available compression algorithms for op compression.
  * @legacy
@@ -1581,20 +1587,13 @@ export class ContainerRuntime extends TypedEventEmitter {
                 : inboundResult.type === "batchStartingMessage"
                     ? { batchStart: true, batchEnd: false }
                     : { batchStart: false, batchEnd: inboundResult.batchEnd === true };
-            this.processInboundMessages(messagesWithPendingState, locationInBatch, local, savedOp, runtimeBatch);
+            this.processInboundMessages(messagesWithPendingState, locationInBatch, local, savedOp, runtimeBatch, inboundResult.type === "fullBatch"
+                ? inboundResult.groupedBatch
+                : false /* groupedBatch */);
         }
         else {
-            if (!runtimeBatch) {
-                // The DeltaManager used to do this, but doesn't anymore as of Loader v2.4
-                // Anyone listening to our "op" event would expect the contents to be parsed per this same logic
-                if (typeof messageCopy.contents === "string" &&
-                    messageCopy.contents !== "" &&
-                    messageCopy.type !== MessageType.ClientLeave) {
-                    messageCopy.contents = JSON.parse(messageCopy.contents);
-                }
-            }
             this.processInboundMessages([{ message: messageCopy, localOpMetadata: undefined }], { batchStart: true, batchEnd: true }, // Single message
-            local, savedOp, runtimeBatch);
+            local, savedOp, runtimeBatch, false /* groupedBatch */);
         }
         if (local) {
             // If we have processed a local op, this means that the container is
@@ -1605,35 +1604,93 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     /**
      * Processes inbound message(s). It calls schedule manager according to the messages' location in the batch.
-     * @param messages - messages to process.
+     * @param messagesWithMetadata - messages to process along with their metadata.
      * @param locationInBatch - Are we processing the start and/or end of a batch?
      * @param local - true if the messages were originally generated by the client receiving it.
      * @param savedOp - true if the message is a replayed saved op.
      * @param runtimeBatch - true if these are runtime messages.
+     * @param groupedBatch - true if these messages are part of a grouped op batch.
      */
-    processInboundMessages(messages, locationInBatch, local, savedOp, runtimeBatch) {
+    processInboundMessages(messagesWithMetadata, locationInBatch, local, savedOp, runtimeBatch, groupedBatch) {
         if (locationInBatch.batchStart) {
-            const firstMessage = messages[0]?.message;
+            const firstMessage = messagesWithMetadata[0]?.message;
             assert(firstMessage !== undefined, 0xa31 /* Batch must have at least one message */);
             this.scheduleManager.batchBegin(firstMessage);
         }
         let error;
         try {
-            messages.forEach(({ message, localOpMetadata }) => {
-                this.ensureNoDataModelChanges(() => {
-                    if (runtimeBatch) {
-                        this.validateAndProcessRuntimeMessage({
-                            message: message,
-                            local,
-                            savedOp,
-                            localOpMetadata,
-                        });
-                    }
-                    else {
+            if (!runtimeBatch) {
+                messagesWithMetadata.forEach(({ message }) => {
+                    this.ensureNoDataModelChanges(() => {
                         this.observeNonRuntimeMessage(message);
-                    }
+                    });
                 });
-            });
+                return;
+            }
+            // Helper that updates a message's minimum sequence number to the minimum sequence number that container
+            // runtime is tracking and sets _processedClientSequenceNumber. It returns the updated message.
+            const updateSequenceNumbers = (message) => {
+                // Set the minimum sequence number to the containerRuntime's understanding of minimum sequence number.
+                message.minimumSequenceNumber =
+                    this.useDeltaManagerOpsProxy &&
+                        this.deltaManager.minimumSequenceNumber < message.minimumSequenceNumber
+                        ? this.deltaManager.minimumSequenceNumber
+                        : message.minimumSequenceNumber;
+                this._processedClientSequenceNumber = message.clientSequenceNumber;
+                return message;
+            };
+            // Non-grouped batch messages are processed one at a time.
+            if (!groupedBatch) {
+                for (const { message, localOpMetadata } of messagesWithMetadata) {
+                    updateSequenceNumbers(message);
+                    this.ensureNoDataModelChanges(() => {
+                        this.validateAndProcessRuntimeMessages(message, [
+                            {
+                                contents: message.contents,
+                                localOpMetadata,
+                                clientSequenceNumber: message.clientSequenceNumber,
+                            },
+                        ], local, savedOp);
+                        this.emit("op", message, true /* runtimeMessage */);
+                    });
+                }
+                return;
+            }
+            let bunchedMessagesContent = [];
+            let previousMessage;
+            // Helper that processes the previous bunch of messages.
+            const sendBunchedMessages = () => {
+                assert(previousMessage !== undefined, 0xa67 /* previous message must exist */);
+                this.ensureNoDataModelChanges(() => {
+                    this.validateAndProcessRuntimeMessages(
+                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                    previousMessage, bunchedMessagesContent, local, savedOp);
+                });
+                bunchedMessagesContent = [];
+            };
+            /**
+             * For grouped batch messages, bunch contiguous messages of the same type and process them together.
+             * This is an optimization mainly for DDSes, where it can process a bunch of ops together. DDSes
+             * like merge tree or shared tree can process ops more efficiently when they are bunched together.
+             */
+            for (const { message, localOpMetadata } of messagesWithMetadata) {
+                const currentMessage = updateSequenceNumbers(message);
+                if (previousMessage && previousMessage.type !== currentMessage.type) {
+                    sendBunchedMessages();
+                }
+                previousMessage = currentMessage;
+                bunchedMessagesContent.push({
+                    contents: message.contents,
+                    localOpMetadata,
+                    clientSequenceNumber: message.clientSequenceNumber,
+                });
+            }
+            // Process the last bunch of messages.
+            sendBunchedMessages();
+            // Send the "op" events for the messages now that the ops have been processed.
+            for (const { message } of messagesWithMetadata) {
+                this.emit("op", message, true /* runtimeMessage */);
+            }
         }
         catch (e) {
             error = e;
@@ -1641,7 +1698,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         finally {
             if (locationInBatch.batchEnd) {
-                const lastMessage = messages[messages.length - 1]?.message;
+                const lastMessage = messagesWithMetadata[messagesWithMetadata.length - 1]?.message;
                 assert(lastMessage !== undefined, 0xa32 /* Batch must have at least one message */);
                 this.scheduleManager.batchEnd(error, lastMessage);
             }
@@ -1662,65 +1719,60 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (!this.hasPendingMessages()) {
             this.updateDocumentDirtyState(false);
         }
+        // The DeltaManager used to do this, but doesn't anymore as of Loader v2.4
+        // Anyone listening to our "op" event would expect the contents to be parsed per this same logic
+        if (typeof message.contents === "string" &&
+            message.contents !== "" &&
+            message.type !== MessageType.ClientLeave) {
+            message.contents = JSON.parse(message.contents);
+        }
         this.emit("op", message, false /* runtimeMessage */);
     }
     /**
-     * Assuming the given message is also a TypedContainerRuntimeMessage,
-     * checks its type and dispatches the message to the appropriate handler in the runtime.
+     * Process runtime messages. The messages here are contiguous messages in a batch.
+     * Assuming the messages in the given bunch are also a TypedContainerRuntimeMessage, checks its type and dispatch
+     * the messages to the appropriate handler in the runtime.
      * Throws a DataProcessingError if the message looks like but doesn't conform to a known TypedContainerRuntimeMessage type.
+     * @param message - The core message with common properties for all the messages.
+     * @param messageContents - The contents, local metadata and clientSequenceNumbers of the messages.
+     * @param local - true if the messages were originally generated by the client receiving it.
+     * @param savedOp - true if the message is a replayed saved op.
+     *
      */
-    validateAndProcessRuntimeMessage(messageWithContext) {
-        const { local, message, savedOp, localOpMetadata } = messageWithContext;
-        // Set the minimum sequence number to the containerRuntime's understanding of minimum sequence number.
-        if (this.useDeltaManagerOpsProxy &&
-            this.deltaManager.minimumSequenceNumber < message.minimumSequenceNumber) {
-            message.minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
-        }
-        this._processedClientSequenceNumber = message.clientSequenceNumber;
+    validateAndProcessRuntimeMessages(message, messagesContent, local, savedOp) {
         // If there are no more pending messages after processing a local message,
         // the document is no longer dirty.
         if (!this.hasPendingMessages()) {
             this.updateDocumentDirtyState(false);
         }
+        // Get the contents without the localOpMetadata because not all message types know about localOpMetadata.
+        const contents = messagesContent.map((c) => c.contents);
         switch (message.type) {
+            case ContainerMessageType.FluidDataStoreOp:
             case ContainerMessageType.Attach:
             case ContainerMessageType.Alias:
-            case ContainerMessageType.FluidDataStoreOp:
-                this.channelCollection.process(message, local, localOpMetadata);
+                // Remove the metadata from the message before sending it to the channel collection. The metadata
+                // is added by the container runtime and is not part of the message that the channel collection and
+                // layers below it expect.
+                this.channelCollection.processMessages({ envelope: message, messagesContent, local });
                 break;
             case ContainerMessageType.BlobAttach:
-                this.blobManager.processBlobAttachOp(message, local);
+                this.blobManager.processBlobAttachMessage(message, local);
                 break;
             case ContainerMessageType.IdAllocation:
-                // Don't re-finalize the range if we're processing a "savedOp" in
-                // stashed ops flow. The compressor is stashed with these ops already processed.
-                // That said, in idCompressorMode === "delayed", we might not serialize ID compressor, and
-                // thus we need to process all the ops.
-                if (!(this.skipSavedCompressorOps && savedOp === true)) {
-                    const range = message.contents;
-                    // Some other client turned on the id compressor. If we have not turned it on,
-                    // put it in a pending queue and delay finalization.
-                    if (this._idCompressor === undefined) {
-                        assert(this.idCompressorMode !== undefined, 0x93c /* id compressor should be enabled */);
-                        this.pendingIdCompressorOps.push(range);
-                    }
-                    else {
-                        assert(this.pendingIdCompressorOps.length === 0, 0x979 /* there should be no pending ops! */);
-                        this._idCompressor.finalizeCreationRange(range);
-                    }
-                }
+                this.processIdCompressorMessages(contents, savedOp);
                 break;
             case ContainerMessageType.GC:
-                this.garbageCollector.processMessage(message, message.timestamp, local);
+                this.garbageCollector.processMessages(contents, message.timestamp, local);
                 break;
             case ContainerMessageType.ChunkedOp:
-                // From observability POV, we should not exppse the rest of the system (including "op" events on object) to these messages.
+                // From observability POV, we should not expose the rest of the system (including "op" events on object) to these messages.
                 // Also resetReconnectCount() would be wrong - see comment that was there before this change was made.
                 assert(false, 0x93d /* should not even get here */);
             case ContainerMessageType.Rejoin:
                 break;
             case ContainerMessageType.DocumentSchemaChange:
-                this.documentsSchemaController.processDocumentSchemaOp(message.contents, local, message.sequenceNumber);
+                this.documentsSchemaController.processDocumentSchemaMessages(contents, local, message.sequenceNumber);
                 break;
             default: {
                 const error = getUnknownMessageTypeError(message.type, "validateAndProcessRuntimeMessage" /* codePath */, message);
@@ -1728,7 +1780,26 @@ export class ContainerRuntime extends TypedEventEmitter {
                 throw error;
             }
         }
-        this.emit("op", message, true /* runtimeMessage */);
+    }
+    processIdCompressorMessages(messageContents, savedOp) {
+        for (const range of messageContents) {
+            // Don't re-finalize the range if we're processing a "savedOp" in
+            // stashed ops flow. The compressor is stashed with these ops already processed.
+            // That said, in idCompressorMode === "delayed", we might not serialize ID compressor, and
+            // thus we need to process all the ops.
+            if (!(this.skipSavedCompressorOps && savedOp === true)) {
+                // Some other client turned on the id compressor. If we have not turned it on,
+                // put it in a pending queue and delay finalization.
+                if (this._idCompressor === undefined) {
+                    assert(this.idCompressorMode !== undefined, 0x93c /* id compressor should be enabled */);
+                    this.pendingIdCompressorOps.push(range);
+                }
+                else {
+                    assert(this.pendingIdCompressorOps.length === 0, 0x979 /* there should be no pending ops! */);
+                    this._idCompressor.finalizeCreationRange(range);
+                }
+            }
+        }
     }
     /**
      * Emits the Signal event and update the perf signal data.
@@ -2304,7 +2375,17 @@ export class ContainerRuntime extends TypedEventEmitter {
                 },
             },
         });
-        assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
+        // legacy: assert 0x3d1
+        if (!this.outbox.isEmpty) {
+            throw DataProcessingError.create("Can't trigger summary in the middle of a batch", "submitSummary", undefined, {
+                summaryNumber,
+                pendingMessages: this.pendingMessagesCount,
+                outboxLength: this.outbox.messageCount,
+                mainBatchLength: this.outbox.mainBatchMessageCount,
+                blobAttachBatchLength: this.outbox.blobAttachBatchMessageCount,
+                idAllocationBatchLength: this.outbox.idAllocationBatchMessageCount,
+            });
+        }
         // If the container is dirty, i.e., there are pending unacked ops, the summary will not be eventual consistent
         // and it may even be incorrect. So, wait for the container to be saved with a timeout. If the container is not
         // saved within the timeout, check if it should be failed or can continue.