@fluidframework/container-runtime 2.0.0-dev-rc.2.0.0.246488 → 2.0.0-dev-rc.3.0.0.253463

package/src/opLifecycle/outbox.ts

@@ -3,18 +3,20 @@
  * Licensed under the MIT License.
  */
 
+import { ICriticalContainerError } from "@fluidframework/container-definitions";
+import { IBatchMessage } from "@fluidframework/container-definitions/internal";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { assert } from "@fluidframework/core-utils/internal";
 import {
-	createChildMonitoringContext,
 	GenericError,
 	MonitoringContext,
 	UsageError,
-} from "@fluidframework/telemetry-utils";
-import { assert } from "@fluidframework/core-utils";
-import { IBatchMessage, ICriticalContainerError } from "@fluidframework/container-definitions";
-import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+	createChildMonitoringContext,
+} from "@fluidframework/telemetry-utils/internal";
+
 import { ICompressionRuntimeOptions } from "../containerRuntime.js";
 import { IPendingBatchMessage, PendingStateManager } from "../pendingStateManager.js";
-import { ContainerMessageType } from "../messageTypes.js";
+
 import {
 	BatchManager,
 	BatchSequenceNumbers,
@@ -187,20 +189,12 @@ export class Outbox {
 	}
 
 	public submit(message: BatchMessage) {
-		assert(
-			message.type !== ContainerMessageType.IdAllocation,
-			0x8f8 /* Allocation message submitted to mainBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		this.addMessageToBatchManager(this.mainBatch, message);
 	}
 
 	public submitAttach(message: BatchMessage) {
-		assert(
-			message.type === ContainerMessageType.Attach,
-			0x8f9 /* Non attach message submitted to attachFlowBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		if (
@@ -232,10 +226,6 @@ export class Outbox {
 	}
 
 	public submitBlobAttach(message: BatchMessage) {
-		assert(
-			message.type === ContainerMessageType.BlobAttach,
-			0x8fa /* Non blobAttach message submitted to blobAttachBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		this.addMessageToBatchManager(this.blobAttachBatch, message);
@@ -254,10 +244,6 @@ export class Outbox {
 	}
 
 	public submitIdAllocation(message: BatchMessage) {
-		assert(
-			message.type === ContainerMessageType.IdAllocation,
-			0x8fb /* Non allocation message submitted to idAllocationBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		if (
@@ -328,10 +314,9 @@ export class Outbox {
 		}
 
 		const rawBatch = batchManager.popBatch();
-		if (
-			rawBatch.hasReentrantOps === true &&
-			this.params.groupingManager.shouldGroup(rawBatch)
-		) {
+		const shouldGroup =
+			!disableGroupedBatching && this.params.groupingManager.shouldGroup(rawBatch);
+		if (rawBatch.hasReentrantOps === true && shouldGroup) {
 			assert(!this.rebasing, 0x6fa /* A rebased batch should never have reentrant ops */);
 			// If a batch contains reentrant ops (ops created as a result from processing another op)
 			// it needs to be rebased so that we can ensure consistent reference sequence numbers
@@ -344,7 +329,9 @@ export class Outbox {
 		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
 		// Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
 		if (this.params.shouldSend()) {
-			const processedBatch = this.compressBatch(rawBatch, disableGroupedBatching);
+			const processedBatch = this.compressBatch(
+				shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch,
+			);
 			this.sendBatch(processedBatch);
 		}
 
@@ -390,7 +377,7 @@ export class Outbox {
 		return this.params.opReentrancy() && !this.rebasing;
 	}
 
-	private compressBatch(batch: IBatch, disableGroupedBatching: boolean): IBatch {
+	private compressBatch(batch: IBatch): IBatch {
 		if (
 			batch.content.length === 0 ||
 			this.params.config.compressionOptions === undefined ||
@@ -399,12 +386,10 @@ export class Outbox {
 			this.params.submitBatchFn === undefined
 		) {
 			// Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
-			return disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch);
+			return batch;
 		}
 
-		const compressedBatch = this.params.compressor.compressBatch(
-			disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch),
-		);
+		const compressedBatch = this.params.compressor.compressBatch(batch);
 
 		if (this.params.splitter.isBatchChunkingEnabled) {
 			return compressedBatch.contentSizeInBytes <= this.params.splitter.chunkSizeInBytes

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -4,6 +4,7 @@
  */
 
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
+
 import {
 	ContainerMessageType,
 	type InboundContainerRuntimeMessage,
@@ -11,9 +12,10 @@ import {
 	type InboundSequencedContainerRuntimeMessageOrSystemMessage,
 	type InboundSequencedRecentlyAddedContainerRuntimeMessage,
 } from "../messageTypes.js";
+
 import { OpDecompressor } from "./opDecompressor.js";
-import { OpGroupingManager } from "./opGroupingManager.js";
-import { OpSplitter } from "./opSplitter.js";
+import { OpGroupingManager, isGroupedBatch } from "./opGroupingManager.js";
+import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
 
 /**
  * Stateful class for processing incoming remote messages as the virtualization measures are unwrapped,
@@ -44,6 +46,14 @@ export class RemoteMessageProcessor {
 	 * depends on this object instance.
 	 * Note remoteMessageCopy.contents (and other object props) MUST not be modified,
 	 * but may be overwritten (as is the case with contents).
+	 *
+	 * Incoming messages will always have compression, chunking, and grouped batching happen in a defined order and that order cannot be changed.
+	 * When processing these messages, the order is:
+	 * 1. If chunked, process the chunk and only continue if this is a final chunk
+	 * 2. If compressed, decompress the message and store for further unrolling of the decompressed content
+	 * 3. If grouped, ungroup the message
+	 * For more details, see https://github.com/microsoft/FluidFramework/blob/main/packages/runtime/container-runtime/src/opLifecycle/README.md#inbound
+	 *
 	 * @returns the unchunked, decompressed, ungrouped, unpacked SequencedContainerRuntimeMessages encapsulated in the remote message.
 	 * For ops that weren't virtualized (e.g. System ops that the ContainerRuntime will ultimately ignore),
 	 * a singleton array [remoteMessageCopy] is returned
@@ -51,63 +61,38 @@ export class RemoteMessageProcessor {
 	public process(
 		remoteMessageCopy: ISequencedDocumentMessage,
 	): InboundSequencedContainerRuntimeMessageOrSystemMessage[] {
-		const result: InboundSequencedContainerRuntimeMessageOrSystemMessage[] = [];
-
-		ensureContentsDeserialized(remoteMessageCopy);
-
-		// Ungroup before and after decompression for back-compat (cleanup tracked by AB#4371)
-		for (const ungroupedMessage of this.opGroupingManager.ungroupOp(remoteMessageCopy)) {
-			const message = this.opDecompressor.processMessage(ungroupedMessage).message;
-
-			for (let ungroupedMessage2 of this.opGroupingManager.ungroupOp(message)) {
-				// unpack and unchunk the ungrouped message in place
-				unpackRuntimeMessage(ungroupedMessage2);
-				const chunkProcessingResult =
-					this.opSplitter.processRemoteMessage(ungroupedMessage2);
-				ungroupedMessage2 = chunkProcessingResult.message;
-
-				// if the splitter is still rebuilding the original message, there is no need to continue processing
-				if (chunkProcessingResult.state === "Accepted") {
-					continue;
-				}
-
-				// If the message is not chunked there is no need to continue processing
-				if (chunkProcessingResult.state !== "Processed") {
-					result.push(
-						ungroupedMessage2 as InboundSequencedContainerRuntimeMessageOrSystemMessage,
-					);
-					continue;
-				}
-
-				// Ungroup before and after decompression for back-compat (cleanup tracked by AB#4371)
-				for (const ungroupedMessageAfterChunking of this.opGroupingManager.ungroupOp(
-					ungroupedMessage2,
-				)) {
-					const decompressionAfterChunking = this.opDecompressor.processMessage(
-						ungroupedMessageAfterChunking,
-					);
-
-					for (const ungroupedMessageAfterChunking2 of this.opGroupingManager.ungroupOp(
-						decompressionAfterChunking.message,
-					)) {
-						if (decompressionAfterChunking.state === "Skipped") {
-							// After chunking, if the original message was not compressed,
-							// there is no need to continue processing
-							result.push(
-								ungroupedMessageAfterChunking2 as InboundSequencedContainerRuntimeMessageOrSystemMessage,
-							);
-							continue;
-						}
-
-						// The message needs to be unpacked after chunking + decompression
-						unpack(ungroupedMessageAfterChunking2);
-						result.push(ungroupedMessageAfterChunking2);
-					}
-				}
+		let message = remoteMessageCopy;
+		ensureContentsDeserialized(message);
+
+		if (isChunkedMessage(message)) {
+			const chunkProcessingResult = this.opSplitter.processChunk(message);
+			// Only continue further if current chunk is the final chunk
+			if (!chunkProcessingResult.isFinalChunk) {
+				return [];
 			}
+			// This message will always be compressed
+			message = chunkProcessingResult.message;
+		}
+
+		if (this.opDecompressor.isCompressedMessage(message)) {
+			this.opDecompressor.decompressAndStore(message);
+		}
+
+		if (this.opDecompressor.currentlyUnrolling) {
+			message = this.opDecompressor.unroll(message);
+			// Need to unpack after unrolling if not a groupedBatch
+			if (!isGroupedBatch(message)) {
+				unpack(message);
+			}
+		}
+
+		if (isGroupedBatch(message)) {
+			return this.opGroupingManager.ungroupOp(message).map(unpack);
 		}
 
-		return result;
+		// Do a final unpack of runtime messages in case the message was not grouped, compressed, or chunked
+		unpackRuntimeMessage(message);
+		return [message as InboundSequencedContainerRuntimeMessageOrSystemMessage];
 	}
 }
 
@@ -128,9 +113,7 @@ function ensureContentsDeserialized(mutableMessage: ISequencedDocumentMessage):
  * becomes a InboundSequencedContainerRuntimeMessage by the time the function returns
  * (but there is no runtime validation of the 'type' or 'compatDetails' values).
  */
-function unpack(
-	message: ISequencedDocumentMessage,
-): asserts message is InboundSequencedContainerRuntimeMessage {
+function unpack(message: ISequencedDocumentMessage): InboundSequencedContainerRuntimeMessage {
 	// We assume the contents is an InboundContainerRuntimeMessage (the message is "packed")
 	const contents = message.contents as InboundContainerRuntimeMessage;
 
@@ -143,6 +126,7 @@ function unpack(
 		(messageUnpacked as InboundSequencedRecentlyAddedContainerRuntimeMessage).compatDetails =
 			contents.compatDetails;
 	}
+	return messageUnpacked;
 }
 
 /**

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev-rc.2.0.0.246488";
+export const pkgVersion = "2.0.0-dev-rc.3.0.0.253463";

package/src/pendingStateManager.ts

@@ -3,17 +3,17 @@
  * Licensed under the MIT License.
  */
 
-import Deque from "double-ended-queue";
-
-import { IDisposable } from "@fluidframework/core-interfaces";
-import { assert, Lazy } from "@fluidframework/core-utils";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
+import { IDisposable } from "@fluidframework/core-interfaces";
+import { assert, Lazy } from "@fluidframework/core-utils/internal";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { DataProcessingError, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+import { DataProcessingError } from "@fluidframework/telemetry-utils/internal";
+import Deque from "double-ended-queue";
 
 import { InboundSequencedContainerRuntimeMessage } from "./messageTypes.js";
-import { pkgVersion } from "./packageVersion.js";
 import { IBatchMetadata } from "./metadata.js";
+import { pkgVersion } from "./packageVersion.js";
 
 /**
  * This represents a message that has been submitted and is added to the pending queue when `submit` is called on the

package/src/scheduleManager.ts

@@ -3,22 +3,23 @@
  * Licensed under the MIT License.
  */
 
+import type { EventEmitter } from "@fluid-internal/client-utils";
+import { performance } from "@fluid-internal/client-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";
+import { assert } from "@fluidframework/core-utils/internal";
+import { isRuntimeMessage } from "@fluidframework/driver-utils/internal";
 import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import {
-	createChildLogger,
 	DataCorruptionError,
 	DataProcessingError,
+	createChildLogger,
 	extractSafePropertiesFromMessage,
-	ITelemetryLoggerExt,
-} from "@fluidframework/telemetry-utils";
-import { assert } from "@fluidframework/core-utils";
-import type { EventEmitter } from "@fluid-internal/client-utils";
-import { performance } from "@fluid-internal/client-utils";
-import { isRuntimeMessage } from "@fluidframework/driver-utils";
+} from "@fluidframework/telemetry-utils/internal";
+
 import { DeltaScheduler } from "./deltaScheduler.js";
-import { pkgVersion } from "./packageVersion.js";
 import { IBatchMetadata } from "./metadata.js";
+import { pkgVersion } from "./packageVersion.js";
 
 type IRuntimeMessageMetadata =
 	| undefined

package/src/storageServiceWithAttachBlobs.ts

@@ -6,8 +6,8 @@
 import {
 	IDocumentStorageService,
 	IDocumentStorageServicePolicies,
-} from "@fluidframework/driver-definitions";
-import { DocumentStorageServiceProxy } from "@fluidframework/driver-utils";
+} from "@fluidframework/driver-definitions/internal";
+import { DocumentStorageServiceProxy } from "@fluidframework/driver-utils/internal";
 
 /**
  * IDocumentStorageService proxy which intercepts requests if they can be satisfied by the blobs received in the