@fluidframework/container-runtime 2.20.0 → 2.21.0

package/src/gc/gcTelemetry.ts

@@ -14,10 +14,13 @@ import {
 } from "@fluidframework/telemetry-utils/internal";
 
 import { RuntimeHeaderData } from "../containerRuntime.js";
+// eslint-disable-next-line import/no-deprecated
 import { ICreateContainerMetadata } from "../summary/index.js";
 
 import {
+	// eslint-disable-next-line import/no-deprecated
 	GCFeatureMatrix,
+	// eslint-disable-next-line import/no-deprecated
 	GCNodeType,
 	IGarbageCollectorConfigs,
 	UnreferencedState,
@@ -41,6 +44,7 @@ interface ICommonProps {
 /**
  * The event that is logged when unreferenced node is used after a certain time.
  */
+// eslint-disable-next-line import/no-deprecated
 interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps {
 	/**
 	 * The id that GC uses to track the node. May or may not match id
@@ -52,11 +56,13 @@ interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps
 	 */
 	id: Tagged<string>;
 	fromId?: Tagged<string>;
+	// eslint-disable-next-line import/no-deprecated
 	type: GCNodeType;
 	unrefTime: number;
 	age: number;
 	// Expanding GC feature matrix. Without doing this, the configs cannot be logged in telemetry directly.
 	gcConfigs: Omit<IGarbageCollectorConfigs, "persistedGcFeatureMatrix"> & {
+		// eslint-disable-next-line import/no-deprecated
 		[K in keyof GCFeatureMatrix]: GCFeatureMatrix[K];
 	};
 	timeout?: number;
@@ -123,7 +129,9 @@ export class GCTelemetryTracker {
 		private readonly mc: MonitoringContext,
 		private readonly configs: IGarbageCollectorConfigs,
 		private readonly isSummarizerClient: boolean,
+		// eslint-disable-next-line import/no-deprecated
 		private readonly createContainerMetadata: ICreateContainerMetadata,
+		// eslint-disable-next-line import/no-deprecated
 		private readonly getNodeType: (nodeId: string) => GCNodeType,
 		private readonly getNodeStateTracker: (
 			nodeId: string,
@@ -144,6 +152,7 @@ export class GCTelemetryTracker {
 	 * 2. An event is logged only once per container instance per event per node.
 	 */
 	private shouldLogNonActiveEvent(
+		// eslint-disable-next-line import/no-deprecated
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
 		nodeStateTracker: UnreferencedStateTracker,
@@ -153,12 +162,14 @@ export class GCTelemetryTracker {
 			return false;
 		}
 
+		// eslint-disable-next-line import/no-deprecated
 		if (nodeType === GCNodeType.Other) {
 			return false;
 		}
 
 		// For sub data store (DDS) nodes, if they are changed, its data store will also be changed,
 		// so skip logging to make the telemetry less noisy.
+		// eslint-disable-next-line import/no-deprecated
 		if (nodeType === GCNodeType.SubDataStore && usageType === "Changed") {
 			return false;
 		}
@@ -196,17 +207,21 @@ export class GCTelemetryTracker {
 
 		const timeout = (() => {
 			switch (nodeStateTracker?.state) {
-				case UnreferencedState.Inactive:
+				case UnreferencedState.Inactive: {
 					return this.configs.inactiveTimeoutMs;
-				case UnreferencedState.TombstoneReady:
+				}
+				case UnreferencedState.TombstoneReady: {
 					return this.configs.tombstoneTimeoutMs;
-				case UnreferencedState.SweepReady:
+				}
+				case UnreferencedState.SweepReady: {
 					return (
 						this.configs.tombstoneTimeoutMs &&
 						this.configs.tombstoneTimeoutMs + this.configs.sweepGracePeriodMs
 					);
-				default:
+				}
+				default: {
 					return undefined;
+				}
 			}
 		})();
 		const { persistedGcFeatureMatrix, ...configs } = this.configs;
@@ -215,9 +230,9 @@ export class GCTelemetryTracker {
 			type: nodeType,
 			unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
 			age:
-				nodeStateTracker !== undefined
-					? currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs
-					: -1,
+				nodeStateTracker === undefined
+					? -1
+					: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
 			timeout,
 			isTombstoned,
 			...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
@@ -290,6 +305,7 @@ export class GCTelemetryTracker {
 	 */
 	private logTombstoneUsageTelemetry(
 		unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
+		// eslint-disable-next-line import/no-deprecated
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
 		packagePath?: readonly string[],
@@ -357,6 +373,7 @@ export class GCTelemetryTracker {
 			for (const route of currentOutboundRoutes) {
 				const nodeType = this.getNodeType(route);
 				if (
+					// eslint-disable-next-line import/no-deprecated
 					(nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) &&
 					!nodeId.startsWith(route) &&
 					!previousRoutes.includes(route) &&
@@ -411,10 +428,8 @@ export class GCTelemetryTracker {
 			const active =
 				nodeStateTracker === undefined || nodeStateTracker.state === UnreferencedState.Active;
 			if ((usageType === "Revived") === active) {
-				const pkg = await this.getNodePackagePath(eventProps.id.value);
-				const fromPkg = eventProps.fromId
-					? await this.getNodePackagePath(eventProps.fromId.value)
-					: undefined;
+				const pkg = await this.getNodePackagePath(id.value);
+				const fromPkg = fromId ? await this.getNodePackagePath(fromId.value) : undefined;
 				const event = {
 					eventName: `${state}Object_${usageType}`,
 					id,

package/src/inboundBatchAggregator.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { performance } from "@fluid-internal/client-utils";
+import { performanceNow } from "@fluid-internal/client-utils";
 import { IDeltaManagerFull } from "@fluidframework/container-definitions/internal";
 import { assert } from "@fluidframework/core-utils/internal";
 import { ISequencedDocumentMessage } from "@fluidframework/driver-definitions/internal";
@@ -114,7 +114,7 @@ export class InboundBatchAggregator {
 	 */
 	private readonly trackPending = (message: ISequencedDocumentMessage): void => {
 		assert(
-			this.deltaManager.inbound.length !== 0,
+			this.deltaManager.inbound.length > 0,
 			0x298 /* "we have something in the queue that generates this event" */,
 		);
 
@@ -227,14 +227,14 @@ export class InboundBatchAggregator {
 	private pauseQueue(): void {
 		assert(!this.localPaused, 0x297 /* "always called from resumed state" */);
 		this.localPaused = true;
-		this.timePaused = performance.now();
+		this.timePaused = performanceNow();
 		// eslint-disable-next-line @typescript-eslint/no-floating-promises
 		this.deltaManager.inbound.pause();
 	}
 
 	private resumeQueue(startBatch: number, messageEndBatch: ISequencedDocumentMessage): void {
 		const endBatch = messageEndBatch.sequenceNumber;
-		const duration = this.localPaused ? performance.now() - this.timePaused : undefined;
+		const duration = this.localPaused ? performanceNow() - this.timePaused : undefined;
 
 		this.batchCount++;
 		if (this.batchCount % 1000 === 1) {

package/src/layerCompatState.ts

@@ -0,0 +1,75 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	checkLayerCompatibility,
+	type ILayerCompatDetails,
+	type ILayerCompatSupportRequirements,
+} from "@fluid-internal/client-utils";
+import type { ICriticalContainerError } from "@fluidframework/container-definitions";
+import { UsageError } from "@fluidframework/telemetry-utils/internal";
+
+import { pkgVersion } from "./packageVersion.js";
+
+/**
+ * Runtime's compatibility details that is exposed to the Loader layer.
+ */
+export const RuntimeCompatDetails: ILayerCompatDetails = {
+	/**
+	 * The package version of the Runtime layer.
+	 */
+	pkgVersion,
+	/**
+	 * The current generation of the Runtime layer.
+	 */
+	generation: 1,
+	/**
+	 * The features supported by the Runtime layer across the Runtime / Loader boundary.
+	 */
+	supportedFeatures: new Set<string>(),
+};
+
+/**
+ * The requirements that the Loader layer must meet to be compatible with this Runtime.
+ */
+export const LoaderSupportRequirements: ILayerCompatSupportRequirements = {
+	/**
+	 * Minimum generation that Loader must be at to be compatible with Runtime.
+	 */
+	minSupportedGeneration: 0,
+	/**
+	 * The features that the Loader must support to be compatible with Runtime. Note that 0 is used here for
+	 * Loader layers before the introduction of the layer compatibility enforcement.
+	 */
+	requiredFeatures: [],
+};
+
+/**
+ * Validates that the Loader layer is compatible with this Runtime.
+ */
+export function validateLoaderCompatibility(
+	maybeLoaderCompatDetails: ILayerCompatDetails | undefined,
+	disposeFn: (error?: ICriticalContainerError) => void,
+): void {
+	const layerCheckResult = checkLayerCompatibility(
+		LoaderSupportRequirements,
+		maybeLoaderCompatDetails,
+	);
+	if (!layerCheckResult.isCompatible) {
+		const error = new UsageError("Runtime is not compatible with Loader", {
+			errorDetails: JSON.stringify({
+				runtimeVersion: RuntimeCompatDetails.pkgVersion,
+				loaderVersion: maybeLoaderCompatDetails?.pkgVersion,
+				runtimeGeneration: RuntimeCompatDetails.generation,
+				loaderGeneration: maybeLoaderCompatDetails?.generation,
+				minSupportedGeneration: LoaderSupportRequirements.minSupportedGeneration,
+				isGenerationCompatible: layerCheckResult.isGenerationCompatible,
+				unsupportedFeatures: layerCheckResult.unsupportedFeatures,
+			}),
+		});
+		disposeFn(error);
+		throw error;
+	}
+}

package/src/messageTypes.ts

@@ -14,6 +14,7 @@ import {
 import { IDataStoreAliasMessage } from "./dataStore.js";
 import { GarbageCollectionMessage } from "./gc/index.js";
 import { IChunkedOp } from "./opLifecycle/index.js";
+// eslint-disable-next-line import/no-deprecated
 import { IDocumentSchemaChangeMessage } from "./summary/index.js";
 
 /**
@@ -114,6 +115,7 @@ export type ContainerRuntimeGCMessage = TypedContainerRuntimeMessage<
 >;
 export type ContainerRuntimeDocumentSchemaMessage = TypedContainerRuntimeMessage<
 	ContainerMessageType.DocumentSchemaChange,
+	// eslint-disable-next-line import/no-deprecated
 	IDocumentSchemaChangeMessage
 >;
 

package/src/opLifecycle/README.md

@@ -2,21 +2,23 @@
 
 ## Table of contents
 
--   [Configs and feature gates for solving the 1MB limit.](#configs-and-feature-gates-for-solving-the-1mb-limit)
-    -   [Table of contents](#table-of-contents)
-    -   [Introduction](#introduction)
-        -   [How batching works](#how-batching-works)
-    -   [Compression](#compression)
-    -   [Grouped batching](#grouped-batching)
-        -   [Changes in op semantics](#changes-in-op-semantics)
-    -   [Chunking for compression](#chunking-for-compression)
-    -   [Configuration](#configuration)
-    -   [Note about performance and latency](#note-about-performance-and-latency)
-    -   [How it works](#how-it-works)
-    -   [How it works (Grouped Batching disabled)](#how-it-works-grouped-batching-disabled)
-    -   [How the overall op flow works](#how-the-overall-op-flow-works)
-        -   [Outbound](#outbound)
-        -   [Inbound](#inbound)
+- [Configs and feature gates for solving the 1MB limit.](#configs-and-feature-gates-for-solving-the-1mb-limit)
+  - [Table of contents](#table-of-contents)
+  - [Introduction](#introduction)
+    - [How batching works](#how-batching-works)
+  - [Grouped batching](#grouped-batching)
+    - [Changes in op semantics](#changes-in-op-semantics)
+  - [Compression](#compression)
+    - [Only single-message batches are compressed](#only-single-message-batches-are-compressed)
+  - [Chunking for compression](#chunking-for-compression)
+  - [Configuration](#configuration)
+  - [Note about performance and latency](#note-about-performance-and-latency)
+  - [How it works](#how-it-works)
+  - [Legacy behavior - How it used to work (Compression+Chunking without Grouped Batching)](#legacy-behavior---how-it-used-to-work-compressionchunking-without-grouped-batching)
+    - [IMPORTANT - As of 2.20.0, we no longer compress ungrouped batches, but we do need to read such ops - read on to learn what these legacy ops look like](#important---as-of-2200-we-no-longer-compress-ungrouped-batches-but-we-do-need-to-read-such-ops---read-on-to-learn-what-these-legacy-ops-look-like)
+  - [How the overall op flow works](#how-the-overall-op-flow-works)
+    - [Outbound](#outbound)
+    - [Inbound](#inbound)
 
 ## Introduction
 
@@ -56,20 +58,6 @@ What this means is that `FlushMode.Immediate` will send each op in its own paylo
 
 As `FlushMode.TurnBased` accumulates ops, it is the most vulnerable to run into the 1MB socket limit.
 
-## Compression
-
-**Compression targets payloads which exceed the max batch size and it is enabled by default.**. The `IContainerRuntimeOptions.compressionOptions` property, of type `ICompressionRuntimeOptions` is the configuration governing how compression works.
-
-`ICompressionRuntimeOptions` has two properties:
-
--   `minimumBatchSizeInBytes` – the minimum size of the batch for which compression should kick in. If the payload is too small, compression may not yield too many benefits. To target the original 1MB issue, a good value here would be to match the default maxBatchSizeInBytes (972800), however, experimentally, a good lower value could be at around 614400 bytes. Setting this value to `Number.POSITIVE_INFINITY` will disable compression.
--   `compressionAlgorithm` – currently, only `lz4` is supported.
-
-Compression is relevant for both `FlushMode.TurnBased` and `FlushMode.Immediate` as it only targets the contents of the ops and not the number of ops in a batch. Compression is opaque to the server and implementations of the Fluid protocol do not need to alter their behavior to support this client feature.
-
-Compressing a batch yields a batch with the same number of messages. It compresses all the content, shifting the compressed payload into the first op,
-leaving the rest of the batch's messages as empty placeholders to reserve sequence numbers for the compressed messages.
-
 ## Grouped batching
 
 With Grouped Batching enabled (it's on by default), all batch messages are combined under a single "grouped" message _before compression_. Upon receiving this new "grouped" message, the batch messages will be extracted, and they each will be given the same sequence number - that of the parent "grouped" message.
@@ -78,9 +66,7 @@ The purpose for enabling grouped batching before compression is to eliminate the
 
 Grouped batching is only relevant for `FlushMode.TurnBased`, since `OpGroupingManagerConfig.opCountThreshold` defaults to 2. Grouped batching is opaque to the server and implementations of the Fluid protocol do not need to alter their behavior to support this client feature.
 
-Grouped Batching can be disabled by setting `IContainerRuntimeOptions.enableGroupedBatching` to `false`.
-
-See [below](#how-grouped-batching-works) for an example.
+See [below](#how-it-works) for an example.
 
 ### Changes in op semantics
 
@@ -92,6 +78,27 @@ Grouped Batching changed a couple of expectations around message structure and r
 -   All ops in a batch must also have the same reference sequence number to ensure eventualy consistency of the model. The runtime will "rebase" ops in a batch with different ref sequence number to satisfy that requirement.
     -   What causes ops in a single JS turn (and thus in a batch) to have different reference sequence number? "Op reentrancy", where changes are made to a DDS inside a DDS 'onChanged' event handler.
 
+## Compression
+
+**Compression targets payloads which exceed the max batch size and it is enabled by default.**. The `IContainerRuntimeOptions.compressionOptions` property, of type `ICompressionRuntimeOptions` is the configuration governing how compression works.
+
+`ICompressionRuntimeOptions` has two properties:
+
+-   `minimumBatchSizeInBytes` – the minimum size of the batch for which compression should kick in. If the payload is too small, compression may not yield too many benefits. To target the original 1MB issue, a good value here would be to match the default maxBatchSizeInBytes (972800), however, experimentally, a good lower value could be at around 614400 bytes. Setting this value to `Number.POSITIVE_INFINITY` will disable compression.
+-   `compressionAlgorithm` – currently, only `lz4` is supported.
+
+Compression is relevant for both `FlushMode.TurnBased` and `FlushMode.Immediate` as it only targets the contents of the ops and not the number of ops in a batch. Compression is opaque to the server and implementations of the Fluid protocol do not need to alter their behavior to support this client feature.
+
+### Only single-message batches are compressed
+
+The batch to compress has to have only one message and it yields a batch with a single message. It compresses all the content, replacing the op's original contents with the compressed payload and the appropriate metadata to indicate it's compressed.
+
+Compression is only enabled if Grouped Batching is enabled.
+
+Legacy compressed batches could contain multiple messages, compressing all the content and shifting the compressed payload into the first op, leaving the rest of the batch's messages as empty placeholders to reserve sequence numbers for the compressed messages.
+
+While it remains possible to read compressed batches with empty ops, the system will no longer create said type of batches.
+
 ## Chunking for compression
 
 **Op chunking for compression targets payloads which exceed the max batch size after compression.** So, only payloads which are already compressed. By default, the feature is enabled.
@@ -200,7 +207,9 @@ Ungrouped batch:
 +-----------------+-----------------+-----------------+-----------------+-----------------+
 ```
 
-## How it works (Grouped Batching disabled)
+## Legacy behavior - How it used to work (Compression+Chunking without Grouped Batching)
+
+### IMPORTANT - As of 2.20.0, we no longer compress ungrouped batches, but we do need to read such ops - read on to learn what these legacy ops look like
 
 If we have a batch with a size larger than the configured minimum required for compression (in the example let’s say it’s 850 bytes), as following:
 
@@ -311,7 +320,7 @@ stateDiagram-v2
     groupBatch --> if_compression
 	flushInternal --> if_compression
     if_compression --> post
-	if_compression --> compress: if compression is enabled
+	if_compression --> compress: if compression and grouped batching are enabled
 	compress --> post
 	compress --> opSplitter.split: if the compressed payload is larger than the chunk size
 	opSplitter.split --> post

package/src/opLifecycle/duplicateBatchDetector.ts

@@ -85,12 +85,12 @@ export class DuplicateBatchDetector {
 	 * since the batch start has been processed by all clients, and local batches are deduped and the forked client would close.
 	 */
 	private clearOldBatchIds(msn: number): void {
-		this.batchIdsBySeqNum.forEach((batchId, sequenceNumber) => {
+		for (const [sequenceNumber, batchId] of this.batchIdsBySeqNum) {
 			if (sequenceNumber < msn) {
 				this.batchIdsBySeqNum.delete(sequenceNumber);
 				this.batchIdsAll.delete(batchId);
 			}
-		});
+		}
 	}
 
 	/**

package/src/opLifecycle/opCompressor.ts

@@ -34,13 +34,14 @@ export class OpCompressor {
 	 * Combines the contents of the batch into a single JSON string and compresses it, putting
 	 * the resulting string as the first message of the batch. The rest of the messages are
 	 * empty placeholders to reserve sequence numbers.
+	 * This should only take a single message batch and compress it.
 	 * @param batch - The batch to compress
 	 * @returns A batch of the same length as the input batch, containing a single compressed message followed by empty placeholders
 	 */
-	public compressBatch(batch: IBatch): IBatch {
+	public compressBatch(batch: IBatch): IBatch<[BatchMessage]> {
 		assert(
-			batch.contentSizeInBytes > 0 && batch.messages.length > 0,
-			0x5a4 /* Batch should not be empty */,
+			batch.contentSizeInBytes > 0 && batch.messages.length === 1,
+			0x5a4 /* Batch should not be empty and should contain a single message */,
 		);
 
 		const compressionStart = Date.now();
@@ -49,24 +50,16 @@ export class OpCompressor {
 		const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
 		const duration = Date.now() - compressionStart;
 
-		const messages: BatchMessage[] = [];
-		messages.push({
-			...batch.messages[0],
-			contents: JSON.stringify({ packedContents: compressedContent }),
-			metadata: batch.messages[0].metadata,
-			compression: CompressionAlgorithms.lz4,
-		});
+		const messages: [BatchMessage] = [
+			{
+				...batch.messages[0],
+				contents: JSON.stringify({ packedContents: compressedContent }),
+				metadata: batch.messages[0].metadata,
+				compression: CompressionAlgorithms.lz4,
+			},
+		];
 
-		// Add empty placeholder messages to reserve the sequence numbers
-		for (const message of batch.messages.slice(1)) {
-			messages.push({
-				localOpMetadata: message.localOpMetadata,
-				metadata: message.metadata,
-				referenceSequenceNumber: message.referenceSequenceNumber,
-			});
-		}
-
-		const compressedBatch: IBatch = {
+		const compressedBatch = {
 			contentSizeInBytes: compressedContent.length,
 			messages,
 			referenceSequenceNumber: batch.referenceSequenceNumber,
@@ -93,8 +86,8 @@ export class OpCompressor {
 		try {
 			// Yields a valid JSON array, since each message.contents is already serialized to JSON
 			return `[${batch.messages.map(({ contents }) => contents).join(",")}]`;
-		} catch (e: unknown) {
-			if ((e as Partial<Error>).message === "Invalid string length") {
+		} catch (newError: unknown) {
+			if ((newError as Partial<Error>).message === "Invalid string length") {
 				// This is how JSON.stringify signals that
 				// the content size exceeds its capacity
 				const error = new UsageError("Payload too large");
@@ -109,7 +102,7 @@ export class OpCompressor {
 				throw error;
 			}
 
-			throw e;
+			throw newError;
 		}
 	}
 }

package/src/opLifecycle/opDecompressor.ts

@@ -30,6 +30,9 @@ interface IPackedContentsContents {
  * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
  * 3. The final message of a batch will have batch metadata set to false
  * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ *
+ * Compressed batches from current code are always a single message but this class needs to handle a legacy compressed batch with multiple messages
+ * because we need that functionality for back compat.
  */
 export class OpDecompressor {
 	private activeBatch = false;
@@ -79,7 +82,7 @@ export class OpDecompressor {
 				});
 				return true;
 			}
-		} catch (err) {
+		} catch {
 			return false;
 		}
 

package/src/opLifecycle/opGroupingManager.ts

@@ -40,7 +40,6 @@ export function isGroupedBatch(op: ISequencedDocumentMessage): boolean {
 
 export interface OpGroupingManagerConfig {
 	readonly groupedBatchingEnabled: boolean;
-	readonly opCountThreshold: number;
 }
 
 export class OpGroupingManager {
@@ -102,7 +101,6 @@ export class OpGroupingManager {
 			this.logger.sendTelemetryEvent({
 				eventName: "GroupLargeBatch",
 				length: batch.messages.length,
-				threshold: this.config.opCountThreshold,
 				reentrant: batch.hasReentrantOps,
 				referenceSequenceNumber: batch.messages[0].referenceSequenceNumber,
 			});
@@ -159,10 +157,13 @@ export class OpGroupingManager {
 		return (
 			// Grouped batching must be enabled
 			this.config.groupedBatchingEnabled &&
-			// The number of ops in the batch must surpass the configured threshold
+			// The number of ops in the batch must be 2 or more
 			// or be empty (to allow for empty batches to be grouped)
-			(batch.messages.length === 0 || batch.messages.length >= this.config.opCountThreshold)
+			batch.messages.length !== 1
 			// Support for reentrant batches will be on by default
 		);
 	}
+	public groupedBatchingEnabled(): boolean {
+		return this.config.groupedBatchingEnabled;
+	}
 }

package/src/opLifecycle/opSplitter.ts

@@ -99,25 +99,28 @@ export class OpSplitter {
 	 * Splits the first op of a compressed batch in chunks, sends the chunks separately and
 	 * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
 	 *
-	 * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
-	 * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
-	 * uncompressed ops at ingestion in the runtime.
+	 * A compressed batch is formed by one large op at the first position.
 	 *
-	 * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+	 * If the op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
 	 * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
 	 *
-	 * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
-	 * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
-	 * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
-	 * (as it is the last chunk).
+	 * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch.
+	 * This will ensure that the batch semantics of the original (non-compressed) batch are preserved, as the original chunked op
+	 * will be unrolled by the runtime when the first message in the batch is processed (as it is the last chunk).
 	 *
-	 * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+	 * To handle legacy compressed batches with empty placeholders this method can attach the empty placeholder ops at the end
+	 * of the result batch, ensuring that the batch semantics are preserved.
+	 *
+	 * To illustrate the current functionality, if the input is `[largeOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+	 * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4]` will be returned.
+	 *
+	 * With the legacy code, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
 	 * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
 	 *
 	 * @remarks - A side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.
 	 *
 	 * @param batch - the compressed batch which needs to be processed
-	 * @returns A new adjusted batch (last chunk + empty placeholders) which can be sent over the wire
+	 * @returns A batch with the last chunk of the original message
 	 */
 	public splitFirstBatchMessage(batch: IBatch): IBatch {
 		assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
@@ -282,7 +285,7 @@ export const splitOp = (
 	for (let chunkId = 1; chunkId <= chunkCount; chunkId++) {
 		const chunk: IChunkedOp = {
 			chunkId,
-			contents: op.contents.substr(offset, chunkSizeInBytes),
+			contents: op.contents.slice(offset, offset + chunkSizeInBytes),
 			totalChunks: chunkCount,
 		};
 

package/src/opLifecycle/outbox.ts

@@ -135,7 +135,9 @@ export class Outbox {
 			this.params.config.compressionOptions.minimumBatchSizeInBytes !==
 			Number.POSITIVE_INFINITY;
 		// We need to allow infinite size batches if we enable compression
-		const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
+		const hardLimit = isCompressionEnabled
+			? Number.POSITIVE_INFINITY
+			: this.params.config.maxBatchSizeInBytes;
 
 		this.mainBatch = new BatchManager({ hardLimit, canRebase: true });
 		this.blobAttachBatch = new BatchManager({ hardLimit, canRebase: true });
@@ -229,18 +231,6 @@ export class Outbox {
 		this.maybeFlushPartialBatch();
 
 		this.addMessageToBatchManager(this.blobAttachBatch, message);
-
-		// If compression is enabled, we will always successfully receive
-		// blobAttach ops and compress then send them at the next JS turn, regardless
-		// of the overall size of the accumulated ops in the batch.
-		// However, it is more efficient to flush these ops faster, preferably
-		// after they reach a size which would benefit from compression.
-		if (
-			this.blobAttachBatch.contentSizeInBytes >=
-			this.params.config.compressionOptions.minimumBatchSizeInBytes
-		) {
-			this.flushInternal(this.blobAttachBatch);
-		}
 	}
 
 	public submitIdAllocation(message: BatchMessage): void {
@@ -357,9 +347,11 @@ export class Outbox {
 		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
 		// Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
 		if (this.params.shouldSend()) {
-			const processedBatch = this.compressBatch(
-				shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch,
-			);
+			const processedBatch = disableGroupedBatching
+				? rawBatch
+				: this.compressAndChunkBatch(
+						shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch,
+					);
 			clientSequenceNumber = this.sendBatch(processedBatch);
 			assert(
 				clientSequenceNumber === undefined || clientSequenceNumber >= 0,
@@ -420,16 +412,17 @@ export class Outbox {
 	 * @remarks - If chunking happens, a side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.
 	 *
 	 * @param batch - Raw or Grouped batch to consider for compression/chunking
-	 * @returns Either (A) the original batch, (B) a compressed batch (same length as original),
-	 * or (C) a batch containing the last chunk (plus empty placeholders from compression if applicable).
+	 * @returns Either (A) the original batch, (B) a compressed batch (same length as original)
+	 * or (C) a batch containing the last chunk.
 	 */
-	private compressBatch(batch: IBatch): IBatch {
+	private compressAndChunkBatch(batch: IBatch): IBatch {
 		if (
 			batch.messages.length === 0 ||
 			this.params.config.compressionOptions === undefined ||
 			this.params.config.compressionOptions.minimumBatchSizeInBytes >
 				batch.contentSizeInBytes ||
-			this.params.submitBatchFn === undefined
+			this.params.submitBatchFn === undefined ||
+			!this.params.groupingManager.groupedBatchingEnabled()
 		) {
 			// Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
 			return batch;

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -165,7 +165,9 @@ export class RemoteMessageProcessor {
 			// We should be awaiting a new batch (batchInProgress false)
 			assert(!this.batchInProgress, 0x9d3 /* Grouped batch interrupting another batch */);
 			const batchId = asBatchMetadata(message.metadata)?.batchId;
-			const groupedMessages = this.opGroupingManager.ungroupOp(message).map(unpack);
+			const groupedMessages = this.opGroupingManager
+				.ungroupOp(message)
+				.map((innerMessage) => unpack(innerMessage));
 
 			return {
 				type: "fullBatch",

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.20.0";
+export const pkgVersion = "2.21.0";