@fluidframework/container-runtime 2.0.0-internal.3.0.5 → 2.0.0-internal.3.1.1

package/src/garbageCollectionConstants.ts

@@ -33,6 +33,9 @@ export const throwOnTombstoneLoadKey = "Fluid.GarbageCollection.ThrowOnTombstone
 export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
 // Feature gate to enable GC version upgrade.
 export const gcVersionUpgradeToV2Key = "Fluid.GarbageCollection.GCVersionUpgradeToV2";
+// Feature gate to enable GC sweep for datastores.
+// TODO: Remove Test from the flag when we are confident to turn on sweep
+export const sweepDatastoresKey = "Fluid.GarbageCollection.Test.SweepDataStores";
 
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;

package/src/garbageCollectionHelpers.ts

@@ -0,0 +1,61 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
+import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
+import { MonitoringContext } from "@fluidframework/telemetry-utils";
+import {
+	disableTombstoneKey,
+	runSweepKey,
+	throwOnTombstoneLoadKey,
+	throwOnTombstoneUsageKey,
+} from "./garbageCollectionConstants";
+
+/**
+ * Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
+ * tombstoned or deleted object is loaded.
+ */
+export function sendGCUnexpectedUsageEvent(
+	mc: MonitoringContext,
+	event: ITelemetryGenericEvent & {
+		category: "error" | "generic";
+		gcTombstoneEnforcementAllowed: boolean | undefined;
+	},
+	packagePath: readonly string[] | undefined,
+	error?: unknown,
+) {
+	event.pkg = packagePathToTelemetryProperty(packagePath);
+	event.tombstoneFlags = JSON.stringify({
+		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
+		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
+		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
+	});
+	event.sweepFlags = JSON.stringify({
+		EnableSweepFlag: mc.config.getBoolean(runSweepKey),
+	});
+
+	mc.logger.sendTelemetryEvent(event, error);
+}
+
+/**
+ * In order to protect old documents that were created at a time when known bugs exist that violate GC's invariants
+ * such that enforcing GC (Fail on Tombstone load/usage, GC Sweep) would cause legitimate data loss,
+ * the container author may increment the generation value for Tombstone such that containers created
+ * with a different value will not be subjected to GC enforcement.
+ * If no generation is provided at runtime, this defaults to return true to maintain expected default behavior
+ * @param persistedGeneration - The persisted feature support value
+ * @param currentGeneration - The current app-provided feature support value
+ * @returns true if GC Enforcement (Fail on Tombstone load/usage) should be allowed
+ */
+export function shouldAllowGcTombstoneEnforcement(
+	persistedGeneration: number | undefined,
+	currentGeneration: number | undefined,
+): boolean {
+	// If no Generation value is provided for this session, then we should default to letting Tombstone feature behave as intended.
+	if (currentGeneration === undefined) {
+		return true;
+	}
+	return persistedGeneration === currentGeneration;
+}

package/src/gcSweepReadyUsageDetection.ts

@@ -6,10 +6,10 @@
 import { ITelemetryProperties } from "@fluidframework/common-definitions";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import {
-    IConfigProvider,
-    IFluidErrorBase,
-    LoggingError,
-    MonitoringContext,
+	IConfigProvider,
+	IFluidErrorBase,
+	LoggingError,
+	MonitoringContext,
 } from "@fluidframework/telemetry-utils";
 import { oneDayMs } from "./garbageCollectionConstants";
 
@@ -17,30 +17,33 @@ import { oneDayMs } from "./garbageCollectionConstants";
  * Feature Gate Key -
  * How many days between closing the container from this error (avoids locking user out of their file altogether)
  */
-export const skipClosureForXDaysKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
+export const skipClosureForXDaysKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
 
 /**
  * LocalStorage key (NOT via feature gate / monitoring context)
  * A map from docId to info about the last time we closed due to this error
  */
-export const closuresMapLocalStorageKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
+export const closuresMapLocalStorageKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
 
 /**
  * Feature gate key to enable closing the container if SweepReady objects are used.
  * Value should contain keywords "interactiveClient" and/or "summarizer" to enable detection in each container type
  */
 const sweepReadyUsageDetectionSetting = {
-    read(config: IConfigProvider) {
-        const sweepReadyUsageDetectionKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
-        const value = config.getString(sweepReadyUsageDetectionKey);
-        if (value === undefined) {
-            return { interactiveClient: false, summarizer: false };
-        }
-        return {
-            interactiveClient: value.includes("interactiveClient"),
-            summarizer: value.includes("summarizer"),
-        };
-    },
+	read(config: IConfigProvider) {
+		const sweepReadyUsageDetectionKey =
+			"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
+		const value = config.getString(sweepReadyUsageDetectionKey);
+		if (value === undefined) {
+			return { interactiveClient: false, summarizer: false };
+		}
+		return {
+			interactiveClient: value.includes("interactiveClient"),
+			summarizer: value.includes("summarizer"),
+		};
+	},
 };
 
 /**
@@ -52,8 +55,8 @@ const sweepReadyUsageDetectionSetting = {
  * since only the Summarizer has the latest truth about unreferenced node tracking
  */
 export class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {
-    /** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
-    public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
+	/** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
+	public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
 }
 
 /**
@@ -65,75 +68,78 @@ export class SweepReadyUsageError extends LoggingError implements IFluidErrorBas
  * (via skipClosureForXDaysKey above.  Uses localStorage and closuresMapLocalStorageKey to implement this behavior)
  */
 export class SweepReadyUsageDetectionHandler {
-    private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
+	private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
 
-    constructor(
-        private readonly uniqueContainerKey: string,
-        private readonly mc: MonitoringContext,
-        private readonly closeFn: (error?: ICriticalContainerError) => void,
-        localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
-    ) {
-        const noopStorage = { getItem: () => null, setItem: () => {} };
-        // localStorage is not defined in Node environment, so fall back to noopStorage if needed.
-        this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
+	constructor(
+		private readonly uniqueContainerKey: string,
+		private readonly mc: MonitoringContext,
+		private readonly closeFn: (error?: ICriticalContainerError) => void,
+		localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
+	) {
+		const noopStorage = { getItem: () => null, setItem: () => {} };
+		// localStorage is not defined in Node environment, so fall back to noopStorage if needed.
+		this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
 
-        if (this.localStorage === noopStorage) {
-            // This means the Skip Closure Period logic will not work.
-            this.mc.logger.sendTelemetryEvent({ eventName: "SweepReadyUsageDetectionHandlerNoopStorage" });
-        }
-    }
+		if (this.localStorage === noopStorage) {
+			// This means the Skip Closure Period logic will not work.
+			this.mc.logger.sendTelemetryEvent({
+				eventName: "SweepReadyUsageDetectionHandlerNoopStorage",
+			});
+		}
+	}
 
-    /**
-      * If SweepReady Usage Detection is enabled, close the interactive container.
-      * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
-      *
-      * Once Sweep is fully implemented, this will be removed since the objects will be gone
-      * and errors will arise elsewhere in the runtime
-     */
-    public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
-        if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
-            return;
-        }
+	/**
+	 * If SweepReady Usage Detection is enabled, close the interactive container.
+	 * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
+	 *
+	 * Once Sweep is fully implemented, this will be removed since the objects will be gone
+	 * and errors will arise elsewhere in the runtime
+	 */
+	public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
+		if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
+			return;
+		}
 
-        // Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
-        // However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
-        // which will only allow the container to close once during that period, to avoid locking users out.
-        let shouldClose: boolean = true;
-        let pastClosuresMap: Record<string, { lastCloseTime: number; } | undefined> = {};
-        let lastCloseTime: number | undefined;
-        const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
-        if (skipClosureForXDays !== undefined) {
-            // Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
-            try {
-                const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
-                const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
-                if (typeof parsedValue === "object") {
-                    pastClosuresMap = parsedValue;
-                }
-            } catch (e) {
-            }
-            lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
+		// Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
+		// However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
+		// which will only allow the container to close once during that period, to avoid locking users out.
+		let shouldClose: boolean = true;
+		let pastClosuresMap: Record<string, { lastCloseTime: number } | undefined> = {};
+		let lastCloseTime: number | undefined;
+		const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
+		if (skipClosureForXDays !== undefined) {
+			// Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
+			try {
+				const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
+				const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
+				if (typeof parsedValue === "object") {
+					pastClosuresMap = parsedValue;
+				}
+			} catch (e) {}
+			lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
 
-            // Don't close if we did already within the Skip Closure Period
-            if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs) {
-                shouldClose = false;
-            }
-        }
+			// Don't close if we did already within the Skip Closure Period
+			if (
+				lastCloseTime !== undefined &&
+				Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs
+			) {
+				shouldClose = false;
+			}
+		}
 
-        const error = new SweepReadyUsageError(
-            "SweepReady object used in Non-Summarizer Client",
-            { errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }) },
-        );
-        if (shouldClose) {
-            // Update closures map in localStorage before closing
-            // Note there is a race condition between different tabs updating localStorage and overwriting
-            // each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
-            pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
-            this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
+		const error = new SweepReadyUsageError("SweepReady object used in Non-Summarizer Client", {
+			errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }),
+		});
+		if (shouldClose) {
+			// Update closures map in localStorage before closing
+			// Note there is a race condition between different tabs updating localStorage and overwriting
+			// each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
+			pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
+			this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
 
-            this.closeFn(error);
-        } else {
-            this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
-        }
-    }
+			this.closeFn(error);
+		} else {
+			this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
+		}
+	}
 }

package/src/index.ts

@@ -4,79 +4,80 @@
  */
 
 export {
-    ContainerMessageType,
-    ContainerRuntimeMessage,
-    IGCRuntimeOptions,
-    ISummaryRuntimeOptions,
-    ISummaryBaseConfiguration,
-    ISummaryConfigurationHeuristics,
-    ISummaryConfigurationDisableSummarizer,
-    ISummaryConfigurationDisableHeuristics,
-    IContainerRuntimeOptions,
-    IRootSummaryTreeWithStats,
-    isRuntimeMessage,
-    RuntimeMessage,
-    agentSchedulerId,
-    ContainerRuntime,
-    RuntimeHeaders,
-    AllowTombstoneRequestHeaderKey,
-    TombstoneResponseHeaderKey,
-    ISummaryConfiguration,
-    DefaultSummaryConfiguration,
-    ICompressionRuntimeOptions,
-    CompressionAlgorithms,
+	ContainerMessageType,
+	ContainerRuntimeMessage,
+	IGCRuntimeOptions,
+	ISummaryRuntimeOptions,
+	ISummaryBaseConfiguration,
+	ISummaryConfigurationHeuristics,
+	ISummaryConfigurationDisableSummarizer,
+	ISummaryConfigurationDisableHeuristics,
+	IContainerRuntimeOptions,
+	IRootSummaryTreeWithStats,
+	isRuntimeMessage,
+	RuntimeMessage,
+	agentSchedulerId,
+	ContainerRuntime,
+	RuntimeHeaders,
+	AllowTombstoneRequestHeaderKey,
+	TombstoneResponseHeaderKey,
+	ISummaryConfiguration,
+	DefaultSummaryConfiguration,
+	ICompressionRuntimeOptions,
+	CompressionAlgorithms,
 } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
+export { IGCStats } from "./garbageCollection";
 export {
-    IGCStats,
-} from "./garbageCollection";
-export {
-    IPendingFlush,
-    IPendingLocalState,
-    IPendingMessage,
-    IPendingState,
+	IPendingFlush,
+	IPendingLocalState,
+	IPendingMessage,
+	IPendingState,
 } from "./pendingStateManager";
 export { Summarizer } from "./summarizer";
 export {
-    EnqueueSummarizeResult,
-    IAckSummaryResult,
-    IBaseSummarizeResult,
-    IBroadcastSummaryResult,
-    ICancellationToken,
-    IConnectableRuntime,
-    IEnqueueSummarizeOptions,
-    IGenerateSummaryTreeResult,
-    IGeneratedSummaryStats,
-    INackSummaryResult,
-    IOnDemandSummarizeOptions,
-    IProvideSummarizer,
-    IRefreshSummaryAckOptions,
-    ISubmitSummaryOpResult,
-    ISubmitSummaryOptions,
-    ISummarizeOptions,
-    ISummarizeResults,
-    ISummarizer,
-    ISummarizerEvents,
-    ISummarizerInternalsProvider,
-    ISummarizerRuntime,
-    ISummarizingWarning,
-    ISummaryCancellationToken,
-    IUploadSummaryResult,
-    SubmitSummaryResult,
-    SummarizeResultPart,
-    SummarizerStopReason,
+	EnqueueSummarizeResult,
+	IAckSummaryResult,
+	IBaseSummarizeResult,
+	IBroadcastSummaryResult,
+	ICancellationToken,
+	IConnectableRuntime,
+	IEnqueueSummarizeOptions,
+	IGenerateSummaryTreeResult,
+	IGeneratedSummaryStats,
+	INackSummaryResult,
+	IOnDemandSummarizeOptions,
+	IProvideSummarizer,
+	IRefreshSummaryAckOptions,
+	ISubmitSummaryOpResult,
+	ISubmitSummaryOptions,
+	ISummarizeOptions,
+	ISummarizeResults,
+	ISummarizer,
+	ISummarizerEvents,
+	ISummarizerInternalsProvider,
+	ISummarizerRuntime,
+	ISummarizingWarning,
+	ISummaryCancellationToken,
+	IUploadSummaryResult,
+	SubmitSummaryResult,
+	SummarizeResultPart,
+	SummarizerStopReason,
 } from "./summarizerTypes";
 export {
-    IAckedSummary,
-    IClientSummaryWatcher,
-    ISummary,
-    ISummaryCollectionOpEvents,
-    ISummaryAckMessage,
-    ISummaryNackMessage,
-    ISummaryOpMessage,
-    OpActionEventListener,
-    OpActionEventName,
-    SummaryCollection,
+	IAckedSummary,
+	IClientSummaryWatcher,
+	ISummary,
+	ISummaryCollectionOpEvents,
+	ISummaryAckMessage,
+	ISummaryNackMessage,
+	ISummaryOpMessage,
+	OpActionEventListener,
+	OpActionEventName,
+	SummaryCollection,
 } from "./summaryCollection";
-export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
+export {
+	ICancellableSummarizerController,
+	neverCancelledSummaryToken,
+} from "./runWhileConnectedCoordinator";
 export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";

package/src/opLifecycle/README.md

@@ -0,0 +1,152 @@
+# Configs and feature gates for solving the 1MB limit.
+
+## Introduction
+
+There is a current limitation regarding the size of the payload a Fluid client can send and receive. [The limit is 1MB per payload](https://github.com/microsoft/FluidFramework/issues/9023) and it is currently enforced explicitly with the `BatchTooLarge` error which closes the container.
+
+There are two features which can be used to work around this size limit, batch compression and compressed batch chunking. This document describes how to enable/disable them, along with a brief description of how they work.
+
+**The features are still considered experimental and for safety and back-compat reasons they are disabled by default.**
+
+## Table of contents
+
+-   [Introduction](#introduction)
+    -   [Enabling compression](#enabling-compression)
+    -   [Enabling chunking for compression](#enabling-chunking-for-compression)
+    -   [Disabling in case of emergency](#disabling-in-case-of-emergency)
+    -   [Example configs](#example-configs)
+    -   [How it works](#how-it-works)
+
+## Enabling compression
+
+**Compression targets payloads which exceed the max batch size (1MB) and it is disabled by default.** To enable compression, use the `IContainerRuntimeOptions.compressionOptions` property, of type `ICompressionRuntimeOptions`.
+
+`ICompressionRuntimeOptions` has two properties:
+
+-   `minimumBatchSizeInBytes` – the minimum size of the batch for which compression should kick in. If the payload is too small, compression may not yield too many benefits. To target the original 1MB issue, a good value here would be to match the default maxBatchSizeInBytes (972800), however, experimentally, a good lower value could be at around 614400 bytes.
+-   `compressionAlgorithm` – currently, only `lz4` is supported.
+
+## Enabling chunking for compression
+
+**Op chunking for compression targets payloads which exceed the max batch size (1MB) after compression.** So, only payloads which are already compressed. By default, the feature is disabled.
+
+To enable, use the `IContainerRuntimeOptions.chunkSizeInBytes` property, which represents the size of the chunked ops, when chunking is necessary. When chunking is performed, the large op is split into smaller ops (chunks). This config represents the size of the chunks. The value enables a trade-off between large chunks / few ops and small chunks / many ops. A good value for this would be at around 614400.
+
+This config would govern chunking compressed batches only. We will not be enabling chunking across all types of ops/batches but **only when compression is enabled and when the batch is compressed**, and its payload size is more than `maxBatchSizeInBytes`. Therefore, for this feature to be working, it is required that compression is enabled using `IContainerRuntimeOptions.compressionOptions`.
+
+It is recommended to also change the `maxBatchSizeInBytes` property in `IContainerRuntimeOptions`. By default, if unspecified it is 972800. We recommend a lower value such as `716800`, to account for any overhead on the server side. The reason for this is that chunking will only kick in after this configuration limit is exceeded. If the limit is too high, we might allow batches which are under 1MB but can increase in size due to overhead after they reach the server.
+
+## Disabling in case of emergency
+
+If the features are enabled using the configs, they can be disabled at runtime via feature gates as following:
+
+-   `Fluid.ContainerRuntime.DisableCompression` - if set to true, will disable compression (this has a side effect of also disabling chunking, as chunking is invoked only for compressed payloads).
+-   `Fluid.ContainerRuntime.DisableCompressionChunking` - if set to true, will disable chunking for compression.
+
+## Example configs
+
+Enable only compression:
+
+```
+    const runtimeOptions: IContainerRuntimeOptions = {
+        compressionOptions: {
+            minimumBatchSizeInBytes: 614400,
+            compressionAlgorithm: CompressionAlgorithms.lz4,
+        },
+        maxBatchSizeInBytes: 716800,
+    }
+```
+
+Enable compression and chunking:
+
+```
+    const runtimeOptions: IContainerRuntimeOptions = {
+        compressionOptions: {
+            minimumBatchSizeInBytes: 614400,
+            compressionAlgorithm: CompressionAlgorithms.lz4,
+        },
+        chunkSizeInBytes: 614400,
+        maxBatchSizeInBytes: 716800,
+    }
+```
+
+## How it works
+
+Compression currently works as a runtime layer over the regular op sending/receiving pipeline.
+
+If we have a batch with a size larger than the configured minimum required for compression (in the example let’s say it’s 850 bytes), as following:
+
+```
++-----------+-----------+-----------+-----------+
+| Op 1      | Op 2      | Op 3      | Op 4      |
+| SeqNum: 1 | SeqNum: 2 | SeqNum: 3 | SeqNum: 4 |
+| Size: 100 | Size: 150 | Size: 200 | Size: 400 |
++-----------+-----------+-----------+-----------+
+```
+
+The total size of the batch is 850 bytes. The client which needs to send the batch would compress the batch to a smaller size (200 bytes) and will send a new batch like the following:
+
+```
++--------------------+-----------+-----------+-----------+
+| Op 1               | Op 2      | Op 3      | Op 4      |
+| SeqNum: 1          | SeqNum: 2 | SeqNum: 3 | SeqNum: 4 |
+| Size: 200          | Size: 0   | Size: 0   | Size: 0   |
+| Compression: 'lz4' |           |           |           |
++--------------------+-----------+-----------+-----------+
+```
+
+The first op in the batch is the only one with content (which is opaque due to it being compressed), the rest of the ops serve only to reserve the sequence numbers so that the state machine which rebuilds the original batch on the receiving client can reconstruct the original batch.
+
+When the batch is received by a client, it will detect the first op as being compressed, it will decompress it and store it in memory. For each empty op subsequently received, it will fetch the uncompressed content from memory and rebuild the original ops. The original ops are then processed by the runtime and applied accordingly.
+So, compression virtualizes the batch.
+
+After compression, the first op in the batch can exceed 1MB, therefore it would still be rejected. In this case, another layer of virtualization is added after compression (and before decompression, symmetrically on the receiving end).
+
+The first op in the compressed batch can be chunked into smaller ops which can be sent outside the original batch. However, to conveniently maintain the batch semantics, the last chunk (the chunk which triggers rebuilding the original op) is the first op in the new batch.
+
+To illustrate, let’s take the large batch below:
+
+```
++--------------------+-----------+-----------+-----------+
+| Op 1               | Op 2      | Op 3      | Op 4      |
+| SeqNum: 1          | SeqNum: 2 | SeqNum: 3 | SeqNum: 4 |
+| Size: 900          | Size: 0   | Size: 0   | Size: 0   |
+| Compression: 'lz4' |           |           |           |
++--------------------+-----------+-----------+-----------+
+```
+
+This will produce the following batches:
+
+```
++-----------+
+| Chunk 1/3 |
+| SeqNum: 1 |
+| Size: 300 |
++-----------+
+
+```
+
+```
++-----------+
+| Chunk 2/3 |
+| SeqNum: 2 |
+| Size: 300 |
++-----------+
+
+```
+
+```
++-----------+-----------+-----------+-----------+
+| Chunk 3/3 | Op 2      | Op 3      | Op 4      |
+| SeqNum: 3 | SeqNum: 4 | SeqNum: 5 | SeqNum: 6 |
+| Size: 300 | Size: 0   | Size: 0   | Size: 0   |
++-----------+-----------+-----------+-----------+
+```
+
+The first 2 chunks are sent in their own batches, while the last chunk is the first op in the last batch which contains the ops reserving the required sequence numbers.
+
+Notice that the sequence numbers don’t matter here, as all ops will be based off the same reference sequence number, so the sequence number will be recalculated for all, without additional work.
+
+Additionally, as compression preserves the original uncompressed batch layout in terms of the number of ops by using empty ops to reserve the sequence numbers, this ensures that the clients will always receive the exact count of ops to rebuild the uncompressed batch sequentially.
+
+On the receiving end, the client will accumulate chunks 1 and 2 and keep them in memory. When chunk 3 is received, the original large, decompressed op will be rebuilt, and the runtime will then process the batch as if it is a compressed batch.