@fluidframework/container-runtime 2.0.0-internal.3.0.2 → 2.0.0-internal.3.2.0

package/src/gc/gcSweepReadyUsageDetection.ts

@@ -0,0 +1,145 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryProperties } from "@fluidframework/common-definitions";
+import { ICriticalContainerError } from "@fluidframework/container-definitions";
+import {
+	IConfigProvider,
+	IFluidErrorBase,
+	LoggingError,
+	MonitoringContext,
+} from "@fluidframework/telemetry-utils";
+import { oneDayMs } from "./gcDefinitions";
+
+/**
+ * Feature Gate Key -
+ * How many days between closing the container from this error (avoids locking user out of their file altogether)
+ */
+export const skipClosureForXDaysKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
+
+/**
+ * LocalStorage key (NOT via feature gate / monitoring context)
+ * A map from docId to info about the last time we closed due to this error
+ */
+export const closuresMapLocalStorageKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
+
+/**
+ * Feature gate key to enable closing the container if SweepReady objects are used.
+ * Value should contain keywords "interactiveClient" and/or "summarizer" to enable detection in each container type
+ */
+const sweepReadyUsageDetectionSetting = {
+	read(config: IConfigProvider) {
+		const sweepReadyUsageDetectionKey =
+			"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
+		const value = config.getString(sweepReadyUsageDetectionKey);
+		if (value === undefined) {
+			return { interactiveClient: false, summarizer: false };
+		}
+		return {
+			interactiveClient: value.includes("interactiveClient"),
+			summarizer: value.includes("summarizer"),
+		};
+	},
+};
+
+/**
+ * Error class raised when a SweepReady object is used, indicating a bug in how
+ * references are managed in the container by the application, or a bug in how
+ * GC tracks those references.
+ *
+ * There's a chance for false positives when this error is raised by an Interactive Container,
+ * since only the Summarizer has the latest truth about unreferenced node tracking
+ */
+export class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {
+	/** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
+	public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
+}
+
+/**
+ * This class encapsulates the logic around what to do when a SweepReady object is used.
+ * There are several tactics we plan to use in Dogfood environments to aid diagnosis of these cases:
+ * - Closing the interactive container when either the interactive or summarizer client detects this kind of violation
+ * (via sweepReadyUsageDetectionSetting above)
+ * - Throttling the frequency of these crashes via a "Skip Closure Period" per container per device
+ * (via skipClosureForXDaysKey above.  Uses localStorage and closuresMapLocalStorageKey to implement this behavior)
+ */
+export class SweepReadyUsageDetectionHandler {
+	private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
+
+	constructor(
+		private readonly uniqueContainerKey: string,
+		private readonly mc: MonitoringContext,
+		private readonly closeFn: (error?: ICriticalContainerError) => void,
+		localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
+	) {
+		const noopStorage = { getItem: () => null, setItem: () => {} };
+		// localStorage is not defined in Node environment, so fall back to noopStorage if needed.
+		this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
+
+		if (this.localStorage === noopStorage) {
+			// This means the Skip Closure Period logic will not work.
+			this.mc.logger.sendTelemetryEvent({
+				eventName: "SweepReadyUsageDetectionHandlerNoopStorage",
+			});
+		}
+	}
+
+	/**
+	 * If SweepReady Usage Detection is enabled, close the interactive container.
+	 * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
+	 *
+	 * Once Sweep is fully implemented, this will be removed since the objects will be gone
+	 * and errors will arise elsewhere in the runtime
+	 */
+	public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
+		if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
+			return;
+		}
+
+		// Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
+		// However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
+		// which will only allow the container to close once during that period, to avoid locking users out.
+		let shouldClose: boolean = true;
+		let pastClosuresMap: Record<string, { lastCloseTime: number } | undefined> = {};
+		let lastCloseTime: number | undefined;
+		const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
+		if (skipClosureForXDays !== undefined) {
+			// Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
+			try {
+				const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
+				const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
+				if (typeof parsedValue === "object") {
+					pastClosuresMap = parsedValue;
+				}
+			} catch (e) {}
+			lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
+
+			// Don't close if we did already within the Skip Closure Period
+			if (
+				lastCloseTime !== undefined &&
+				Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs
+			) {
+				shouldClose = false;
+			}
+		}
+
+		const error = new SweepReadyUsageError("SweepReady object used in Non-Summarizer Client", {
+			errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }),
+		});
+		if (shouldClose) {
+			// Update closures map in localStorage before closing
+			// Note there is a race condition between different tabs updating localStorage and overwriting
+			// each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
+			pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
+			this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
+
+			this.closeFn(error);
+		} else {
+			this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
+		}
+	}
+}

package/src/gc/gcUnreferencedStateTracker.ts

@@ -0,0 +1,114 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert, Timer } from "@fluidframework/common-utils";
+import { UnreferencedState } from "./gcDefinitions";
+
+/** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
+class TimerWithNoDefaultTimeout extends Timer {
+	constructor(private readonly callback: () => void) {
+		// The default timeout/handlers will never be used since start/restart pass overrides below
+		super(0, () => {
+			throw new Error("DefaultHandler should not be used");
+		});
+	}
+
+	start(timeoutMs: number) {
+		super.start(timeoutMs, this.callback);
+	}
+
+	restart(timeoutMs: number): void {
+		super.restart(timeoutMs, this.callback);
+	}
+}
+
+/**
+ * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
+ * be deleted by the sweep phase.
+ */
+export class UnreferencedStateTracker {
+	private _state: UnreferencedState = UnreferencedState.Active;
+	public get state(): UnreferencedState {
+		return this._state;
+	}
+
+	/** Timer to indicate when an unreferenced object is considered Inactive */
+	private readonly inactiveTimer: TimerWithNoDefaultTimeout;
+	/** Timer to indicate when an unreferenced object is Sweep-Ready */
+	private readonly sweepTimer: TimerWithNoDefaultTimeout;
+
+	constructor(
+		public readonly unreferencedTimestampMs: number,
+		/** The time after which node transitions to Inactive state. */
+		private readonly inactiveTimeoutMs: number,
+		/** The current reference timestamp used to track how long this node has been unreferenced for. */
+		currentReferenceTimestampMs: number,
+		/** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
+		private readonly sweepTimeoutMs: number | undefined,
+	) {
+		if (this.sweepTimeoutMs !== undefined) {
+			assert(
+				this.inactiveTimeoutMs <= this.sweepTimeoutMs,
+				0x3b0 /* inactive timeout must not be greater than the sweep timeout */,
+			);
+		}
+
+		this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
+			this._state = UnreferencedState.SweepReady;
+			assert(
+				!this.inactiveTimer.hasTimer,
+				0x3b1 /* inactiveTimer still running after sweepTimer fired! */,
+			);
+		});
+
+		this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
+			this._state = UnreferencedState.Inactive;
+
+			// After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
+			if (this.sweepTimeoutMs !== undefined) {
+				this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
+			}
+		});
+		this.updateTracking(currentReferenceTimestampMs);
+	}
+
+	/* Updates the unreferenced state based on the provided timestamp. */
+	public updateTracking(currentReferenceTimestampMs: number) {
+		const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
+
+		// If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
+		if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
+			this._state = UnreferencedState.SweepReady;
+			this.clearTimers();
+			return;
+		}
+
+		// If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
+		// Also, start a timer for the sweep timeout.
+		if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
+			this._state = UnreferencedState.Inactive;
+			this.inactiveTimer.clear();
+
+			if (this.sweepTimeoutMs !== undefined) {
+				this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
+			}
+			return;
+		}
+
+		// The node is still active. Ensure the inactive timer is running with the proper remaining duration.
+		this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
+	}
+
+	private clearTimers() {
+		this.inactiveTimer.clear();
+		this.sweepTimer.clear();
+	}
+
+	/** Stop tracking this node. Reset the unreferenced timers and state, if any. */
+	public stopTracking() {
+		this.clearTimers();
+		this._state = UnreferencedState.Active;
+	}
+}

package/src/gc/index.ts

@@ -0,0 +1,40 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export { GarbageCollector } from "./garbageCollection";
+export {
+	currentGCVersion,
+	defaultInactiveTimeoutMs,
+	defaultSessionExpiryDurationMs,
+	disableSweepLogKey,
+	GCNodeType,
+	gcTestModeKey,
+	gcTombstoneGenerationOptionName,
+	GCVersion,
+	gcVersionUpgradeToV2Key,
+	IGarbageCollectionRuntime,
+	IGarbageCollector,
+	IGarbageCollectorCreateParams,
+	IGCMetadata,
+	IGCStats,
+	oneDayMs,
+	runGCKey,
+	runSessionExpiryKey,
+	runSweepKey,
+	stableGCVersion,
+	sweepAttachmentBlobsKey,
+	sweepDatastoresKey,
+	throwOnTombstoneLoadKey,
+	throwOnTombstoneUsageKey,
+	UnreferencedState,
+} from "./gcDefinitions";
+export { sendGCUnexpectedUsageEvent, shouldAllowGcTombstoneEnforcement } from "./gcHelpers";
+export { GCSummaryStateTracker } from "./gcSummaryStateTracker";
+export {
+	skipClosureForXDaysKey,
+	closuresMapLocalStorageKey,
+	SweepReadyUsageDetectionHandler,
+} from "./gcSweepReadyUsageDetection";
+export { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";

package/src/index.ts

@@ -4,79 +4,76 @@
  */
 
 export {
-    ContainerMessageType,
-    ContainerRuntimeMessage,
-    IGCRuntimeOptions,
-    ISummaryRuntimeOptions,
-    ISummaryBaseConfiguration,
-    ISummaryConfigurationHeuristics,
-    ISummaryConfigurationDisableSummarizer,
-    ISummaryConfigurationDisableHeuristics,
-    IContainerRuntimeOptions,
-    IRootSummaryTreeWithStats,
-    isRuntimeMessage,
-    RuntimeMessage,
-    agentSchedulerId,
-    ContainerRuntime,
-    RuntimeHeaders,
-    AllowTombstoneRequestHeaderKey,
-    TombstoneResponseHeaderKey,
-    ISummaryConfiguration,
-    DefaultSummaryConfiguration,
-    ICompressionRuntimeOptions,
-    CompressionAlgorithms,
+	ContainerMessageType,
+	ContainerRuntimeMessage,
+	IGCRuntimeOptions,
+	ISummaryRuntimeOptions,
+	ISummaryBaseConfiguration,
+	ISummaryConfigurationHeuristics,
+	ISummaryConfigurationDisableSummarizer,
+	ISummaryConfigurationDisableHeuristics,
+	IContainerRuntimeOptions,
+	IRootSummaryTreeWithStats,
+	isRuntimeMessage,
+	RuntimeMessage,
+	agentSchedulerId,
+	ContainerRuntime,
+	RuntimeHeaders,
+	AllowTombstoneRequestHeaderKey,
+	TombstoneResponseHeaderKey,
+	ISummaryConfiguration,
+	DefaultSummaryConfiguration,
+	ICompressionRuntimeOptions,
+	CompressionAlgorithms,
 } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
+export { IGCStats } from "./gc";
 export {
-    IGCStats,
-} from "./garbageCollection";
-export {
-    IPendingFlush,
-    IPendingLocalState,
-    IPendingMessage,
-    IPendingState,
+	IPendingFlush,
+	IPendingLocalState,
+	IPendingMessage,
+	IPendingState,
 } from "./pendingStateManager";
-export { Summarizer } from "./summarizer";
-export {
-    EnqueueSummarizeResult,
-    IAckSummaryResult,
-    IBaseSummarizeResult,
-    IBroadcastSummaryResult,
-    ICancellationToken,
-    IConnectableRuntime,
-    IEnqueueSummarizeOptions,
-    IGenerateSummaryTreeResult,
-    IGeneratedSummaryStats,
-    INackSummaryResult,
-    IOnDemandSummarizeOptions,
-    IProvideSummarizer,
-    IRefreshSummaryAckOptions,
-    ISubmitSummaryOpResult,
-    ISubmitSummaryOptions,
-    ISummarizeOptions,
-    ISummarizeResults,
-    ISummarizer,
-    ISummarizerEvents,
-    ISummarizerInternalsProvider,
-    ISummarizerRuntime,
-    ISummarizingWarning,
-    ISummaryCancellationToken,
-    IUploadSummaryResult,
-    SubmitSummaryResult,
-    SummarizeResultPart,
-    SummarizerStopReason,
-} from "./summarizerTypes";
 export {
-    IAckedSummary,
-    IClientSummaryWatcher,
-    ISummary,
-    ISummaryCollectionOpEvents,
-    ISummaryAckMessage,
-    ISummaryNackMessage,
-    ISummaryOpMessage,
-    OpActionEventListener,
-    OpActionEventName,
-    SummaryCollection,
-} from "./summaryCollection";
-export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
+	IAckedSummary,
+	ISummarizer,
+	ISummarizeResults,
+	ISummaryCancellationToken,
+	neverCancelledSummaryToken,
+	Summarizer,
+	SummarizerStopReason,
+	SummaryCollection,
+	EnqueueSummarizeResult,
+	IAckSummaryResult,
+	IBaseSummarizeResult,
+	IBroadcastSummaryResult,
+	ICancellationToken,
+	IConnectableRuntime,
+	IEnqueueSummarizeOptions,
+	IGenerateSummaryTreeResult,
+	IGeneratedSummaryStats,
+	INackSummaryResult,
+	IOnDemandSummarizeOptions,
+	IProvideSummarizer,
+	IRefreshSummaryAckOptions,
+	ISubmitSummaryOpResult,
+	ISubmitSummaryOptions,
+	ISummarizeOptions,
+	ISummarizerEvents,
+	ISummarizerInternalsProvider,
+	ISummarizerRuntime,
+	ISummarizingWarning,
+	IUploadSummaryResult,
+	SubmitSummaryResult,
+	SummarizeResultPart,
+	IClientSummaryWatcher,
+	ISummary,
+	ISummaryCollectionOpEvents,
+	ISummaryAckMessage,
+	ISummaryNackMessage,
+	ISummaryOpMessage,
+	OpActionEventListener,
+	OpActionEventName,
+	ICancellableSummarizerController,
+} from "./summary";
 export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";

package/src/opLifecycle/README.md

@@ -0,0 +1,152 @@
+# Configs and feature gates for solving the 1MB limit.
+
+## Introduction
+
+There is a current limitation regarding the size of the payload a Fluid client can send and receive. [The limit is 1MB per payload](https://github.com/microsoft/FluidFramework/issues/9023) and it is currently enforced explicitly with the `BatchTooLarge` error which closes the container.
+
+There are two features which can be used to work around this size limit, batch compression and compressed batch chunking. This document describes how to enable/disable them, along with a brief description of how they work.
+
+**The features are still considered experimental and for safety and back-compat reasons they are disabled by default.**
+
+## Table of contents
+
+-   [Introduction](#introduction)
+    -   [Enabling compression](#enabling-compression)
+    -   [Enabling chunking for compression](#enabling-chunking-for-compression)
+    -   [Disabling in case of emergency](#disabling-in-case-of-emergency)
+    -   [Example configs](#example-configs)
+    -   [How it works](#how-it-works)
+
+## Enabling compression
+
+**Compression targets payloads which exceed the max batch size (1MB) and it is disabled by default.** To enable compression, use the `IContainerRuntimeOptions.compressionOptions` property, of type `ICompressionRuntimeOptions`.
+
+`ICompressionRuntimeOptions` has two properties:
+
+-   `minimumBatchSizeInBytes` – the minimum size of the batch for which compression should kick in. If the payload is too small, compression may not yield too many benefits. To target the original 1MB issue, a good value here would be to match the default maxBatchSizeInBytes (972800), however, experimentally, a good lower value could be at around 614400 bytes.
+-   `compressionAlgorithm` – currently, only `lz4` is supported.
+
+## Enabling chunking for compression
+
+**Op chunking for compression targets payloads which exceed the max batch size (1MB) after compression.** So, only payloads which are already compressed. By default, the feature is disabled.
+
+To enable, use the `IContainerRuntimeOptions.chunkSizeInBytes` property, which represents the size of the chunked ops, when chunking is necessary. When chunking is performed, the large op is split into smaller ops (chunks). This config represents the size of the chunks. The value enables a trade-off between large chunks / few ops and small chunks / many ops. A good value for this would be at around 614400.
+
+This config would govern chunking compressed batches only. We will not be enabling chunking across all types of ops/batches but **only when compression is enabled and when the batch is compressed**, and its payload size is more than `maxBatchSizeInBytes`. Therefore, for this feature to be working, it is required that compression is enabled using `IContainerRuntimeOptions.compressionOptions`.
+
+It is recommended to also change the `maxBatchSizeInBytes` property in `IContainerRuntimeOptions`. By default, if unspecified it is 972800. We recommend a lower value such as `716800`, to account for any overhead on the server side. The reason for this is that chunking will only kick in after this configuration limit is exceeded. If the limit is too high, we might allow batches which are under 1MB but can increase in size due to overhead after they reach the server.
+
+## Disabling in case of emergency
+
+If the features are enabled using the configs, they can be disabled at runtime via feature gates as following:
+
+-   `Fluid.ContainerRuntime.DisableCompression` - if set to true, will disable compression (this has a side effect of also disabling chunking, as chunking is invoked only for compressed payloads).
+-   `Fluid.ContainerRuntime.DisableCompressionChunking` - if set to true, will disable chunking for compression.
+
+## Example configs
+
+Enable only compression:
+
+```
+    const runtimeOptions: IContainerRuntimeOptions = {
+        compressionOptions: {
+            minimumBatchSizeInBytes: 614400,
+            compressionAlgorithm: CompressionAlgorithms.lz4,
+        },
+        maxBatchSizeInBytes: 716800,
+    }
+```
+
+Enable compression and chunking:
+
+```
+    const runtimeOptions: IContainerRuntimeOptions = {
+        compressionOptions: {
+            minimumBatchSizeInBytes: 614400,
+            compressionAlgorithm: CompressionAlgorithms.lz4,
+        },
+        chunkSizeInBytes: 614400,
+        maxBatchSizeInBytes: 716800,
+    }
+```
+
+## How it works
+
+Compression currently works as a runtime layer over the regular op sending/receiving pipeline.
+
+If we have a batch with a size larger than the configured minimum required for compression (in the example let’s say it’s 850 bytes), as following:
+
+```
++-----------+-----------+-----------+-----------+
+| Op 1      | Op 2      | Op 3      | Op 4      |
+| SeqNum: 1 | SeqNum: 2 | SeqNum: 3 | SeqNum: 4 |
+| Size: 100 | Size: 150 | Size: 200 | Size: 400 |
++-----------+-----------+-----------+-----------+
+```
+
+The total size of the batch is 850 bytes. The client which needs to send the batch would compress the batch to a smaller size (200 bytes) and will send a new batch like the following:
+
+```
++--------------------+-----------+-----------+-----------+
+| Op 1               | Op 2      | Op 3      | Op 4      |
+| SeqNum: 1          | SeqNum: 2 | SeqNum: 3 | SeqNum: 4 |
+| Size: 200          | Size: 0   | Size: 0   | Size: 0   |
+| Compression: 'lz4' |           |           |           |
++--------------------+-----------+-----------+-----------+
+```
+
+The first op in the batch is the only one with content (which is opaque due to it being compressed), the rest of the ops serve only to reserve the sequence numbers so that the state machine which rebuilds the original batch on the receiving client can reconstruct the original batch.
+
+When the batch is received by a client, it will detect the first op as being compressed, it will decompress it and store it in memory. For each empty op subsequently received, it will fetch the uncompressed content from memory and rebuild the original ops. The original ops are then processed by the runtime and applied accordingly.
+So, compression virtualizes the batch.
+
+After compression, the first op in the batch can exceed 1MB, therefore it would still be rejected. In this case, another layer of virtualization is added after compression (and before decompression, symmetrically on the receiving end).
+
+The first op in the compressed batch can be chunked into smaller ops which can be sent outside the original batch. However, to conveniently maintain the batch semantics, the last chunk (the chunk which triggers rebuilding the original op) is the first op in the new batch.
+
+To illustrate, let’s take the large batch below:
+
+```
++--------------------+-----------+-----------+-----------+
+| Op 1               | Op 2      | Op 3      | Op 4      |
+| SeqNum: 1          | SeqNum: 2 | SeqNum: 3 | SeqNum: 4 |
+| Size: 900          | Size: 0   | Size: 0   | Size: 0   |
+| Compression: 'lz4' |           |           |           |
++--------------------+-----------+-----------+-----------+
+```
+
+This will produce the following batches:
+
+```
++-----------+
+| Chunk 1/3 |
+| SeqNum: 1 |
+| Size: 300 |
++-----------+
+
+```
+
+```
++-----------+
+| Chunk 2/3 |
+| SeqNum: 2 |
+| Size: 300 |
++-----------+
+
+```
+
+```
++-----------+-----------+-----------+-----------+
+| Chunk 3/3 | Op 2      | Op 3      | Op 4      |
+| SeqNum: 3 | SeqNum: 4 | SeqNum: 5 | SeqNum: 6 |
+| Size: 300 | Size: 0   | Size: 0   | Size: 0   |
++-----------+-----------+-----------+-----------+
+```
+
+The first 2 chunks are sent in their own batches, while the last chunk is the first op in the last batch which contains the ops reserving the required sequence numbers.
+
+Notice that the sequence numbers don’t matter here, as all ops will be based off the same reference sequence number, so the sequence number will be recalculated for all, without additional work.
+
+Additionally, as compression preserves the original uncompressed batch layout in terms of the number of ops by using empty ops to reserve the sequence numbers, this ensures that the clients will always receive the exact count of ops to rebuild the uncompressed batch sequentially.
+
+On the receiving end, the client will accumulate chunks 1 and 2 and keep them in memory. When chunk 3 is received, the original large, decompressed op will be rebuilt, and the runtime will then process the batch as if it is a compressed batch.