@fluidframework/container-runtime 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.225277

package/src/opLifecycle/opCompressor.ts

@@ -0,0 +1,99 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { createChildLogger, UsageError } from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer } from "@fluid-internal/client-utils";
+import { compress } from "lz4js";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { CompressionAlgorithms } from "../containerRuntime";
+import { estimateSocketSize } from "./batchManager";
+import { IBatch, BatchMessage } from "./definitions";
+
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+export class OpCompressor {
+	private readonly logger;
+
+	constructor(logger: ITelemetryBaseLogger) {
+		this.logger = createChildLogger({ logger, namespace: "OpCompressor" });
+	}
+
+	public compressBatch(batch: IBatch): IBatch {
+		assert(
+			batch.contentSizeInBytes > 0 && batch.content.length > 0,
+			0x5a4 /* Batch should not be empty */,
+		);
+
+		const compressionStart = Date.now();
+		const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
+		const compressedContents = compress(contentsAsBuffer);
+		const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
+		const duration = Date.now() - compressionStart;
+
+		const messages: BatchMessage[] = [];
+		messages.push({
+			...batch.content[0],
+			contents: JSON.stringify({ packedContents: compressedContent }),
+			metadata: batch.content[0].metadata,
+			compression: CompressionAlgorithms.lz4,
+		});
+
+		// Add empty placeholder messages to reserve the sequence numbers
+		for (const message of batch.content.slice(1)) {
+			messages.push({
+				type: message.type,
+				localOpMetadata: message.localOpMetadata,
+				metadata: message.metadata,
+				referenceSequenceNumber: message.referenceSequenceNumber,
+			});
+		}
+
+		const compressedBatch: IBatch = {
+			contentSizeInBytes: compressedContent.length,
+			content: messages,
+			referenceSequenceNumber: batch.referenceSequenceNumber,
+		};
+
+		if (batch.contentSizeInBytes > 200000) {
+			this.logger.sendPerformanceEvent({
+				eventName: "CompressedBatch",
+				duration,
+				sizeBeforeCompression: batch.contentSizeInBytes,
+				sizeAfterCompression: compressedBatch.contentSizeInBytes,
+				opCount: compressedBatch.content.length,
+				socketSize: estimateSocketSize(compressedBatch),
+			});
+		}
+
+		return compressedBatch;
+	}
+
+	private serializeBatch(batch: IBatch): string {
+		try {
+			return `[${batch.content.map((message) => message.contents).join(",")}]`;
+		} catch (e: any) {
+			if (e.message === "Invalid string length") {
+				// This is how JSON.stringify signals that
+				// the content size exceeds its capacity
+				const error = new UsageError("Payload too large");
+				this.logger.sendErrorEvent(
+					{
+						eventName: "BatchTooLarge",
+						size: batch.contentSizeInBytes,
+						length: batch.content.length,
+					},
+					error,
+				);
+				throw error;
+			}
+
+			throw e;
+		}
+	}
+}

package/src/opLifecycle/opDecompressor.ts

@@ -0,0 +1,190 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { decompress } from "lz4js";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer, Uint8ArrayToString } from "@fluid-internal/client-utils";
+import { createChildLogger } from "@fluidframework/telemetry-utils";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { CompressionAlgorithms } from "../containerRuntime";
+import { IBatchMetadata } from "../metadata";
+import { IMessageProcessingResult } from "./definitions";
+
+/**
+ * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
+ */
+interface IPackedContentsContents {
+	packedContents: string;
+}
+
+/**
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
+ * This class relies on some implicit contracts defined below:
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
+ * 3. The final message of a batch will have batch metadata set to false
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ */
+export class OpDecompressor {
+	private activeBatch = false;
+	private rootMessageContents: any | undefined;
+	private processedCount = 0;
+	private readonly logger;
+
+	constructor(logger: ITelemetryBaseLogger) {
+		this.logger = createChildLogger({ logger, namespace: "OpDecompressor" });
+	}
+
+	public processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
+		assert(
+			message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
+			0x511 /* Only lz4 compression is supported */,
+		);
+
+		if (
+			(message.metadata as IBatchMetadata | undefined)?.batch === true &&
+			this.isCompressed(message)
+		) {
+			// Beginning of a compressed batch
+			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+			this.activeBatch = true;
+
+			const contents = IsoBuffer.from(
+				(message.contents as IPackedContentsContents).packedContents,
+				"base64",
+			);
+			const decompressedMessage = decompress(contents);
+			const intoString = Uint8ArrayToString(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+			this.rootMessageContents = asObj;
+
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (
+			this.rootMessageContents !== undefined &&
+			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
+			this.activeBatch
+		) {
+			assert(message.contents === undefined, 0x512 /* Expecting empty message */);
+
+			// Continuation of compressed batch
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (
+			this.rootMessageContents !== undefined &&
+			(message.metadata as IBatchMetadata | undefined)?.batch === false
+		) {
+			// End of compressed batch
+			const returnMessage = newMessage(
+				message,
+				this.rootMessageContents[this.processedCount++],
+			);
+
+			this.activeBatch = false;
+			this.rootMessageContents = undefined;
+			this.processedCount = 0;
+
+			return {
+				message: returnMessage,
+				state: "Processed",
+			};
+		}
+
+		if (
+			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
+			this.isCompressed(message)
+		) {
+			// Single compressed message
+			assert(
+				this.activeBatch === false,
+				0x4ba /* shouldn't receive compressed message in middle of a batch */,
+			);
+
+			const contents = IsoBuffer.from(
+				(message.contents as IPackedContentsContents).packedContents,
+				"base64",
+			);
+			const decompressedMessage = decompress(contents);
+			const intoString = new TextDecoder().decode(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+
+			return {
+				message: newMessage(message, asObj[0]),
+				state: "Processed",
+			};
+		}
+
+		return {
+			message,
+			state: "Skipped",
+		};
+	}
+
+	private isCompressed(message: ISequencedDocumentMessage) {
+		if (message.compression === CompressionAlgorithms.lz4) {
+			return true;
+		}
+
+		/**
+		 * Back-compat self healing mechanism for ADO:3538, as loaders from
+		 * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not
+		 * support adding the proper compression metadata to compressed messages submitted
+		 * by the runtime. Should be removed after the loader reaches sufficient saturation
+		 * for a version greater or equal than client_v2.0.0-internal.2.2.0.
+		 *
+		 * The condition holds true for compressed messages, regardless of metadata. We are ultimately
+		 * looking for a message with a single property `packedContents` inside `contents`, of type 'string'
+		 * with a base64 encoded value.
+		 */
+		try {
+			if (
+				message.contents !== null &&
+				typeof message.contents === "object" &&
+				Object.keys(message.contents).length === 1 &&
+				typeof (message.contents as { packedContents?: unknown }).packedContents ===
+					"string" &&
+				(message.contents as IPackedContentsContents).packedContents.length > 0 &&
+				IsoBuffer.from(
+					(message.contents as IPackedContentsContents).packedContents,
+					"base64",
+				).toString("base64") ===
+					(message.contents as IPackedContentsContents).packedContents
+			) {
+				this.logger.sendTelemetryEvent({
+					eventName: "LegacyCompression",
+					type: message.type,
+					batch: (message.metadata as IBatchMetadata | undefined)?.batch,
+				});
+				return true;
+			}
+		} catch (err) {
+			return false;
+		}
+
+		return false;
+	}
+}
+
+// We should not be mutating the input message nor its metadata
+const newMessage = (
+	originalMessage: ISequencedDocumentMessage,
+	contents: any,
+): ISequencedDocumentMessage => ({
+	...originalMessage,
+	contents,
+	compression: undefined,
+	// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+	metadata: { ...(originalMessage.metadata as any) },
+});

package/src/opLifecycle/opGroupingManager.ts

@@ -0,0 +1,133 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { createChildLogger } from "@fluidframework/telemetry-utils";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { ContainerMessageType } from "../messageTypes";
+import { IBatch } from "./definitions";
+
+/**
+ * Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
+ */
+interface IGroupedBatchMessageContents {
+	type: typeof OpGroupingManager.groupedBatchOp;
+	contents: IGroupedMessage[];
+}
+
+interface IGroupedMessage {
+	contents?: unknown;
+	metadata?: Record<string, unknown>;
+	compression?: string;
+}
+
+function isGroupContents(opContents: any): opContents is IGroupedBatchMessageContents {
+	return opContents?.type === OpGroupingManager.groupedBatchOp;
+}
+
+export interface OpGroupingManagerConfig {
+	readonly groupedBatchingEnabled: boolean;
+	readonly opCountThreshold: number;
+	readonly reentrantBatchGroupingEnabled: boolean;
+}
+
+export class OpGroupingManager {
+	static readonly groupedBatchOp = "groupedBatch";
+	private readonly logger;
+
+	constructor(
+		private readonly config: OpGroupingManagerConfig,
+		logger: ITelemetryBaseLogger,
+	) {
+		this.logger = createChildLogger({ logger, namespace: "OpGroupingManager" });
+	}
+
+	public groupBatch(batch: IBatch): IBatch {
+		if (!this.shouldGroup(batch)) {
+			return batch;
+		}
+
+		if (batch.content.length >= 1000) {
+			this.logger.sendTelemetryEvent({
+				eventName: "GroupLargeBatch",
+				length: batch.content.length,
+				threshold: this.config.opCountThreshold,
+				reentrant: batch.hasReentrantOps,
+				referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
+			});
+		}
+
+		for (const message of batch.content) {
+			if (message.metadata) {
+				const keys = Object.keys(message.metadata);
+				assert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
+				assert(
+					keys.length === 0 || keys[0] === "batch",
+					0x5de /* unexpected op metadata */,
+				);
+			}
+		}
+
+		const serializedContent = JSON.stringify({
+			type: OpGroupingManager.groupedBatchOp,
+			contents: batch.content.map<IGroupedMessage>((message) => ({
+				contents: message.contents === undefined ? undefined : JSON.parse(message.contents),
+				metadata: message.metadata,
+				compression: message.compression,
+			})),
+		});
+
+		const groupedBatch: IBatch = {
+			...batch,
+			content: [
+				{
+					localOpMetadata: undefined,
+					metadata: undefined,
+					referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
+					contents: serializedContent,
+					type: OpGroupingManager.groupedBatchOp as ContainerMessageType,
+				},
+			],
+		};
+		return groupedBatch;
+	}
+
+	public ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
+		let fakeCsn = 1;
+		if (!isGroupContents(op.contents)) {
+			// Align the worlds of what clientSequenceNumber represents when grouped batching is enabled
+			if (this.config.groupedBatchingEnabled) {
+				return [
+					{
+						...op,
+						clientSequenceNumber: fakeCsn,
+					},
+				];
+			}
+			return [op];
+		}
+
+		const messages = op.contents.contents;
+		return messages.map((subMessage) => ({
+			...op,
+			clientSequenceNumber: fakeCsn++,
+			contents: subMessage.contents,
+			metadata: subMessage.metadata,
+			compression: subMessage.compression,
+		}));
+	}
+
+	public shouldGroup(batch: IBatch): boolean {
+		return (
+			// Grouped batching must be enabled
+			this.config.groupedBatchingEnabled &&
+			// The number of ops in the batch must surpass the configured threshold
+			batch.content.length >= this.config.opCountThreshold &&
+			// Support for reentrant batches must be explicitly enabled
+			(this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true)
+		);
+	}
+}

package/src/opLifecycle/opSplitter.ts

@@ -0,0 +1,279 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	createChildLogger,
+	DataCorruptionError,
+	extractSafePropertiesFromMessage,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IBatchMessage } from "@fluidframework/container-definitions";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { ContainerMessageType, ContainerRuntimeChunkedOpMessage } from "../messageTypes";
+import { estimateSocketSize } from "./batchManager";
+import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
+
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+export class OpSplitter {
+	// Local copy of incomplete received chunks.
+	private readonly chunkMap: Map<string, string[]>;
+	private readonly logger;
+
+	constructor(
+		chunks: [string, string[]][],
+		private readonly submitBatchFn:
+			| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)
+			| undefined,
+		public readonly chunkSizeInBytes: number,
+		private readonly maxBatchSizeInBytes: number,
+		logger: ITelemetryBaseLogger,
+	) {
+		this.chunkMap = new Map<string, string[]>(chunks);
+		this.logger = createChildLogger({ logger, namespace: "OpSplitter" });
+	}
+
+	public get isBatchChunkingEnabled(): boolean {
+		return this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;
+	}
+
+	public get chunks(): ReadonlyMap<string, string[]> {
+		return this.chunkMap;
+	}
+
+	public processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
+		if (message.type !== ContainerMessageType.ChunkedOp) {
+			return {
+				message,
+				state: "Skipped",
+			};
+		}
+
+		// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
+		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+		const clientId = message.clientId as string;
+		const chunkedContent = message.contents as IChunkedOp;
+		this.addChunk(clientId, chunkedContent, message);
+
+		if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+			// We are processing the op in chunks but haven't reached
+			// the last chunk yet in order to reconstruct the original op
+			return {
+				message,
+				state: "Accepted",
+			};
+		}
+
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		const serializedContent = this.chunkMap.get(clientId)!.join("");
+		this.clearPartialChunks(clientId);
+
+		const newMessage = { ...message };
+		newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+		newMessage.type = chunkedContent.originalType;
+		newMessage.metadata = chunkedContent.originalMetadata;
+		newMessage.compression = chunkedContent.originalCompression;
+		return {
+			message: newMessage,
+			state: "Processed",
+		};
+	}
+
+	public clearPartialChunks(clientId: string) {
+		if (this.chunkMap.has(clientId)) {
+			this.chunkMap.delete(clientId);
+		}
+	}
+
+	private addChunk(
+		clientId: string,
+		chunkedContent: IChunkedOp,
+		originalMessage: ISequencedDocumentMessage,
+	) {
+		let map = this.chunkMap.get(clientId);
+		if (map === undefined) {
+			map = [];
+			this.chunkMap.set(clientId, map);
+		}
+
+		if (chunkedContent.chunkId !== map.length + 1) {
+			// We are expecting the chunks to be processed sequentially, in the same order as they are sent.
+			// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
+			// holding the existing chunks for that particular clientId.
+			throw new DataCorruptionError("Chunk Id mismatch", {
+				...extractSafePropertiesFromMessage(originalMessage),
+				chunkMapLength: map.length,
+				chunkId: chunkedContent.chunkId,
+				totalChunks: chunkedContent.totalChunks,
+			});
+		}
+
+		map.push(chunkedContent.contents);
+	}
+
+	/**
+	 * Splits the first op of a compressed batch in chunks, sends the chunks separately and
+	 * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
+	 *
+	 * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
+	 * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
+	 * uncompressed ops at ingestion in the runtime.
+	 *
+	 * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+	 * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
+	 *
+	 * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
+	 * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
+	 * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
+	 * (as it is the last chunk).
+	 *
+	 * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+	 * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
+	 *
+	 * @param batch - the compressed batch which needs to be processed
+	 * @returns A new adjusted batch which can be sent over the wire
+	 */
+	public splitFirstBatchMessage(batch: IBatch): IBatch {
+		assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
+		assert(
+			batch.contentSizeInBytes > 0 && batch.content.length > 0,
+			0x514 /* Batch needs to be non-empty */,
+		);
+		assert(
+			batch.referenceSequenceNumber !== undefined,
+			0x58a /* Batch must have a reference sequence number if non-empty */,
+		);
+		assert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
+		assert(
+			this.chunkSizeInBytes < this.maxBatchSizeInBytes,
+			0x516 /* Chunk size needs to be smaller than the max batch size */,
+		);
+
+		const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
+		assert(
+			(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,
+			0x518 /* First message in the batch needs to be chunkable */,
+		);
+
+		const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
+		const socketSize = estimateSocketSize(batch);
+		const chunks = splitOp(
+			firstMessage,
+			this.chunkSizeInBytes,
+			// If we estimate that the socket batch size will exceed the batch limit
+			// we will inject an empty op to minimize the risk of the payload failing due to
+			// the overhead from the trailing empty ops in the batch.
+			socketSize >= this.maxBatchSizeInBytes,
+		);
+
+		assert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
+		// Send the first N-1 chunks immediately
+		for (const chunk of chunks.slice(0, -1)) {
+			this.submitBatchFn(
+				[chunkToBatchMessage(chunk, batch.referenceSequenceNumber)],
+				batch.referenceSequenceNumber,
+			);
+		}
+
+		// The last chunk will be part of the new batch and needs to
+		// preserve the batch metadata of the original batch
+		const lastChunk = chunkToBatchMessage(
+			chunks[chunks.length - 1],
+			batch.referenceSequenceNumber,
+			{ batch: firstMessage.metadata?.batch },
+		);
+
+		this.logger.sendPerformanceEvent({
+			// Used to be "Chunked compressed batch"
+			eventName: "CompressedChunkedBatch",
+			length: batch.content.length,
+			sizeInBytes: batch.contentSizeInBytes,
+			chunks: chunks.length,
+			chunkSizeInBytes: this.chunkSizeInBytes,
+			socketSize,
+		});
+
+		return {
+			content: [lastChunk, ...restOfMessages],
+			contentSizeInBytes: lastChunk.contents?.length ?? 0,
+			referenceSequenceNumber: batch.referenceSequenceNumber,
+		};
+	}
+}
+
+const chunkToBatchMessage = (
+	chunk: IChunkedOp,
+	referenceSequenceNumber: number,
+	metadata: Record<string, unknown> | undefined = undefined,
+): BatchMessage => {
+	const payload: ContainerRuntimeChunkedOpMessage = {
+		type: ContainerMessageType.ChunkedOp,
+		contents: chunk,
+	};
+	return {
+		contents: JSON.stringify(payload),
+		type: payload.type,
+		metadata,
+		localOpMetadata: undefined,
+		referenceSequenceNumber,
+	};
+};
+
+/**
+ * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.
+ *
+ * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload
+ * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.
+ * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.
+ *
+ * @param op - the op to be split
+ * @param chunkSizeInBytes - how large should the chunks be
+ * @param extraOp - should an extra empty op be added to the result
+ * @returns an array of chunked ops
+ */
+export const splitOp = (
+	op: BatchMessage,
+	chunkSizeInBytes: number,
+	extraOp: boolean = false,
+): IChunkedOp[] => {
+	const chunks: IChunkedOp[] = [];
+	assert(
+		op.contents !== undefined && op.contents !== null,
+		0x51a /* We should have something to chunk */,
+	);
+
+	const contentLength = op.contents.length;
+	const chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);
+	let offset = 0;
+	for (let chunkId = 1; chunkId <= chunkCount; chunkId++) {
+		const chunk: IChunkedOp = {
+			chunkId,
+			contents: op.contents.substr(offset, chunkSizeInBytes),
+			originalType: op.type,
+			totalChunks: chunkCount,
+		};
+
+		if (chunkId === chunkCount) {
+			// We don't need to port these to all the chunks,
+			// as we rebuild the original op when we process the
+			// last chunk, therefore it is the only one that needs it.
+			chunk.originalMetadata = op.metadata;
+			chunk.originalCompression = op.compression;
+		}
+
+		chunks.push(chunk);
+		offset += chunkSizeInBytes;
+		assert(
+			chunkId >= chunkCount - 1 || offset <= contentLength,
+			0x58b /* Content offset within bounds */,
+		);
+	}
+
+	assert(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);
+	assert(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);
+	return chunks;
+};