@fluidframework/container-runtime 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.224419

package/src/opLifecycle/outbox.ts

@@ -0,0 +1,471 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	createChildMonitoringContext,
+	GenericError,
+	MonitoringContext,
+	UsageError,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IBatchMessage, ICriticalContainerError } from "@fluidframework/container-definitions";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { ICompressionRuntimeOptions } from "../containerRuntime";
+import { IPendingBatchMessage, PendingStateManager } from "../pendingStateManager";
+import {
+	BatchManager,
+	BatchSequenceNumbers,
+	estimateSocketSize,
+	sequenceNumbersMatch,
+} from "./batchManager";
+import { BatchMessage, IBatch } from "./definitions";
+import { OpCompressor } from "./opCompressor";
+import { OpGroupingManager } from "./opGroupingManager";
+import { OpSplitter } from "./opSplitter";
+
+export interface IOutboxConfig {
+	readonly compressionOptions: ICompressionRuntimeOptions;
+	// The maximum size of a batch that we can send over the wire.
+	readonly maxBatchSizeInBytes: number;
+	readonly disablePartialFlush: boolean;
+}
+
+export interface IOutboxParameters {
+	readonly shouldSend: () => boolean;
+	readonly pendingStateManager: PendingStateManager;
+	readonly submitBatchFn:
+		| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)
+		| undefined;
+	readonly legacySendBatchFn: (batch: IBatch) => void;
+	readonly config: IOutboxConfig;
+	readonly compressor: OpCompressor;
+	readonly splitter: OpSplitter;
+	readonly logger: ITelemetryBaseLogger;
+	readonly groupingManager: OpGroupingManager;
+	readonly getCurrentSequenceNumbers: () => BatchSequenceNumbers;
+	readonly reSubmit: (message: IPendingBatchMessage) => void;
+	readonly opReentrancy: () => boolean;
+	readonly closeContainer: (error?: ICriticalContainerError) => void;
+}
+
+/**
+ * Temporarily increase the stack limit while executing the provided action.
+ * If a negative value is provided for `length`, no stack frames will be collected.
+ * If Infinity is provided, all frames will be collected.
+ *
+ * ADO:4663 - add this to the common packages.
+ *
+ * @param action - action which returns an error
+ * @param length - number of stack frames to collect, 50 if unspecified.
+ * @returns the result of the action provided
+ */
+export function getLongStack<T>(action: () => T, length: number = 50): T {
+	const errorObj = Error as any;
+	if (
+		/* eslint-disable @typescript-eslint/prefer-nullish-coalescing */
+		// ?? is not logically equivalent when the first clause returns false.
+		(
+			Object.getOwnPropertyDescriptor(errorObj, "stackTraceLimit") ||
+			Object.getOwnPropertyDescriptor(Object.getPrototypeOf(errorObj), "stackTraceLimit")
+		)?.writable !== true
+		/* eslint-enable @typescript-eslint/prefer-nullish-coalescing */
+	) {
+		return action();
+	}
+
+	const originalStackTraceLimit = errorObj.stackTraceLimit;
+	try {
+		errorObj.stackTraceLimit = length;
+		return action();
+	} finally {
+		errorObj.stackTraceLimit = originalStackTraceLimit;
+	}
+}
+
+export class Outbox {
+	private readonly mc: MonitoringContext;
+	private readonly attachFlowBatch: BatchManager;
+	private readonly mainBatch: BatchManager;
+	private readonly blobAttachBatch: BatchManager;
+	private readonly idAllocationBatch: BatchManager;
+	private readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;
+	private batchRebasesToReport = 5;
+	private rebasing = false;
+
+	/**
+	 * Track the number of ops which were detected to have a mismatched
+	 * reference sequence number, in order to self-throttle the telemetry events.
+	 *
+	 * This should be removed as part of ADO:2322
+	 */
+	private readonly maxMismatchedOpsToReport = 3;
+	private mismatchedOpsReported = 0;
+
+	constructor(private readonly params: IOutboxParameters) {
+		this.mc = createChildMonitoringContext({ logger: params.logger, namespace: "Outbox" });
+		const isCompressionEnabled =
+			this.params.config.compressionOptions.minimumBatchSizeInBytes !==
+			Number.POSITIVE_INFINITY;
+		// We need to allow infinite size batches if we enable compression
+		const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
+		const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
+
+		this.attachFlowBatch = new BatchManager({ hardLimit, softLimit });
+		this.mainBatch = new BatchManager({ hardLimit });
+		this.blobAttachBatch = new BatchManager({ hardLimit });
+		this.idAllocationBatch = new BatchManager({ hardLimit });
+	}
+
+	public get messageCount(): number {
+		return this.attachFlowBatch.length + this.mainBatch.length + this.blobAttachBatch.length;
+	}
+
+	public get isEmpty(): boolean {
+		return this.messageCount === 0;
+	}
+
+	/**
+	 * If we detect that the reference sequence number of the incoming message does not match
+	 * what was already in the batch managers, this means that batching has been interrupted so
+	 * we will flush the accumulated messages to account for that and create a new batch with the new
+	 * message as the first message.
+	 */
+	private maybeFlushPartialBatch() {
+		const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
+		const attachFlowBatchSeqNums = this.attachFlowBatch.sequenceNumbers;
+		const blobAttachSeqNums = this.blobAttachBatch.sequenceNumbers;
+		assert(
+			this.params.config.disablePartialFlush ||
+				(sequenceNumbersMatch(mainBatchSeqNums, attachFlowBatchSeqNums) &&
+					sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums)),
+			0x58d /* Reference sequence numbers from both batches must be in sync */,
+		);
+
+		const currentSequenceNumbers = this.params.getCurrentSequenceNumbers();
+
+		if (
+			sequenceNumbersMatch(mainBatchSeqNums, currentSequenceNumbers) &&
+			sequenceNumbersMatch(attachFlowBatchSeqNums, currentSequenceNumbers) &&
+			sequenceNumbersMatch(blobAttachSeqNums, currentSequenceNumbers)
+		) {
+			// The reference sequence numbers are stable, there is nothing to do
+			return;
+		}
+
+		if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
+			this.mc.logger.sendTelemetryEvent(
+				{
+					category: this.params.config.disablePartialFlush ? "error" : "generic",
+					eventName: "ReferenceSequenceNumberMismatch",
+					mainReferenceSequenceNumber: mainBatchSeqNums.referenceSequenceNumber,
+					mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
+					attachReferenceSequenceNumber: attachFlowBatchSeqNums.referenceSequenceNumber,
+					attachClientSequenceNumber: attachFlowBatchSeqNums.clientSequenceNumber,
+					blobAttachReferenceSequenceNumber: blobAttachSeqNums.referenceSequenceNumber,
+					blobAttachClientSequenceNumber: blobAttachSeqNums.clientSequenceNumber,
+					currentReferenceSequenceNumber: currentSequenceNumbers.referenceSequenceNumber,
+					currentClientSequenceNumber: currentSequenceNumbers.clientSequenceNumber,
+				},
+				getLongStack(() => new UsageError("Submission of an out of order message")),
+			);
+		}
+
+		if (!this.params.config.disablePartialFlush) {
+			this.flushAll();
+		}
+	}
+
+	public submit(message: BatchMessage) {
+		this.maybeFlushPartialBatch();
+
+		this.addMessageToBatchManager(this.mainBatch, message);
+	}
+
+	public submitAttach(message: BatchMessage) {
+		this.maybeFlushPartialBatch();
+
+		if (
+			!this.attachFlowBatch.push(
+				message,
+				this.isContextReentrant(),
+				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
+			)
+		) {
+			// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
+			// when queue is not empty.
+			// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
+			this.flushInternal(this.attachFlowBatch);
+
+			this.addMessageToBatchManager(this.attachFlowBatch, message);
+		}
+
+		// If compression is enabled, we will always successfully receive
+		// attach ops and compress then send them at the next JS turn, regardless
+		// of the overall size of the accumulated ops in the batch.
+		// However, it is more efficient to flush these ops faster, preferably
+		// after they reach a size which would benefit from compression.
+		if (
+			this.attachFlowBatch.contentSizeInBytes >=
+			this.params.config.compressionOptions.minimumBatchSizeInBytes
+		) {
+			this.flushInternal(this.attachFlowBatch);
+		}
+	}
+
+	public submitBlobAttach(message: BatchMessage) {
+		this.maybeFlushPartialBatch();
+
+		this.addMessageToBatchManager(this.blobAttachBatch, message);
+
+		// If compression is enabled, we will always successfully receive
+		// blobAttach ops and compress then send them at the next JS turn, regardless
+		// of the overall size of the accumulated ops in the batch.
+		// However, it is more efficient to flush these ops faster, preferably
+		// after they reach a size which would benefit from compression.
+		if (
+			this.blobAttachBatch.contentSizeInBytes >=
+			this.params.config.compressionOptions.minimumBatchSizeInBytes
+		) {
+			this.flushInternal(this.blobAttachBatch);
+		}
+	}
+
+	public submitIdAllocation(message: BatchMessage) {
+		this.maybeFlushPartialBatch();
+
+		if (
+			!this.idAllocationBatch.push(
+				message,
+				this.isContextReentrant(),
+				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
+			)
+		) {
+			// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
+			// when queue is not empty.
+			// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
+			this.flushInternal(this.idAllocationBatch);
+
+			this.addMessageToBatchManager(this.idAllocationBatch, message);
+		}
+
+		// If compression is enabled, we will always successfully receive
+		// attach ops and compress then send them at the next JS turn, regardless
+		// of the overall size of the accumulated ops in the batch.
+		// However, it is more efficient to flush these ops faster, preferably
+		// after they reach a size which would benefit from compression.
+		if (
+			this.idAllocationBatch.contentSizeInBytes >=
+			this.params.config.compressionOptions.minimumBatchSizeInBytes
+		) {
+			this.flushInternal(this.idAllocationBatch);
+		}
+	}
+
+	private addMessageToBatchManager(batchManager: BatchManager, message: BatchMessage) {
+		if (
+			!batchManager.push(
+				message,
+				this.isContextReentrant(),
+				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
+			)
+		) {
+			throw new GenericError("BatchTooLarge", /* error */ undefined, {
+				opSize: message.contents?.length ?? 0,
+				batchSize: batchManager.contentSizeInBytes,
+				count: batchManager.length,
+				limit: batchManager.options.hardLimit,
+			});
+		}
+	}
+
+	public flush() {
+		if (this.isContextReentrant()) {
+			const error = new UsageError("Flushing is not supported inside DDS event handlers");
+			this.params.closeContainer(error);
+			throw error;
+		}
+
+		this.flushAll();
+	}
+
+	private flushAll() {
+		this.flushInternal(this.idAllocationBatch);
+		this.flushInternal(this.attachFlowBatch);
+		this.flushInternal(this.blobAttachBatch, true /* disableGroupedBatching */);
+		this.flushInternal(this.mainBatch);
+	}
+
+	private flushInternal(batchManager: BatchManager, disableGroupedBatching: boolean = false) {
+		if (batchManager.empty) {
+			return;
+		}
+
+		const rawBatch = batchManager.popBatch();
+		if (
+			rawBatch.hasReentrantOps === true &&
+			this.params.groupingManager.shouldGroup(rawBatch)
+		) {
+			assert(!this.rebasing, 0x6fa /* A rebased batch should never have reentrant ops */);
+			// If a batch contains reentrant ops (ops created as a result from processing another op)
+			// it needs to be rebased so that we can ensure consistent reference sequence numbers
+			// and eventual consistency at the DDS level.
+			this.rebase(rawBatch, batchManager);
+			return;
+		}
+
+		const processedBatch = this.compressBatch(rawBatch, disableGroupedBatching);
+		this.sendBatch(processedBatch);
+
+		this.persistBatch(rawBatch.content);
+	}
+
+	/**
+	 * Rebases a batch. All the ops in the batch are resubmitted to the runtime and
+	 * they will end up back in the same batch manager they were flushed from and subsequently flushed.
+	 *
+	 * @param rawBatch - the batch to be rebased
+	 */
+	private rebase(rawBatch: IBatch, batchManager: BatchManager) {
+		assert(!this.rebasing, 0x6fb /* Reentrancy */);
+
+		this.rebasing = true;
+		for (const message of rawBatch.content) {
+			this.params.reSubmit({
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				content: message.contents!,
+				localOpMetadata: message.localOpMetadata,
+				opMetadata: message.metadata,
+			});
+		}
+
+		if (this.batchRebasesToReport > 0) {
+			this.mc.logger.sendTelemetryEvent(
+				{
+					eventName: "BatchRebase",
+					length: rawBatch.content.length,
+					referenceSequenceNumber: rawBatch.referenceSequenceNumber,
+				},
+				new UsageError("BatchRebase"),
+			);
+			this.batchRebasesToReport--;
+		}
+
+		this.flushInternal(batchManager);
+		this.rebasing = false;
+	}
+
+	private isContextReentrant(): boolean {
+		return this.params.opReentrancy() && !this.rebasing;
+	}
+
+	private compressBatch(batch: IBatch, disableGroupedBatching: boolean): IBatch {
+		if (
+			batch.content.length === 0 ||
+			this.params.config.compressionOptions === undefined ||
+			this.params.config.compressionOptions.minimumBatchSizeInBytes >
+				batch.contentSizeInBytes ||
+			this.params.submitBatchFn === undefined
+		) {
+			// Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
+			return disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch);
+		}
+
+		const compressedBatch = this.params.compressor.compressBatch(
+			disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch),
+		);
+
+		if (this.params.splitter.isBatchChunkingEnabled) {
+			return compressedBatch.contentSizeInBytes <= this.params.splitter.chunkSizeInBytes
+				? compressedBatch
+				: this.params.splitter.splitFirstBatchMessage(compressedBatch);
+		}
+
+		if (compressedBatch.contentSizeInBytes >= this.params.config.maxBatchSizeInBytes) {
+			throw new GenericError("BatchTooLarge", /* error */ undefined, {
+				batchSize: batch.contentSizeInBytes,
+				compressedBatchSize: compressedBatch.contentSizeInBytes,
+				count: compressedBatch.content.length,
+				limit: this.params.config.maxBatchSizeInBytes,
+				chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
+				compressionOptions: JSON.stringify(this.params.config.compressionOptions),
+				socketSize: estimateSocketSize(batch),
+			});
+		}
+
+		return compressedBatch;
+	}
+
+	/**
+	 * Sends the batch object to the container context to be sent over the wire.
+	 *
+	 * @param batch - batch to be sent
+	 */
+	private sendBatch(batch: IBatch) {
+		const length = batch.content.length;
+
+		// Did we disconnect in the middle of turn-based batch?
+		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
+		if (length === 0 || !this.params.shouldSend()) {
+			return;
+		}
+
+		const socketSize = estimateSocketSize(batch);
+		if (socketSize >= this.params.config.maxBatchSizeInBytes) {
+			this.mc.logger.sendPerformanceEvent({
+				eventName: "LargeBatch",
+				length: batch.content.length,
+				sizeInBytes: batch.contentSizeInBytes,
+				socketSize,
+			});
+		}
+
+		if (this.params.submitBatchFn === undefined) {
+			// Legacy path - supporting old loader versions. Can be removed only when LTS moves above
+			// version that has support for batches (submitBatchFn)
+			assert(
+				batch.content[0].compression === undefined,
+				0x5a6 /* Compression should not have happened if the loader does not support it */,
+			);
+
+			this.params.legacySendBatchFn(batch);
+		} else {
+			assert(
+				batch.referenceSequenceNumber !== undefined,
+				0x58e /* Batch must not be empty */,
+			);
+			this.params.submitBatchFn(
+				batch.content.map((message) => ({
+					contents: message.contents,
+					metadata: message.metadata,
+					compression: message.compression,
+					referenceSequenceNumber: message.referenceSequenceNumber,
+				})),
+				batch.referenceSequenceNumber,
+			);
+		}
+	}
+
+	private persistBatch(batch: BatchMessage[]) {
+		// Let the PendingStateManager know that a message was submitted.
+		// In future, need to shift toward keeping batch as a whole!
+		for (const message of batch) {
+			this.params.pendingStateManager.onSubmitMessage(
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				message.contents!,
+				message.referenceSequenceNumber,
+				message.localOpMetadata,
+				message.metadata,
+			);
+		}
+	}
+
+	public checkpoint() {
+		return {
+			mainBatch: this.mainBatch.checkpoint(),
+			attachFlowBatch: this.attachFlowBatch.checkpoint(),
+			blobAttachBatch: this.blobAttachBatch.checkpoint(),
+		};
+	}
+}

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -0,0 +1,175 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
+import {
+	ContainerMessageType,
+	type InboundContainerRuntimeMessage,
+	type InboundSequencedContainerRuntimeMessage,
+	type InboundSequencedContainerRuntimeMessageOrSystemMessage,
+	type InboundSequencedRecentlyAddedContainerRuntimeMessage,
+} from "../messageTypes";
+import { OpDecompressor } from "./opDecompressor";
+import { OpGroupingManager } from "./opGroupingManager";
+import { OpSplitter } from "./opSplitter";
+
+/**
+ * Stateful class for processing incoming remote messages as the virtualization measures are unwrapped,
+ * potentially across numerous inbound ops.
+ *
+ * @internal
+ */
+export class RemoteMessageProcessor {
+	constructor(
+		private readonly opSplitter: OpSplitter,
+		private readonly opDecompressor: OpDecompressor,
+		private readonly opGroupingManager: OpGroupingManager,
+	) {}
+
+	public get partialMessages(): ReadonlyMap<string, string[]> {
+		return this.opSplitter.chunks;
+	}
+
+	public clearPartialMessagesFor(clientId: string) {
+		this.opSplitter.clearPartialChunks(clientId);
+	}
+
+	/**
+	 * Ungroups and Unchunks the runtime ops encapsulated by the single remoteMessage received over the wire
+	 * @param remoteMessageCopy - A shallow copy of a message from another client, possibly virtualized
+	 * (grouped, compressed, and/or chunked).
+	 * Being a shallow copy, it's considered mutable, meaning no other Container or other parallel procedure
+	 * depends on this object instance.
+	 * Note remoteMessageCopy.contents (and other object props) MUST not be modified,
+	 * but may be overwritten (as is the case with contents).
+	 * @returns the unchunked, decompressed, ungrouped, unpacked SequencedContainerRuntimeMessages encapsulated in the remote message.
+	 * For ops that weren't virtualized (e.g. System ops that the ContainerRuntime will ultimately ignore),
+	 * a singleton array [remoteMessageCopy] is returned
+	 */
+	public process(
+		remoteMessageCopy: ISequencedDocumentMessage,
+	): InboundSequencedContainerRuntimeMessageOrSystemMessage[] {
+		const result: InboundSequencedContainerRuntimeMessageOrSystemMessage[] = [];
+
+		ensureContentsDeserialized(remoteMessageCopy);
+
+		// Ungroup before and after decompression for back-compat (cleanup tracked by AB#4371)
+		for (const ungroupedMessage of this.opGroupingManager.ungroupOp(remoteMessageCopy)) {
+			const message = this.opDecompressor.processMessage(ungroupedMessage).message;
+
+			for (let ungroupedMessage2 of this.opGroupingManager.ungroupOp(message)) {
+				// unpack and unchunk the ungrouped message in place
+				unpackRuntimeMessage(ungroupedMessage2);
+				const chunkProcessingResult =
+					this.opSplitter.processRemoteMessage(ungroupedMessage2);
+				ungroupedMessage2 = chunkProcessingResult.message;
+
+				if (chunkProcessingResult.state !== "Processed") {
+					// If the message is not chunked or if the splitter is still rebuilding the original message,
+					// there is no need to continue processing
+					result.push(
+						ungroupedMessage2 as InboundSequencedContainerRuntimeMessageOrSystemMessage,
+					);
+					continue;
+				}
+
+				// Ungroup before and after decompression for back-compat (cleanup tracked by AB#4371)
+				for (const ungroupedMessageAfterChunking of this.opGroupingManager.ungroupOp(
+					ungroupedMessage2,
+				)) {
+					const decompressionAfterChunking = this.opDecompressor.processMessage(
+						ungroupedMessageAfterChunking,
+					);
+
+					for (const ungroupedMessageAfterChunking2 of this.opGroupingManager.ungroupOp(
+						decompressionAfterChunking.message,
+					)) {
+						if (decompressionAfterChunking.state === "Skipped") {
+							// After chunking, if the original message was not compressed,
+							// there is no need to continue processing
+							result.push(
+								ungroupedMessageAfterChunking2 as InboundSequencedContainerRuntimeMessageOrSystemMessage,
+							);
+							continue;
+						}
+
+						// The message needs to be unpacked after chunking + decompression
+						unpack(ungroupedMessageAfterChunking2);
+						result.push(ungroupedMessageAfterChunking2);
+					}
+				}
+			}
+		}
+
+		return result;
+	}
+}
+
+/** Takes an incoming message and if the contents is a string, JSON.parse's it in place */
+function ensureContentsDeserialized(mutableMessage: ISequencedDocumentMessage): void {
+	// back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
+	// System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
+	// Old ops may contain empty string (I assume noops).
+	if (typeof mutableMessage.contents === "string" && mutableMessage.contents !== "") {
+		mutableMessage.contents = JSON.parse(mutableMessage.contents);
+	}
+}
+
+/**
+ * For a given message, it moves the nested InboundContainerRuntimeMessage props one level up.
+ *
+ * The return type illustrates the assumption that the message param
+ * becomes a InboundSequencedContainerRuntimeMessage by the time the function returns
+ * (but there is no runtime validation of the 'type' or 'compatDetails' values).
+ */
+function unpack(
+	message: ISequencedDocumentMessage,
+): asserts message is InboundSequencedContainerRuntimeMessage {
+	// We assume the contents is an InboundContainerRuntimeMessage (the message is "packed")
+	const contents = message.contents as InboundContainerRuntimeMessage;
+
+	// We're going to unpack message in-place (promoting those properties of contents up to message itself)
+	const messageUnpacked = message as InboundSequencedContainerRuntimeMessage;
+
+	messageUnpacked.type = contents.type;
+	messageUnpacked.contents = contents.contents;
+	if ("compatDetails" in contents) {
+		(messageUnpacked as InboundSequencedRecentlyAddedContainerRuntimeMessage).compatDetails =
+			contents.compatDetails;
+	}
+}
+
+/**
+ * Unpacks runtime messages.
+ *
+ * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
+ * @param message - message (as it observed in storage / service)
+ * @returns whether the given message was unpacked
+ *
+ * @internal
+ */
+export function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolean {
+	if (message.type !== MessageType.Operation) {
+		// Legacy format, but it's already "unpacked",
+		// i.e. message.type is actually ContainerMessageType.
+		// Or it's non-runtime message.
+		// Nothing to do in such case.
+		return false;
+	}
+
+	// legacy op format?
+	// TODO: Unsure if this is a real format we should be concerned with. There doesn't appear to be anything prepared to handle the address member.
+	if (
+		(message.contents as { address?: unknown }).address !== undefined &&
+		(message.contents as { type?: unknown }).type === undefined
+	) {
+		message.type = ContainerMessageType.FluidDataStoreOp;
+	} else {
+		// new format
+		unpack(message);
+	}
+
+	return true;
+}

package/src/opProperties.ts

@@ -0,0 +1,21 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	ISequencedDocumentMessage,
+	ISequencedDocumentSystemMessage,
+} from "@fluidframework/protocol-definitions";
+
+export const opSize = (op: ISequencedDocumentMessage): number => {
+	// Some messages may already have string contents,
+	// so stringifying them again will add inaccurate overhead.
+	const content =
+		typeof op.contents === "string" ? op.contents : JSON.stringify(op.contents) ?? "";
+	const data = opHasData(op) ? op.data : "";
+	return content.length + data.length;
+};
+
+const opHasData = (op: ISequencedDocumentMessage): op is ISequencedDocumentSystemMessage =>
+	(op as ISequencedDocumentSystemMessage).data !== undefined;

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "1.4.0-121020";
+export const pkgVersion = "2.0.0-dev-rc.1.0.0.224419";