@fluidframework/container-runtime 2.0.0-internal.3.0.2 → 2.0.0-internal.3.2.0

package/src/opLifecycle/outbox.ts

@@ -6,8 +6,13 @@
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert } from "@fluidframework/common-utils";
 import { IContainerContext } from "@fluidframework/container-definitions";
-import { GenericError } from "@fluidframework/container-utils";
+import { GenericError, UsageError } from "@fluidframework/container-utils";
 import { MessageType } from "@fluidframework/protocol-definitions";
+import {
+	ChildLogger,
+	loggerToMonitoringContext,
+	MonitoringContext,
+} from "@fluidframework/telemetry-utils";
 import { ICompressionRuntimeOptions } from "../containerRuntime";
 import { PendingStateManager } from "../pendingStateManager";
 import { BatchManager } from "./batchManager";
@@ -16,203 +21,254 @@ import { OpCompressor } from "./opCompressor";
 import { OpSplitter } from "./opSplitter";
 
 export interface IOutboxConfig {
-    readonly compressionOptions: ICompressionRuntimeOptions;
-    // The maximum size of a batch that we can send over the wire.
-    readonly maxBatchSizeInBytes: number;
-    readonly enableOpReentryCheck?: boolean;
-};
+	readonly compressionOptions: ICompressionRuntimeOptions;
+	// The maximum size of a batch that we can send over the wire.
+	readonly maxBatchSizeInBytes: number;
+	readonly disablePartialFlush: boolean;
+}
 
 export interface IOutboxParameters {
-    readonly shouldSend: () => boolean,
-    readonly pendingStateManager: PendingStateManager,
-    readonly containerContext: IContainerContext,
-    readonly config: IOutboxConfig,
-    readonly compressor: OpCompressor;
-    readonly splitter: OpSplitter;
-    readonly logger: ITelemetryLogger;
+	readonly shouldSend: () => boolean;
+	readonly pendingStateManager: PendingStateManager;
+	readonly containerContext: IContainerContext;
+	readonly config: IOutboxConfig;
+	readonly compressor: OpCompressor;
+	readonly splitter: OpSplitter;
+	readonly logger: ITelemetryLogger;
 }
 
 export class Outbox {
-    private readonly attachFlowBatch: BatchManager;
-    private readonly mainBatch: BatchManager;
-    private readonly defaultAttachFlowSoftLimitInBytes = 64 * 1024;
-
-    constructor(private readonly params: IOutboxParameters) {
-        const isCompressionEnabled = this.params.config.compressionOptions.minimumBatchSizeInBytes !== Number.POSITIVE_INFINITY;
-        // We need to allow infinite size batches if we enable compression
-        const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
-        const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
-
-        this.attachFlowBatch = new BatchManager({
-            hardLimit,
-            softLimit,
-            enableOpReentryCheck: params.config.enableOpReentryCheck,
-        }, params.logger);
-        this.mainBatch = new BatchManager({
-            hardLimit,
-            enableOpReentryCheck: params.config.enableOpReentryCheck,
-        }, params.logger);
-    }
-
-    public get isEmpty(): boolean {
-        return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
-    }
-
-    public submit(message: BatchMessage) {
-        if (!this.mainBatch.push(message)) {
-            throw new GenericError(
-                "BatchTooLarge",
-                /* error */ undefined,
-                {
-                    opSize: (message.contents?.length) ?? 0,
-                    count: this.mainBatch.length,
-                    limit: this.mainBatch.options.hardLimit,
-                });
-        }
-    }
-
-    public submitAttach(message: BatchMessage) {
-        if (!this.attachFlowBatch.push(message)) {
-            // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
-            // when queue is not empty.
-            // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
-            this.flushInternal(this.attachFlowBatch.popBatch());
-            if (!this.attachFlowBatch.push(message)) {
-                throw new GenericError(
-                    "BatchTooLarge",
-                    /* error */ undefined,
-                    {
-                        opSize: (message.contents?.length) ?? 0,
-                        count: this.attachFlowBatch.length,
-                        limit: this.attachFlowBatch.options.hardLimit,
-                    });
-            }
-        }
-
-        // If compression is enabled, we will always successfully receive
-        // attach ops and compress then send them at the next JS turn, regardless
-        // of the overall size of the accumulated ops in the batch.
-        // However, it is more efficient to flush these ops faster, preferably
-        // after they reach a size which would benefit from compression.
-        if (this.attachFlowBatch.contentSizeInBytes >= this.params.config.compressionOptions.minimumBatchSizeInBytes) {
-            this.flushInternal(this.attachFlowBatch.popBatch());
-        }
-    }
-
-    public flush() {
-        this.flushInternal(this.attachFlowBatch.popBatch());
-        this.flushInternal(this.mainBatch.popBatch());
-    }
-
-    private flushInternal(rawBatch: IBatch) {
-        const processedBatch = this.compressBatch(rawBatch);
-        const clientSequenceNumber = this.sendBatch(processedBatch);
-
-        this.persistBatch(clientSequenceNumber, rawBatch.content);
-    }
-
-    private compressBatch(batch: IBatch): IBatch {
-        if (batch.content.length === 0
-            || this.params.config.compressionOptions === undefined
-            || this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes) {
-            // Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress
-            return batch;
-        }
-
-        const compressedBatch = this.params.compressor.compressBatch(batch);
-        if (compressedBatch.contentSizeInBytes <= this.params.config.maxBatchSizeInBytes) {
-            // If we don't reach the maximum supported size of a batch, it can safely be sent as is
-            return compressedBatch;
-        }
-
-        if (this.params.splitter.isBatchChunkingEnabled) {
-            return this.params.splitter.splitCompressedBatch(compressedBatch);
-        }
-
-        // If we've reached this point, the runtime would attempt to send a batch larger than the allowed size
-        throw new GenericError(
-            "BatchTooLarge",
-            /* error */ undefined,
-            {
-                opSize: batch.contentSizeInBytes,
-                count: batch.content.length,
-                limit: this.params.config.maxBatchSizeInBytes,
-                compressed: true,
-            });
-    }
-
-    /**
-     * Sends the batch object to the container context to be sent over the wire.
-     *
-     * @param batch - batch to be sent
-     * @returns the client sequence number of the last batched op which was sent and
-     * -1 if there are no ops or the container cannot send ops.
-     */
-    private sendBatch(batch: IBatch): number {
-        let clientSequenceNumber: number = -1;
-        const length = batch.content.length;
-
-        // Did we disconnect in the middle of turn-based batch?
-        // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
-        if (length === 0 || !this.params.shouldSend()) {
-            return clientSequenceNumber;
-        }
-
-        if (this.params.containerContext.submitBatchFn === undefined) {
-            // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
-            // version that has support for batches (submitBatchFn)
-            for (const message of batch.content) {
-                // Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
-                if (message.metadata?.compressed) {
-                    delete message.metadata.compressed;
-                }
-
-                clientSequenceNumber = this.params.containerContext.submitFn(
-                    MessageType.Operation,
-                    message.deserializedContent,
-                    true, // batch
-                    message.metadata);
-            }
-
-            this.params.containerContext.deltaManager.flush();
-        } else {
-            // returns clientSequenceNumber of last message in a batch
-            clientSequenceNumber = this.params.containerContext.submitBatchFn(
-                batch.content.map((message) => ({
-                    contents: message.contents,
-                    metadata: message.metadata,
-                    compression: message.compression,
-                })));
-        }
-
-        // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
-        clientSequenceNumber -= length - 1;
-        assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
-        return clientSequenceNumber;
-    }
-
-    private persistBatch(initialClientSequenceNumber: number, batch: BatchMessage[]) {
-        let clientSequenceNumber = initialClientSequenceNumber;
-        // Let the PendingStateManager know that a message was submitted.
-        // In future, need to shift toward keeping batch as a whole!
-        for (const message of batch) {
-            this.params.pendingStateManager.onSubmitMessage(
-                message.deserializedContent.type,
-                clientSequenceNumber,
-                message.referenceSequenceNumber,
-                message.deserializedContent.contents,
-                message.localOpMetadata,
-                message.metadata,
-            );
-
-            clientSequenceNumber++;
-        }
-    }
-
-    public checkpoint() {
-        return {
-            mainBatch: this.mainBatch.checkpoint(),
-            attachFlowBatch: this.attachFlowBatch.checkpoint(),
-        };
-    }
+	private readonly mc: MonitoringContext;
+	private readonly attachFlowBatch: BatchManager;
+	private readonly mainBatch: BatchManager;
+	private readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;
+
+	/**
+	 * Track the number of ops which were detected to have a mismatched
+	 * reference sequence number, in order to self-throttle the telemetry events.
+	 *
+	 * This should be removed as part of ADO:2322
+	 */
+	private readonly maxMismatchedOpsToReport = 3;
+	private mismatchedOpsReported = 0;
+
+	constructor(private readonly params: IOutboxParameters) {
+		this.mc = loggerToMonitoringContext(ChildLogger.create(params.logger, "Outbox"));
+		const isCompressionEnabled =
+			this.params.config.compressionOptions.minimumBatchSizeInBytes !==
+			Number.POSITIVE_INFINITY;
+		// We need to allow infinite size batches if we enable compression
+		const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
+		const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
+
+		this.attachFlowBatch = new BatchManager({ hardLimit, softLimit });
+		this.mainBatch = new BatchManager({ hardLimit });
+	}
+
+	public get isEmpty(): boolean {
+		return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
+	}
+
+	/**
+	 * If we detect that the reference sequence number of the incoming message does not match
+	 * what was already in the batch managers, this means that batching has been interrupted so
+	 * we will flush the accumulated messages to account for that and create a new batch with the new
+	 * message as the first message.
+	 *
+	 * @param message - the incoming message
+	 */
+	private maybeFlushPartialBatch(message: BatchMessage) {
+		const mainBatchReference = this.mainBatch.referenceSequenceNumber;
+		const attachFlowBatchReference = this.attachFlowBatch.referenceSequenceNumber;
+		assert(
+			this.params.config.disablePartialFlush ||
+				mainBatchReference === undefined ||
+				attachFlowBatchReference === undefined ||
+				mainBatchReference === attachFlowBatchReference,
+			0x58d /* Reference sequence numbers from both batches must be in sync */,
+		);
+
+		if (
+			(mainBatchReference === undefined ||
+				mainBatchReference === message.referenceSequenceNumber) &&
+			(attachFlowBatchReference === undefined ||
+				attachFlowBatchReference === message.referenceSequenceNumber)
+		) {
+			// The reference sequence numbers are stable, there is nothing to do
+			return;
+		}
+
+		if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
+			this.mc.logger.sendErrorEvent(
+				{
+					eventName: "ReferenceSequenceNumberMismatch",
+					mainReferenceSequenceNumber: mainBatchReference,
+					attachReferenceSequenceNumber: attachFlowBatchReference,
+					messageReferenceSequenceNumber: message.referenceSequenceNumber,
+				},
+				new UsageError("Submission of an out of order message"),
+			);
+		}
+
+		if (!this.params.config.disablePartialFlush) {
+			this.flush();
+		}
+	}
+
+	public submit(message: BatchMessage) {
+		this.maybeFlushPartialBatch(message);
+
+		if (!this.mainBatch.push(message)) {
+			throw new GenericError("BatchTooLarge", /* error */ undefined, {
+				opSize: message.contents?.length ?? 0,
+				batchSize: this.mainBatch.contentSizeInBytes,
+				count: this.mainBatch.length,
+				limit: this.mainBatch.options.hardLimit,
+			});
+		}
+	}
+
+	public submitAttach(message: BatchMessage) {
+		this.maybeFlushPartialBatch(message);
+
+		if (!this.attachFlowBatch.push(message)) {
+			// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
+			// when queue is not empty.
+			// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
+			this.flushInternal(this.attachFlowBatch.popBatch());
+			if (!this.attachFlowBatch.push(message)) {
+				throw new GenericError("BatchTooLarge", /* error */ undefined, {
+					opSize: message.contents?.length ?? 0,
+					batchSize: this.attachFlowBatch.contentSizeInBytes,
+					count: this.attachFlowBatch.length,
+					limit: this.attachFlowBatch.options.hardLimit,
+				});
+			}
+		}
+
+		// If compression is enabled, we will always successfully receive
+		// attach ops and compress then send them at the next JS turn, regardless
+		// of the overall size of the accumulated ops in the batch.
+		// However, it is more efficient to flush these ops faster, preferably
+		// after they reach a size which would benefit from compression.
+		if (
+			this.attachFlowBatch.contentSizeInBytes >=
+			this.params.config.compressionOptions.minimumBatchSizeInBytes
+		) {
+			this.flushInternal(this.attachFlowBatch.popBatch());
+		}
+	}
+
+	public flush() {
+		this.flushInternal(this.attachFlowBatch.popBatch());
+		this.flushInternal(this.mainBatch.popBatch());
+	}
+
+	private flushInternal(rawBatch: IBatch) {
+		const processedBatch = this.compressBatch(rawBatch);
+		this.sendBatch(processedBatch);
+
+		this.persistBatch(rawBatch.content);
+	}
+
+	private compressBatch(batch: IBatch): IBatch {
+		if (
+			batch.content.length === 0 ||
+			this.params.config.compressionOptions === undefined ||
+			this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes
+		) {
+			// Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress
+			return batch;
+		}
+
+		const compressedBatch = this.params.compressor.compressBatch(batch);
+		if (compressedBatch.contentSizeInBytes <= this.params.config.maxBatchSizeInBytes) {
+			// If we don't reach the maximum supported size of a batch, it can safely be sent as is
+			return compressedBatch;
+		}
+
+		if (this.params.splitter.isBatchChunkingEnabled) {
+			return this.params.splitter.splitCompressedBatch(compressedBatch);
+		}
+
+		// If we've reached this point, the runtime would attempt to send a batch larger than the allowed size
+		throw new GenericError("BatchTooLarge", /* error */ undefined, {
+			batchSize: batch.contentSizeInBytes,
+			compressedBatchSize: compressedBatch.contentSizeInBytes,
+			count: compressedBatch.content.length,
+			limit: this.params.config.maxBatchSizeInBytes,
+			chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
+			compressionOptions: JSON.stringify(this.params.config.compressionOptions),
+		});
+	}
+
+	/**
+	 * Sends the batch object to the container context to be sent over the wire.
+	 *
+	 * @param batch - batch to be sent
+	 */
+	private sendBatch(batch: IBatch) {
+		const length = batch.content.length;
+
+		// Did we disconnect in the middle of turn-based batch?
+		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
+		if (length === 0 || !this.params.shouldSend()) {
+			return;
+		}
+
+		if (this.params.containerContext.submitBatchFn === undefined) {
+			// Legacy path - supporting old loader versions. Can be removed only when LTS moves above
+			// version that has support for batches (submitBatchFn)
+			for (const message of batch.content) {
+				// Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
+				if (message.metadata?.compressed) {
+					delete message.metadata.compressed;
+				}
+
+				this.params.containerContext.submitFn(
+					MessageType.Operation,
+					message.deserializedContent,
+					true, // batch
+					message.metadata,
+				);
+			}
+
+			this.params.containerContext.deltaManager.flush();
+		} else {
+			assert(
+				batch.referenceSequenceNumber !== undefined,
+				0x58e /* Batch must not be empty */,
+			);
+			this.params.containerContext.submitBatchFn(
+				batch.content.map((message) => ({
+					contents: message.contents,
+					metadata: message.metadata,
+					compression: message.compression,
+					referenceSequenceNumber: message.referenceSequenceNumber,
+				})),
+				batch.referenceSequenceNumber,
+			);
+		}
+	}
+
+	private persistBatch(batch: BatchMessage[]) {
+		// Let the PendingStateManager know that a message was submitted.
+		// In future, need to shift toward keeping batch as a whole!
+		for (const message of batch) {
+			this.params.pendingStateManager.onSubmitMessage(
+				message.deserializedContent.type,
+				message.referenceSequenceNumber,
+				message.deserializedContent.contents,
+				message.localOpMetadata,
+				message.metadata,
+			);
+		}
+	}
+
+	public checkpoint() {
+		return {
+			mainBatch: this.mainBatch.checkpoint(),
+			attachFlowBatch: this.attachFlowBatch.checkpoint(),
+		};
+	}
 }

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -9,61 +9,61 @@ import { OpDecompressor } from "./opDecompressor";
 import { OpSplitter } from "./opSplitter";
 
 export class RemoteMessageProcessor {
-    constructor(
-        private readonly opSplitter: OpSplitter,
-        private readonly opDecompressor: OpDecompressor,
-    ) { }
+	constructor(
+		private readonly opSplitter: OpSplitter,
+		private readonly opDecompressor: OpDecompressor,
+	) {}
 
-    public get partialMessages(): ReadonlyMap<string, string[]> {
-        return this.opSplitter.chunks;
-    }
+	public get partialMessages(): ReadonlyMap<string, string[]> {
+		return this.opSplitter.chunks;
+	}
 
-    public clearPartialMessagesFor(clientId: string) {
-        this.opSplitter.clearPartialChunks(clientId);
-    }
+	public clearPartialMessagesFor(clientId: string) {
+		this.opSplitter.clearPartialChunks(clientId);
+	}
 
-    public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage {
-        let message = copy(remoteMessage);
-        message = this.opDecompressor.processMessage(message).message;
-        unpackRuntimeMessage(message);
+	public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage {
+		let message = copy(remoteMessage);
+		message = this.opDecompressor.processMessage(message).message;
+		unpackRuntimeMessage(message);
 
-        const chunkProcessingResult = this.opSplitter.processRemoteMessage(message);
-        message = chunkProcessingResult.message;
-        if (chunkProcessingResult.state !== "Processed") {
-            // If the message is not chunked or if the splitter is still rebuilding the original message,
-            // there is no need to continue processing
-            return message;
-        }
+		const chunkProcessingResult = this.opSplitter.processRemoteMessage(message);
+		message = chunkProcessingResult.message;
+		if (chunkProcessingResult.state !== "Processed") {
+			// If the message is not chunked or if the splitter is still rebuilding the original message,
+			// there is no need to continue processing
+			return message;
+		}
 
-        const decompressionAfterChunking = this.opDecompressor.processMessage(message);
-        message = decompressionAfterChunking.message;
-        if (decompressionAfterChunking.state === "Skipped") {
-            // After chunking, if the original message was not compressed,
-            // there is no need to continue processing
-            return message;
-        }
+		const decompressionAfterChunking = this.opDecompressor.processMessage(message);
+		message = decompressionAfterChunking.message;
+		if (decompressionAfterChunking.state === "Skipped") {
+			// After chunking, if the original message was not compressed,
+			// there is no need to continue processing
+			return message;
+		}
 
-        // The message needs to be unpacked after chunking + decompression
-        unpack(message);
-        return message;
-    }
+		// The message needs to be unpacked after chunking + decompression
+		unpack(message);
+		return message;
+	}
 }
 
 const copy = (remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage => {
-    // Do shallow copy of message, as the processing flow will modify it.
-    // There might be multiple container instances receiving same message
-    // We do not need to make deep copy, as each layer will just replace message.content itself,
-    // but would not modify contents details
-    const message = { ...remoteMessage };
+	// Do shallow copy of message, as the processing flow will modify it.
+	// There might be multiple container instances receiving same message
+	// We do not need to make deep copy, as each layer will just replace message.content itself,
+	// but would not modify contents details
+	const message = { ...remoteMessage };
 
-    // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
-    // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
-    // Old ops may contain empty string (I assume noops).
-    if (typeof message.contents === "string" && message.contents !== "") {
-        message.contents = JSON.parse(message.contents);
-    }
+	// back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
+	// System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
+	// Old ops may contain empty string (I assume noops).
+	if (typeof message.contents === "string" && message.contents !== "") {
+		message.contents = JSON.parse(message.contents);
+	}
 
-    return message;
+	return message;
 };
 
 /**
@@ -71,9 +71,9 @@ const copy = (remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessa
  *
  */
 const unpack = (message: ISequencedDocumentMessage) => {
-    const innerContents = message.contents as ContainerRuntimeMessage;
-    message.type = innerContents.type;
-    message.contents = innerContents.contents;
+	const innerContents = message.contents as ContainerRuntimeMessage;
+	message.type = innerContents.type;
+	message.contents = innerContents.contents;
 };
 
 /**
@@ -86,21 +86,21 @@ const unpack = (message: ISequencedDocumentMessage) => {
  * @internal
  */
 export function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolean {
-    if (message.type !== MessageType.Operation) {
-        // Legacy format, but it's already "unpacked",
-        // i.e. message.type is actually ContainerMessageType.
-        // Or it's non-runtime message.
-        // Nothing to do in such case.
-        return false;
-    }
+	if (message.type !== MessageType.Operation) {
+		// Legacy format, but it's already "unpacked",
+		// i.e. message.type is actually ContainerMessageType.
+		// Or it's non-runtime message.
+		// Nothing to do in such case.
+		return false;
+	}
 
-    // legacy op format?
-    if (message.contents.address !== undefined && message.contents.type === undefined) {
-        message.type = ContainerMessageType.FluidDataStoreOp;
-    } else {
-        // new format
-        unpack(message);
-    }
+	// legacy op format?
+	if (message.contents.address !== undefined && message.contents.type === undefined) {
+		message.type = ContainerMessageType.FluidDataStoreOp;
+	} else {
+		// new format
+		unpack(message);
+	}
 
-    return true;
+	return true;
 }