@fluidframework/container-runtime 2.0.0-dev.4.4.0.162574 → 2.0.0-dev.5.3.2.178189

package/src/opLifecycle/outbox.ts

@@ -3,18 +3,18 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert } from "@fluidframework/common-utils";
-import { IContainerContext } from "@fluidframework/container-definitions";
-import { GenericError, UsageError } from "@fluidframework/container-utils";
-import { MessageType } from "@fluidframework/protocol-definitions";
 import {
+	ITelemetryLoggerExt,
 	ChildLogger,
 	loggerToMonitoringContext,
 	MonitoringContext,
 } from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/common-utils";
+import { IContainerContext, ICriticalContainerError } from "@fluidframework/container-definitions";
+import { GenericError, UsageError } from "@fluidframework/container-utils";
+import { MessageType } from "@fluidframework/protocol-definitions";
 import { ICompressionRuntimeOptions } from "../containerRuntime";
-import { PendingStateManager } from "../pendingStateManager";
+import { IPendingBatchMessage, PendingStateManager } from "../pendingStateManager";
 import {
 	BatchManager,
 	BatchSequenceNumbers,
@@ -31,6 +31,7 @@ export interface IOutboxConfig {
 	// The maximum size of a batch that we can send over the wire.
 	readonly maxBatchSizeInBytes: number;
 	readonly disablePartialFlush: boolean;
+	readonly enableGroupedBatching: boolean;
 }
 
 export interface IOutboxParameters {
@@ -40,21 +41,43 @@ export interface IOutboxParameters {
 	readonly config: IOutboxConfig;
 	readonly compressor: OpCompressor;
 	readonly splitter: OpSplitter;
-	readonly logger: ITelemetryLogger;
+	readonly logger: ITelemetryLoggerExt;
 	readonly groupingManager: OpGroupingManager;
 	readonly getCurrentSequenceNumbers: () => BatchSequenceNumbers;
+	readonly reSubmit: (message: IPendingBatchMessage) => void;
+	readonly opReentrancy: () => boolean;
+	readonly closeContainer: (error?: ICriticalContainerError) => void;
 }
 
-function getLongStack(action: () => Error): Error {
-	// Increase the stack trace limit temporarily, so as to debug better in case it occurs.
+/**
+ * Temporarily increase the stack limit while executing the provided action.
+ * If a negative value is provided for `length`, no stack frames will be collected.
+ * If Infinity is provided, all frames will be collected.
+ *
+ * ADO:4663 - add this to the common packages.
+ *
+ * @param action - action which returns an error
+ * @param length - number of stack frames to collect, 50 if unspecified.
+ * @returns the result of the action provided
+ */
+export function getLongStack<T>(action: () => T, length: number = 50): T {
+	const errorObj = Error as any;
+	if (
+		(
+			Object.getOwnPropertyDescriptor(errorObj, "stackTraceLimit") ||
+			Object.getOwnPropertyDescriptor(Object.getPrototypeOf(errorObj), "stackTraceLimit") ||
+			{}
+		).writable !== true
+	) {
+		return action();
+	}
+
+	const originalStackTraceLimit = errorObj.stackTraceLimit;
 	try {
-		const originalStackTraceLimit = (Error as any).stackTraceLimit;
-		(Error as any).stackTraceLimit = 50;
-		const result = action();
-		(Error as any).stackTraceLimit = originalStackTraceLimit;
-		return result;
-	} catch (error) {
+		errorObj.stackTraceLimit = length;
 		return action();
+	} finally {
+		errorObj.stackTraceLimit = originalStackTraceLimit;
 	}
 }
 
@@ -62,7 +85,10 @@ export class Outbox {
 	private readonly mc: MonitoringContext;
 	private readonly attachFlowBatch: BatchManager;
 	private readonly mainBatch: BatchManager;
+	private readonly blobAttachBatch: BatchManager;
 	private readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;
+	private batchRebasesToReport = 5;
+	private rebasing = false;
 
 	/**
 	 * Track the number of ops which were detected to have a mismatched
@@ -84,10 +110,15 @@ export class Outbox {
 
 		this.attachFlowBatch = new BatchManager({ hardLimit, softLimit });
 		this.mainBatch = new BatchManager({ hardLimit });
+		this.blobAttachBatch = new BatchManager({ hardLimit });
 	}
 
 	public get isEmpty(): boolean {
-		return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
+		return (
+			this.attachFlowBatch.length === 0 &&
+			this.mainBatch.length === 0 &&
+			this.blobAttachBatch.length === 0
+		);
 	}
 
 	/**
@@ -99,9 +130,11 @@ export class Outbox {
 	private maybeFlushPartialBatch() {
 		const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
 		const attachFlowBatchSeqNums = this.attachFlowBatch.sequenceNumbers;
+		const blobAttachSeqNums = this.blobAttachBatch.sequenceNumbers;
 		assert(
 			this.params.config.disablePartialFlush ||
-				sequenceNumbersMatch(mainBatchSeqNums, attachFlowBatchSeqNums),
+				(sequenceNumbersMatch(mainBatchSeqNums, attachFlowBatchSeqNums) &&
+					sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums)),
 			0x58d /* Reference sequence numbers from both batches must be in sync */,
 		);
 
@@ -109,7 +142,8 @@ export class Outbox {
 
 		if (
 			sequenceNumbersMatch(mainBatchSeqNums, currentSequenceNumbers) &&
-			sequenceNumbersMatch(attachFlowBatchSeqNums, currentSequenceNumbers)
+			sequenceNumbersMatch(attachFlowBatchSeqNums, currentSequenceNumbers) &&
+			sequenceNumbersMatch(blobAttachSeqNums, currentSequenceNumbers)
 		) {
 			// The reference sequence numbers are stable, there is nothing to do
 			return;
@@ -124,6 +158,8 @@ export class Outbox {
 					mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
 					attachReferenceSequenceNumber: attachFlowBatchSeqNums.referenceSequenceNumber,
 					attachClientSequenceNumber: attachFlowBatchSeqNums.clientSequenceNumber,
+					blobAttachReferenceSequenceNumber: blobAttachSeqNums.referenceSequenceNumber,
+					blobAttachClientSequenceNumber: blobAttachSeqNums.clientSequenceNumber,
 					currentReferenceSequenceNumber: currentSequenceNumbers.referenceSequenceNumber,
 					currentClientSequenceNumber: currentSequenceNumbers.clientSequenceNumber,
 				},
@@ -132,26 +168,14 @@ export class Outbox {
 		}
 
 		if (!this.params.config.disablePartialFlush) {
-			this.flush();
+			this.flushAll();
 		}
 	}
 
 	public submit(message: BatchMessage) {
 		this.maybeFlushPartialBatch();
 
-		if (
-			!this.mainBatch.push(
-				message,
-				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
-			)
-		) {
-			throw new GenericError("BatchTooLarge", /* error */ undefined, {
-				opSize: message.contents?.length ?? 0,
-				batchSize: this.mainBatch.contentSizeInBytes,
-				count: this.mainBatch.length,
-				limit: this.mainBatch.options.hardLimit,
-			});
-		}
+		this.addMessageToBatchManager(this.mainBatch, message);
 	}
 
 	public submitAttach(message: BatchMessage) {
@@ -160,26 +184,16 @@ export class Outbox {
 		if (
 			!this.attachFlowBatch.push(
 				message,
+				this.isContextReentrant(),
 				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
 			)
 		) {
 			// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
 			// when queue is not empty.
 			// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
-			this.flushInternal(this.attachFlowBatch.popBatch());
-			if (
-				!this.attachFlowBatch.push(
-					message,
-					this.params.getCurrentSequenceNumbers().clientSequenceNumber,
-				)
-			) {
-				throw new GenericError("BatchTooLarge", /* error */ undefined, {
-					opSize: message.contents?.length ?? 0,
-					batchSize: this.attachFlowBatch.contentSizeInBytes,
-					count: this.attachFlowBatch.length,
-					limit: this.attachFlowBatch.options.hardLimit,
-				});
-			}
+			this.flushInternal(this.attachFlowBatch);
+
+			this.addMessageToBatchManager(this.attachFlowBatch, message);
 		}
 
 		// If compression is enabled, we will always successfully receive
@@ -191,23 +205,122 @@ export class Outbox {
 			this.attachFlowBatch.contentSizeInBytes >=
 			this.params.config.compressionOptions.minimumBatchSizeInBytes
 		) {
-			this.flushInternal(this.attachFlowBatch.popBatch());
+			this.flushInternal(this.attachFlowBatch);
+		}
+	}
+
+	public submitBlobAttach(message: BatchMessage) {
+		this.maybeFlushPartialBatch();
+
+		this.addMessageToBatchManager(this.blobAttachBatch, message);
+
+		// If compression is enabled, we will always successfully receive
+		// blobAttach ops and compress then send them at the next JS turn, regardless
+		// of the overall size of the accumulated ops in the batch.
+		// However, it is more efficient to flush these ops faster, preferably
+		// after they reach a size which would benefit from compression.
+		if (
+			this.blobAttachBatch.contentSizeInBytes >=
+			this.params.config.compressionOptions.minimumBatchSizeInBytes
+		) {
+			this.flushInternal(this.blobAttachBatch);
+		}
+	}
+
+	private addMessageToBatchManager(batchManager: BatchManager, message: BatchMessage) {
+		if (
+			!batchManager.push(
+				message,
+				this.isContextReentrant(),
+				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
+			)
+		) {
+			throw new GenericError("BatchTooLarge", /* error */ undefined, {
+				opSize: message.contents?.length ?? 0,
+				batchSize: batchManager.contentSizeInBytes,
+				count: batchManager.length,
+				limit: batchManager.options.hardLimit,
+			});
 		}
 	}
 
 	public flush() {
-		this.flushInternal(this.attachFlowBatch.popBatch());
-		this.flushInternal(this.mainBatch.popBatch());
+		if (this.isContextReentrant()) {
+			const error = new UsageError("Flushing is not supported inside DDS event handlers");
+			this.params.closeContainer(error);
+			throw error;
+		}
+
+		this.flushAll();
+	}
+
+	private flushAll() {
+		this.flushInternal(this.attachFlowBatch);
+		this.flushInternal(this.blobAttachBatch, true /* disableGroupedBatching */);
+		this.flushInternal(this.mainBatch);
 	}
 
-	private flushInternal(rawBatch: IBatch) {
-		const processedBatch = this.compressBatch(rawBatch);
+	private flushInternal(batchManager: BatchManager, disableGroupedBatching: boolean = false) {
+		if (batchManager.empty) {
+			return;
+		}
+
+		const rawBatch = batchManager.popBatch();
+		if (rawBatch.hasReentrantOps === true && this.params.config.enableGroupedBatching) {
+			assert(!this.rebasing, 0x6fa /* A rebased batch should never have reentrant ops */);
+			// If a batch contains reentrant ops (ops created as a result from processing another op)
+			// it needs to be rebased so that we can ensure consistent reference sequence numbers
+			// and eventual consistency at the DDS level.
+			this.rebase(rawBatch, batchManager);
+			return;
+		}
+
+		const processedBatch = this.compressBatch(rawBatch, disableGroupedBatching);
 		this.sendBatch(processedBatch);
 
 		this.persistBatch(rawBatch.content);
 	}
 
-	private compressBatch(batch: IBatch): IBatch {
+	/**
+	 * Rebases a batch. All the ops in the batch are resubmitted to the runtime and
+	 * they will end up back in the same batch manager they were flushed from and subsequently flushed.
+	 *
+	 * @param rawBatch - the batch to be rebased
+	 */
+	private rebase(rawBatch: IBatch, batchManager: BatchManager) {
+		assert(!this.rebasing, 0x6fb /* Reentrancy */);
+
+		this.rebasing = true;
+		for (const message of rawBatch.content) {
+			this.params.reSubmit({
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				content: message.contents!,
+				localOpMetadata: message.localOpMetadata,
+				opMetadata: message.metadata,
+			});
+		}
+
+		if (this.batchRebasesToReport > 0) {
+			this.mc.logger.sendTelemetryEvent(
+				{
+					eventName: "BatchRebase",
+					length: rawBatch.content.length,
+					referenceSequenceNumber: rawBatch.referenceSequenceNumber,
+				},
+				new UsageError("BatchRebase"),
+			);
+			this.batchRebasesToReport--;
+		}
+
+		this.flushInternal(batchManager);
+		this.rebasing = false;
+	}
+
+	private isContextReentrant(): boolean {
+		return this.params.opReentrancy() && !this.rebasing;
+	}
+
+	private compressBatch(batch: IBatch, disableGroupedBatching: boolean): IBatch {
 		if (
 			batch.content.length === 0 ||
 			this.params.config.compressionOptions === undefined ||
@@ -216,11 +329,11 @@ export class Outbox {
 			this.params.containerContext.submitBatchFn === undefined
 		) {
 			// Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
-			return this.params.groupingManager.groupBatch(batch);
+			return disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch);
 		}
 
 		const compressedBatch = this.params.compressor.compressBatch(
-			this.params.groupingManager.groupBatch(batch),
+			disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch),
 		);
 
 		if (this.params.splitter.isBatchChunkingEnabled) {
@@ -279,7 +392,8 @@ export class Outbox {
 			for (const message of batch.content) {
 				this.params.containerContext.submitFn(
 					MessageType.Operation,
-					message.deserializedContent,
+					// For back-compat (submitFn only works on deserialized content)
+					message.contents === undefined ? undefined : JSON.parse(message.contents),
 					true, // batch
 					message.metadata,
 				);
@@ -308,9 +422,9 @@ export class Outbox {
 		// In future, need to shift toward keeping batch as a whole!
 		for (const message of batch) {
 			this.params.pendingStateManager.onSubmitMessage(
-				message.deserializedContent.type,
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				message.contents!,
 				message.referenceSequenceNumber,
-				message.deserializedContent.contents,
 				message.localOpMetadata,
 				message.metadata,
 			);
@@ -321,6 +435,7 @@ export class Outbox {
 		return {
 			mainBatch: this.mainBatch.checkpoint(),
 			attachFlowBatch: this.attachFlowBatch.checkpoint(),
+			blobAttachBatch: this.blobAttachBatch.checkpoint(),
 		};
 	}
 }

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -120,7 +120,11 @@ export function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolea
 	}
 
 	// legacy op format?
-	if (message.contents.address !== undefined && message.contents.type === undefined) {
+	// TODO: Unsure if this is a real format we should be concerned with. There doesn't appear to be anything prepared to handle the address member.
+	if (
+		(message.contents as { address?: unknown }).address !== undefined &&
+		(message.contents as { type?: unknown }).type === undefined
+	) {
 		message.type = ContainerMessageType.FluidDataStoreOp;
 	} else {
 		// new format

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev.4.4.0.162574";
+export const pkgVersion = "2.0.0-dev.5.3.2.178189";