@fluidframework/container-runtime 2.0.0-internal.6.3.3 → 2.0.0-internal.6.4.0

package/src/opLifecycle/README.md

@@ -212,8 +212,6 @@ On the receiving end, the client will accumulate chunks 1 and 2 and keep them in
 
 ## How grouped batching works
 
-**Note: There are plans to replace empty ops with something more efficient when doing grouped batching AB#4092**
-
 Given the following baseline batch:
 
 ```
@@ -223,68 +221,70 @@ Given the following baseline batch:
 +---------------+---------------+---------------+---------------+---------------+
 ```
 
-Compressed batch:
+Grouped batch:
 
 ```
-+--------------------+-----------------+-----------------+-----------------+-----------------+
-| Op 1               | Op 2            | Op 3            | Op 4            | Op 5            |
-| Contents: "abcde"  | Contents: empty | Contents: empty | Contents: empty | Contents: empty |
-| Compression: 'lz4' |                 |                 |                 |                 |
-+--------------------+-----------------+-----------------+-----------------+-----------------+
++---------------------------------------------------------------------------------------------------------------------+
+| Op 1                   Contents: +----------------+---------------+---------------+---------------+---------------+ |
+| Type: "groupedBatch"             | Op 1           | Op 2          | Op 3          | Op 4          | Op 5          | |
+|                                  | Contents: "a"  | Contents: "b" | Contents: "c" | Contents: "d" | Contents: "e" | |
+|                                  +----------------+---------------+---------------+---------------+---------------+ |
++---------------------------------------------------------------------------------------------------------------------+
 ```
 
-Grouped batch:
+Compressed batch:
 
 ```
-+---------------------------------------------------------------------------------------------------------------------------------+
-| Op 1                   Contents: +--------------------+-----------------+-----------------+-----------------+-----------------+ |
-| SeqNum: 1                        | Op 1               | Op 2            | Op 3            | Op 4            | Op 5            | |
-| Type: "groupedBatch"             | Contents: "abcde"  | Contents: empty | Contents: empty | Contents: empty | Contents: empty | |
-|                                  | Compression: 'lz4' |                 |                 |                 |                 | |
-|                                  +--------------------+-----------------+-----------------+-----------------+-----------------+ |
-+---------------------------------------------------------------------------------------------------------------------------------+
++-------------------------------------------------------------------------------------------------------------------------+
+| Op 1                   Contents: +------------------------------------------------------------------------------------+ |
+| Compression: 'lz4'               | Type: "groupedBatch"                                                               | |
+|                                  | +----------------+---------------+---------------+---------------+---------------+ | |
+|                                  | | Op 1           | Op 2          | Op 3          | Op 4          | Op 5          | | |
+|                                  | | Contents: "a"  | Contents: "b" | Contents: "c" | Contents: "d" | Contents: "e" | | |
+|                                  | +----------------+---------------+---------------+---------------+---------------+ | |
+|                                  +------------------------------------------------------------------------------------+ |
++-------------------------------------------------------------------------------------------------------------------------+
 ```
 
 Can produce the following chunks:
 
 ```
-+-------------------------------------------------+
-| Chunk 1/2    Contents: +----------------------+ |
-| SeqNum: 1              |  +-----------------+ | |
-|                        |  | Contents: "abc" | | |
-|                        |  +-----------------+ | |
-|                        +----------------------+ |
-+-------------------------------------------------+
++------------------------------------------------+
+| Chunk 1/2    Contents: +---------------------+ |
+|                        | +-----------------+ | |
+|                        | | Contents: "abc" | | |
+|                        | +-----------------+ | |
+|                        +---------------------+ |
++------------------------------------------------+
 ```
 
 ```
-+--------------------------------------------------------------------------------------------------------------------------+
-| Chunk 2/2    Contents: +---------------------------------------------------------------------------------------------+ | |
-| SeqNum: 2              |  +----------------+-----------------+-----------------+-----------------+-----------------+ | | |
-|                        |  | Contents: "de" | Contents: empty | Contents: empty | Contents: empty | Contents: empty | | | |
-|                        |  +----------------+-----------------+-----------------+-----------------+-----------------+ | | |
-|                        +---------------------------------------------------------------------------------------------+ | |
-+--------------------------------------------------------------------------------------------------------------------------+
++-----------------------------------------------+
+| Chunk 2/2    Contents: +--------------------+ |
+|                        | +----------------+ | |
+|                        | | Contents: "de" | | |
+|                        | +----------------+ | |
+|                        +--------------------+ |
++-----------------------------------------------+
 ```
 
 -   Send to service
 -   Service acks ops sent
 -   Receive chunks from service
--   Recompile to the grouped batch step
+-   Recompile to the compression step
 
-Ungrouped batch:
+Decompressed batch:
 
 ```
-+--------------------+-----------------+-----------------+-----------------+-----------------+
-| Op 1               | Op 2            | Op 3            | Op 4            | Op 5            |
-| Contents: "abcde"  | Contents: empty | Contents: empty | Contents: empty | Contents: empty |
-| SeqNum: 2          | SeqNum: 2       | SeqNum: 2       | SeqNum: 2       | SeqNum: 2       |
-| ClientSeqNum: 1    | ClientSeqNum: 2 | ClientSeqNum: 3 | ClientSeqNum: 4 | ClientSeqNum: 5 |
-| Compression: 'lz4' |                 |                 |                 |                 |
-+--------------------+-----------------+-----------------+-----------------+-----------------+
++---------------------------------------------------------------------------------------------------------------------+
+| Op 1                   Contents: +----------------+---------------+---------------+---------------+---------------+ |
+| SeqNum: 2                        | Op 1           | Op 2            | Op 3        | Op 4          | Op 5          | |
+| Type: "groupedBatch"             | Contents: "a"  | Contents: "b" | Contents: "c" | Contents: "d" | Contents: "e" | |
+|                                  +----------------+---------------+---------------+---------------+---------------+ |
++---------------------------------------------------------------------------------------------------------------------+
 ```
 
-Uncompressed batch:
+Ungrouped batch:
 
 ```
 +-----------------+-----------------+-----------------+-----------------+-----------------+

package/src/opLifecycle/definitions.ts

@@ -5,7 +5,8 @@
 
 import { IBatchMessage } from "@fluidframework/container-definitions";
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
-import { CompressionAlgorithms, ContainerMessageType } from "..";
+import { CompressionAlgorithms } from "..";
+import { ContainerMessageType } from "../messageTypes";
 
 /**
  * Batch message type used internally by the runtime

package/src/opLifecycle/opDecompressor.ts

@@ -50,14 +50,6 @@ export class OpDecompressor {
 		) {
 			// Beginning of a compressed batch
 			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
-			if (message.compression) {
-				// lz4 is the only supported compression algorithm for now
-				assert(
-					message.compression === CompressionAlgorithms.lz4,
-					0x4b9 /* lz4 is currently the only supported compression algorithm */,
-				);
-			}
-
 			this.activeBatch = true;
 
 			const contents = IsoBuffer.from(

package/src/opLifecycle/opGroupingManager.ts

@@ -5,7 +5,7 @@
 
 import { assert } from "@fluidframework/core-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ContainerMessageType } from "..";
+import { ContainerMessageType } from "../messageTypes";
 import { IBatch } from "./definitions";
 
 /**
@@ -22,6 +22,10 @@ interface IGroupedMessage {
 	compression?: string;
 }
 
+function isGroupContents(opContents: any): opContents is IGroupedBatchMessageContents {
+	return opContents?.type === OpGroupingManager.groupedBatchOp;
+}
+
 export class OpGroupingManager {
 	static readonly groupedBatchOp = "groupedBatch";
 
@@ -68,14 +72,11 @@ export class OpGroupingManager {
 	}
 
 	public ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
-		if (
-			(op.contents as { type?: unknown } | undefined)?.type !==
-			OpGroupingManager.groupedBatchOp
-		) {
+		if (!isGroupContents(op.contents)) {
 			return [op];
 		}
 
-		const messages = (op.contents as IGroupedBatchMessageContents).contents;
+		const messages = op.contents.contents;
 		let fakeCsn = 1;
 		return messages.map((subMessage) => ({
 			...op,

package/src/opLifecycle/opSplitter.ts

@@ -12,7 +12,7 @@ import { assert } from "@fluidframework/core-utils";
 import { IBatchMessage } from "@fluidframework/container-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
-import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
+import { ContainerMessageType, ContainerRuntimeChunkedOpMessage } from "../messageTypes";
 import { estimateSocketSize } from "./batchManager";
 import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
 
@@ -210,7 +210,7 @@ const chunkToBatchMessage = (
 	referenceSequenceNumber: number,
 	metadata: Record<string, unknown> | undefined = undefined,
 ): BatchMessage => {
-	const payload: ContainerRuntimeMessage = {
+	const payload: ContainerRuntimeChunkedOpMessage = {
 		type: ContainerMessageType.ChunkedOp,
 		contents: chunk,
 	};

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -6,13 +6,21 @@
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
 import {
 	ContainerMessageType,
-	ContainerRuntimeMessage,
-	SequencedContainerRuntimeMessage,
-} from "../containerRuntime";
+	type InboundContainerRuntimeMessage,
+	type InboundSequencedContainerRuntimeMessage,
+	type InboundSequencedContainerRuntimeMessageOrSystemMessage,
+	type InboundSequencedRecentlyAddedContainerRuntimeMessage,
+} from "../messageTypes";
 import { OpDecompressor } from "./opDecompressor";
 import { OpGroupingManager } from "./opGroupingManager";
 import { OpSplitter } from "./opSplitter";
 
+/**
+ * Stateful class for processing incoming remote messages as the virtualization measures are unwrapped,
+ * potentially across numerous inbound ops.
+ *
+ * @internal
+ */
 export class RemoteMessageProcessor {
 	constructor(
 		private readonly opSplitter: OpSplitter,
@@ -30,14 +38,25 @@ export class RemoteMessageProcessor {
 
 	/**
 	 * Ungroups and Unchunks the runtime ops encapsulated by the single remoteMessage received over the wire
-	 * @param remoteMessage - A message from another client, likely a chunked/grouped op
-	 * @returns the ungrouped, unchunked, unpacked SequencedContainerRuntimeMessage encapsulated in the remote message
+	 * @param remoteMessageCopy - A shallow copy of a message from another client, possibly virtualized
+	 * (grouped, compressed, and/or chunked).
+	 * Being a shallow copy, it's considered mutable, meaning no other Container or other parallel procedure
+	 * depends on this object instance.
+	 * Note remoteMessageCopy.contents (and other object props) MUST not be modified,
+	 * but may be overwritten (as is the case with contents).
+	 * @returns the unchunked, decompressed, ungrouped, unpacked SequencedContainerRuntimeMessages encapsulated in the remote message.
+	 * For ops that weren't virtualized (e.g. System ops that the ContainerRuntime will ultimately ignore),
+	 * a singleton array [remoteMessageCopy] is returned
 	 */
-	public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
-		const result: ISequencedDocumentMessage[] = [];
+	public process(
+		remoteMessageCopy: ISequencedDocumentMessage,
+	): InboundSequencedContainerRuntimeMessageOrSystemMessage[] {
+		const result: InboundSequencedContainerRuntimeMessageOrSystemMessage[] = [];
+
+		ensureContentsDeserialized(remoteMessageCopy);
 
 		// Ungroup before and after decompression for back-compat (cleanup tracked by AB#4371)
-		for (const ungroupedMessage of this.opGroupingManager.ungroupOp(copy(remoteMessage))) {
+		for (const ungroupedMessage of this.opGroupingManager.ungroupOp(remoteMessageCopy)) {
 			const message = this.opDecompressor.processMessage(ungroupedMessage).message;
 
 			for (let ungroupedMessage2 of this.opGroupingManager.ungroupOp(message)) {
@@ -50,7 +69,9 @@ export class RemoteMessageProcessor {
 				if (chunkProcessingResult.state !== "Processed") {
 					// If the message is not chunked or if the splitter is still rebuilding the original message,
 					// there is no need to continue processing
-					result.push(ungroupedMessage2);
+					result.push(
+						ungroupedMessage2 as InboundSequencedContainerRuntimeMessageOrSystemMessage,
+					);
 					continue;
 				}
 
@@ -68,7 +89,9 @@ export class RemoteMessageProcessor {
 						if (decompressionAfterChunking.state === "Skipped") {
 							// After chunking, if the original message was not compressed,
 							// there is no need to continue processing
-							result.push(ungroupedMessageAfterChunking2);
+							result.push(
+								ungroupedMessageAfterChunking2 as InboundSequencedContainerRuntimeMessageOrSystemMessage,
+							);
 							continue;
 						}
 
@@ -84,40 +107,38 @@ export class RemoteMessageProcessor {
 	}
 }
 
-const copy = (remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage => {
-	// Do shallow copy of message, as the processing flow will modify it.
-	// There might be multiple container instances receiving same message
-	// We do not need to make deep copy, as each layer will just replace message.content itself,
-	// but would not modify contents details
-	const message = { ...remoteMessage };
-
+/** Takes an incoming message and if the contents is a string, JSON.parse's it in place */
+function ensureContentsDeserialized(mutableMessage: ISequencedDocumentMessage): void {
 	// back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
 	// System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
 	// Old ops may contain empty string (I assume noops).
-	if (typeof message.contents === "string" && message.contents !== "") {
-		message.contents = JSON.parse(message.contents);
+	if (typeof mutableMessage.contents === "string" && mutableMessage.contents !== "") {
+		mutableMessage.contents = JSON.parse(mutableMessage.contents);
 	}
-
-	return message;
-};
+}
 
 /**
- * For a given message, it moves the nested ContainerRuntimeMessage props one level up.
+ * For a given message, it moves the nested InboundContainerRuntimeMessage props one level up.
  *
  * The return type illustrates the assumption that the message param
- * becomes a ContainerRuntimeMessage by the time the function returns
- * (but there is no runtime validation of the 'type' or 'compatDetails' values)
+ * becomes a InboundSequencedContainerRuntimeMessage by the time the function returns
+ * (but there is no runtime validation of the 'type' or 'compatDetails' values).
  */
 function unpack(
 	message: ISequencedDocumentMessage,
-): asserts message is SequencedContainerRuntimeMessage {
-	const innerContents = message.contents as ContainerRuntimeMessage;
-
-	// We're going to turn message into a SequencedContainerRuntimeMessage in-place
-	const sequencedContainerRuntimeMessage = message as SequencedContainerRuntimeMessage;
-	sequencedContainerRuntimeMessage.type = innerContents.type;
-	sequencedContainerRuntimeMessage.contents = innerContents.contents;
-	sequencedContainerRuntimeMessage.compatDetails = innerContents.compatDetails;
+): asserts message is InboundSequencedContainerRuntimeMessage {
+	// We assume the contents is an InboundContainerRuntimeMessage (the message is "packed")
+	const contents = message.contents as InboundContainerRuntimeMessage;
+
+	// We're going to unpack message in-place (promoting those properties of contents up to message itself)
+	const messageUnpacked = message as InboundSequencedContainerRuntimeMessage;
+
+	messageUnpacked.type = contents.type;
+	messageUnpacked.contents = contents.contents;
+	if ("compatDetails" in contents) {
+		(messageUnpacked as InboundSequencedRecentlyAddedContainerRuntimeMessage).compatDetails =
+			contents.compatDetails;
+	}
 }
 
 /**

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.6.3.3";
+export const pkgVersion = "2.0.0-internal.6.4.0";

package/src/pendingStateManager.ts

@@ -11,7 +11,7 @@ import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { DataProcessingError, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 
-import { ContainerMessageType, SequencedContainerRuntimeMessage } from "./containerRuntime";
+import { ContainerMessageType, InboundSequencedContainerRuntimeMessage } from "./messageTypes";
 import { pkgVersion } from "./packageVersion";
 import { IBatchMetadata } from "./metadata";
 
@@ -69,6 +69,26 @@ export interface IRuntimeStateHandler {
 	isActiveConnection: () => boolean;
 }
 
+/** Union of keys of T */
+type KeysOfUnion<T extends object> = T extends T ? keyof T : never;
+/** *Partial* type all possible combinations of properties and values of union T.
+ * This loosens typing allowing access to all possible properties without
+ * narrowing.
+ */
+type AnyComboFromUnion<T extends object> = { [P in KeysOfUnion<T>]?: T[P] };
+
+function buildPendingMessageContent(
+	// AnyComboFromUnion is needed need to gain access to compatDetails that
+	// is only defined for some cases.
+	message: AnyComboFromUnion<InboundSequencedContainerRuntimeMessage>,
+): string {
+	// IMPORTANT: Order matters here, this must match the order of the properties used
+	// when submitting the message.
+	const { type, contents, compatDetails } = message;
+	// Any properties that are not defined, won't be emitted by stringify.
+	return JSON.stringify({ type, contents, compatDetails });
+}
+
 /**
  * PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
  * acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
@@ -242,7 +262,7 @@ export class PendingStateManager implements IDisposable {
 	 * the batch information was preserved for batch messages.
 	 * @param message - The message that got ack'd and needs to be processed.
 	 */
-	public processPendingLocalMessage(message: SequencedContainerRuntimeMessage): unknown {
+	public processPendingLocalMessage(message: InboundSequencedContainerRuntimeMessage): unknown {
 		// Pre-processing part - This may be the start of a batch.
 		this.maybeProcessBatchBegin(message);
 
@@ -256,10 +276,7 @@ export class PendingStateManager implements IDisposable {
 
 		this.pendingMessages.shift();
 
-		// IMPORTANT: Order matters here, this must match the order of the properties used
-		// when submitting the message.
-		const { type, contents, compatDetails } = message;
-		const messageContent = JSON.stringify({ type, contents, compatDetails });
+		const messageContent = buildPendingMessageContent(message);
 
 		// Stringified content should match
 		if (pendingMessage.content !== messageContent) {

package/src/summary/runningSummarizer.ts

@@ -308,7 +308,8 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 						// latest version with which we will refresh the state. However in case of single commit
 						// summary, we might me missing a summary ack, so in that case we are still fine as the
 						// code in `submitSummary` function in container runtime, will refresh the latest state
-						// by calling `refreshLatestSummaryAckFromServer` and we will be fine.
+						// by calling `prefetchLatestSummaryThenClose`. We will load the next summarizer from the
+						// updated state and be fine.
 						const isIgnoredError =
 							isFluidError(error) &&
 							error.errorType === DriverErrorTypes.fileNotFoundOrAccessDeniedError;
@@ -407,6 +408,7 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 			!this.heuristicRunnerMicroTaskExists
 		) {
 			this.heuristicRunnerMicroTaskExists = true;
+			// eslint-disable-next-line @typescript-eslint/no-floating-promises
 			Promise.resolve()
 				.then(() => {
 					this.heuristicRunner?.run();
@@ -519,7 +521,7 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 	 * @param before - set of instructions to run before running the action.
 	 * @param action - action to perform.
 	 * @param after - set of instructions to run after running the action.
-	 * @returns - result of action.
+	 * @returns The result of the action.
 	 */
 	private async lockedSummaryAction<T>(
 		before: () => void,

package/src/summary/summarizer.ts

@@ -216,7 +216,7 @@ export class Summarizer extends TypedEventEmitter<ISummarizerEvents> implements
 	 * Should we try to run a last summary for the given stop reason?
 	 * Currently only allows "parentNotConnected"
 	 * @param stopReason - SummarizerStopReason
-	 * @returns - true if the stop reason can run a last summary
+	 * @returns `true` if the stop reason can run a last summary, otherwise `false`.
 	 */
 	public static stopReasonCanRunLastSummary(stopReason: SummarizerStopReason): boolean {
 		return stopReason === "parentNotConnected";
@@ -231,7 +231,7 @@ export class Summarizer extends TypedEventEmitter<ISummarizerEvents> implements
 	 * @param onBehalfOf - ID of the client that requested that the summarizer start
 	 * @param runCoordinator - cancellation token
 	 * @param newConfig - Summary configuration to override the existing config when invoking the RunningSummarizer.
-	 * @returns - Promise that is fulfilled when the RunningSummarizer is ready
+	 * @returns A promise that is fulfilled when the RunningSummarizer is ready.
 	 */
 	private async start(
 		onBehalfOf: string,

package/src/summary/summarizerNode/summarizerNode.ts

@@ -424,7 +424,7 @@ export class SummarizerNode implements IRootSummarizerNode {
 	 * Called when we get an ack from the server for a summary we've just sent. Updates the reference state of this node
 	 * from the state in the pending summary queue.
 	 * @param proposalHandle - Handle for the current proposal.
-	 * @param referenceSequenceNumber -  reference sequence number of sent summary.
+	 * @param referenceSequenceNumber - Reference sequence number of sent summary.
 	 */
 	protected refreshLatestSummaryFromPending(
 		proposalHandle: string,

package/src/summary/summarizerTypes.ts

@@ -104,7 +104,8 @@ export interface ISummarizeOptions {
 	/**
 	 * True to ask the server what the latest summary is first; defaults to false
 	 *
-	 * @deprecated - Summarize will not refresh latest snapshot state anymore.
+	 * @deprecated - Summarize will not refresh latest snapshot state anymore. Instead it updates the cache and closes
+	 * It's expected a new summarizer client will be created, likely by the same parent.
 	 */
 	readonly refreshLatestAck?: boolean;
 }

package/src/summary/summaryFormat.ts

@@ -187,7 +187,9 @@ export const dataStoreAttributesBlobName = ".component";
  * @param summarizeResult - Summary tree and stats to modify
  *
  * @example
+ *
  * Converts from:
+ *
  * ```typescript
  * {
  *     type: SummaryType.Tree,
@@ -208,6 +210,7 @@ export const dataStoreAttributesBlobName = ".component";
  *     },
  * }
  * ```
+ *
  * And adds +1 to treeNodeCount in stats.
  */
 export function wrapSummaryInChannelsTree(summarizeResult: ISummaryTreeWithStats): void {