@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258

package/src/pendingStateManager.ts

@@ -3,17 +3,20 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable } from "@fluidframework/common-definitions";
-import { assert, Lazy } from "@fluidframework/common-utils";
+import Deque from "double-ended-queue";
+
+import { IDisposable } from "@fluidframework/core-interfaces";
+import { assert, Lazy } from "@fluidframework/core-utils";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { DataProcessingError } from "@fluidframework/container-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import Deque from "double-ended-queue";
-import { ContainerMessageType } from "./containerRuntime";
+import { DataProcessingError, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+
+import { ContainerMessageType, SequencedContainerRuntimeMessage } from "./containerRuntime";
 import { pkgVersion } from "./packageVersion";
+import { IBatchMetadata } from "./metadata";
 
 /**
- * ! TODO: Remove this interface in "2.0.0-internal.7.0.0" once we only read IPendingMessageNew
+ * ! TODO: Remove this interface in "2.0.0-internal.7.0.0" once we only read IPendingMessageNew (AB#4763)
  */
 export interface IPendingMessageOld {
 	type: "message";
@@ -39,7 +42,7 @@ export interface IPendingMessageNew {
 }
 
 /**
- * ! TODO: Remove this type in "2.0.0-internal.7.0.0"
+ * ! TODO: Remove this type in "2.0.0-internal.7.0.0" (AB#4763)
  */
 export type IPendingState = IPendingMessageOld | IPendingMessageNew;
 
@@ -50,17 +53,20 @@ export interface IPendingLocalState {
 	pendingStates: IPendingState[];
 }
 
+export interface IPendingBatchMessage {
+	content: string;
+	localOpMetadata: unknown;
+	opMetadata: Record<string, unknown> | undefined;
+}
+
 export interface IRuntimeStateHandler {
 	connected(): boolean;
 	clientId(): string | undefined;
 	close(error?: ICriticalContainerError): void;
 	applyStashedOp(content: string): Promise<unknown>;
-	reSubmit(
-		content: string | undefined,
-		localOpMetadata: unknown,
-		opMetadata: Record<string, unknown> | undefined,
-	): void;
-	orderSequentially(callback: () => void): void;
+	reSubmit(message: IPendingBatchMessage): void;
+	reSubmitBatch(batch: IPendingBatchMessage[]): void;
+	isActiveConnection: () => boolean;
 }
 
 /**
@@ -75,15 +81,17 @@ export interface IRuntimeStateHandler {
 export class PendingStateManager implements IDisposable {
 	private readonly pendingMessages = new Deque<IPendingMessageNew>();
 	private readonly initialMessages = new Deque<IPendingMessageNew>();
+
+	/**
+	 * Sequenced local ops that are saved when stashing since pending ops may depend on them
+	 */
+	private savedOps: IPendingMessageNew[] = [];
+
 	private readonly disposeOnce = new Lazy<void>(() => {
 		this.initialMessages.clear();
 		this.pendingMessages.clear();
 	});
 
-	public get pendingMessagesCount(): number {
-		return this.pendingMessages.length;
-	}
-
 	// Indicates whether we are processing a batch.
 	private isProcessingBatch: boolean = false;
 
@@ -93,12 +101,20 @@ export class PendingStateManager implements IDisposable {
 
 	private clientId: string | undefined;
 
+	/**
+	 * The pending messages count. Includes `pendingMessages` and `initialMessages` to keep in sync with
+	 * 'hasPendingMessages'.
+	 */
+	public get pendingMessagesCount(): number {
+		return this.pendingMessages.length + this.initialMessages.length;
+	}
+
 	/**
 	 * Called to check if there are any pending messages in the pending message queue.
 	 * @returns A boolean indicating whether there are messages or not.
 	 */
 	public hasPendingMessages(): boolean {
-		return !this.pendingMessages.isEmpty() || !this.initialMessages.isEmpty();
+		return this.pendingMessagesCount !== 0;
 	}
 
 	public getLocalState(): IPendingLocalState | undefined {
@@ -108,23 +124,22 @@ export class PendingStateManager implements IDisposable {
 		);
 		if (!this.pendingMessages.isEmpty()) {
 			return {
-				pendingStates: this.pendingMessages.toArray().map((message) => {
-					// ! TODO: Remove conversion to IPendingMessageOld in "2.0.0-internal.6.0.0" AB#3826
-					const content = JSON.parse(message.content);
-					// IdAllocations need their localOpMetadata stashed in the contents
-					// of the op to correctly resume the session when processing stashed ops
-					if (content.type === ContainerMessageType.IdAllocation) {
-						content.contents.stashedState = message.localOpMetadata;
-					}
-					return {
-						...message,
-						messageType: content.type,
-						content: content.contents,
+				pendingStates: [...this.savedOps, ...this.pendingMessages.toArray()].map(
+					(message) => {
+						let content = message.content;
+						const parsedContent = JSON.parse(content);
+						// IdAllocations need their localOpMetadata stashed in the contents
+						// of the op to correctly resume the session when processing stashed ops
+						if (parsedContent.type === ContainerMessageType.IdAllocation) {
+							parsedContent.contents.stashedState = message.localOpMetadata;
+							content = JSON.stringify(parsedContent);
+						}
+
 						// delete localOpMetadata since it may not be serializable
 						// and will be regenerated by applyStashedOp()
-						localOpMetadata: undefined,
-					};
-				}),
+						return { ...message, content, localOpMetadata: undefined };
+					},
+				),
 			};
 		}
 	}
@@ -132,10 +147,11 @@ export class PendingStateManager implements IDisposable {
 	constructor(
 		private readonly stateHandler: IRuntimeStateHandler,
 		initialLocalState: IPendingLocalState | undefined,
+		private readonly logger: ITelemetryLoggerExt | undefined,
 	) {
 		/**
 		 * Convert old local state format to the new format (IPendingMessageOld to IPendingMessageNew)
-		 * ! TODO: Remove this conversion in "2.0.0-internal.7.0.0"
+		 * ! TODO: Remove this conversion in "2.0.0-internal.7.0.0" (AB#4763)
 		 */
 		if (initialLocalState?.pendingStates) {
 			for (const initialState of initialLocalState.pendingStates) {
@@ -207,9 +223,13 @@ export class PendingStateManager implements IDisposable {
 				}
 			}
 
-			// applyStashedOp will cause the DDS to behave as if it has sent the op but not actually send it
-			const localOpMetadata = await this.stateHandler.applyStashedOp(nextMessage.content);
-			nextMessage.localOpMetadata = localOpMetadata;
+			try {
+				// applyStashedOp will cause the DDS to behave as if it has sent the op but not actually send it
+				const localOpMetadata = await this.stateHandler.applyStashedOp(nextMessage.content);
+				nextMessage.localOpMetadata = localOpMetadata;
+			} catch (error) {
+				throw DataProcessingError.wrapIfUnrecognized(error, "applyStashedOp", nextMessage);
+			}
 
 			// then we push onto pendingMessages which will cause PendingStateManager to resubmit when we connect
 			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -222,7 +242,7 @@ export class PendingStateManager implements IDisposable {
 	 * the batch information was preserved for batch messages.
 	 * @param message - The message that got ack'd and needs to be processed.
 	 */
-	public processPendingLocalMessage(message: ISequencedDocumentMessage): unknown {
+	public processPendingLocalMessage(message: SequencedContainerRuntimeMessage): unknown {
 		// Pre-processing part - This may be the start of a batch.
 		this.maybeProcessBatchBegin(message);
 
@@ -232,10 +252,16 @@ export class PendingStateManager implements IDisposable {
 			pendingMessage !== undefined,
 			0x169 /* "No pending message found for this remote message" */,
 		);
+		this.savedOps.push(pendingMessage);
+
 		this.pendingMessages.shift();
 
-		const messageContent = JSON.stringify({ type: message.type, contents: message.contents });
-		// Stringified content does not match
+		// IMPORTANT: Order matters here, this must match the order of the properties used
+		// when submitting the message.
+		const { type, contents, compatDetails } = message;
+		const messageContent = JSON.stringify({ type, contents, compatDetails });
+
+		// Stringified content should match
 		if (pendingMessage.content !== messageContent) {
 			this.stateHandler.close(
 				DataProcessingError.create(
@@ -262,7 +288,7 @@ export class PendingStateManager implements IDisposable {
 	 */
 	private maybeProcessBatchBegin(message: ISequencedDocumentMessage) {
 		// This message is the first in a batch if the "batch" property on the metadata is set to true
-		if (message.metadata?.batch) {
+		if ((message.metadata as IBatchMetadata | undefined)?.batch) {
 			// We should not already be processing a batch and there should be no pending batch begin message.
 			assert(
 				!this.isProcessingBatch && this.pendingBatchBeginMessage === undefined,
@@ -290,10 +316,12 @@ export class PendingStateManager implements IDisposable {
 			0x16d /* "There is no pending batch begin message" */,
 		);
 
-		const batchEndMetadata = message.metadata?.batch;
+		const batchEndMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;
 		if (this.pendingMessages.isEmpty() || batchEndMetadata === false) {
 			// Get the batch begin metadata from the first message in the batch.
-			const batchBeginMetadata = this.pendingBatchBeginMessage.metadata?.batch;
+			const batchBeginMetadata = (
+				this.pendingBatchBeginMessage.metadata as IBatchMetadata | undefined
+			)?.batch;
 
 			// There could be just a single message in the batch. If so, it should not have any batch metadata. If there
 			// are multiple messages in the batch, verify that we got the correct batch begin and end metadata.
@@ -311,7 +339,10 @@ export class PendingStateManager implements IDisposable {
 							message,
 							{
 								runtimeVersion: pkgVersion,
-								batchClientId: this.pendingBatchBeginMessage.clientId,
+								batchClientId:
+									this.pendingBatchBeginMessage.clientId === null
+										? "null"
+										: this.pendingBatchBeginMessage.clientId,
 								clientId: this.stateHandler.clientId(),
 								hasBatchStart: batchBeginMetadata === true,
 								hasBatchEnd: batchEndMetadata === false,
@@ -351,18 +382,16 @@ export class PendingStateManager implements IDisposable {
 			0x174 /* "initial states should be empty before replaying pending" */,
 		);
 
-		let pendingMessagesCount = this.pendingMessages.length;
-		if (pendingMessagesCount === 0) {
-			return;
-		}
+		const initialPendingMessagesCount = this.pendingMessages.length;
+		let remainingPendingMessagesCount = this.pendingMessages.length;
 
 		// Process exactly `pendingMessagesCount` items in the queue as it represents the number of messages that were
 		// pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
 		// which must not be replayed.
-		while (pendingMessagesCount > 0) {
+		while (remainingPendingMessagesCount > 0) {
 			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 			let pendingMessage = this.pendingMessages.shift()!;
-			pendingMessagesCount--;
+			remainingPendingMessagesCount--;
 			assert(
 				pendingMessage.opMetadata?.batch !== false,
 				0x41b /* We cannot process batches in chunks */,
@@ -375,40 +404,56 @@ export class PendingStateManager implements IDisposable {
 			 */
 			if (pendingMessage.opMetadata?.batch) {
 				assert(
-					pendingMessagesCount > 0,
+					remainingPendingMessagesCount > 0,
 					0x554 /* Last pending message cannot be a batch begin */,
 				);
 
-				this.stateHandler.orderSequentially(() => {
-					while (pendingMessagesCount >= 0) {
-						// check is >= because batch end may be last pending message
-						this.stateHandler.reSubmit(
-							pendingMessage.content,
-							pendingMessage.localOpMetadata,
-							pendingMessage.opMetadata,
-						);
-
-						if (pendingMessage.opMetadata?.batch === false) {
-							break;
-						}
-						assert(pendingMessagesCount > 0, 0x555 /* No batch end found */);
-
-						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-						pendingMessage = this.pendingMessages.shift()!;
-						pendingMessagesCount--;
-						assert(
-							pendingMessage.opMetadata?.batch !== true,
-							0x556 /* Batch start needs a corresponding batch end */,
-						);
+				const batch: IPendingBatchMessage[] = [];
+
+				// check is >= because batch end may be last pending message
+				while (remainingPendingMessagesCount >= 0) {
+					batch.push({
+						content: pendingMessage.content,
+						localOpMetadata: pendingMessage.localOpMetadata,
+						opMetadata: pendingMessage.opMetadata,
+					});
+
+					if (pendingMessage.opMetadata?.batch === false) {
+						break;
 					}
-				});
+					assert(remainingPendingMessagesCount > 0, 0x555 /* No batch end found */);
+
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					pendingMessage = this.pendingMessages.shift()!;
+					remainingPendingMessagesCount--;
+					assert(
+						pendingMessage.opMetadata?.batch !== true,
+						0x556 /* Batch start needs a corresponding batch end */,
+					);
+				}
+
+				this.stateHandler.reSubmitBatch(batch);
 			} else {
-				this.stateHandler.reSubmit(
-					pendingMessage.content,
-					pendingMessage.localOpMetadata,
-					pendingMessage.opMetadata,
-				);
+				this.stateHandler.reSubmit({
+					content: pendingMessage.content,
+					localOpMetadata: pendingMessage.localOpMetadata,
+					opMetadata: pendingMessage.opMetadata,
+				});
 			}
 		}
+
+		// pending ops should no longer depend on previous sequenced local ops after resubmit
+		this.savedOps = [];
+
+		// We replayPendingStates on read connections too - we expect these to get nack'd though, and to then reconnect
+		// on a write connection and replay again. This filters out the replay that happens on the read connection so
+		// we only see the replays on write connections (that have a chance to go through).
+		if (this.stateHandler.isActiveConnection()) {
+			this.logger?.sendTelemetryEvent({
+				eventName: "PendingStatesReplayed",
+				count: initialPendingMessagesCount,
+				clientId: this.stateHandler.clientId(),
+			});
+		}
 	}
 }

package/src/scheduleManager.ts

@@ -5,16 +5,19 @@
 import { EventEmitter } from "events";
 import { IDeltaManager } from "@fluidframework/container-definitions";
 import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, performance } from "@fluidframework/common-utils";
-import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import {
+	createChildLogger,
 	DataCorruptionError,
 	DataProcessingError,
 	extractSafePropertiesFromMessage,
-} from "@fluidframework/container-utils";
+	ITelemetryLoggerExt,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { performance } from "@fluid-internal/client-utils";
+import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import { DeltaScheduler } from "./deltaScheduler";
 import { pkgVersion } from "./packageVersion";
+import { IBatchMetadata } from "./metadata";
 
 type IRuntimeMessageMetadata =
 	| undefined
@@ -44,7 +47,7 @@ export class ScheduleManager {
 	) {
 		this.deltaScheduler = new DeltaScheduler(
 			this.deltaManager,
-			ChildLogger.create(this.logger, "DeltaScheduler"),
+			createChildLogger({ logger: this.logger, namespace: "DeltaScheduler" }),
 		);
 		void new ScheduleManagerCore(deltaManager, getClientId, logger);
 	}
@@ -61,7 +64,9 @@ export class ScheduleManager {
 			this.deltaScheduler.batchBegin(message);
 
 			const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
-			this.batchClientId = batch ? message.clientId : undefined;
+			// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
+			// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+			this.batchClientId = batch ? (message.clientId as string) : undefined;
 		}
 	}
 
@@ -127,7 +132,9 @@ class ScheduleManagerCore {
 
 			// Set the batch flag to false on the last message to indicate the end of the send batch
 			const lastMessage = messages[messages.length - 1];
-			lastMessage.metadata = { ...lastMessage.metadata, batch: false };
+			// TODO: It's not clear if this shallow clone is required, as opposed to just setting "batch" to false.
+			// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+			lastMessage.metadata = { ...(lastMessage.metadata as any), batch: false };
 		});
 
 		// Listen for updates and peek at the inbound
@@ -185,7 +192,7 @@ class ScheduleManagerCore {
 					{
 						type: message.type,
 						contentType: typeof message.contents,
-						batch: message.metadata?.batch,
+						batch: (message.metadata as IBatchMetadata | undefined)?.batch,
 						compression: message.compression,
 						pauseSeqNum: this.pauseSequenceNumber,
 					},
@@ -263,7 +270,8 @@ class ScheduleManagerCore {
 					message,
 					{
 						runtimeVersion: pkgVersion,
-						batchClientId: this.currentBatchClientId,
+						batchClientId:
+							this.currentBatchClientId === null ? "null" : this.currentBatchClientId,
 						pauseSequenceNumber: this.pauseSequenceNumber,
 						localBatch: this.currentBatchClientId === this.getClientId(),
 						messageType: message.type,
@@ -297,7 +305,8 @@ class ScheduleManagerCore {
 		) {
 			throw new DataCorruptionError("OpBatchIncomplete", {
 				runtimeVersion: pkgVersion,
-				batchClientId: this.currentBatchClientId,
+				batchClientId:
+					this.currentBatchClientId === null ? "null" : this.currentBatchClientId,
 				pauseSequenceNumber: this.pauseSequenceNumber,
 				localBatch: this.currentBatchClientId === this.getClientId(),
 				localMessage: message.clientId === this.getClientId(),
@@ -321,7 +330,9 @@ class ScheduleManagerCore {
 				0x29f /* "we should be processing ops when there is no active batch" */,
 			);
 			this.pauseSequenceNumber = message.sequenceNumber;
-			this.currentBatchClientId = message.clientId;
+			// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
+			// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+			this.currentBatchClientId = message.clientId as string;
 			// Start of the batch
 			// Only pause processing if queue has no other ops!
 			// If there are any other ops in the queue, processing will be stopped when they are processed!

package/src/summary/index.ts

@@ -11,7 +11,7 @@ export {
 	OrderedClientCollection,
 	OrderedClientElection,
 } from "./orderedClientElection";
-export { RunningSummarizer } from "./runningSummarizer";
+export { defaultMaxAttemptsForSubmitFailures, RunningSummarizer } from "./runningSummarizer";
 export {
 	ICancellableSummarizerController,
 	neverCancelledSummaryToken,
@@ -28,10 +28,9 @@ export { SummarizeHeuristicData, SummarizeHeuristicRunner } from "./summarizerHe
 export {
 	createRootSummarizerNode,
 	createRootSummarizerNodeWithGC,
-	IFetchSnapshotResult,
+	IRefreshSummaryResult,
 	IRootSummarizerNode,
 	IRootSummarizerNodeWithGC,
-	RefreshSummaryResult,
 } from "./summarizerNode";
 export {
 	IConnectableRuntime,
@@ -62,6 +61,10 @@ export {
 	ISummarizingWarning,
 	IUploadSummaryResult,
 	SummarizeResultPart,
+	SubmitSummaryFailureData,
+	SummaryStage,
+	IRetriableFailureResult,
+	ISummarizeEventProps,
 } from "./summarizerTypes";
 export {
 	IAckedSummary,
@@ -96,7 +99,7 @@ export {
 	wrapSummaryInChannelsTree,
 	idCompressorBlobName,
 } from "./summaryFormat";
-export { getFailMessage, SummarizeReason } from "./summaryGenerator";
+export { getFailMessage, RetriableSummaryError, SummarizeReason } from "./summaryGenerator";
 export {
 	IConnectedEvents,
 	IConnectedState,

package/src/summary/orderedClientElection.ts

@@ -3,11 +3,15 @@
  * Licensed under the MIT License.
  */
 /* eslint-disable @rushstack/no-new-null */
-import { IEvent, IEventProvider } from "@fluidframework/common-definitions";
-import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, TypedEventEmitter } from "@fluidframework/common-utils";
+import { IEvent, IEventProvider, ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import {
+	ITelemetryLoggerExt,
+	createChildLogger,
+	UsageError,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { TypedEventEmitter } from "@fluid-internal/client-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";
-import { UsageError } from "@fluidframework/container-utils";
 import { IClient, IQuorumClients, ISequencedClient } from "@fluidframework/protocol-definitions";
 import { summarizerClientType } from "./summarizerClientElection";
 
@@ -101,12 +105,12 @@ export class OrderedClientCollection
 	}
 
 	constructor(
-		logger: ITelemetryLoggerExt,
+		logger: ITelemetryBaseLogger,
 		deltaManager: Pick<IDeltaManager<unknown, unknown>, "lastSequenceNumber">,
 		quorum: Pick<IQuorumClients, "getMembers" | "on">,
 	) {
 		super();
-		this.logger = ChildLogger.create(logger, "OrderedClientCollection");
+		this.logger = createChildLogger({ logger, namespace: "OrderedClientCollection" });
 		const members = quorum.getMembers();
 		for (const [clientId, client] of members) {
 			this.addClient(clientId, client);

package/src/summary/runWhileConnectedCoordinator.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { assert, Deferred } from "@fluidframework/common-utils";
+import { assert, Deferred } from "@fluidframework/core-utils";
 import {
 	SummarizerStopReason,
 	IConnectableRuntime,