@fluidframework/container-runtime 2.0.0-dev.5.3.2.178189 → 2.0.0-dev.6.4.0.191258

package/src/pendingStateManager.ts

@@ -3,19 +3,20 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable } from "@fluidframework/common-definitions";
-import { assert } from "@fluidframework/common-utils";
+import Deque from "double-ended-queue";
+
+import { IDisposable } from "@fluidframework/core-interfaces";
+import { assert, Lazy } from "@fluidframework/core-utils";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { DataProcessingError } from "@fluidframework/container-utils";
-import { Lazy } from "@fluidframework/core-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import Deque from "double-ended-queue";
-import { ContainerMessageType } from "./containerRuntime";
+import { DataProcessingError, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+
+import { ContainerMessageType, SequencedContainerRuntimeMessage } from "./containerRuntime";
 import { pkgVersion } from "./packageVersion";
 import { IBatchMetadata } from "./metadata";
 
 /**
- * ! TODO: Remove this interface in "2.0.0-internal.7.0.0" once we only read IPendingMessageNew
+ * ! TODO: Remove this interface in "2.0.0-internal.7.0.0" once we only read IPendingMessageNew (AB#4763)
  */
 export interface IPendingMessageOld {
 	type: "message";
@@ -41,7 +42,7 @@ export interface IPendingMessageNew {
 }
 
 /**
- * ! TODO: Remove this type in "2.0.0-internal.7.0.0"
+ * ! TODO: Remove this type in "2.0.0-internal.7.0.0" (AB#4763)
  */
 export type IPendingState = IPendingMessageOld | IPendingMessageNew;
 
@@ -65,6 +66,7 @@ export interface IRuntimeStateHandler {
 	applyStashedOp(content: string): Promise<unknown>;
 	reSubmit(message: IPendingBatchMessage): void;
 	reSubmitBatch(batch: IPendingBatchMessage[]): void;
+	isActiveConnection: () => boolean;
 }
 
 /**
@@ -79,15 +81,17 @@ export interface IRuntimeStateHandler {
 export class PendingStateManager implements IDisposable {
 	private readonly pendingMessages = new Deque<IPendingMessageNew>();
 	private readonly initialMessages = new Deque<IPendingMessageNew>();
+
+	/**
+	 * Sequenced local ops that are saved when stashing since pending ops may depend on them
+	 */
+	private savedOps: IPendingMessageNew[] = [];
+
 	private readonly disposeOnce = new Lazy<void>(() => {
 		this.initialMessages.clear();
 		this.pendingMessages.clear();
 	});
 
-	public get pendingMessagesCount(): number {
-		return this.pendingMessages.length;
-	}
-
 	// Indicates whether we are processing a batch.
 	private isProcessingBatch: boolean = false;
 
@@ -97,12 +101,20 @@ export class PendingStateManager implements IDisposable {
 
 	private clientId: string | undefined;
 
+	/**
+	 * The pending messages count. Includes `pendingMessages` and `initialMessages` to keep in sync with
+	 * 'hasPendingMessages'.
+	 */
+	public get pendingMessagesCount(): number {
+		return this.pendingMessages.length + this.initialMessages.length;
+	}
+
 	/**
 	 * Called to check if there are any pending messages in the pending message queue.
 	 * @returns A boolean indicating whether there are messages or not.
 	 */
 	public hasPendingMessages(): boolean {
-		return !this.pendingMessages.isEmpty() || !this.initialMessages.isEmpty();
+		return this.pendingMessagesCount !== 0;
 	}
 
 	public getLocalState(): IPendingLocalState | undefined {
@@ -112,23 +124,22 @@ export class PendingStateManager implements IDisposable {
 		);
 		if (!this.pendingMessages.isEmpty()) {
 			return {
-				pendingStates: this.pendingMessages.toArray().map((message) => {
-					// ! TODO: Remove conversion to IPendingMessageOld in "2.0.0-internal.6.0.0" AB#3826
-					const content = JSON.parse(message.content);
-					// IdAllocations need their localOpMetadata stashed in the contents
-					// of the op to correctly resume the session when processing stashed ops
-					if (content.type === ContainerMessageType.IdAllocation) {
-						content.contents.stashedState = message.localOpMetadata;
-					}
-					return {
-						...message,
-						messageType: content.type,
-						content: content.contents,
+				pendingStates: [...this.savedOps, ...this.pendingMessages.toArray()].map(
+					(message) => {
+						let content = message.content;
+						const parsedContent = JSON.parse(content);
+						// IdAllocations need their localOpMetadata stashed in the contents
+						// of the op to correctly resume the session when processing stashed ops
+						if (parsedContent.type === ContainerMessageType.IdAllocation) {
+							parsedContent.contents.stashedState = message.localOpMetadata;
+							content = JSON.stringify(parsedContent);
+						}
+
 						// delete localOpMetadata since it may not be serializable
 						// and will be regenerated by applyStashedOp()
-						localOpMetadata: undefined,
-					};
-				}),
+						return { ...message, content, localOpMetadata: undefined };
+					},
+				),
 			};
 		}
 	}
@@ -136,10 +147,11 @@ export class PendingStateManager implements IDisposable {
 	constructor(
 		private readonly stateHandler: IRuntimeStateHandler,
 		initialLocalState: IPendingLocalState | undefined,
+		private readonly logger: ITelemetryLoggerExt | undefined,
 	) {
 		/**
 		 * Convert old local state format to the new format (IPendingMessageOld to IPendingMessageNew)
-		 * ! TODO: Remove this conversion in "2.0.0-internal.7.0.0"
+		 * ! TODO: Remove this conversion in "2.0.0-internal.7.0.0" (AB#4763)
 		 */
 		if (initialLocalState?.pendingStates) {
 			for (const initialState of initialLocalState.pendingStates) {
@@ -230,7 +242,7 @@ export class PendingStateManager implements IDisposable {
 	 * the batch information was preserved for batch messages.
 	 * @param message - The message that got ack'd and needs to be processed.
 	 */
-	public processPendingLocalMessage(message: ISequencedDocumentMessage): unknown {
+	public processPendingLocalMessage(message: SequencedContainerRuntimeMessage): unknown {
 		// Pre-processing part - This may be the start of a batch.
 		this.maybeProcessBatchBegin(message);
 
@@ -240,10 +252,16 @@ export class PendingStateManager implements IDisposable {
 			pendingMessage !== undefined,
 			0x169 /* "No pending message found for this remote message" */,
 		);
+		this.savedOps.push(pendingMessage);
+
 		this.pendingMessages.shift();
 
-		const messageContent = JSON.stringify({ type: message.type, contents: message.contents });
-		// Stringified content does not match
+		// IMPORTANT: Order matters here, this must match the order of the properties used
+		// when submitting the message.
+		const { type, contents, compatDetails } = message;
+		const messageContent = JSON.stringify({ type, contents, compatDetails });
+
+		// Stringified content should match
 		if (pendingMessage.content !== messageContent) {
 			this.stateHandler.close(
 				DataProcessingError.create(
@@ -364,18 +382,16 @@ export class PendingStateManager implements IDisposable {
 			0x174 /* "initial states should be empty before replaying pending" */,
 		);
 
-		let pendingMessagesCount = this.pendingMessages.length;
-		if (pendingMessagesCount === 0) {
-			return;
-		}
+		const initialPendingMessagesCount = this.pendingMessages.length;
+		let remainingPendingMessagesCount = this.pendingMessages.length;
 
 		// Process exactly `pendingMessagesCount` items in the queue as it represents the number of messages that were
 		// pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
 		// which must not be replayed.
-		while (pendingMessagesCount > 0) {
+		while (remainingPendingMessagesCount > 0) {
 			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 			let pendingMessage = this.pendingMessages.shift()!;
-			pendingMessagesCount--;
+			remainingPendingMessagesCount--;
 			assert(
 				pendingMessage.opMetadata?.batch !== false,
 				0x41b /* We cannot process batches in chunks */,
@@ -388,14 +404,14 @@ export class PendingStateManager implements IDisposable {
 			 */
 			if (pendingMessage.opMetadata?.batch) {
 				assert(
-					pendingMessagesCount > 0,
+					remainingPendingMessagesCount > 0,
 					0x554 /* Last pending message cannot be a batch begin */,
 				);
 
 				const batch: IPendingBatchMessage[] = [];
 
 				// check is >= because batch end may be last pending message
-				while (pendingMessagesCount >= 0) {
+				while (remainingPendingMessagesCount >= 0) {
 					batch.push({
 						content: pendingMessage.content,
 						localOpMetadata: pendingMessage.localOpMetadata,
@@ -405,11 +421,11 @@ export class PendingStateManager implements IDisposable {
 					if (pendingMessage.opMetadata?.batch === false) {
 						break;
 					}
-					assert(pendingMessagesCount > 0, 0x555 /* No batch end found */);
+					assert(remainingPendingMessagesCount > 0, 0x555 /* No batch end found */);
 
 					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 					pendingMessage = this.pendingMessages.shift()!;
-					pendingMessagesCount--;
+					remainingPendingMessagesCount--;
 					assert(
 						pendingMessage.opMetadata?.batch !== true,
 						0x556 /* Batch start needs a corresponding batch end */,
@@ -425,5 +441,19 @@ export class PendingStateManager implements IDisposable {
 				});
 			}
 		}
+
+		// pending ops should no longer depend on previous sequenced local ops after resubmit
+		this.savedOps = [];
+
+		// We replayPendingStates on read connections too - we expect these to get nack'd though, and to then reconnect
+		// on a write connection and replay again. This filters out the replay that happens on the read connection so
+		// we only see the replays on write connections (that have a chance to go through).
+		if (this.stateHandler.isActiveConnection()) {
+			this.logger?.sendTelemetryEvent({
+				eventName: "PendingStatesReplayed",
+				count: initialPendingMessagesCount,
+				clientId: this.stateHandler.clientId(),
+			});
+		}
 	}
 }

package/src/scheduleManager.ts

@@ -5,14 +5,16 @@
 import { EventEmitter } from "events";
 import { IDeltaManager } from "@fluidframework/container-definitions";
 import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, performance } from "@fluidframework/common-utils";
-import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import {
+	createChildLogger,
 	DataCorruptionError,
 	DataProcessingError,
 	extractSafePropertiesFromMessage,
-} from "@fluidframework/container-utils";
+	ITelemetryLoggerExt,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { performance } from "@fluid-internal/client-utils";
+import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import { DeltaScheduler } from "./deltaScheduler";
 import { pkgVersion } from "./packageVersion";
 import { IBatchMetadata } from "./metadata";
@@ -45,7 +47,7 @@ export class ScheduleManager {
 	) {
 		this.deltaScheduler = new DeltaScheduler(
 			this.deltaManager,
-			ChildLogger.create(this.logger, "DeltaScheduler"),
+			createChildLogger({ logger: this.logger, namespace: "DeltaScheduler" }),
 		);
 		void new ScheduleManagerCore(deltaManager, getClientId, logger);
 	}

package/src/summary/index.ts

@@ -11,7 +11,7 @@ export {
 	OrderedClientCollection,
 	OrderedClientElection,
 } from "./orderedClientElection";
-export { RunningSummarizer } from "./runningSummarizer";
+export { defaultMaxAttemptsForSubmitFailures, RunningSummarizer } from "./runningSummarizer";
 export {
 	ICancellableSummarizerController,
 	neverCancelledSummaryToken,
@@ -28,10 +28,9 @@ export { SummarizeHeuristicData, SummarizeHeuristicRunner } from "./summarizerHe
 export {
 	createRootSummarizerNode,
 	createRootSummarizerNodeWithGC,
-	IFetchSnapshotResult,
+	IRefreshSummaryResult,
 	IRootSummarizerNode,
 	IRootSummarizerNodeWithGC,
-	RefreshSummaryResult,
 } from "./summarizerNode";
 export {
 	IConnectableRuntime,
@@ -64,6 +63,8 @@ export {
 	SummarizeResultPart,
 	SubmitSummaryFailureData,
 	SummaryStage,
+	IRetriableFailureResult,
+	ISummarizeEventProps,
 } from "./summarizerTypes";
 export {
 	IAckedSummary,

package/src/summary/orderedClientElection.ts

@@ -3,11 +3,15 @@
  * Licensed under the MIT License.
  */
 /* eslint-disable @rushstack/no-new-null */
-import { IEvent, IEventProvider } from "@fluidframework/common-definitions";
-import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, TypedEventEmitter } from "@fluidframework/common-utils";
+import { IEvent, IEventProvider, ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import {
+	ITelemetryLoggerExt,
+	createChildLogger,
+	UsageError,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { TypedEventEmitter } from "@fluid-internal/client-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";
-import { UsageError } from "@fluidframework/container-utils";
 import { IClient, IQuorumClients, ISequencedClient } from "@fluidframework/protocol-definitions";
 import { summarizerClientType } from "./summarizerClientElection";
 
@@ -101,12 +105,12 @@ export class OrderedClientCollection
 	}
 
 	constructor(
-		logger: ITelemetryLoggerExt,
+		logger: ITelemetryBaseLogger,
 		deltaManager: Pick<IDeltaManager<unknown, unknown>, "lastSequenceNumber">,
 		quorum: Pick<IQuorumClients, "getMembers" | "on">,
 	) {
 		super();
-		this.logger = ChildLogger.create(logger, "OrderedClientCollection");
+		this.logger = createChildLogger({ logger, namespace: "OrderedClientCollection" });
 		const members = quorum.getMembers();
 		for (const [clientId, client] of members) {
 			this.addClient(clientId, client);

package/src/summary/runWhileConnectedCoordinator.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { assert, Deferred } from "@fluidframework/common-utils";
+import { assert, Deferred } from "@fluidframework/core-utils";
 import {
 	SummarizerStopReason,
 	IConnectableRuntime,