@fluidframework/container-runtime 2.0.0-internal.6.3.3 → 2.0.0-internal.6.4.0

package/src/containerRuntime.ts

@@ -88,7 +88,6 @@ import {
 	IIdCompressorCore,
 	IdCreationRange,
 	IdCreationRangeWithStashedState,
-	IAttachMessage,
 } from "@fluidframework/runtime-definitions";
 import {
 	addBlobToSummary,
@@ -184,92 +183,42 @@ import {
 } from "./opLifecycle";
 import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy";
 import { IBatchMetadata } from "./metadata";
-
-export enum ContainerMessageType {
-	// An op to be delivered to store
-	FluidDataStoreOp = "component",
-
-	// Creates a new store
-	Attach = "attach",
-
-	// Chunked operation.
-	ChunkedOp = "chunkedOp",
-
-	// Signifies that a blob has been attached and should not be garbage collected by storage
-	BlobAttach = "blobAttach",
-
-	// Ties our new clientId to our old one on reconnect
-	Rejoin = "rejoin",
-
-	// Sets the alias of a root data store
-	Alias = "alias",
-
-	/**
-	 * An op containing an IdRange of Ids allocated using the runtime's IdCompressor since
-	 * the last allocation op was sent.
-	 * See the [IdCompressor README](./id-compressor/README.md) for more details.
-	 */
-	IdAllocation = "idAllocation",
-}
-
-/**
- * How should an older client handle an unrecognized remote op type?
- *
- * @internal
- */
-export type CompatModeBehavior =
-	/** Ignore the op. It won't be persisted if this client summarizes */
-	| "Ignore"
-	/** Fail processing immediately. (The container will close) */
-	| "FailToProcess";
-
-/**
- * All the info an older client would need to know how to handle an unrecognized remote op type
- *
- * @internal
- */
-export interface IContainerRuntimeMessageCompatDetails {
-	/** How should an older client handle an unrecognized remote op type? */
-	behavior: CompatModeBehavior;
-}
+import {
+	ContainerMessageType,
+	type InboundSequencedContainerRuntimeMessage,
+	type InboundSequencedContainerRuntimeMessageOrSystemMessage,
+	type ContainerRuntimeIdAllocationMessage,
+	type LocalContainerRuntimeIdAllocationMessage,
+	type LocalContainerRuntimeMessage,
+	type OutboundContainerRuntimeMessage,
+	type UnknownContainerRuntimeMessage,
+} from "./messageTypes";
 
 /**
  * Utility to implement compat behaviors given an unknown message type
  * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
  *
- * @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
+ * @param _unknownContainerRuntimeMessageType - Typed as something unexpected, to ensure all known types have been
  * handled before calling this function (e.g. in a switch statement).
  * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
  */
 function compatBehaviorAllowsMessageType(
-	_unknownContainerRuntimeMessageType: never,
+	_unknownContainerRuntimeMessageType: UnknownContainerRuntimeMessage["type"],
 	compatBehavior: "Ignore" | "FailToProcess" | undefined,
 ): boolean {
 	// undefined defaults to same behavior as "FailToProcess"
 	return compatBehavior === "Ignore";
 }
 
-/**
- * The unpacked runtime message / details to be handled or dispatched by the ContainerRuntime
- *
- * IMPORTANT: when creating one to be serialized, set the properties in the order they appear here.
- * This way stringified values can be compared.
- */
-export interface ContainerRuntimeMessage {
-	/** Type of the op, within the ContainerRuntime's domain */
-	type: ContainerMessageType;
-	/** Domain-specific contents, interpreted according to the type */
-	contents: any;
-	/** Info describing how to handle this op in case the type is unrecognized (default: fail to process) */
-	compatDetails?: IContainerRuntimeMessageCompatDetails;
+function prepareLocalContainerRuntimeIdAllocationMessageForTransit(
+	message: LocalContainerRuntimeIdAllocationMessage | ContainerRuntimeIdAllocationMessage,
+): asserts message is ContainerRuntimeIdAllocationMessage {
+	// Remove the stashedState from the op if it's a stashed op
+	if ("stashedState" in message.contents) {
+		delete message.contents.stashedState;
+	}
 }
 
-/**
- * An unpacked ISequencedDocumentMessage with the inner ContainerRuntimeMessage type/contents/etc
- * promoted up to the outer object
- */
-export type SequencedContainerRuntimeMessage = ISequencedDocumentMessage & ContainerRuntimeMessage;
-
 export interface ISummaryBaseConfiguration {
 	/**
 	 * Delay before first attempt to spawn summarizing container.
@@ -597,7 +546,7 @@ export const defaultPendingOpsRetryDelayMs = 1000;
 const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
 
 /**
- * @deprecated - use ContainerRuntimeMessage instead
+ * @deprecated - use ContainerRuntimeMessageType instead
  */
 export enum RuntimeMessage {
 	FluidDataStoreOp = "component",
@@ -660,6 +609,23 @@ export const makeLegacySendBatchFn =
 		deltaManager.flush();
 	};
 
+/** Helper type for type constraints passed through several functions.
+ * message - The unpacked message. Likely a TypedContainerRuntimeMessage, but could also be a system op
+ * modernRuntimeMessage - Does this appear like a current TypedContainerRuntimeMessage?
+ * local - Did this client send the op?
+ */
+type MessageWithContext =
+	| {
+			message: InboundSequencedContainerRuntimeMessage;
+			modernRuntimeMessage: true;
+			local: boolean;
+	  }
+	| {
+			message: InboundSequencedContainerRuntimeMessageOrSystemMessage;
+			modernRuntimeMessage: false;
+			local: boolean;
+	  };
+
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
@@ -889,10 +855,9 @@ export class ContainerRuntime
 			initializeEntryPoint,
 		);
 
-		await runtime.blobManager.processStashedChanges();
-		// It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
-		// at 1, so we won't see a replayed saved op with a sequence number of 0.
-		await runtime.pendingStateManager.applyStashedOpsAt(0);
+		// Apply stashed ops with a reference sequence number equal to the sequence number of the snapshot,
+		// or zero. This must be done before Container replays saved ops.
+		await runtime.pendingStateManager.applyStashedOpsAt(runtimeSequenceNumber ?? 0);
 
 		// Initialize the base state of the runtime before it's returned.
 		await runtime.initializeBaseState();
@@ -1638,35 +1603,6 @@ export class ContainerRuntime
 			}
 		}
 
-		this.deltaManager.on("readonly", (readonly: boolean) => {
-			// we accumulate ops while being in read-only state.
-			// once user gets write permissions and we have active connection, flush all pending ops.
-			// Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-			assert(
-				readonly === this.innerDeltaManager.readOnlyInfo.readonly,
-				0x124 /* "inconsistent readonly property/event state" */,
-			);
-
-			// We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
-			// when we either never send an op, or attempted to send it but we know for sure it was not
-			// sequenced by server and will never be sequenced (i.e. was lost)
-			// For loss of connection, we wait for our own "join" op and use it a a barrier to know all the
-			// ops that made it from previous connection, before switching clientId and raising "connected" event
-			// But with read-only permissions, if we transition between read-only and r/w states while on same
-			// connection, then we have no good signal to tell us when it's safe to send ops we accumulated while
-			// being in read-only state.
-			// For that reason, we support getting to read-only state only when disconnected. This ensures that we
-			// can rely on same safety mechanism and resend ops only when we establish new connection.
-			// This is applicable for read-only permissions (event is raised before connection is properly registered),
-			// but it's an extra requirement for Container.forceReadonly() API
-			assert(
-				!readonly || !this.connected,
-				0x125 /* "Unsafe to transition to read-only state!" */,
-			);
-
-			this.replayPendingStates();
-		});
-
 		// logging hardware telemetry
 		logger.sendTelemetryEvent({
 			eventName: "DeviceSpec",
@@ -1826,7 +1762,10 @@ export class ContainerRuntime
 		return this.dataStores.aliases.get(maybeAlias) ?? maybeAlias;
 	}
 
-	private async getDataStoreFromRequest(id: string, request: IRequest): Promise<IFluidRouter> {
+	private async getDataStoreFromRequest(
+		id: string,
+		request: IRequest,
+	): Promise<IFluidDataStoreChannel> {
 		const headerData: RuntimeHeaderData = {};
 		if (typeof request.headers?.[RuntimeHeaders.wait] === "boolean") {
 			headerData.wait = request.headers[RuntimeHeaders.wait];
@@ -2015,28 +1954,28 @@ export class ContainerRuntime
 	 * Parse an op's type and actual content from given serialized content
 	 * ! Note: this format needs to be in-line with what is set in the "ContainerRuntime.submit(...)" method
 	 */
-	private parseOpContent(serializedContent?: string): ContainerRuntimeMessage {
-		assert(serializedContent !== undefined, 0x6d5 /* content must be defined */);
-		const { type, contents, compatDetails }: ContainerRuntimeMessage =
-			JSON.parse(serializedContent);
-		assert(type !== undefined, 0x6d6 /* incorrect op content format */);
-		return { type, contents, compatDetails };
+	// TODO: markfields: confirm Local- versus Outbound- ContainerRuntimeMessage typing
+	private parseLocalOpContent(serializedContents?: string): LocalContainerRuntimeMessage {
+		assert(serializedContents !== undefined, 0x6d5 /* content must be defined */);
+		const message: LocalContainerRuntimeMessage = JSON.parse(serializedContents);
+		assert(message.type !== undefined, 0x6d6 /* incorrect op content format */);
+		return message;
 	}
 
-	private async applyStashedOp(op: string): Promise<unknown> {
+	private async applyStashedOp(serializedOpContent: string): Promise<unknown> {
 		// Need to parse from string for back-compat
-		const { type, contents, compatDetails } = this.parseOpContent(op);
-		switch (type) {
+		const opContents = this.parseLocalOpContent(serializedOpContent);
+		switch (opContents.type) {
 			case ContainerMessageType.FluidDataStoreOp:
-				return this.dataStores.applyStashedOp(contents as IEnvelope);
+				return this.dataStores.applyStashedOp(opContents.contents);
 			case ContainerMessageType.Attach:
-				return this.dataStores.applyStashedAttachOp(contents as IAttachMessage);
+				return this.dataStores.applyStashedAttachOp(opContents.contents);
 			case ContainerMessageType.IdAllocation:
 				assert(
 					this.idCompressor !== undefined,
 					0x67b /* IdCompressor should be defined if enabled */,
 				);
-				return this.applyStashedIdAllocationOp(contents as IdCreationRangeWithStashedState);
+				return this.applyStashedIdAllocationOp(opContents.contents);
 			case ContainerMessageType.Alias:
 			case ContainerMessageType.BlobAttach:
 				return;
@@ -2048,15 +1987,15 @@ export class ContainerRuntime
 				// This should be extremely rare for stashed ops.
 				// It would require a newer runtime stashing ops and then an older one applying them,
 				// e.g. if an app rolled back its container version
-				const compatBehavior = compatDetails?.behavior;
-				if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
+				const compatBehavior = opContents.compatDetails?.behavior;
+				if (!compatBehaviorAllowsMessageType(opContents.type, compatBehavior)) {
 					const error = DataProcessingError.create(
 						"Stashed runtime message of unknown type",
 						"applyStashedOp",
 						undefined /* sequencedMessage */,
 						{
 							messageDetails: JSON.stringify({
-								type,
+								type: opContents.type,
 								compatBehavior,
 							}),
 						},
@@ -2078,6 +2017,30 @@ export class ContainerRuntime
 			return;
 		}
 
+		// If there are stashed blobs in the pending state, we need to delay
+		// propagation of the "connected" event until we have uploaded them to
+		// ensure we don't submit ops referencing a blob that has not been uploaded
+		const connecting = connected && !this._connected;
+		if (connecting && this.blobManager.hasPendingStashedBlobs()) {
+			assert(
+				!this.delayConnectClientId,
+				0x791 /* Connect event delay must be canceled before subsequent connect event */,
+			);
+			assert(!!clientId, 0x792 /* Must have clientId when connecting */);
+			this.delayConnectClientId = clientId;
+			this.blobManager.processStashedChanges().then(
+				() => {
+					// make sure we didn't reconnect before the promise resolved
+					if (this.delayConnectClientId === clientId && !this.disposed) {
+						this.delayConnectClientId = undefined;
+						this.setConnectionStateCore(connected, clientId);
+					}
+				},
+				(error) => this.closeFn(error),
+			);
+			return;
+		}
+
 		this.setConnectionStateCore(connected, clientId);
 	}
 
@@ -2156,9 +2119,25 @@ export class ContainerRuntime
 		const modernRuntimeMessage = messageArg.type === MessageType.Operation;
 
 		// Do shallow copy of message, as the processing flow will modify it.
+		// There might be multiple container instances receiving the same message.
+		// We do not need to make a deep copy. Each layer will just replace message.contents itself,
+		// but will not modify the contents object (likely it will replace it on the message).
 		const messageCopy = { ...messageArg };
 		for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-			this.processCore(message, local, modernRuntimeMessage);
+			if (modernRuntimeMessage) {
+				this.processCore({
+					// Cast it since we expect it to be this based on modernRuntimeMessage computation above.
+					// There is nothing really ensuring that anytime original message.type is Operation that
+					// the result messages will be so. In the end modern bool being true only directs to
+					// throw error if ultimately unrecognized without compat details saying otherwise.
+					message: message as InboundSequencedContainerRuntimeMessage,
+					local,
+					modernRuntimeMessage,
+				});
+			} else {
+				// Unrecognized message will be ignored.
+				this.processCore({ message, local, modernRuntimeMessage });
+			}
 		}
 	}
 
@@ -2166,15 +2145,9 @@ export class ContainerRuntime
 
 	/**
 	 * Direct the message to the correct subsystem for processing, and implement other side effects
-	 * @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
-	 * @param local - Did this client send the op?
-	 * @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
-	 */
-	private processCore(
-		message: ISequencedDocumentMessage | SequencedContainerRuntimeMessage,
-		local: boolean,
-		modernRuntimeMessage: boolean,
-	) {
+	 */
+	private processCore(messageWithContext: MessageWithContext) {
+		const { message, local } = messageWithContext;
 		// Surround the actual processing of the operation with messages to the schedule manager indicating
 		// the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
 		// messages once a batch has been fully processed.
@@ -2184,9 +2157,13 @@ export class ContainerRuntime
 
 		try {
 			let localOpMetadata: unknown;
-			if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
+			if (
+				local &&
+				messageWithContext.modernRuntimeMessage &&
+				message.type !== ContainerMessageType.ChunkedOp
+			) {
 				localOpMetadata = this.pendingStateManager.processPendingLocalMessage(
-					message as SequencedContainerRuntimeMessage,
+					messageWithContext.message,
 				);
 			}
 
@@ -2196,14 +2173,9 @@ export class ContainerRuntime
 				this.updateDocumentDirtyState(false);
 			}
 
-			this.validateAndProcessRuntimeMessage(
-				message,
-				localOpMetadata,
-				local,
-				modernRuntimeMessage,
-			);
+			this.validateAndProcessRuntimeMessage(messageWithContext, localOpMetadata);
 
-			this.emit("op", message, modernRuntimeMessage);
+			this.emit("op", message, messageWithContext.modernRuntimeMessage);
 
 			this.scheduleManager.afterOpProcessing(undefined, message);
 
@@ -2219,39 +2191,43 @@ export class ContainerRuntime
 		}
 	}
 	/**
-	 * Assuming the given message is also a ContainerRuntimeMessage,
+	 * Assuming the given message is also a TypedContainerRuntimeMessage,
 	 * checks its type and dispatches the message to the appropriate handler in the runtime.
-	 * Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
+	 * Throws a DataProcessingError if the message looks like but doesn't conform to a known TypedContainerRuntimeMessage type.
 	 */
 	private validateAndProcessRuntimeMessage(
-		message: ISequencedDocumentMessage,
+		messageWithContext: MessageWithContext,
 		localOpMetadata: unknown,
-		local: boolean,
-		expectRuntimeMessageType: boolean,
-	): asserts message is SequencedContainerRuntimeMessage {
-		// Optimistically extract ContainerRuntimeMessage-specific props from the message
-		const { type: maybeContainerMessageType, compatDetails } =
-			message as ContainerRuntimeMessage;
-
-		switch (maybeContainerMessageType) {
+	): void {
+		// TODO: destructure message and modernRuntimeMessage once using typescript 5.2.2+
+		const { local } = messageWithContext;
+		switch (messageWithContext.message.type) {
 			case ContainerMessageType.Attach:
-				this.dataStores.processAttachMessage(message, local);
+				this.dataStores.processAttachMessage(messageWithContext.message, local);
 				break;
 			case ContainerMessageType.Alias:
-				this.dataStores.processAliasMessage(message, localOpMetadata, local);
+				this.dataStores.processAliasMessage(
+					messageWithContext.message,
+					localOpMetadata,
+					local,
+				);
 				break;
 			case ContainerMessageType.FluidDataStoreOp:
-				this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
+				this.dataStores.processFluidDataStoreOp(
+					messageWithContext.message,
+					local,
+					localOpMetadata,
+				);
 				break;
 			case ContainerMessageType.BlobAttach:
-				this.blobManager.processBlobAttachOp(message, local);
+				this.blobManager.processBlobAttachOp(messageWithContext.message, local);
 				break;
 			case ContainerMessageType.IdAllocation:
 				assert(
 					this.idCompressor !== undefined,
 					0x67c /* IdCompressor should be defined if enabled */,
 				);
-				this.idCompressor.finalizeCreationRange(message.contents as IdCreationRange);
+				this.idCompressor.finalizeCreationRange(messageWithContext.message.contents);
 				break;
 			case ContainerMessageType.ChunkedOp:
 			case ContainerMessageType.Rejoin:
@@ -2259,12 +2235,18 @@ export class ContainerRuntime
 			default: {
 				// If we didn't necessarily expect a runtime message type, then no worries - just return
 				// e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
-				if (!expectRuntimeMessageType) {
+				if (!messageWithContext.modernRuntimeMessage) {
 					return;
 				}
 
-				const compatBehavior = compatDetails?.behavior;
-				if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
+				const compatBehavior = messageWithContext.message.compatDetails?.behavior;
+				if (
+					!compatBehaviorAllowsMessageType(
+						messageWithContext.message.type,
+						compatBehavior,
+					)
+				) {
+					const { message } = messageWithContext;
 					const error = DataProcessingError.create(
 						// Former assert 0x3ce
 						"Runtime message of unknown type",
@@ -2434,8 +2416,8 @@ export class ContainerRuntime
 	/**
 	 * Returns the aliased data store's entryPoint, given the alias.
 	 * @param alias - The alias for the data store.
-	 * @returns - The data store's entry point (IFluidHandle) if it exists and is aliased. Returns undefined if no
-	 * data store has been assigned the given alias.
+	 * @returns The data store's entry point ({@link @fluidframework/core-interfaces#IFluidHandle}) if it exists and is aliased.
+	 * Returns undefined if no data store has been assigned the given alias.
 	 */
 	public async getAliasedDataStoreEntryPoint(
 		alias: string,
@@ -2535,7 +2517,7 @@ export class ContainerRuntime
 		return this.dirtyContainer;
 	}
 
-	private isContainerMessageDirtyable({ type, contents }: ContainerRuntimeMessage) {
+	private isContainerMessageDirtyable({ type, contents }: OutboundContainerRuntimeMessage) {
 		// For legacy purposes, exclude the old built-in AgentScheduler from dirty consideration as a special-case.
 		// Ultimately we should have no special-cases from the ContainerRuntime's perspective.
 		if (type === ContainerMessageType.Attach) {
@@ -2797,7 +2779,7 @@ export class ContainerRuntime
 	/**
 	 * After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
 	 * @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
-	 * @returns - The routes of nodes that were deleted.
+	 * @returns The routes of nodes that were deleted.
 	 */
 	public deleteSweepReadyNodes(sweepReadyRoutes: string[]): string[] {
 		const { dataStoreRoutes, blobManagerRoutes } =
@@ -2868,7 +2850,7 @@ export class ContainerRuntime
 	/**
 	 * From a given list of routes, separate and return routes that belong to blob manager and data stores.
 	 * @param routes - A list of routes that can belong to data stores or blob manager.
-	 * @returns - Two route lists - One that contains routes for blob manager and another one that contains routes
+	 * @returns Two route lists - One that contains routes for blob manager and another one that contains routes
 	 * for data stores.
 	 */
 	private getDataStoreAndBlobManagerRoutes(routes: string[]) {
@@ -2937,19 +2919,15 @@ export class ContainerRuntime
 
 		assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
 
+		// We close the summarizer and download a new snapshot and reload the container
 		let latestSnapshotVersionId: string | undefined;
-		if (refreshLatestAck) {
-			const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(
+		if (refreshLatestAck === true) {
+			return this.prefetchLatestSummaryThenClose(
 				createChildLogger({
 					logger: summaryNumberLogger,
 					properties: { all: { safeSummary: true } },
 				}),
 			);
-			const latestSnapshotRefSeq = latestSnapshotInfo.latestSnapshotRefSeq;
-			latestSnapshotVersionId = latestSnapshotInfo.latestSnapshotVersionId;
-
-			// We might need to catch up to the latest summary's reference sequence number before pausing.
-			await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
 		}
 
 		// If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
@@ -3397,7 +3375,7 @@ export class ContainerRuntime
 			}
 
 			if (idRange !== undefined) {
-				const idAllocationMessage: ContainerRuntimeMessage = {
+				const idAllocationMessage: ContainerRuntimeIdAllocationMessage = {
 					type: ContainerMessageType.IdAllocation,
 					contents: idRange,
 				};
@@ -3417,7 +3395,7 @@ export class ContainerRuntime
 	}
 
 	private submit(
-		containerRuntimeMessage: ContainerRuntimeMessage,
+		containerRuntimeMessage: OutboundContainerRuntimeMessage,
 		localOpMetadata: unknown = undefined,
 		metadata: Record<string, unknown> | undefined = undefined,
 	): void {
@@ -3613,39 +3591,36 @@ export class ContainerRuntime
 
 	private reSubmit(message: IPendingBatchMessage) {
 		// Need to parse from string for back-compat
-		const containerRuntimeMessage = this.parseOpContent(message.content);
+		const containerRuntimeMessage = this.parseLocalOpContent(message.content);
 		this.reSubmitCore(containerRuntimeMessage, message.localOpMetadata, message.opMetadata);
 	}
 
 	/**
 	 * Finds the right store and asks it to resubmit the message. This typically happens when we
 	 * reconnect and there are pending messages.
-	 * @param message - The original ContainerRuntimeMessage.
+	 * @param message - The original LocalContainerRuntimeMessage.
 	 * @param localOpMetadata - The local metadata associated with the original message.
 	 */
 	private reSubmitCore(
-		message: ContainerRuntimeMessage,
+		message: LocalContainerRuntimeMessage,
 		localOpMetadata: unknown,
 		opMetadata: Record<string, unknown> | undefined,
 	) {
-		const contents = message.contents;
 		switch (message.type) {
 			case ContainerMessageType.FluidDataStoreOp:
 				// For Operations, call resubmitDataStoreOp which will find the right store
 				// and trigger resubmission on it.
-				this.dataStores.resubmitDataStoreOp(contents, localOpMetadata);
+				this.dataStores.resubmitDataStoreOp(message.contents, localOpMetadata);
 				break;
 			case ContainerMessageType.Attach:
 			case ContainerMessageType.Alias:
 				this.submit(message, localOpMetadata);
 				break;
-			case ContainerMessageType.IdAllocation:
-				// Remove the stashedState from the op if it's a stashed op
-				if (contents.stashedState !== undefined) {
-					delete contents.stashedState;
-				}
+			case ContainerMessageType.IdAllocation: {
+				prepareLocalContainerRuntimeIdAllocationMessageForTransit(message);
 				this.submit(message, localOpMetadata);
 				break;
+			}
 			case ContainerMessageType.ChunkedOp:
 				throw new Error(`chunkedOp not expected here`);
 			case ContainerMessageType.BlobAttach:
@@ -3684,7 +3659,7 @@ export class ContainerRuntime
 
 	private rollback(content: string | undefined, localOpMetadata: unknown) {
 		// Need to parse from string for back-compat
-		const { type, contents } = this.parseOpContent(content);
+		const { type, contents } = this.parseLocalOpContent(content);
 		switch (type) {
 			case ContainerMessageType.FluidDataStoreOp:
 				// For operations, call rollbackDataStoreOp which will find the right store
@@ -3697,26 +3672,6 @@ export class ContainerRuntime
 		}
 	}
 
-	private async waitForDeltaManagerToCatchup(
-		latestSnapshotRefSeq: number,
-		summaryLogger: ITelemetryLoggerExt,
-	): Promise<void> {
-		if (latestSnapshotRefSeq > this.deltaManager.lastSequenceNumber) {
-			// We need to catch up to the latest summary's reference sequence number before proceeding.
-			await PerformanceEvent.timedExecAsync(
-				summaryLogger,
-				{
-					eventName: "WaitingForSeq",
-					lastSequenceNumber: this.deltaManager.lastSequenceNumber,
-					targetSequenceNumber: latestSnapshotRefSeq,
-					lastKnownSeqNumber: this.deltaManager.lastKnownSeqNumber,
-				},
-				async () => waitForSeq(this.deltaManager, latestSnapshotRefSeq),
-				{ start: true, end: true, cancel: "error" }, // definitely want start event
-			);
-		}
-	}
-
 	/** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
 	public async refreshLatestSummaryAck(options: IRefreshSummaryAckOptions) {
 		const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
@@ -3780,16 +3735,19 @@ export class ContainerRuntime
 	}
 
 	/**
-	 * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
-	 * internal state as it should be considered the latest summary ack.
+	 * Fetches the latest snapshot from storage to refresh the cache as a performance optimization and closes the
+	 * summarizer to reload from new state.
 	 * @param summaryLogger - logger to use when fetching snapshot from storage
-	 * @returns downloaded snapshot's reference sequence number
+	 * @returns a generic summarization error
 	 */
-	private async refreshLatestSummaryAckFromServer(
+	private async prefetchLatestSummaryThenClose(
 		summaryLogger: ITelemetryLoggerExt,
-	): Promise<{ latestSnapshotRefSeq: number; latestSnapshotVersionId: string | undefined }> {
+	): Promise<IBaseSummarizeResult> {
 		const readAndParseBlob = async <T>(id: string) => readAndParse<T>(this.storage, id);
-		const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(
+
+		// This is a performance optimization as the same parent is likely to be elected again, and would use its
+		// cache to fetch the snapshot instead of the network.
+		await this.fetchSnapshotFromStorage(
 			summaryLogger,
 			{
 				eventName: "RefreshLatestSummaryFromServerFetch",
@@ -3800,7 +3758,12 @@ export class ContainerRuntime
 
 		await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
 
-		return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
+		return {
+			stage: "base",
+			error: "summary state stale - Unsupported option 'refreshLatestAck'",
+			referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+			minimumSequenceNumber: this.deltaManager.minimumSequenceNumber,
+		};
 	}
 
 	private async closeStaleSummarizer(codePath: string): Promise<void> {
@@ -3853,7 +3816,7 @@ export class ContainerRuntime
 				const versions = await this.storage.getVersions(
 					versionId,
 					1,
-					"refreshLatestSummaryAckFromServer",
+					"prefetchLatestSummaryBeforeClose",
 					versionId === null ? FetchSource.noCache : undefined,
 				);
 				assert(
@@ -3896,17 +3859,17 @@ export class ContainerRuntime
 				if (this._orderSequentiallyCalls !== 0) {
 					throw new UsageError("can't get state during orderSequentially");
 				}
-				const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(
-					waitBlobsToAttach,
-				);
-				const pending = this.pendingStateManager.getLocalState();
-				if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
-					return; // no pending state to save
-				}
 				// Flush pending batch.
 				// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
 				// to close current batch.
 				this.flush();
+				const pendingAttachmentBlobs = waitBlobsToAttach
+					? await this.blobManager.attachAndGetPendingBlobs()
+					: undefined;
+				const pending = this.pendingStateManager.getLocalState();
+				if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+					return; // no pending state to save
+				}
 
 				const pendingState: IPendingRuntimeState = {
 					pending,
@@ -4003,30 +3966,3 @@ export class ContainerRuntime
 		return killSwitch !== true && this.runtimeOptions.enableGroupedBatching;
 	}
 }
-
-/**
- * Wait for a specific sequence number. Promise should resolve when we reach that number,
- * or reject if closed.
- */
-const waitForSeq = async (
-	deltaManager: IDeltaManager<Pick<ISequencedDocumentMessage, "sequenceNumber">, unknown>,
-	targetSeq: number,
-): Promise<void> =>
-	new Promise<void>((resolve, reject) => {
-		// TODO: remove cast to any when actual event is determined
-		deltaManager.on("closed" as any, reject);
-		deltaManager.on("disposed" as any, reject);
-
-		// If we already reached target sequence number, simply resolve the promise.
-		if (deltaManager.lastSequenceNumber >= targetSeq) {
-			resolve();
-		} else {
-			const handleOp = (message: Pick<ISequencedDocumentMessage, "sequenceNumber">) => {
-				if (message.sequenceNumber >= targetSeq) {
-					resolve();
-					deltaManager.off("op", handleOp);
-				}
-			};
-			deltaManager.on("op", handleOp);
-		}
-	});