@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258

package/src/containerRuntime.ts

@@ -2,8 +2,9 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { ITelemetryBaseLogger, ITelemetryGenericEvent } from "@fluidframework/common-definitions";
 import {
+	ITelemetryBaseLogger,
+	ITelemetryGenericEvent,
 	FluidObject,
 	IFluidHandle,
 	IFluidHandleContext,
@@ -13,6 +14,7 @@ import {
 } from "@fluidframework/core-interfaces";
 import {
 	IAudience,
+	IBatchMessage,
 	IContainerContext,
 	IDeltaManager,
 	IRuntime,
@@ -25,23 +27,22 @@ import {
 	IContainerRuntime,
 	IContainerRuntimeEvents,
 } from "@fluidframework/container-runtime-definitions";
+import { assert, delay, LazyPromise } from "@fluidframework/core-utils";
+import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
 import {
-	assert,
-	delay,
-	LazyPromise,
-	Trace,
-	TypedEventEmitter,
-	unreachableCase,
-} from "@fluidframework/common-utils";
-import {
-	ChildLogger,
+	createChildLogger,
+	createChildMonitoringContext,
+	DataCorruptionError,
+	DataProcessingError,
+	GenericError,
 	raiseConnectedEvent,
 	PerformanceEvent,
+	// eslint-disable-next-line import/no-deprecated
 	TaggedLoggerAdapter,
 	MonitoringContext,
-	loggerToMonitoringContext,
 	wrapError,
 	ITelemetryLoggerExt,
+	UsageError,
 } from "@fluidframework/telemetry-utils";
 import {
 	DriverHeader,
@@ -50,12 +51,6 @@ import {
 	ISummaryContext,
 } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
-import {
-	DataCorruptionError,
-	DataProcessingError,
-	GenericError,
-	UsageError,
-} from "@fluidframework/container-utils";
 import {
 	IClientDetails,
 	IDocumentMessage,
@@ -113,7 +108,11 @@ import { v4 as uuid } from "uuid";
 import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
 import { ReportOpPerfTelemetry, IPerfSignalReport } from "./connectionTelemetry";
-import { IPendingLocalState, PendingStateManager } from "./pendingStateManager";
+import {
+	IPendingBatchMessage,
+	IPendingLocalState,
+	PendingStateManager,
+} from "./pendingStateManager";
 import { pkgVersion } from "./packageVersion";
 import { BlobManager, IBlobManagerLoadInfo, IPendingBlobs } from "./blobManager";
 import { DataStores, getSummaryForDatastores } from "./dataStores";
@@ -127,7 +126,6 @@ import {
 	IContainerRuntimeMetadata,
 	ICreateContainerMetadata,
 	idCompressorBlobName,
-	IFetchSnapshotResult,
 	IRootSummarizerNodeWithGC,
 	ISummaryMetadataMessage,
 	metadataBlobName,
@@ -149,6 +147,14 @@ import {
 	ISummarizerRuntime,
 	IRefreshSummaryAckOptions,
 	RunWhileConnectedCoordinator,
+	IGenerateSummaryTreeResult,
+	RetriableSummaryError,
+	IOnDemandSummarizeOptions,
+	ISummarizeResults,
+	IEnqueueSummarizeOptions,
+	EnqueueSummarizeResult,
+	ISummarizerEvents,
+	IBaseSummarizeResult,
 } from "./summary";
 import { formExponentialFn, Throttler } from "./throttler";
 import {
@@ -166,6 +172,7 @@ import { BindBatchTracker } from "./batchTracker";
 import { ScheduleManager } from "./scheduleManager";
 import {
 	BatchMessage,
+	IBatch,
 	IBatchCheckpoint,
 	OpCompressor,
 	OpDecompressor,
@@ -173,8 +180,10 @@ import {
 	OpSplitter,
 	RemoteMessageProcessor,
 	OpGroupingManager,
+	getLongStack,
 } from "./opLifecycle";
 import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy";
+import { IBatchMetadata } from "./metadata";
 
 export enum ContainerMessageType {
 	// An op to be delivered to store
@@ -203,11 +212,64 @@ export enum ContainerMessageType {
 	IdAllocation = "idAllocation",
 }
 
+/**
+ * How should an older client handle an unrecognized remote op type?
+ *
+ * @internal
+ */
+export type CompatModeBehavior =
+	/** Ignore the op. It won't be persisted if this client summarizes */
+	| "Ignore"
+	/** Fail processing immediately. (The container will close) */
+	| "FailToProcess";
+
+/**
+ * All the info an older client would need to know how to handle an unrecognized remote op type
+ *
+ * @internal
+ */
+export interface IContainerRuntimeMessageCompatDetails {
+	/** How should an older client handle an unrecognized remote op type? */
+	behavior: CompatModeBehavior;
+}
+
+/**
+ * Utility to implement compat behaviors given an unknown message type
+ * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
+ *
+ * @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
+ * handled before calling this function (e.g. in a switch statement).
+ * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
+ */
+function compatBehaviorAllowsMessageType(
+	_unknownContainerRuntimeMessageType: never,
+	compatBehavior: "Ignore" | "FailToProcess" | undefined,
+): boolean {
+	// undefined defaults to same behavior as "FailToProcess"
+	return compatBehavior === "Ignore";
+}
+
+/**
+ * The unpacked runtime message / details to be handled or dispatched by the ContainerRuntime
+ *
+ * IMPORTANT: when creating one to be serialized, set the properties in the order they appear here.
+ * This way stringified values can be compared.
+ */
 export interface ContainerRuntimeMessage {
-	contents: any;
+	/** Type of the op, within the ContainerRuntime's domain */
 	type: ContainerMessageType;
+	/** Domain-specific contents, interpreted according to the type */
+	contents: any;
+	/** Info describing how to handle this op in case the type is unrecognized (default: fail to process) */
+	compatDetails?: IContainerRuntimeMessageCompatDetails;
 }
 
+/**
+ * An unpacked ISequencedDocumentMessage with the inner ContainerRuntimeMessage type/contents/etc
+ * promoted up to the outer object
+ */
+export type SequencedContainerRuntimeMessage = ISequencedDocumentMessage & ContainerRuntimeMessage;
+
 export interface ISummaryBaseConfiguration {
 	/**
 	 * Delay before first attempt to spawn summarizing container.
@@ -434,14 +496,6 @@ export interface IContainerRuntimeOptions {
 	readonly enableGroupedBatching?: boolean;
 }
 
-/**
- * The summary tree returned by the root node. It adds state relevant to the root of the tree.
- */
-export interface IRootSummaryTreeWithStats extends ISummaryTreeWithStats {
-	/** The garbage collection stats if GC ran, undefined otherwise. */
-	gcStats?: IGCStats;
-}
-
 /**
  * Accepted header keys for requests coming to the runtime.
  */
@@ -454,9 +508,13 @@ export enum RuntimeHeaders {
 
 /** True if a tombstoned object should be returned without erroring */
 export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
+/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
+export const AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
 
 /** Tombstone error responses will have this header set to true */
 export const TombstoneResponseHeaderKey = "isTombstoned";
+/** Inactive error responses will have this header set to true */
+export const InactiveResponseHeaderKey = "isInactive";
 
 /**
  * The full set of parsed header data that may be found on Runtime requests
@@ -497,7 +555,7 @@ interface OldContainerContextWithLogger extends Omit<IContainerContext, "taggedL
  * instantiated runtime in a new instance of the container, so it can load to the
  * same state
  */
-interface IPendingRuntimeState {
+export interface IPendingRuntimeState {
 	/**
 	 * Pending ops from PendingStateManager
 	 */
@@ -526,12 +584,17 @@ const defaultCompressionConfig = {
 
 const defaultChunkSizeInBytes = 204800;
 
+/** The default time to wait for pending ops to be processed during summarization */
+export const defaultPendingOpsWaitTimeoutMs = 1000;
+/** The default time to delay a summarization retry attempt when there are pending ops */
+export const defaultPendingOpsRetryDelayMs = 1000;
+
 /**
  * Instead of refreshing from latest because we do not have 100% confidence in the state
  * of the current system, we should close the summarizer and let it recover.
  * This delay's goal is to prevent tight restart loops
  */
-const defaultCloseSummarizerDelayMs = 10000; // 10 seconds
+const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
 
 /**
  * @deprecated - use ContainerRuntimeMessage instead
@@ -573,14 +636,41 @@ export function getDeviceSpec() {
 	return {};
 }
 
+/**
+ * Older loader doesn't have a submitBatchFn member, this is the older way of submitting a batch.
+ * Rather than exposing the submitFn (now deprecated) and IDeltaManager (dangerous to hand out) to the Outbox,
+ * we can provide a partially-applied function to keep those items private to the ContainerRuntime.
+ */
+export const makeLegacySendBatchFn =
+	(
+		submitFn: (type: MessageType, contents: any, batch: boolean, appData?: any) => number,
+		deltaManager: Pick<IDeltaManager<unknown, unknown>, "flush">,
+	) =>
+	(batch: IBatch) => {
+		for (const message of batch.content) {
+			submitFn(
+				MessageType.Operation,
+				// For back-compat (submitFn only works on deserialized content)
+				message.contents === undefined ? undefined : JSON.parse(message.contents),
+				true, // batch
+				message.metadata,
+			);
+		}
+
+		deltaManager.flush();
+	};
+
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
  */
 export class ContainerRuntime
-	extends TypedEventEmitter<IContainerRuntimeEvents>
+	extends TypedEventEmitter<IContainerRuntimeEvents & ISummarizerEvents>
 	implements IContainerRuntime, IRuntime, ISummarizerRuntime, ISummarizerInternalsProvider
 {
+	/**
+	 * @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
+	 */
 	public get IFluidRouter() {
 		return this;
 	}
@@ -639,10 +729,10 @@ export class ContainerRuntime
 		context: IContainerContext;
 		registryEntries: NamedFluidDataStoreRegistryEntries;
 		existing: boolean;
-		requestHandler?: (request: IRequest, runtime: IContainerRuntime) => Promise<IResponse>;
 		runtimeOptions?: IContainerRuntimeOptions;
 		containerScope?: FluidObject;
 		containerRuntimeCtor?: typeof ContainerRuntime;
+		requestHandler?: (request: IRequest, runtime: IContainerRuntime) => Promise<IResponse>;
 		initializeEntryPoint?: (containerRuntime: IContainerRuntime) => Promise<FluidObject>;
 	}): Promise<ContainerRuntime> {
 		const {
@@ -653,18 +743,32 @@ export class ContainerRuntime
 			runtimeOptions = {},
 			containerScope = {},
 			containerRuntimeCtor = ContainerRuntime,
-			initializeEntryPoint,
 		} = params;
 
+		const initializeEntryPoint =
+			params.initializeEntryPoint ??
+			(async (containerRuntime: IContainerRuntime) => ({
+				get IFluidRouter() {
+					return this;
+				},
+				async request(req) {
+					return containerRuntime.request(req);
+				},
+			}));
+
 		// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
 		// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
 		const backCompatContext: IContainerContext | OldContainerContextWithLogger = context;
 		const passLogger =
 			backCompatContext.taggedLogger ??
+			// eslint-disable-next-line import/no-deprecated
 			new TaggedLoggerAdapter((backCompatContext as OldContainerContextWithLogger).logger);
-		const logger = ChildLogger.create(passLogger, undefined, {
-			all: {
-				runtimeVersion: pkgVersion,
+		const logger = createChildLogger({
+			logger: passLogger,
+			properties: {
+				all: {
+					runtimeVersion: pkgVersion,
+				},
 			},
 		});
 
@@ -705,8 +809,6 @@ export class ContainerRuntime
 				tryFetchBlob<SerializedIdCompressorWithNoSession>(idCompressorBlobName),
 			]);
 
-		const loadExisting = existing === true || context.existing === true;
-
 		// read snapshot blobs needed for BlobManager to load
 		const blobManagerSnapshot = await BlobManager.load(
 			context.baseSnapshot?.trees[blobsTreeName],
@@ -741,9 +843,7 @@ export class ContainerRuntime
 				if (loadSequenceNumberVerification === "log") {
 					logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
 				} else {
-					// Call both close and dispose as closeFn implementation will no longer dispose runtime in future
 					context.closeFn(error);
-					context.disposeFn?.(error);
 				}
 			}
 		}
@@ -756,7 +856,7 @@ export class ContainerRuntime
 			idCompressor =
 				serializedIdCompressor !== undefined
 					? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
-					: new IdCompressor(createSessionId(), logger);
+					: IdCompressor.create(logger);
 		}
 
 		const runtime = new containerRuntimeCtor(
@@ -780,7 +880,7 @@ export class ContainerRuntime
 			},
 			containerScope,
 			logger,
-			loadExisting,
+			existing,
 			blobManagerSnapshot,
 			context.storage,
 			idCompressor,
@@ -789,6 +889,7 @@ export class ContainerRuntime
 			initializeEntryPoint,
 		);
 
+		await runtime.blobManager.processStashedChanges();
 		// It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
 		// at 1, so we won't see a replayed saved op with a sequence number of 0.
 		await runtime.pendingStateManager.applyStashedOpsAt(0);
@@ -799,44 +900,49 @@ export class ContainerRuntime
 		return runtime;
 	}
 
-	public get options(): ILoaderOptions {
-		return this.context.options;
-	}
+	public readonly options: ILoaderOptions;
 
+	private readonly _getClientId: () => string | undefined;
 	public get clientId(): string | undefined {
-		return this.context.clientId;
+		return this._getClientId();
 	}
 
-	public get clientDetails(): IClientDetails {
-		return this.context.clientDetails;
-	}
+	public readonly clientDetails: IClientDetails;
 
 	public get storage(): IDocumentStorageService {
 		return this._storage;
 	}
 
-	public get reSubmitFn(): (
-		type: ContainerMessageType,
-		content: any,
-		localOpMetadata: unknown,
-		opMetadata: Record<string, unknown> | undefined,
-	) => void {
-		// eslint-disable-next-line @typescript-eslint/unbound-method
-		return this.reSubmitCore;
+	/** @deprecated - The functionality is no longer exposed publicly */
+	public get reSubmitFn() {
+		return (
+			type: ContainerMessageType,
+			contents: any,
+			localOpMetadata: unknown,
+			opMetadata: Record<string, unknown> | undefined,
+		) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
+		// Note: compatDetails is not included in this deprecated API
 	}
 
-	public get disposeFn(): (error?: ICriticalContainerError) => void {
-		// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
-		return this.context.disposeFn ?? this.context.closeFn;
-	}
-
-	public get closeFn(): (error?: ICriticalContainerError) => void {
-		// Also call disposeFn to retain functionality of runtime being disposed on close
-		return (error?: ICriticalContainerError) => {
-			this.context.closeFn(error);
-			this.context.disposeFn?.(error);
-		};
-	}
+	private readonly submitFn: (
+		type: MessageType,
+		contents: any,
+		batch: boolean,
+		appData?: any,
+	) => number;
+	/**
+	 * Although current IContainerContext guarantees submitBatchFn, it is not available on older loaders.
+	 */
+	private readonly submitBatchFn:
+		| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)
+		| undefined;
+	private readonly submitSummaryFn: (
+		summaryOp: ISummaryContent,
+		referenceSequenceNumber?: number,
+	) => number;
+	private readonly submitSignalFn: (contents: any) => void;
+	public readonly disposeFn: (error?: ICriticalContainerError) => void;
+	public readonly closeFn: (error?: ICriticalContainerError) => void;
 
 	public get flushMode(): FlushMode {
 		return this._flushMode;
@@ -850,8 +956,9 @@ export class ContainerRuntime
 		return this.registry;
 	}
 
+	private readonly _getAttachState: () => AttachState;
 	public get attachState(): AttachState {
-		return this.context.attachState;
+		return this._getAttachState();
 	}
 
 	public idCompressor: (IIdCompressor & IIdCompressorCore) | undefined;
@@ -950,11 +1057,16 @@ export class ContainerRuntime
 	private emitDirtyDocumentEvent = true;
 	private readonly enableOpReentryCheck: boolean;
 	private readonly disableAttachReorder: boolean | undefined;
-	private readonly summaryStateUpdateMethod: string | undefined;
 	private readonly closeSummarizerDelayMs: number;
+	/**
+	 * If true, summary generated is validate before uploading it to the server. With single commit summaries,
+	 * summaries will be accepted once uploaded, so they should be validated before upload. However, this can
+	 * currently be controlled via a feature flag as its a new functionality.
+	 */
+	private readonly validateSummaryBeforeUpload: boolean;
 
 	private readonly defaultTelemetrySignalSampleCount = 100;
-	private _perfSignalData: IPerfSignalReport = {
+	private readonly _perfSignalData: IPerfSignalReport = {
 		signalsLost: 0,
 		signalSequenceNumber: 0,
 		signalTimestamp: 0,
@@ -1036,11 +1148,21 @@ export class ContainerRuntime
 	 */
 	private readonly idCompressorEnabled: boolean;
 
+	/**
+	 * Whether this client is the summarizer client itself (type is summarizerClientType)
+	 */
+	private readonly isSummarizerClient: boolean;
+
+	/**
+	 * The id of the version used to initially load this runtime, or undefined if it's newly created.
+	 */
+	private readonly loadedFromVersionId: string | undefined;
+
 	/**
 	 * @internal
 	 */
 	protected constructor(
-		private readonly context: IContainerContext,
+		context: IContainerContext,
 		private readonly registry: IFluidDataStoreRegistry,
 		metadata: IContainerRuntimeMetadata | undefined,
 		electedSummarizerData: ISerializedElection | undefined,
@@ -1067,10 +1189,64 @@ export class ContainerRuntime
 	) {
 		super();
 
-		this.innerDeltaManager = context.deltaManager;
-		this.deltaManager = new DeltaManagerSummarizerProxy(context.deltaManager);
+		const {
+			options,
+			clientDetails,
+			connected,
+			baseSnapshot,
+			submitFn,
+			submitBatchFn,
+			submitSummaryFn,
+			submitSignalFn,
+			disposeFn,
+			closeFn,
+			deltaManager,
+			quorum,
+			audience,
+			loader,
+			pendingLocalState,
+			supportedFeatures,
+		} = context;
+
+		this.innerDeltaManager = deltaManager;
+		this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
+
+		// Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
+		// This makes ContainerRuntime the final gatekeeper for outgoing messages.
+		this.submitFn = submitFn;
+		this.submitBatchFn = submitBatchFn;
+		this.submitSummaryFn = submitSummaryFn;
+		this.submitSignalFn = submitSignalFn;
+
+		this.options = options;
+		this.clientDetails = clientDetails;
+		this.isSummarizerClient = this.clientDetails.type === summarizerClientType;
+		this.loadedFromVersionId = context.getLoadedFromVersion()?.id;
+		this._getClientId = () => context.clientId;
+		this._getAttachState = () => context.attachState;
+		this.getAbsoluteUrl = async (relativeUrl: string) => {
+			if (context.getAbsoluteUrl === undefined) {
+				throw new Error("Driver does not implement getAbsoluteUrl");
+			}
+			if (this.attachState !== AttachState.Attached) {
+				return undefined;
+			}
+			return context.getAbsoluteUrl(relativeUrl);
+		};
+		// TODO: Consider that the Container could just listen to these events itself, or even more appropriately maybe the
+		// customer should observe dirty state on the runtime (the owner of dirty state) directly, rather than on the IContainer.
+		this.on("dirty", () => context.updateDirtyContainerState(true));
+		this.on("saved", () => context.updateDirtyContainerState(false));
+
+		// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
+		this.disposeFn = disposeFn ?? closeFn;
+		// In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
+		this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
 
-		this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
+		this.mc = createChildMonitoringContext({
+			logger: this.logger,
+			namespace: "ContainerRuntime",
+		});
 
 		let loadSummaryNumber: number;
 		// Get the container creation metadata. For new container, we initialize these. For existing containers,
@@ -1102,7 +1278,9 @@ export class ContainerRuntime
 
 		this.messageAtLastSummary = metadata?.message;
 
-		this._connected = this.context.connected;
+		// Note that we only need to pull the *initial* connected state from the context.
+		// Later updates come through calls to setConnectionState.
+		this._connected = connected;
 
 		this.gcTombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement(
 			metadata?.gcFeatureMatrix?.tombstoneGeneration /* persisted */,
@@ -1131,7 +1309,7 @@ export class ContainerRuntime
 
 		const opSplitter = new OpSplitter(
 			chunks,
-			this.context.submitBatchFn,
+			this.submitBatchFn,
 			disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes,
 			runtimeOptions.maxBatchSizeInBytes,
 			this.mc.logger,
@@ -1172,7 +1350,7 @@ export class ContainerRuntime
 
 		if (
 			runtimeOptions.flushMode === (FlushModeExperimental.Async as unknown as FlushMode) &&
-			context.supportedFeatures?.get("referenceSequenceNumbers") !== true
+			supportedFeatures?.get("referenceSequenceNumbers") !== true
 		) {
 			// The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
 			this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
@@ -1181,7 +1359,7 @@ export class ContainerRuntime
 			this._flushMode = runtimeOptions.flushMode;
 		}
 
-		const pendingRuntimeState = context.pendingLocalState as IPendingRuntimeState | undefined;
+		const pendingRuntimeState = pendingLocalState as IPendingRuntimeState | undefined;
 
 		const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
 		if (
@@ -1197,16 +1375,15 @@ export class ContainerRuntime
 		this.garbageCollector = GarbageCollector.create({
 			runtime: this,
 			gcOptions: this.runtimeOptions.gcOptions,
-			baseSnapshot: context.baseSnapshot,
+			baseSnapshot,
 			baseLogger: this.mc.logger,
 			existing,
 			metadata,
 			createContainerMetadata: this.createContainerMetadata,
-			isSummarizerClient: this.context.clientDetails.type === summarizerClientType,
+			isSummarizerClient: this.isSummarizerClient,
 			getNodePackagePath: async (nodePath: string) => this.getGCNodePackagePath(nodePath),
 			getLastSummaryTimestampMs: () => this.messageAtLastSummary?.timestamp,
 			readAndParseBlob: async <T>(id: string) => readAndParse<T>(this.storage, id),
-			getContainerDiagnosticId: () => this.context.id,
 			// GC runs in summarizer client and needs access to the real (non-proxy) active information. The proxy
 			// delta manager would always return false for summarizer client.
 			activeConnection: () => this.innerDeltaManager.active,
@@ -1214,14 +1391,14 @@ export class ContainerRuntime
 
 		const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
 		this.summarizerNode = createRootSummarizerNodeWithGC(
-			ChildLogger.create(this.logger, "SummarizerNode"),
+			createChildLogger({ logger: this.logger, namespace: "SummarizerNode" }),
 			// Summarize function to call when summarize is called. Summarizer node always tracks summary state.
 			async (fullTree: boolean, trackState: boolean, telemetryContext?: ITelemetryContext) =>
 				this.summarizeInternal(fullTree, trackState, telemetryContext),
 			// Latest change sequence number, no changes since summary applied yet
 			loadedFromSequenceNumber,
 			// Summary reference sequence number, undefined if no summary yet
-			context.baseSnapshot ? loadedFromSequenceNumber : undefined,
+			baseSnapshot !== undefined ? loadedFromSequenceNumber : undefined,
 			{
 				// Must set to false to prevent sending summary handle which would be pointing to
 				// a summary with an older protocol state.
@@ -1238,14 +1415,14 @@ export class ContainerRuntime
 			async () => this.garbageCollector.getBaseGCDetails(),
 		);
 
-		if (context.baseSnapshot) {
-			this.summarizerNode.updateBaseSummaryState(context.baseSnapshot);
+		if (baseSnapshot) {
+			this.summarizerNode.updateBaseSummaryState(baseSnapshot);
 		}
 
 		this.dataStores = new DataStores(
-			getSummaryForDatastores(context.baseSnapshot, metadata),
+			getSummaryForDatastores(baseSnapshot, metadata),
 			this,
-			(attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg),
+			(attachMsg) => this.submit({ type: ContainerMessageType.Attach, contents: attachMsg }),
 			(id: string, createParam: CreateChildSummarizerNodeParam) =>
 				(
 					summarizeInternal: SummarizeInternalFn,
@@ -1272,10 +1449,14 @@ export class ContainerRuntime
 			() => this.storage,
 			(localId: string, blobId?: string) => {
 				if (!this.disposed) {
-					this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
-						localId,
-						blobId,
-					});
+					this.submit(
+						{ type: ContainerMessageType.BlobAttach, contents: undefined },
+						undefined,
+						{
+							localId,
+							blobId,
+						},
+					);
 				}
 			},
 			(blobPath: string) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
@@ -1286,10 +1467,10 @@ export class ContainerRuntime
 		);
 
 		this.scheduleManager = new ScheduleManager(
-			context.deltaManager,
+			this.innerDeltaManager,
 			this,
 			() => this.clientId,
-			ChildLogger.create(this.logger, "ScheduleManager"),
+			createChildLogger({ logger: this.logger, namespace: "ScheduleManager" }),
 		);
 
 		this.pendingStateManager = new PendingStateManager(
@@ -1299,9 +1480,11 @@ export class ContainerRuntime
 				close: this.closeFn,
 				connected: () => this.connected,
 				reSubmit: this.reSubmit.bind(this),
-				orderSequentially: this.orderSequentially.bind(this),
+				reSubmitBatch: this.reSubmitBatch.bind(this),
+				isActiveConnection: () => this.innerDeltaManager.active,
 			},
 			pendingRuntimeState?.pending,
+			this.logger,
 		);
 
 		const disableCompression = this.mc.config.getBoolean(
@@ -1318,16 +1501,21 @@ export class ContainerRuntime
 		const disablePartialFlush = this.mc.config.getBoolean(
 			"Fluid.ContainerRuntime.DisablePartialFlush",
 		);
+
+		const legacySendBatchFn = makeLegacySendBatchFn(this.submitFn, this.innerDeltaManager);
+
 		this.outbox = new Outbox({
 			shouldSend: () => this.canSendOps(),
 			pendingStateManager: this.pendingStateManager,
-			containerContext: this.context,
+			submitBatchFn: this.submitBatchFn,
+			legacySendBatchFn,
 			compressor: new OpCompressor(this.mc.logger),
 			splitter: opSplitter,
 			config: {
 				compressionOptions,
 				maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
 				disablePartialFlush: disablePartialFlush === true,
+				enableGroupedBatching: this.groupedBatchingEnabled,
 			},
 			logger: this.mc.logger,
 			groupingManager: opGroupingManager,
@@ -1335,40 +1523,48 @@ export class ContainerRuntime
 				referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
 				clientSequenceNumber: this._processedClientSequenceNumber,
 			}),
+			reSubmit: this.reSubmit.bind(this),
+			opReentrancy: () => this.ensureNoDataModelChangesCalls > 0,
+			closeContainer: this.closeFn,
 		});
 
-		this.context.quorum.on("removeMember", (clientId: string) => {
+		this._quorum = quorum;
+		this._quorum.on("removeMember", (clientId: string) => {
 			this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
 		});
 
-		this.summaryStateUpdateMethod = this.mc.config.getString(
-			"Fluid.ContainerRuntime.Test.SummaryStateUpdateMethod",
-		);
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		this._audience = audience!;
+
 		const closeSummarizerDelayOverride = this.mc.config.getNumber(
 			"Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs",
 		);
 		this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
+		this.validateSummaryBeforeUpload =
+			this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
 
 		this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
 
 		this.dirtyContainer =
-			this.context.attachState !== AttachState.Attached ||
-			this.pendingStateManager.hasPendingMessages();
-		this.context.updateDirtyContainerState(this.dirtyContainer);
+			this.attachState !== AttachState.Attached || this.hasPendingMessages();
+		context.updateDirtyContainerState(this.dirtyContainer);
 
 		if (this.summariesDisabled) {
 			this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
 		} else {
-			const orderedClientLogger = ChildLogger.create(this.logger, "OrderedClientElection");
+			const orderedClientLogger = createChildLogger({
+				logger: this.logger,
+				namespace: "OrderedClientElection",
+			});
 			const orderedClientCollection = new OrderedClientCollection(
 				orderedClientLogger,
-				this.context.deltaManager,
-				this.context.quorum,
+				this.innerDeltaManager,
+				this._quorum,
 			);
 			const orderedClientElectionForSummarizer = new OrderedClientElection(
 				orderedClientLogger,
 				orderedClientCollection,
-				electedSummarizerData ?? this.context.deltaManager.lastSequenceNumber,
+				electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber,
 				SummarizerClientElection.isClientEligible,
 			);
 
@@ -1379,7 +1575,7 @@ export class ContainerRuntime
 				this.maxOpsSinceLastSummary,
 			);
 
-			if (this.context.clientDetails.type === summarizerClientType) {
+			if (this.isSummarizerClient) {
 				this._summarizer = new Summarizer(
 					this /* ISummarizerRuntime */,
 					() => this.summaryConfiguration,
@@ -1394,19 +1590,19 @@ export class ContainerRuntime
 							() => this.innerDeltaManager.active,
 						),
 				);
-			} else if (
-				SummarizerClientElection.clientDetailsPermitElection(this.context.clientDetails)
-			) {
+			} else if (SummarizerClientElection.clientDetailsPermitElection(this.clientDetails)) {
 				// Only create a SummaryManager and SummarizerClientElection
 				// if summaries are enabled and we are not the summarizer client.
 				const defaultAction = () => {
 					if (this.summaryCollection.opsSinceLastAck > this.maxOpsSinceLastSummary) {
-						this.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
+						this.mc.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
 						// unregister default to no log on every op after falling behind
 						// and register summary ack handler to re-register this handler
 						// after successful summary
 						this.summaryCollection.once(MessageType.SummaryAck, () => {
-							this.logger.sendTelemetryEvent({ eventName: "SummaryStatus:CaughtUp" });
+							this.mc.logger.sendTelemetryEvent({
+								eventName: "SummaryStatus:CaughtUp",
+							});
 							// we've caught up, so re-register the default action to monitor for
 							// falling behind, and unregister ourself
 							this.summaryCollection.on("default", defaultAction);
@@ -1423,7 +1619,7 @@ export class ContainerRuntime
 					this, // IConnectedState
 					this.summaryCollection,
 					this.logger,
-					this.formRequestSummarizerFn(this.context.loader),
+					this.formRequestSummarizerFn(loader),
 					new Throttler(
 						60 * 1000, // 60 sec delay window
 						30 * 1000, // 30 sec max delay
@@ -1435,6 +1631,9 @@ export class ContainerRuntime
 					},
 					this.heuristicsDisabled,
 				);
+				this.summaryManager.on("summarize", (eventProps) => {
+					this.emit("summarize", eventProps);
+				});
 				this.summaryManager.start();
 			}
 		}
@@ -1474,7 +1673,7 @@ export class ContainerRuntime
 			...getDeviceSpec(),
 		});
 
-		this.logger.sendTelemetryEvent({
+		this.mc.logger.sendTelemetryEvent({
 			eventName: "ContainerLoadStats",
 			...this.createContainerMetadata,
 			...this.dataStores.containerLoadStats,
@@ -1490,18 +1689,17 @@ export class ContainerRuntime
 				disableAttachReorder: this.disableAttachReorder,
 				disablePartialFlush,
 				idCompressorEnabled: this.idCompressorEnabled,
-				summaryStateUpdateMethod: this.summaryStateUpdateMethod,
 				closeSummarizerDelayOverride,
 			}),
 			telemetryDocumentId: this.telemetryDocumentId,
 			groupedBatchingEnabled: this.groupedBatchingEnabled,
 		});
 
-		ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
+		ReportOpPerfTelemetry(this.clientId, this.deltaManager, this.logger);
 		BindBatchTracker(this, this.logger);
 
 		this.entryPoint = new LazyPromise(async () => {
-			if (this.context.clientDetails.type === summarizerClientType) {
+			if (this.isSummarizerClient) {
 				assert(
 					this._summarizer !== undefined,
 					0x5bf /* Summarizer object is undefined in a summarizer client */,
@@ -1525,7 +1723,7 @@ export class ContainerRuntime
 		}
 		this._disposed = true;
 
-		this.logger.sendTelemetryEvent(
+		this.mc.logger.sendTelemetryEvent(
 			{
 				eventName: "ContainerRuntimeDisposed",
 				isDirty: this.isDirty,
@@ -1549,6 +1747,7 @@ export class ContainerRuntime
 	/**
 	 * Notifies this object about the request made to the container.
 	 * @param request - Request made to the handler.
+	 * @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
 	 */
 	public async request(request: IRequest): Promise<IResponse> {
 		try {
@@ -1606,7 +1805,7 @@ export class ContainerRuntime
 					subRequest.url.startsWith("/"),
 					0x126 /* "Expected createSubRequest url to include a leading slash" */,
 				);
-				return dataStore.IFluidRouter.request(subRequest);
+				return dataStore.request(subRequest);
 			}
 
 			return create404Response(request);
@@ -1654,6 +1853,7 @@ export class ContainerRuntime
 			dataStoreContext.packagePath,
 			request?.headers,
 		);
+
 		return dataStoreChannel;
 	}
 
@@ -1748,7 +1948,7 @@ export class ContainerRuntime
 			this.mc.logger.sendTelemetryEvent({
 				eventName: "ReconnectsWithNoProgress",
 				attempts: this.consecutiveReconnects,
-				pendingMessages: this.pendingStateManager.pendingMessagesCount,
+				pendingMessages: this.pendingMessagesCount,
 			});
 		}
 
@@ -1815,19 +2015,17 @@ export class ContainerRuntime
 	 * Parse an op's type and actual content from given serialized content
 	 * ! Note: this format needs to be in-line with what is set in the "ContainerRuntime.submit(...)" method
 	 */
-	private parseOpContent(serializedContent?: string): {
-		type: ContainerMessageType;
-		contents: unknown;
-	} {
+	private parseOpContent(serializedContent?: string): ContainerRuntimeMessage {
 		assert(serializedContent !== undefined, 0x6d5 /* content must be defined */);
-		const parsed = JSON.parse(serializedContent);
-		assert(parsed.type !== undefined, 0x6d6 /* incorrect op content format */);
-		return { type: parsed.type as ContainerMessageType, contents: parsed.contents };
+		const { type, contents, compatDetails }: ContainerRuntimeMessage =
+			JSON.parse(serializedContent);
+		assert(type !== undefined, 0x6d6 /* incorrect op content format */);
+		return { type, contents, compatDetails };
 	}
 
 	private async applyStashedOp(op: string): Promise<unknown> {
 		// Need to parse from string for back-compat
-		const { type, contents } = this.parseOpContent(op);
+		const { type, contents, compatDetails } = this.parseOpContent(op);
 		switch (type) {
 			case ContainerMessageType.FluidDataStoreOp:
 				return this.dataStores.applyStashedOp(contents as IEnvelope);
@@ -1846,8 +2044,27 @@ export class ContainerRuntime
 				throw new Error("chunkedOp not expected here");
 			case ContainerMessageType.Rejoin:
 				throw new Error("rejoin not expected here");
-			default:
-				unreachableCase(type, `Unknown ContainerMessageType: ${type}`);
+			default: {
+				// This should be extremely rare for stashed ops.
+				// It would require a newer runtime stashing ops and then an older one applying them,
+				// e.g. if an app rolled back its container version
+				const compatBehavior = compatDetails?.behavior;
+				if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
+					const error = DataProcessingError.create(
+						"Stashed runtime message of unknown type",
+						"applyStashedOp",
+						undefined /* sequencedMessage */,
+						{
+							messageDetails: JSON.stringify({
+								type,
+								compatBehavior,
+							}),
+						},
+					);
+					this.closeFn(error);
+					throw error;
+				}
+			}
 		}
 	}
 
@@ -1861,32 +2078,6 @@ export class ContainerRuntime
 			return;
 		}
 
-		// If attachment blobs were added while disconnected, we need to delay
-		// propagation of the "connected" event until we have uploaded them to
-		// ensure we don't submit ops referencing a blob that has not been uploaded
-		// Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-		const connecting =
-			connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
-		if (connecting && this.blobManager.hasPendingOfflineUploads) {
-			assert(
-				!this.delayConnectClientId,
-				0x392 /* Connect event delay must be canceled before subsequent connect event */,
-			);
-			assert(!!clientId, 0x393 /* Must have clientId when connecting */);
-			this.delayConnectClientId = clientId;
-			this.blobManager.onConnected().then(
-				() => {
-					// make sure we didn't reconnect before the promise resolved
-					if (this.delayConnectClientId === clientId && !this.disposed) {
-						this.delayConnectClientId = undefined;
-						this.setConnectionStateCore(connected, clientId);
-					}
-				},
-				(error) => this.closeFn(error),
-			);
-			return;
-		}
-
 		this.setConnectionStateCore(connected, clientId);
 	}
 
@@ -1900,6 +2091,14 @@ export class ContainerRuntime
 		// There might be no change of state due to Container calling this API after loading runtime.
 		const changeOfState = this._connected !== connected;
 		const reconnection = changeOfState && !connected;
+
+		// We need to flush the ops currently collected by Outbox to preserve original order.
+		// This flush NEEDS to happen before we set the ContainerRuntime to "connected".
+		// We want these ops to get to the PendingStateManager without sending to service and have them return to the Outbox upon calling "replayPendingStates".
+		if (changeOfState && connected) {
+			this.flush();
+		}
+
 		this._connected = connected;
 
 		if (!connected) {
@@ -1926,7 +2125,7 @@ export class ContainerRuntime
 						{
 							dataLoss: 1,
 							attempts: this.consecutiveReconnects,
-							pendingMessages: this.pendingStateManager.pendingMessagesCount,
+							pendingMessages: this.pendingMessagesCount,
 						},
 					),
 				);
@@ -1951,24 +2150,30 @@ export class ContainerRuntime
 	public process(messageArg: ISequencedDocumentMessage, local: boolean) {
 		this.verifyNotClosed();
 
-		// Whether or not the message is actually a runtime message.
+		// Whether or not the message appears to be a runtime message from an up-to-date client.
 		// It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
 		// or something different, like a system message.
-		const runtimeMessage = messageArg.type === MessageType.Operation;
+		const modernRuntimeMessage = messageArg.type === MessageType.Operation;
 
 		// Do shallow copy of message, as the processing flow will modify it.
 		const messageCopy = { ...messageArg };
 		for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-			this.processCore(message, local, runtimeMessage);
+			this.processCore(message, local, modernRuntimeMessage);
 		}
 	}
 
 	private _processedClientSequenceNumber: number | undefined;
 
+	/**
+	 * Direct the message to the correct subsystem for processing, and implement other side effects
+	 * @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
+	 * @param local - Did this client send the op?
+	 * @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
+	 */
 	private processCore(
-		message: ISequencedDocumentMessage,
+		message: ISequencedDocumentMessage | SequencedContainerRuntimeMessage,
 		local: boolean,
-		runtimeMessage: boolean,
+		modernRuntimeMessage: boolean,
 	) {
 		// Surround the actual processing of the operation with messages to the schedule manager indicating
 		// the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
@@ -1979,8 +2184,10 @@ export class ContainerRuntime
 
 		try {
 			let localOpMetadata: unknown;
-			if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
-				localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
+			if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
+				localOpMetadata = this.pendingStateManager.processPendingLocalMessage(
+					message as SequencedContainerRuntimeMessage,
+				);
 			}
 
 			// If there are no more pending messages after processing a local message,
@@ -1989,53 +2196,14 @@ export class ContainerRuntime
 				this.updateDocumentDirtyState(false);
 			}
 
-			const type = message.type as ContainerMessageType;
-			switch (type) {
-				case ContainerMessageType.Attach:
-					this.dataStores.processAttachMessage(message, local);
-					break;
-				case ContainerMessageType.Alias:
-					this.processAliasMessage(message, localOpMetadata, local);
-					break;
-				case ContainerMessageType.FluidDataStoreOp:
-					this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
-					break;
-				case ContainerMessageType.BlobAttach:
-					this.blobManager.processBlobAttachOp(message, local);
-					break;
-				case ContainerMessageType.IdAllocation:
-					assert(
-						this.idCompressor !== undefined,
-						0x67c /* IdCompressor should be defined if enabled */,
-					);
-					this.idCompressor.finalizeCreationRange(message.contents as IdCreationRange);
-					break;
-				case ContainerMessageType.ChunkedOp:
-				case ContainerMessageType.Rejoin:
-					break;
-				default:
-					if (runtimeMessage) {
-						const error = DataProcessingError.create(
-							// Former assert 0x3ce
-							"Runtime message of unknown type",
-							"OpProcessing",
-							message,
-							{
-								local,
-								type: message.type,
-								contentType: typeof message.contents,
-								batch: message.metadata?.batch,
-								compression: message.compression,
-							},
-						);
-						this.closeFn(error);
-						throw error;
-					}
-			}
+			this.validateAndProcessRuntimeMessage(
+				message,
+				localOpMetadata,
+				local,
+				modernRuntimeMessage,
+			);
 
-			if (runtimeMessage || this.groupedBatchingEnabled) {
-				this.emit("op", message, runtimeMessage);
-			}
+			this.emit("op", message, modernRuntimeMessage);
 
 			this.scheduleManager.afterOpProcessing(undefined, message);
 
@@ -2050,13 +2218,74 @@ export class ContainerRuntime
 			throw e;
 		}
 	}
-
-	private processAliasMessage(
+	/**
+	 * Assuming the given message is also a ContainerRuntimeMessage,
+	 * checks its type and dispatches the message to the appropriate handler in the runtime.
+	 * Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
+	 */
+	private validateAndProcessRuntimeMessage(
 		message: ISequencedDocumentMessage,
 		localOpMetadata: unknown,
 		local: boolean,
-	) {
-		this.dataStores.processAliasMessage(message, localOpMetadata, local);
+		expectRuntimeMessageType: boolean,
+	): asserts message is SequencedContainerRuntimeMessage {
+		// Optimistically extract ContainerRuntimeMessage-specific props from the message
+		const { type: maybeContainerMessageType, compatDetails } =
+			message as ContainerRuntimeMessage;
+
+		switch (maybeContainerMessageType) {
+			case ContainerMessageType.Attach:
+				this.dataStores.processAttachMessage(message, local);
+				break;
+			case ContainerMessageType.Alias:
+				this.dataStores.processAliasMessage(message, localOpMetadata, local);
+				break;
+			case ContainerMessageType.FluidDataStoreOp:
+				this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
+				break;
+			case ContainerMessageType.BlobAttach:
+				this.blobManager.processBlobAttachOp(message, local);
+				break;
+			case ContainerMessageType.IdAllocation:
+				assert(
+					this.idCompressor !== undefined,
+					0x67c /* IdCompressor should be defined if enabled */,
+				);
+				this.idCompressor.finalizeCreationRange(message.contents as IdCreationRange);
+				break;
+			case ContainerMessageType.ChunkedOp:
+			case ContainerMessageType.Rejoin:
+				break;
+			default: {
+				// If we didn't necessarily expect a runtime message type, then no worries - just return
+				// e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
+				if (!expectRuntimeMessageType) {
+					return;
+				}
+
+				const compatBehavior = compatDetails?.behavior;
+				if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
+					const error = DataProcessingError.create(
+						// Former assert 0x3ce
+						"Runtime message of unknown type",
+						"OpProcessing",
+						message,
+						{
+							local,
+							messageDetails: JSON.stringify({
+								type: message.type,
+								contentType: typeof message.contents,
+								compatBehavior,
+								batch: (message.metadata as IBatchMetadata | undefined)?.batch,
+								compression: message.compression,
+							}),
+						},
+					);
+					this.closeFn(error);
+					throw error;
+				}
+			}
+		}
 	}
 
 	/**
@@ -2065,7 +2294,7 @@ export class ContainerRuntime
 	 */
 	private sendSignalTelemetryEvent(clientSignalSequenceNumber: number) {
 		const duration = Date.now() - this._perfSignalData.signalTimestamp;
-		this.logger.sendPerformanceEvent({
+		this.mc.logger.sendPerformanceEvent({
 			eventName: "SignalLatency",
 			duration,
 			signalsLost: this._perfSignalData.signalsLost,
@@ -2093,7 +2322,7 @@ export class ContainerRuntime
 			) {
 				this._perfSignalData.signalsLost++;
 				this._perfSignalData.trackingSignalSequenceNumber = undefined;
-				this.logger.sendErrorEvent({
+				this.mc.logger.sendErrorEvent({
 					eventName: "SignalLost",
 					type: envelope.contents.type,
 					signalsLost: this._perfSignalData.signalsLost,
@@ -2121,6 +2350,12 @@ export class ContainerRuntime
 		this.dataStores.processSignal(envelope.address, transformed, local);
 	}
 
+	/**
+	 * Returns the runtime of the data store.
+	 * @param id - Id supplied during creating the data store.
+	 * @param wait - True if you want to wait for it.
+	 * @deprecated - Use getAliasedDataStoreEntryPoint instead to get an aliased data store's entry point.
+	 */
 	public async getRootDataStore(id: string, wait = true): Promise<IFluidRouter> {
 		return this.getRootDataStoreChannel(id, wait);
 	}
@@ -2196,15 +2431,30 @@ export class ContainerRuntime
 		return result;
 	}
 
-	public async createDataStore(pkg: string | string[]): Promise<IDataStore> {
-		const internalId = uuid();
-		return channelToDataStore(
-			await this._createDataStore(pkg, internalId),
-			internalId,
-			this,
-			this.dataStores,
-			this.mc.logger,
-		);
+	/**
+	 * Returns the aliased data store's entryPoint, given the alias.
+	 * @param alias - The alias for the data store.
+	 * @returns The data store's entry point ({@link @fluidframework/core-interfaces#IFluidHandle}) if it exists and is aliased.
+	 * Returns undefined if no data store has been assigned the given alias.
+	 */
+	public async getAliasedDataStoreEntryPoint(
+		alias: string,
+	): Promise<IFluidHandle<FluidObject> | undefined> {
+		await this.dataStores.waitIfPendingAlias(alias);
+		const internalId = this.internalId(alias);
+		const context = await this.dataStores.getDataStoreIfAvailable(internalId, { wait: false });
+		// If the data store is not available or not an alias, return undefined.
+		if (context === undefined || !(await context.isRoot())) {
+			return undefined;
+		}
+
+		const channel = await context.realize();
+		if (channel.entryPoint === undefined) {
+			throw new UsageError(
+				"entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint",
+			);
+		}
+		return channel.entryPoint;
 	}
 
 	public createDetachedRootDataStore(
@@ -2221,25 +2471,37 @@ export class ContainerRuntime
 		return this.dataStores.createDetachedDataStoreCore(pkg, false);
 	}
 
+	public async createDataStore(pkg: string | string[]): Promise<IDataStore> {
+		const id = uuid();
+		return channelToDataStore(
+			await this.dataStores
+				._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id)
+				.realize(),
+			id,
+			this,
+			this.dataStores,
+			this.mc.logger,
+		);
+	}
+
+	/**
+	 * @deprecated 0.16 Issue #1537, #3631
+	 * @internal
+	 */
 	public async _createDataStoreWithProps(
 		pkg: string | string[],
 		props?: any,
 		id = uuid(),
 	): Promise<IDataStore> {
-		const fluidDataStore = await this.dataStores
-			._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
-			.realize();
-		return channelToDataStore(fluidDataStore, id, this, this.dataStores, this.mc.logger);
-	}
-
-	private async _createDataStore(
-		pkg: string | string[],
-		id = uuid(),
-		props?: any,
-	): Promise<IFluidDataStoreChannel> {
-		return this.dataStores
-			._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
-			.realize();
+		return channelToDataStore(
+			await this.dataStores
+				._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
+				.realize(),
+			id,
+			this,
+			this.dataStores,
+			this.mc.logger,
+		);
 	}
 
 	private canSendOps() {
@@ -2255,13 +2517,14 @@ export class ContainerRuntime
 		return this.flushMode !== FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
 	}
 
+	private readonly _quorum: IQuorumClients;
 	public getQuorum(): IQuorumClients {
-		return this.context.quorum;
+		return this._quorum;
 	}
 
+	private readonly _audience: IAudience;
 	public getAudience(): IAudience {
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		return this.context.audience!;
+		return this._audience;
 	}
 
 	/**
@@ -2272,7 +2535,7 @@ export class ContainerRuntime
 		return this.dirtyContainer;
 	}
 
-	private isContainerMessageDirtyable(type: ContainerMessageType, contents: any) {
+	private isContainerMessageDirtyable({ type, contents }: ContainerRuntimeMessage) {
 		// For legacy purposes, exclude the old built-in AgentScheduler from dirty consideration as a special-case.
 		// Ultimately we should have no special-cases from the ContainerRuntime's perspective.
 		if (type === ContainerMessageType.Attach) {
@@ -2321,12 +2584,12 @@ export class ContainerRuntime
 	public submitSignal(type: string, content: any) {
 		this.verifyNotClosed();
 		const envelope = this.createNewSignalEnvelope(undefined /* address */, type, content);
-		return this.context.submitSignalFn(envelope);
+		return this.submitSignalFn(envelope);
 	}
 
 	public submitDataStoreSignal(address: string, type: string, content: any) {
 		const envelope = this.createNewSignalEnvelope(address, type, content);
-		return this.context.submitSignalFn(envelope);
+		return this.submitSignalFn(envelope);
 	}
 
 	public setAttachState(attachState: AttachState.Attaching | AttachState.Attached): void {
@@ -2378,15 +2641,7 @@ export class ContainerRuntime
 		return summarizeResult.summary;
 	}
 
-	public async getAbsoluteUrl(relativeUrl: string): Promise<string | undefined> {
-		if (this.context.getAbsoluteUrl === undefined) {
-			throw new Error("Driver does not implement getAbsoluteUrl");
-		}
-		if (this.attachState !== AttachState.Attached) {
-			return undefined;
-		}
-		return this.context.getAbsoluteUrl(relativeUrl);
-	}
+	public readonly getAbsoluteUrl: (relativeUrl: string) => Promise<string | undefined>;
 
 	private async summarizeInternal(
 		fullTree: boolean,
@@ -2427,7 +2682,7 @@ export class ContainerRuntime
 		fullGC?: boolean;
 		/** True to run GC sweep phase after the mark phase */
 		runSweep?: boolean;
-	}): Promise<IRootSummaryTreeWithStats> {
+	}): Promise<ISummaryTreeWithStats> {
 		this.verifyNotClosed();
 
 		const {
@@ -2450,9 +2705,8 @@ export class ContainerRuntime
 		});
 
 		try {
-			let gcStats: IGCStats | undefined;
 			if (runGC) {
-				gcStats = await this.collectGarbage(
+				await this.collectGarbage(
 					{ logger: summaryLogger, runSweep, fullGC },
 					telemetryContext,
 				);
@@ -2469,9 +2723,9 @@ export class ContainerRuntime
 				0x12f /* "Container Runtime's summarize should always return a tree" */,
 			);
 
-			return { stats, summary, gcStats };
+			return { stats, summary };
 		} finally {
-			this.logger.sendTelemetryEvent({
+			this.mc.logger.sendTelemetryEvent({
 				eventName: "SummarizeTelemetry",
 				details: telemetryContext.serialize(),
 			});
@@ -2543,7 +2797,7 @@ export class ContainerRuntime
 	/**
 	 * After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
 	 * @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
-	 * @returns - The routes of nodes that were deleted.
+	 * @returns The routes of nodes that were deleted.
 	 */
 	public deleteSweepReadyNodes(sweepReadyRoutes: string[]): string[] {
 		const { dataStoreRoutes, blobManagerRoutes } =
@@ -2614,7 +2868,7 @@ export class ContainerRuntime
 	/**
 	 * From a given list of routes, separate and return routes that belong to blob manager and data stores.
 	 * @param routes - A list of routes that can belong to data stores or blob manager.
-	 * @returns - Two route lists - One that contains routes for blob manager and another one that contains routes
+	 * @returns Two route lists - One that contains routes for blob manager and another one that contains routes
 	 * for data stores.
 	 */
 	private getDataStoreAndBlobManagerRoutes(routes: string[]) {
@@ -2670,12 +2924,15 @@ export class ContainerRuntime
 	 * @param options - options controlling how the summary is generated or submitted
 	 */
 	public async submitSummary(options: ISubmitSummaryOptions): Promise<SubmitSummaryResult> {
-		const { fullTree = false, refreshLatestAck, summaryLogger } = options;
+		const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
 		// The summary number for this summary. This will be updated during the summary process, so get it now and
 		// use it for all events logged during this summary.
 		const summaryNumber = this.nextSummaryNumber;
-		const summaryNumberLogger = ChildLogger.create(summaryLogger, undefined, {
-			all: { summaryNumber },
+		const summaryNumberLogger = createChildLogger({
+			logger: summaryLogger,
+			properties: {
+				all: { summaryNumber },
+			},
 		});
 
 		assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
@@ -2683,7 +2940,10 @@ export class ContainerRuntime
 		let latestSnapshotVersionId: string | undefined;
 		if (refreshLatestAck) {
 			const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(
-				ChildLogger.create(summaryNumberLogger, undefined, { all: { safeSummary: true } }),
+				createChildLogger({
+					logger: summaryNumberLogger,
+					properties: { all: { safeSummary: true } },
+				}),
 			);
 			const latestSnapshotRefSeq = latestSnapshotInfo.latestSnapshotRefSeq;
 			latestSnapshotVersionId = latestSnapshotInfo.latestSnapshotVersionId;
@@ -2692,6 +2952,50 @@ export class ContainerRuntime
 			await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
 		}
 
+		// If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
+		// incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
+		// within the timeout, check if it should be failed or can continue.
+		if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
+			const countBefore = this.pendingMessagesCount;
+			// The timeout for waiting for pending ops can be overridden via configurations.
+			const pendingOpsTimeout =
+				this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
+				defaultPendingOpsWaitTimeoutMs;
+			await new Promise<void>((resolve, reject) => {
+				const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
+				this.once("saved", () => {
+					clearTimeout(timeoutId);
+					resolve();
+				});
+				this.once("dispose", () => {
+					clearTimeout(timeoutId);
+					reject(new Error("Runtime is disposed while summarizing"));
+				});
+			});
+
+			// Log that there are pending ops while summarizing. This will help us gather data on how often this
+			// happens, whether we attempted to wait for these ops to be acked and what was the result.
+			summaryNumberLogger.sendTelemetryEvent({
+				eventName: "PendingOpsWhileSummarizing",
+				saved: this.hasPendingMessages() ? false : true,
+				timeout: pendingOpsTimeout,
+				countBefore,
+				countAfter: this.pendingMessagesCount,
+			});
+
+			// There could still be pending ops. Check if summary should fail or continue.
+			const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(
+				summaryNumberLogger,
+				this.deltaManager.lastSequenceNumber,
+				this.deltaManager.minimumSequenceNumber,
+				finalAttempt,
+				true /* beforeSummaryGeneration */,
+			);
+			if (pendingMessagesFailResult !== undefined) {
+				return pendingMessagesFailResult;
+			}
+		}
+
 		const shouldPauseInboundSignal =
 			this.mc.config.getBoolean(
 				"Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause",
@@ -2762,7 +3066,7 @@ export class ContainerRuntime
 			}
 
 			const trace = Trace.start();
-			let summarizeResult: IRootSummaryTreeWithStats;
+			let summarizeResult: ISummaryTreeWithStats;
 			// If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
 			// state of all the nodes.
 			const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
@@ -2781,6 +3085,38 @@ export class ContainerRuntime
 					error,
 				};
 			}
+
+			// If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
+			if (this.validateSummaryBeforeUpload) {
+				// Validate that the summaries generated by summarize nodes is correct.
+				const validateResult = this.summarizerNode.validateSummary();
+				if (!validateResult.success) {
+					const { success, ...loggingProps } = validateResult;
+					const error = new RetriableSummaryError(
+						validateResult.reason,
+						validateResult.retryAfterSeconds,
+						{ ...loggingProps },
+					);
+					return {
+						stage: "base",
+						referenceSequenceNumber: summaryRefSeqNum,
+						minimumSequenceNumber,
+						error,
+					};
+				}
+
+				const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(
+					summaryNumberLogger,
+					summaryRefSeqNum,
+					minimumSequenceNumber,
+					finalAttempt,
+					false /* beforeSummaryGeneration */,
+				);
+				if (pendingMessagesFailResult !== undefined) {
+					return pendingMessagesFailResult;
+				}
+			}
+
 			const { summary: summaryTree, stats: partialStats } = summarizeResult;
 
 			// Now that we have generated the summary, update the message at last summary to the last message processed.
@@ -2808,7 +3144,7 @@ export class ContainerRuntime
 				summaryNumber,
 				...partialStats,
 			};
-			const generateSummaryData = {
+			const generateSummaryData: Omit<IGenerateSummaryTreeResult, "stage" | "error"> = {
 				referenceSequenceNumber: summaryRefSeqNum,
 				minimumSequenceNumber,
 				summaryTree,
@@ -2840,7 +3176,7 @@ export class ContainerRuntime
 			} else if (lastAck === undefined) {
 				summaryContext = {
 					proposalHandle: undefined,
-					ackHandle: this.context.getLoadedFromVersion()?.id,
+					ackHandle: this.loadedFromVersionId,
 					referenceSequenceNumber: summaryRefSeqNum,
 				};
 			} else {
@@ -2894,7 +3230,15 @@ export class ContainerRuntime
 				submitOpDuration: trace.trace().duration,
 			} as const;
 
-			this.summarizerNode.completeSummary(handle);
+			try {
+				// If validateSummaryBeforeUpload is false, the summary should be validated in this step.
+				this.summarizerNode.completeSummary(
+					handle,
+					!this.validateSummaryBeforeUpload /* validate */,
+				);
+			} catch (error) {
+				return { stage: "upload", ...uploadData, error };
+			}
 			return submitData;
 		} finally {
 			// Cleanup wip summary in case of failure
@@ -2911,8 +3255,78 @@ export class ContainerRuntime
 		}
 	}
 
+	/**
+	 * This helper is called during summarization. If there are pending ops, it will return a failed summarize result
+	 * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
+	 * @param logger - The logger to be used for sending telemetry.
+	 * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
+	 * @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
+	 * @param finalAttempt - Whether this is the final summary attempt.
+	 * @param beforeSummaryGeneration - Whether this is called before summary generation or after.
+	 * @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
+	 */
+	private async shouldFailSummaryOnPendingOps(
+		logger: ITelemetryLoggerExt,
+		referenceSequenceNumber: number,
+		minimumSequenceNumber: number,
+		finalAttempt: boolean,
+		beforeSummaryGeneration: boolean,
+	): Promise<IBaseSummarizeResult | undefined> {
+		if (!this.hasPendingMessages()) {
+			return;
+		}
+
+		// If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
+		// This is a fallback to make progress in documents where there are consistently pending ops in
+		// the summarizer.
+		if (
+			finalAttempt &&
+			this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")
+		) {
+			const error = DataProcessingError.create(
+				"Pending ops during summarization",
+				"submitSummary",
+				undefined,
+				{ pendingMessages: this.pendingMessagesCount },
+			);
+			logger.sendErrorEvent(
+				{
+					eventName: "SkipFailingIncorrectSummary",
+					referenceSequenceNumber,
+					minimumSequenceNumber,
+					beforeGenerate: beforeSummaryGeneration,
+				},
+				error,
+			);
+		} else {
+			// The retry delay when there are pending ops can be overridden via config so that we can adjust it
+			// based on telemetry while we decide on a stable number.
+			const retryDelayMs =
+				this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
+				defaultPendingOpsRetryDelayMs;
+			const error = new RetriableSummaryError(
+				"PendingOpsWhileSummarizing",
+				retryDelayMs / 1000,
+				{
+					count: this.pendingMessagesCount,
+					beforeGenerate: beforeSummaryGeneration,
+				},
+			);
+			return {
+				stage: "base",
+				referenceSequenceNumber,
+				minimumSequenceNumber,
+				error,
+			};
+		}
+	}
+
+	private get pendingMessagesCount(): number {
+		return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
+	}
+
 	private hasPendingMessages() {
-		return this.pendingStateManager.hasPendingMessages() || !this.outbox.isEmpty;
+		return this.pendingMessagesCount !== 0;
 	}
 
 	private updateDocumentDirtyState(dirty: boolean) {
@@ -2933,7 +3347,6 @@ export class ContainerRuntime
 		this.dirtyContainer = dirty;
 		if (this.emitDirtyDocumentEvent) {
 			this.emit(dirty ? "dirty" : "saved");
-			this.context.updateDirtyContainerState(dirty);
 		}
 	}
 
@@ -2946,7 +3359,10 @@ export class ContainerRuntime
 			address: id,
 			contents,
 		};
-		this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
+		this.submit(
+			{ type: ContainerMessageType.FluidDataStoreOp, contents: envelope },
+			localOpMetadata,
+		);
 	}
 
 	public submitDataStoreAliasOp(contents: any, localOpMetadata: unknown): void {
@@ -2955,12 +3371,15 @@ export class ContainerRuntime
 			throw new UsageError("malformedDataStoreAliasMessage");
 		}
 
-		this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
+		this.submit({ type: ContainerMessageType.Alias, contents }, localOpMetadata);
 	}
 
-	public async uploadBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {
+	public async uploadBlob(
+		blob: ArrayBufferLike,
+		signal?: AbortSignal,
+	): Promise<IFluidHandle<ArrayBufferLike>> {
 		this.verifyNotClosed();
-		return this.blobManager.createBlob(blob);
+		return this.blobManager.createBlob(blob, signal);
 	}
 
 	private maybeSubmitIdAllocationOp(type: ContainerMessageType) {
@@ -2974,7 +3393,7 @@ export class ContainerRuntime
 				);
 				idRange = this.idCompressor.takeNextCreationRange();
 				// Don't include the idRange if there weren't any Ids allocated
-				idRange = idRange?.ids?.first !== undefined ? idRange : undefined;
+				idRange = idRange?.ids !== undefined ? idRange : undefined;
 			}
 
 			if (idRange !== undefined) {
@@ -2998,8 +3417,7 @@ export class ContainerRuntime
 	}
 
 	private submit(
-		type: ContainerMessageType,
-		contents: any,
+		containerRuntimeMessage: ContainerRuntimeMessage,
 		localOpMetadata: unknown = undefined,
 		metadata: Record<string, unknown> | undefined = undefined,
 	): void {
@@ -3012,17 +3430,18 @@ export class ContainerRuntime
 			0x132 /* "sending ops in detached container" */,
 		);
 
-		const serializedContent = JSON.stringify({ type, contents });
+		const serializedContent = JSON.stringify(containerRuntimeMessage);
 
 		// Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
 		// container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
 		if (this.innerDeltaManager.readOnlyInfo.readonly) {
-			this.logger.sendTelemetryEvent({
+			this.mc.logger.sendTelemetryEvent({
 				eventName: "SubmitOpInReadonly",
 				connected: this.connected,
 			});
 		}
 
+		const type = containerRuntimeMessage.type;
 		const message: BatchMessage = {
 			contents: serializedContent,
 			type,
@@ -3064,6 +3483,9 @@ export class ContainerRuntime
 				this.disableAttachReorder !== true
 			) {
 				this.outbox.submitAttach(message);
+			} else if (type === ContainerMessageType.BlobAttach) {
+				// BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
+				this.outbox.submitBlobAttach(message);
 			} else {
 				this.outbox.submit(message);
 			}
@@ -3078,7 +3500,7 @@ export class ContainerRuntime
 			throw error;
 		}
 
-		if (this.isContainerMessageDirtyable(type, contents)) {
+		if (this.isContainerMessageDirtyable(containerRuntimeMessage)) {
 			this.updateDocumentDirtyState(true);
 		}
 	}
@@ -3134,9 +3556,9 @@ export class ContainerRuntime
 		assert(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
 
 		// back-compat: ADO #1385: Make this call unconditional in the future
-		return this.context.submitSummaryFn !== undefined
-			? this.context.submitSummaryFn(contents, referenceSequenceNumber)
-			: this.context.submitFn(MessageType.Summarize, contents, false);
+		return this.submitSummaryFn !== undefined
+			? this.submitSummaryFn(contents, referenceSequenceNumber)
+			: this.submitFn(MessageType.Summarize, contents, false);
 	}
 
 	/**
@@ -3157,7 +3579,7 @@ export class ContainerRuntime
 				this.mc.logger.sendTelemetryEvent(
 					{ eventName: "OpReentry" },
 					// We need to capture the call stack in order to inspect the source of this usage pattern
-					new UsageError(errorMessage),
+					getLongStack(() => new UsageError(errorMessage)),
 				);
 				this.opReentryCallsToReport--;
 			}
@@ -3180,44 +3602,49 @@ export class ContainerRuntime
 		}
 	}
 
-	private reSubmit(
-		content: string,
-		localOpMetadata: unknown,
-		opMetadata: Record<string, unknown> | undefined,
-	) {
+	private reSubmitBatch(batch: IPendingBatchMessage[]) {
+		this.orderSequentially(() => {
+			for (const message of batch) {
+				this.reSubmit(message);
+			}
+		});
+		this.flush();
+	}
+
+	private reSubmit(message: IPendingBatchMessage) {
 		// Need to parse from string for back-compat
-		const { contents, type } = this.parseOpContent(content);
-		this.reSubmitCore(type, contents, localOpMetadata, opMetadata);
+		const containerRuntimeMessage = this.parseOpContent(message.content);
+		this.reSubmitCore(containerRuntimeMessage, message.localOpMetadata, message.opMetadata);
 	}
 
 	/**
 	 * Finds the right store and asks it to resubmit the message. This typically happens when we
 	 * reconnect and there are pending messages.
-	 * @param content - The content of the original message.
+	 * @param message - The original ContainerRuntimeMessage.
 	 * @param localOpMetadata - The local metadata associated with the original message.
 	 */
 	private reSubmitCore(
-		type: ContainerMessageType,
-		content: any,
+		message: ContainerRuntimeMessage,
 		localOpMetadata: unknown,
 		opMetadata: Record<string, unknown> | undefined,
 	) {
-		switch (type) {
+		const contents = message.contents;
+		switch (message.type) {
 			case ContainerMessageType.FluidDataStoreOp:
 				// For Operations, call resubmitDataStoreOp which will find the right store
 				// and trigger resubmission on it.
-				this.dataStores.resubmitDataStoreOp(content, localOpMetadata);
+				this.dataStores.resubmitDataStoreOp(contents, localOpMetadata);
 				break;
 			case ContainerMessageType.Attach:
 			case ContainerMessageType.Alias:
-				this.submit(type, content, localOpMetadata);
+				this.submit(message, localOpMetadata);
 				break;
 			case ContainerMessageType.IdAllocation:
 				// Remove the stashedState from the op if it's a stashed op
-				if (content.stashedState !== undefined) {
-					delete content.stashedState;
+				if (contents.stashedState !== undefined) {
+					delete contents.stashedState;
 				}
-				this.submit(type, content, localOpMetadata);
+				this.submit(message, localOpMetadata);
 				break;
 			case ContainerMessageType.ChunkedOp:
 				throw new Error(`chunkedOp not expected here`);
@@ -3225,10 +3652,33 @@ export class ContainerRuntime
 				this.blobManager.reSubmit(opMetadata);
 				break;
 			case ContainerMessageType.Rejoin:
-				this.submit(type, content);
+				this.submit(message);
 				break;
-			default:
-				unreachableCase(type, `Unknown ContainerMessageType: ${type}`);
+			default: {
+				// This case should be very rare - it would imply an op was stashed from a
+				// future version of runtime code and now is being applied on an older version
+				const compatBehavior = message.compatDetails?.behavior;
+				if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
+					this.logger.sendTelemetryEvent({
+						eventName: "resubmitUnrecognizedMessageTypeAllowed",
+						messageDetails: { type: message.type, compatBehavior },
+					});
+				} else {
+					const error = DataProcessingError.create(
+						"Resubmitting runtime message of unknown type",
+						"reSubmitCore",
+						undefined /* sequencedMessage */,
+						{
+							messageDetails: JSON.stringify({
+								type: message.type,
+								compatBehavior,
+							}),
+						},
+					);
+					this.closeFn(error);
+					throw error;
+				}
+			}
 		}
 	}
 
@@ -3242,6 +3692,7 @@ export class ContainerRuntime
 				this.dataStores.rollbackDataStoreOp(contents as IEnvelope, localOpMetadata);
 				break;
 			default:
+				// Don't check message.compatDetails because this is for rolling back a local op so the type will be known
 				throw new Error(`Can't rollback ${type}`);
 		}
 	}
@@ -3269,12 +3720,22 @@ export class ContainerRuntime
 	/** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
 	public async refreshLatestSummaryAck(options: IRefreshSummaryAckOptions) {
 		const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
+		// proposalHandle is always passed from RunningSummarizer.
+		assert(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
 		const readAndParseBlob = async <T>(id: string) => readAndParse<T>(this.storage, id);
-		// The call to fetch the snapshot is very expensive and not always needed.
-		// It should only be done by the summarizerNode, if required.
-		// When fetching from storage we will always get the latest version and do not use the ackHandle.
-		const fetchLatestSnapshot: () => Promise<IFetchSnapshotResult> = async () => {
-			let fetchResult = await this.fetchLatestSnapshotFromStorage(
+		const result = await this.summarizerNode.refreshLatestSummary(
+			proposalHandle,
+			summaryRefSeq,
+		);
+
+		/**
+		 * When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
+		 * current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
+		 * it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
+		 * and then close as the current main client is likely to be re-elected as the parent summarizer again.
+		 */
+		if (!result.isSummaryTracked && result.isSummaryNewer) {
+			const fetchResult = await this.fetchSnapshotFromStorage(
 				summaryLogger,
 				{
 					eventName: "RefreshLatestSummaryAckFetch",
@@ -3282,27 +3743,9 @@ export class ContainerRuntime
 					targetSequenceNumber: summaryRefSeq,
 				},
 				readAndParseBlob,
+				null,
 			);
 
-			/**
-			 * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
-			 * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
-			 * these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
-			 * change that started fetching latest snapshot always.
-			 */
-			if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-				fetchResult = await this.fetchSnapshotFromStorage(
-					summaryLogger,
-					{
-						eventName: "RefreshLatestSummaryAckFetchBackCompat",
-						ackHandle,
-						targetSequenceNumber: summaryRefSeq,
-					},
-					readAndParseBlob,
-					ackHandle,
-				);
-			}
-
 			/**
 			 * If the fetched snapshot is older than the one for which the ack was received, close the container.
 			 * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -3324,33 +3767,16 @@ export class ContainerRuntime
 						fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
 					},
 				);
-				this.closeFn(error);
+				this.disposeFn(error);
 				throw error;
 			}
 
-			// In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
-			// wait for the delta manager to catch up before refreshing the latest Summary.
-			await this.waitForDeltaManagerToCatchup(
-				fetchResult.latestSnapshotRefSeq,
-				summaryLogger,
-			);
-
-			return {
-				snapshotTree: fetchResult.snapshotTree,
-				snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
-			};
-		};
-
-		const result = await this.summarizerNode.refreshLatestSummary(
-			proposalHandle,
-			summaryRefSeq,
-			fetchLatestSnapshot,
-			readAndParseBlob,
-			summaryLogger,
-		);
+			await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
+			return;
+		}
 
 		// Notify the garbage collector so it can update its latest summary state.
-		await this.garbageCollector.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
+		await this.garbageCollector.refreshLatestSummary(result);
 	}
 
 	/**
@@ -3363,51 +3789,49 @@ export class ContainerRuntime
 		summaryLogger: ITelemetryLoggerExt,
 	): Promise<{ latestSnapshotRefSeq: number; latestSnapshotVersionId: string | undefined }> {
 		const readAndParseBlob = async <T>(id: string) => readAndParse<T>(this.storage, id);
-		const { snapshotTree, versionId, latestSnapshotRefSeq } =
-			await this.fetchLatestSnapshotFromStorage(
-				summaryLogger,
-				{
-					eventName: "RefreshLatestSummaryFromServerFetch",
-				},
-				readAndParseBlob,
-			);
-		const fetchLatestSnapshot: IFetchSnapshotResult = {
-			snapshotTree,
-			snapshotRefSeq: latestSnapshotRefSeq,
-		};
-		const result = await this.summarizerNode.refreshLatestSummary(
-			undefined /* proposalHandle */,
-			latestSnapshotRefSeq,
-			async () => fetchLatestSnapshot,
-			readAndParseBlob,
+		const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(
 			summaryLogger,
-		);
-
-		// Notify the garbage collector so it can update its latest summary state.
-		await this.garbageCollector.refreshLatestSummary(
-			undefined /* proposalHandle */,
-			result,
+			{
+				eventName: "RefreshLatestSummaryFromServerFetch",
+			},
 			readAndParseBlob,
+			null,
 		);
 
+		await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
+
 		return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
 	}
 
-	private async fetchLatestSnapshotFromStorage(
-		logger: ITelemetryLoggerExt,
-		event: ITelemetryGenericEvent,
-		readAndParseBlob: ReadAndParseBlob,
-	): Promise<{ snapshotTree: ISnapshotTree; versionId: string; latestSnapshotRefSeq: number }> {
-		return this.fetchSnapshotFromStorage(logger, event, readAndParseBlob, null /* latest */);
+	private async closeStaleSummarizer(codePath: string): Promise<void> {
+		this.mc.logger.sendTelemetryEvent(
+			{
+				eventName: "ClosingSummarizerOnSummaryStale",
+				codePath,
+				message: "Stopping fetch from storage",
+				closeSummarizerDelayMs: this.closeSummarizerDelayMs,
+			},
+			new GenericError("Restarting summarizer instead of refreshing"),
+		);
+
+		// Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
+		await delay(this.closeSummarizerDelayMs);
+		this._summarizer?.stop("latestSummaryStateStale");
+		this.disposeFn();
 	}
 
+	/**
+	 * Downloads snapshot from storage with the given versionId or latest if versionId is null.
+	 * By default, it also closes the container after downloading the snapshot. However, this may be
+	 * overridden via options.
+	 */
 	private async fetchSnapshotFromStorage(
 		logger: ITelemetryLoggerExt,
 		event: ITelemetryGenericEvent,
 		readAndParseBlob: ReadAndParseBlob,
 		versionId: string | null,
 	): Promise<{ snapshotTree: ISnapshotTree; versionId: string; latestSnapshotRefSeq: number }> {
-		const snapshotResults = await PerformanceEvent.timedExecAsync(
+		return PerformanceEvent.timedExecAsync(
 			logger,
 			event,
 			async (perfEvent: {
@@ -3453,77 +3877,75 @@ export class ContainerRuntime
 				};
 			},
 		);
-
-		// We choose to close the summarizer after the snapshot cache is updated to avoid
-		// situations which the main client (which is likely to be re-elected as the leader again)
-		// loads the summarizer from cache.
-		if (this.summaryStateUpdateMethod === "restart") {
-			const error = new GenericError("Restarting summarizer instead of refreshing");
-
-			this.mc.logger.sendTelemetryEvent(
-				{
-					...event,
-					eventName: "ClosingSummarizerOnSummaryStale",
-					codePath: event.eventName,
-					message: "Stopping fetch from storage",
-					versionId: versionId != null ? versionId : undefined,
-					closeSummarizerDelayMs: this.closeSummarizerDelayMs,
-				},
-				error,
-			);
-
-			// Delay 10 seconds before restarting summarizer to prevent the summarizer from restarting too frequently.
-			await delay(this.closeSummarizerDelayMs);
-			this._summarizer?.stop("latestSummaryStateStale");
-			this.closeFn();
-			throw error;
-		}
-
-		return snapshotResults;
 	}
 
 	public notifyAttaching() {} // do nothing (deprecated method)
 
-	public getPendingLocalState(): unknown {
-		if (this._orderSequentiallyCalls !== 0) {
-			throw new UsageError("can't get state during orderSequentially");
-		}
-		// Flush pending batch.
-		// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
-		// to close current batch.
-		this.flush();
+	public async getPendingLocalState(props?: {
+		notifyImminentClosure: boolean;
+	}): Promise<unknown> {
+		return PerformanceEvent.timedExecAsync(
+			this.mc.logger,
+			{
+				eventName: "getPendingLocalState",
+				notifyImminentClosure: props?.notifyImminentClosure,
+			},
+			async (event) => {
+				this.verifyNotClosed();
+				const waitBlobsToAttach = props?.notifyImminentClosure;
+				if (this._orderSequentiallyCalls !== 0) {
+					throw new UsageError("can't get state during orderSequentially");
+				}
+				const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(
+					waitBlobsToAttach,
+				);
+				const pending = this.pendingStateManager.getLocalState();
+				if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+					return; // no pending state to save
+				}
+				// Flush pending batch.
+				// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
+				// to close current batch.
+				this.flush();
 
-		return {
-			pending: this.pendingStateManager.getLocalState(),
-			pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
-		};
+				const pendingState: IPendingRuntimeState = {
+					pending,
+					pendingAttachmentBlobs,
+				};
+				event.end({
+					attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
+					pendingOpsSize: pending?.pendingStates.length,
+				});
+				return pendingState;
+			},
+		);
 	}
 
-	public readonly summarizeOnDemand: ISummarizer["summarizeOnDemand"] = (...args) => {
-		if (this.clientDetails.type === summarizerClientType) {
-			return this.summarizer.summarizeOnDemand(...args);
+	public summarizeOnDemand(options: IOnDemandSummarizeOptions): ISummarizeResults {
+		if (this.isSummarizerClient) {
+			return this.summarizer.summarizeOnDemand(options);
 		} else if (this.summaryManager !== undefined) {
-			return this.summaryManager.summarizeOnDemand(...args);
+			return this.summaryManager.summarizeOnDemand(options);
 		} else {
 			// If we're not the summarizer, and we don't have a summaryManager, we expect that
 			// disableSummaries is turned on. We are throwing instead of returning a failure here,
 			// because it is a misuse of the API rather than an expected failure.
 			throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
 		}
-	};
+	}
 
-	public readonly enqueueSummarize: ISummarizer["enqueueSummarize"] = (...args) => {
-		if (this.clientDetails.type === summarizerClientType) {
-			return this.summarizer.enqueueSummarize(...args);
+	public enqueueSummarize(options: IEnqueueSummarizeOptions): EnqueueSummarizeResult {
+		if (this.isSummarizerClient) {
+			return this.summarizer.enqueueSummarize(options);
 		} else if (this.summaryManager !== undefined) {
-			return this.summaryManager.enqueueSummarize(...args);
+			return this.summaryManager.enqueueSummarize(options);
 		} else {
 			// If we're not the summarizer, and we don't have a summaryManager, we expect that
 			// generateSummaries is turned off. We are throwing instead of returning a failure here,
 			// because it is a misuse of the API rather than an expected failure.
 			throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
 		}
-	};
+	}
 
 	/**
 	 * * Forms a function that will request a Summarizer.