@fluidframework/test-utils 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/src/TestSummaryUtils.ts

@@ -29,10 +29,11 @@ import { timeoutAwait } from "./timeoutUtils";
 const summarizerClientType = "summarizer";
 
 async function createSummarizerCore(
-	absoluteUrl: string | undefined,
+	container: IContainer,
 	loader: IHostLoader,
 	summaryVersion?: string,
 ) {
+	const absoluteUrl = await container.getAbsoluteUrl("");
 	if (absoluteUrl === undefined) {
 		throw new Error("URL could not be resolved");
 	}
@@ -52,10 +53,12 @@ async function createSummarizerCore(
 	const summarizerContainer = await loader.resolve(request);
 	await waitForContainerConnection(summarizerContainer);
 
-	const fluidObject = await requestFluidObject<FluidObject<ISummarizer>>(summarizerContainer, {
-		url: "_summarizer",
-	});
-	if (fluidObject.ISummarizer === undefined) {
+	const fluidObject: FluidObject<ISummarizer> | undefined = summarizerContainer.getEntryPoint
+		? await summarizerContainer.getEntryPoint?.()
+		: await requestFluidObject<FluidObject<ISummarizer>>(summarizerContainer, {
+				url: "_summarizer",
+		  });
+	if (fluidObject?.ISummarizer === undefined) {
 		throw new Error("Fluid object does not implement ISummarizer");
 	}
 
@@ -74,6 +77,11 @@ const defaultSummaryOptions: ISummaryRuntimeOptions = {
 	},
 };
 
+/**
+ * Creates a summarizer client from the given container and data store factory, and returns the summarizer client's
+ * IContainer and ISummarizer.
+ * The ISummarizer can be used to generate on-demand summaries. The IContainer can be used to fetch data stores, etc.
+ */
 export async function createSummarizerFromFactory(
 	provider: ITestObjectProvider,
 	container: IContainer,
@@ -81,7 +89,8 @@ export async function createSummarizerFromFactory(
 	summaryVersion?: string,
 	containerRuntimeFactoryType = ContainerRuntimeFactoryWithDefaultDataStore,
 	registryEntries?: NamedFluidDataStoreRegistryEntries,
-): Promise<ISummarizer> {
+	logger?: ITelemetryBaseLogger,
+): Promise<{ container: IContainer; summarizer: ISummarizer }> {
 	const innerRequestHandler = async (request: IRequest, runtime: IContainerRuntimeBase) =>
 		runtime.IFluidHandleContext.resolveHandle(request);
 	const runtimeFactory = new containerRuntimeFactoryType(
@@ -94,11 +103,15 @@ export async function createSummarizerFromFactory(
 
 	const loader = provider.createLoader([[provider.defaultCodeDetails, runtimeFactory]], {
 		configProvider: mockConfigProvider(),
+		logger,
 	});
-	const absoluteUrl = await container.getAbsoluteUrl("");
-	return (await createSummarizerCore(absoluteUrl, loader, summaryVersion)).summarizer;
+	return createSummarizerCore(container, loader, summaryVersion);
 }
 
+/**
+ * Creates a summarizer client from the given container and returns the summarizer client's IContainer and ISummarizer.
+ * The ISummarizer can be used to generate on-demand summaries. The IContainer can be used to fetch data stores, etc.
+ */
 export async function createSummarizer(
 	provider: ITestObjectProvider,
 	container: IContainer,
@@ -106,27 +119,6 @@ export async function createSummarizer(
 	gcOptions?: IGCRuntimeOptions,
 	configProvider: IConfigProviderBase = mockConfigProvider(),
 	logger?: ITelemetryBaseLogger,
-): Promise<ISummarizer> {
-	const absoluteUrl = await container.getAbsoluteUrl("");
-	return (
-		await createSummarizerWithContainer(
-			provider,
-			absoluteUrl,
-			summaryVersion,
-			gcOptions,
-			configProvider,
-			logger,
-		)
-	).summarizer;
-}
-
-export async function createSummarizerWithContainer(
-	provider: ITestObjectProvider,
-	absoluteUrl: string | undefined,
-	summaryVersion?: string,
-	gcOptions?: IGCRuntimeOptions,
-	configProvider: IConfigProviderBase = mockConfigProvider(),
-	logger?: ITelemetryBaseLogger,
 ): Promise<{ container: IContainer; summarizer: ISummarizer }> {
 	const testContainerConfig: ITestContainerConfig = {
 		runtimeOptions: {
@@ -136,8 +128,9 @@ export async function createSummarizerWithContainer(
 		loaderProps: { configProvider, logger },
 	};
 	const loader = provider.makeTestLoader(testContainerConfig);
-	return createSummarizerCore(absoluteUrl, loader, summaryVersion);
+	return createSummarizerCore(container, loader, summaryVersion);
 }
+
 /**
  * Summarizes on demand and returns the summary tree, the version number and the reference sequence number of the
  * submitted summary.

package/src/containerUtils.ts

@@ -4,23 +4,9 @@
  */
 
 import { IContainer } from "@fluidframework/container-definitions";
-import { ConnectionState, Container } from "@fluidframework/container-loader";
+import { ConnectionState } from "@fluidframework/container-loader";
 import { PromiseExecutor, timeoutPromise, TimeoutWithError } from "./timeoutUtils";
 
-/**
- * Waits for the specified container to emit a 'connected' event.
- *
- * @deprecated Use waitForContainerConnection instead.
- * Note that an upcoming release will change the default parameters on that function to:
- * - failOnContainerClose = true
- * - timeoutOptions.durationMs = 1s
- */
-export async function ensureContainerConnected(container: Container): Promise<void> {
-	if (!container.connected) {
-		return timeoutPromise((resolve) => container.once("connected", () => resolve()));
-	}
-}
-
 /**
  * Utility function to wait for the specified Container to be in Connected state.
  * If the Container is already connected, the Promise returns immediately; otherwise it resolves when the Container emits
@@ -30,20 +16,21 @@ export async function ensureContainerConnected(container: Container): Promise<vo
  * @param container - The container to wait for.
  * @param failOnContainerClose - If true, the returned Promise will be rejected if the container emits a 'closed' event
  * before a 'connected' event.
- * Defaults to false (but this will change in an upcoming version).
+ * Defaults to true.
  * @param timeoutOptions - Options related to the behavior of the timeout.
- * If not provided, no timeout will be applied and the promise will wait indefinitely for the Container to emit its
- * 'connected' (or 'closed, if failOnContainerClose === true) event.
- * @returns A Promise that resolves when the specified container emits a 'connected' event (or immediately if the
- * Container is already connected).
- * If failOnContainerClose === true and the container emits a 'closed' event before a 'connected' event, the Promise
- * is rejected with the error from the 'closed' event, if any.
- * If timeoutOptions is provided, the Promise will reject if the container hasn't emmited a relevant event before
- * timeoutOptions.durationMs (which defaults to 250ms if left undefined).
+ * If provided, the returned Promise will reject if the container hasn't emitted relevant events in timeoutOptions.durationMs.
+ * If not provided, the Promise will wait indefinitely for the Container to emit its 'connected' (or 'closed', if
+ * failOnContainerClose === true) event.
+ *
+ * @returns A Promise that either:
+ * - Resolves when the specified container emits a 'connected' event (or immediately if the Container is already connected).
+ * - Rejects if failOnContainerClose === true and the container emits a 'closed' event before a 'connected' event.
+ * - Rejects after timeoutOptions.durationMs if timeoutOptions !== undefined and the container does not emit relevant
+ * events, within that timeframe.
  */
 export async function waitForContainerConnection(
 	container: IContainer,
-	failOnContainerClose: boolean = false,
+	failOnContainerClose: boolean = true,
 	timeoutOptions?: TimeoutWithError,
 ): Promise<void> {
 	if (container.connectionState !== ConnectionState.Connected) {

package/src/index.ts

@@ -29,12 +29,7 @@ export {
 	ITestObjectProvider,
 	TestObjectProvider,
 } from "./testObjectProvider";
-export {
-	createSummarizer,
-	createSummarizerFromFactory,
-	createSummarizerWithContainer,
-	summarizeNow,
-} from "./TestSummaryUtils";
+export { createSummarizer, createSummarizerFromFactory, summarizeNow } from "./TestSummaryUtils";
 export {
 	defaultTimeoutDurationMs,
 	timeoutAwait,
@@ -42,4 +37,4 @@ export {
 	TimeoutWithError,
 	TimeoutWithValue,
 } from "./timeoutUtils";
-export { ensureContainerConnected, waitForContainerConnection } from "./containerUtils";
+export { waitForContainerConnection } from "./containerUtils";

package/src/loaderContainerTracker.ts

@@ -4,13 +4,14 @@
  */
 import { assert } from "@fluidframework/common-utils";
 import { IContainer, IDeltaQueue, IHostLoader } from "@fluidframework/container-definitions";
-import { Container } from "@fluidframework/container-loader";
+import { ConnectionState } from "@fluidframework/container-loader";
 import { canBeCoalescedByService } from "@fluidframework/driver-utils";
 import {
 	IDocumentMessage,
 	ISequencedDocumentMessage,
 	MessageType,
 } from "@fluidframework/protocol-definitions";
+import { waitForContainerConnection } from "./containerUtils";
 import { debug } from "./debug";
 import { IOpProcessingController } from "./testObjectProvider";
 import { timeoutAwait, timeoutPromise } from "./timeoutUtils";
@@ -24,6 +25,7 @@ interface ContainerRecord {
 
 	// LoaderContainerTracker paused state
 	paused: boolean;
+	pauseP?: Promise<void>; // promise for for the pause that is in progress
 
 	// Tracking trailing no-op that may or may be acked by the server so we can discount them
 	// See issue #5629
@@ -123,7 +125,7 @@ export class LoaderContainerTracker implements IOpProcessingController {
 			// Received the no op back, update the record if we are tracking
 			if (
 				canBeCoalescedByService(message) &&
-				message.clientId === (container as Container).clientId &&
+				message.clientId === container.clientId &&
 				record.trailingNoOps !== 0 &&
 				record.startTrailingNoOps <= message.clientSequenceNumber
 			) {
@@ -162,57 +164,51 @@ export class LoaderContainerTracker implements IOpProcessingController {
 		// REVIEW: do we need to unpatch the loaders?
 	}
 
-	/**
-	 * Ensure all tracked containers are synchronized
-	 */
-	public async ensureSynchronized(...containers: IContainer[]): Promise<void> {
-		await this.processSynchronized(undefined, ...containers);
-	}
-
 	/**
 	 * Ensure all tracked containers are synchronized with a time limit
+	 *
+	 * @deprecated - this method is equivalent to @see {@link LoaderContainerTracker.ensureSynchronized}, please configure the test timeout instead
 	 */
 	public async ensureSynchronizedWithTimeout?(
 		timeoutDuration: number | undefined,
 		...containers: IContainer[]
 	) {
-		await this.processSynchronized(timeoutDuration, ...containers);
+		await this.ensureSynchronized(...containers);
 	}
 
 	/**
 	 * Make sure all the tracked containers are synchronized.
 	 *
 	 * No isDirty (non-readonly) containers
-	 *
 	 * No extra clientId in quorum of any container that is not tracked and still opened.
-	 *
 	 * - i.e. no pending Join/Leave message.
-	 *
 	 * No unresolved proposal (minSeqNum \>= lastProposalSeqNum)
-	 *
 	 * lastSequenceNumber of all container is the same
-	 *
 	 * clientSequenceNumberObserved is the same as clientSequenceNumber sent
-	 *
 	 * - this overlaps with !isDirty, but include task scheduler ops.
-	 *
 	 * - Trailing NoOp is tracked and don't count as pending ops.
+	 *
+	 * Containers that are already pause will resume process and paused again once
+	 * everything is synchronized.  Containers that aren't paused will remain unpaused when this
+	 * function returns.
 	 */
-	private async processSynchronized(
-		timeoutDuration: number | undefined,
-		...containers: IContainer[]
-	) {
+	public async ensureSynchronized(...containers: IContainer[]): Promise<void> {
 		const resumed = this.resumeProcessing(...containers);
 
-		let waitingSequenceNumberSynchronized = false;
+		let waitingSequenceNumberSynchronized: string | undefined;
 		// eslint-disable-next-line no-constant-condition
 		while (true) {
+			// yield a turn to allow side effect of resuming or the ops we just processed execute before we check
+			await new Promise<void>((resolve) => {
+				setTimeout(resolve, 0);
+			});
+
 			const containersToApply = this.getContainers(containers);
 			if (containersToApply.length === 0) {
 				break;
 			}
 
-			// Ignore readonly dirty containers, because it can't sent up and nothing can be done about it being dirty
+			// Ignore readonly dirty containers, because it can't sent ops and nothing can be done about it being dirty
 			const dirtyContainers = containersToApply.filter((c) => {
 				const { deltaManager, isDirty } = c;
 				return deltaManager.readOnlyInfo.readonly !== true && isDirty;
@@ -221,20 +217,22 @@ export class LoaderContainerTracker implements IOpProcessingController {
 				// Wait for all the leave messages
 				const pendingClients = this.getPendingClients(containersToApply);
 				if (pendingClients.length === 0) {
-					if (this.isSequenceNumberSynchronized(containersToApply)) {
+					const needSync = this.needSequenceNumberSynchronize(containersToApply);
+					if (needSync === undefined) {
 						// done, we are in sync
 						break;
 					}
-					if (!waitingSequenceNumberSynchronized) {
-						// Only write it out once
-						waitingSequenceNumberSynchronized = true;
-						debugWait("Waiting for sequence number synchronized");
-						await timeoutAwait(this.waitForAnyInboundOps(containersToApply), {
-							errorMsg: "Timeout on waiting for sequence number synchronized",
-						});
+					if (waitingSequenceNumberSynchronized !== needSync.reason) {
+						// Don't repeat writing to console if it is the same reason
+						waitingSequenceNumberSynchronized = needSync.reason;
+						debugWait(needSync.message);
 					}
+					// Wait for one inbounds ops which might change the state of things
+					await timeoutAwait(this.waitForAnyInboundOps(containersToApply), {
+						errorMsg: `Timeout on ${needSync.message}`,
+					});
 				} else {
-					waitingSequenceNumberSynchronized = false;
+					waitingSequenceNumberSynchronized = undefined;
 					await timeoutAwait(this.waitForPendingClients(pendingClients), {
 						errorMsg: "Timeout on waiting for pending join or leave op",
 					});
@@ -242,12 +240,9 @@ export class LoaderContainerTracker implements IOpProcessingController {
 			} else {
 				// Wait for all the containers to be saved
 				debugWait(
-					`Waiting container to be saved ${dirtyContainers.map(
-						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-						(c) => this.containers.get(c)!.index,
-					)}`,
+					`Waiting container to be saved ${this.containerIndexStrings(dirtyContainers)}`,
 				);
-				waitingSequenceNumberSynchronized = false;
+				waitingSequenceNumberSynchronized = undefined;
 				await Promise.all(
 					dirtyContainers.map(async (c) =>
 						Promise.race([
@@ -259,11 +254,6 @@ export class LoaderContainerTracker implements IOpProcessingController {
 					),
 				);
 			}
-
-			// yield a turn to allow side effect of the ops we just processed execute before we check again
-			await new Promise<void>((resolve) => {
-				setTimeout(resolve, 0);
-			});
 		}
 
 		// Pause all container that was resumed
@@ -287,9 +277,7 @@ export class LoaderContainerTracker implements IOpProcessingController {
 		// All the clientId we track should be a superset of the quorum, otherwise, we are missing
 		// leave messages
 		const openedDocuments = Array.from(this.containers.keys()).filter((c) => !c.closed);
-		const openedClientId = openedDocuments.map(
-			(container) => (container as Container).clientId,
-		);
+		const openedClientId = openedDocuments.map((container) => container.clientId);
 
 		const pendingClients: [IContainer, Set<string>][] = [];
 		containersToApply.forEach((container) => {
@@ -321,48 +309,78 @@ export class LoaderContainerTracker implements IOpProcessingController {
 	 *
 	 * @param containersToApply - the set of containers to check
 	 */
-	private isSequenceNumberSynchronized(containersToApply: IContainer[]) {
+	private needSequenceNumberSynchronize(containersToApply: IContainer[]) {
+		// If there is a pending proposal, wait for it to be accepted
+		const minSeqNum = containersToApply[0].deltaManager.minimumSequenceNumber;
+		if (minSeqNum < this.lastProposalSeqNum) {
+			return {
+				reason: "Proposal",
+				message: `waiting for MSN to advance to proposal at sequence number ${this.lastProposalSeqNum}`,
+			};
+		}
+
 		// clientSequenceNumber check detects ops in flight, both on the wire and in the outbound queue
 		// We need both client sequence number and isDirty check because:
 		// - Currently isDirty flag ignores ops for task scheduler, so we need the client sequence number check
 		// - But isDirty flags include ops during forceReadonly and disconnected, because we don't submit
 		//   the ops in the first place, clientSequenceNumber is not assigned
 
-		const isClientSequenceNumberSynchronized = containersToApply.every((container) => {
+		const containerWithInflightOps = containersToApply.filter((container) => {
 			if (container.deltaManager.readOnlyInfo.readonly === true) {
 				// Ignore readonly container. the clientSeqNum and clientSeqNumObserved might be out of sync
 				// because we transition to readonly when outbound is not empty or the in transit op got lost
-				return true;
+				return false;
 			}
 			// Note that in read only mode, the op won't be submitted
 			let deltaManager = container.deltaManager as any;
 			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 			const { trailingNoOps } = this.containers.get(container)!;
-			// Back-compat: clientSequenceNumber & clientSequenceNumberObserved moved to ConnectionManager in 0.53
+			// Back-compat: lastSubmittedClientId/clientSequenceNumber/clientSequenceNumberObserved moved to ConnectionManager in 0.53
 			if (!("clientSequenceNumber" in deltaManager)) {
 				deltaManager = deltaManager.connectionManager;
 			}
 			assert("clientSequenceNumber" in deltaManager, "no clientSequenceNumber");
 			assert("clientSequenceNumberObserved" in deltaManager, "no clientSequenceNumber");
+			// If last submittedClientId isn't the current clientId, then we haven't send any ops
 			return (
-				deltaManager.clientSequenceNumber ===
-				(deltaManager.clientSequenceNumberObserved as number) + trailingNoOps
+				deltaManager.lastSubmittedClientId === container.clientId &&
+				deltaManager.clientSequenceNumber !==
+					(deltaManager.clientSequenceNumberObserved as number) + trailingNoOps
 			);
 		});
 
-		if (!isClientSequenceNumberSynchronized) {
-			return false;
+		if (containerWithInflightOps.length !== 0) {
+			return {
+				reason: "InflightOps",
+				message: `waiting for containers with inflight ops: ${this.containerIndexStrings(
+					containerWithInflightOps,
+				)}`,
+			};
 		}
 
-		const minSeqNum = containersToApply[0].deltaManager.minimumSequenceNumber;
-		if (minSeqNum < this.lastProposalSeqNum) {
-			// There is an unresolved proposal
-			return false;
+		// Check to see if all the container has process the same number of ops.
+		const maxSeqNum = Math.max(
+			...containersToApply.map((c) => c.deltaManager.lastSequenceNumber),
+		);
+		const containerWithPendingIncoming = containersToApply.filter(
+			(c) => c.deltaManager.lastSequenceNumber !== maxSeqNum,
+		);
+		if (containerWithPendingIncoming.length !== 0) {
+			return {
+				reason: "Pending",
+				message: `waiting for containers with pending incoming ops up to sequence number ${maxSeqNum}: ${this.containerIndexStrings(
+					containerWithPendingIncoming,
+				)}`,
+			};
 		}
+		return undefined;
+	}
 
-		// Check to see if all the container has process the same number of ops.
-		const seqNum = containersToApply[0].deltaManager.lastSequenceNumber;
-		return containersToApply.every((c) => c.deltaManager.lastSequenceNumber === seqNum);
+	private containerIndexStrings(containers: IContainer[]) {
+		return containers.map(
+			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+			(c) => this.containers.get(c)!.index,
+		);
 	}
 
 	/**
@@ -375,7 +393,7 @@ export class LoaderContainerTracker implements IOpProcessingController {
 	 */
 	private async waitForPendingClients(pendingClients: [IContainer, Set<string>][]) {
 		const unconnectedClients = Array.from(this.containers.keys()).filter(
-			(c) => !c.closed && !(c as Container).connected,
+			(c) => !c.closed && c.connectionState !== ConnectionState.Connected,
 		);
 		return Promise.all(
 			pendingClients.map(async ([container, pendingClientId]) => {
@@ -435,6 +453,10 @@ export class LoaderContainerTracker implements IOpProcessingController {
 		const containersToApply = this.getContainers(containers);
 		for (const container of containersToApply) {
 			const record = this.containers.get(container);
+			assert(
+				record?.pauseP === undefined,
+				"Cannot resume container while pausing is in progress",
+			);
 			if (record?.paused === true) {
 				debugWait(`${record.index}: container resumed`);
 				container.deltaManager.inbound.resume();
@@ -449,26 +471,98 @@ export class LoaderContainerTracker implements IOpProcessingController {
 	/**
 	 * Pause all queue activities on the containers given, or all tracked containers
 	 * Any containers given that is not tracked will be ignored.
+	 *
+	 * When a container is paused, it is assumed that we want fine grain control over op
+	 * sequencing.  This function will prepare the container and force it into write mode to
+	 * avoid missing join messages or change the sequence of event when switching from read to
+	 * write mode.
 	 */
 	public async pauseProcessing(...containers: IContainer[]) {
-		const pauseP: Promise<void>[] = [];
+		const waitP: Promise<void>[] = [];
 		const containersToApply = this.getContainers(containers);
 		for (const container of containersToApply) {
 			const record = this.containers.get(container);
 			if (record !== undefined && !record.paused) {
-				debugWait(`${record.index}: container paused`);
-				pauseP.push(container.deltaManager.inbound.pause());
-				pauseP.push(container.deltaManager.outbound.pause());
-				record.paused = true;
+				if (record.pauseP === undefined) {
+					record.pauseP = this.pauseContainer(container, record);
+				}
+				waitP.push(record.pauseP);
 			}
 		}
-		await Promise.all(pauseP);
+		await Promise.all(waitP);
+	}
+
+	/**
+	 * When a container is paused, it is assumed that we want fine grain control over op
+	 * sequencing.  This function will prepare the container and force it into write mode to
+	 * avoid missing join messages or change the sequence of event when switching from read to
+	 * write mode.
+	 *
+	 * @param container - the container to pause
+	 * @param record - the record for the container
+	 */
+	private async pauseContainer(container: IContainer, record: ContainerRecord) {
+		debugWait(`${record.index}: pausing container`);
+		assert(!container.deltaManager.outbound.paused, "Container should not be paused yet");
+		assert(!container.deltaManager.inbound.paused, "Container should not be paused yet");
+
+		// Pause outbound
+		debugWait(`${record.index}: pausing container outbound queues`);
+		await container.deltaManager.outbound.pause();
+
+		// Ensure the container is connected first.
+		if (container.connectionState !== ConnectionState.Connected) {
+			debugWait(`${record.index}: Wait for container connection`);
+			await waitForContainerConnection(container);
+		}
+
+		// Check if the container is in write mode
+		if (!container.deltaManager.active) {
+			let proposalP: Promise<boolean> | undefined;
+			if (container.deltaManager.outbound.idle) {
+				// Need to generate an op to force write mode
+				debugWait(`${record.index}: container force write connection`);
+				const maybeContainer = container as Partial<IContainer>;
+				const codeProposal = maybeContainer.getLoadedCodeDetails
+					? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					  container.getLoadedCodeDetails()!
+					: (container as any).chaincodePackage;
+
+				proposalP = container.proposeCodeDetails(codeProposal);
+			}
+
+			// Wait for nack
+			debugWait(`${record.index}: Wait for container disconnect`);
+			container.deltaManager.outbound.resume();
+			await new Promise<void>((resolve) => container.once("disconnected", resolve));
+			const accepted = proposalP ? await proposalP : false;
+			assert(!accepted, "A proposal in read mode should be rejected");
+			await container.deltaManager.outbound.pause();
+
+			// Ensure the container is reconnect.
+			if (container.connectionState !== ConnectionState.Connected) {
+				debugWait(`${record.index}: Wait for container reconnection`);
+				await waitForContainerConnection(container);
+			}
+		}
+
+		debugWait(`${record.index}: pausing container inbound queues`);
+
+		// Pause inbound
+		await container.deltaManager.inbound.pause();
+
+		debugWait(`${record.index}: container paused`);
+
+		record.pauseP = undefined;
+		record.paused = true;
 	}
 
 	/**
 	 * Pause all queue activities on all tracked containers, and resume only
 	 * inbound to process ops until it is idle. All queues are left in the paused state
-	 * after the function
+	 * after the function.
+	 *
+	 * Pausing will switch the container to write mode. See `pauseProcessing`
 	 */
 	public async processIncoming(...containers: IContainer[]) {
 		return this.processQueue(containers, (container) => container.deltaManager.inbound);
@@ -477,7 +571,9 @@ export class LoaderContainerTracker implements IOpProcessingController {
 	/**
 	 * Pause all queue activities on all tracked containers, and resume only
 	 * outbound to process ops until it is idle. All queues are left in the paused state
-	 * after the function
+	 * after the function.
+	 *
+	 * Pausing will switch the container to write mode. See `pauseProcessing`
 	 */
 	public async processOutgoing(...containers: IContainer[]) {
 		return this.processQueue(containers, (container) => container.deltaManager.outbound);
@@ -494,9 +590,15 @@ export class LoaderContainerTracker implements IOpProcessingController {
 		const resumed: IDeltaQueue<U>[] = [];
 
 		const containersToApply = this.getContainers(containers);
+
 		const inflightTracker = new Map<IContainer, number>();
 		const cleanup: (() => void)[] = [];
 		for (const container of containersToApply) {
+			assert(
+				container.deltaManager.active,
+				"Container should be connected in write mode already",
+			);
+
 			const queue = getQueue(container);
 
 			// track the outgoing ops (if any) to make sure they make the round trip to at least to the same client
@@ -546,7 +648,7 @@ export class LoaderContainerTracker implements IOpProcessingController {
 		const inHandler = (message: ISequencedDocumentMessage) => {
 			if (
 				!canBeCoalescedByService(message) &&
-				message.clientId === (container as Container).clientId &&
+				message.clientId === container.clientId &&
 				inflightTracker.get(container) === message.clientSequenceNumber
 			) {
 				inflightTracker.delete(container);
@@ -600,7 +702,7 @@ export class LoaderContainerTracker implements IOpProcessingController {
 					return `${e.message}: ${e.stack}`;
 				}
 			};
-			debugOp(`${index}: ADD: clientId: ${(container as Container).clientId}`);
+			debugOp(`${index}: ADD: clientId: ${container.clientId}`);
 			container.deltaManager.outbound.on("op", (messages) => {
 				for (const msg of messages) {
 					debugOp(
@@ -614,7 +716,7 @@ export class LoaderContainerTracker implements IOpProcessingController {
 			const getInboundHandler = (type: string) => {
 				return (msg: ISequencedDocumentMessage) => {
 					const clientSeq =
-						msg.clientId === (container as Container).clientId
+						msg.clientId === container.clientId
 							? `cli: ${msg.clientSequenceNumber.toString().padStart(3)}`
 							: "        ";
 					debugOp(

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/test-utils";
-export const pkgVersion = "2.0.0-dev.3.1.0.125672";
+export const pkgVersion = "2.0.0-dev.4.2.0.153917";