@fluidframework/container-runtime 2.32.0 → 2.33.0

package/src/pendingStateManager.ts

@@ -27,6 +27,8 @@ import {
 	getEffectiveBatchId,
 	BatchStartInfo,
 	InboundMessageResult,
+	serializeOp,
+	type LocalEmptyBatchPlaceholder,
 } from "./opLifecycle/index.js";
 
 /**
@@ -38,7 +40,15 @@ import {
 export interface IPendingMessage {
 	type: "message";
 	referenceSequenceNumber: number;
+	/**
+	 * Serialized copy of runtimeOp
+	 */
 	content: string;
+	/**
+	 * The original runtime op that was submitted to the ContainerRuntime
+	 * Unless this pending message came from stashed content, in which case this was roundtripped through string
+	 */
+	runtimeOp?: LocalContainerRuntimeMessage | undefined; // Undefined for empty batches and initial messages before parsing
 	localOpMetadata: unknown;
 	opMetadata: Record<string, unknown> | undefined;
 	sequenceNumber?: number;
@@ -65,6 +75,10 @@ export interface IPendingMessage {
 		 * If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored
 		 */
 		ignoreBatchId?: boolean;
+		/**
+		 * If true, this batch is staged and should not actually be submitted on replayPendingStates.
+		 */
+		staged: boolean;
 	};
 }
 
@@ -94,14 +108,17 @@ export interface IPendingLocalState {
  */
 export type PendingMessageResubmitData = Pick<
 	IPendingMessage,
-	"content" | "localOpMetadata" | "opMetadata"
->;
+	"runtimeOp" | "localOpMetadata" | "opMetadata"
+> & {
+	// Required (it's only missing on IPendingMessage for empty batch, which will be resubmitted as an empty array)
+	runtimeOp: LocalContainerRuntimeMessage;
+};
 
 export interface IRuntimeStateHandler {
 	connected(): boolean;
 	clientId(): string | undefined;
-	applyStashedOp(content: string): Promise<unknown>;
-	reSubmitBatch(batch: PendingMessageResubmitData[], batchId: BatchId): void;
+	applyStashedOp(serializedOp: string): Promise<unknown>;
+	reSubmitBatch(batch: PendingMessageResubmitData[], batchId: BatchId, staged: boolean): void;
 	isActiveConnection: () => boolean;
 	isAttached: () => boolean;
 }
@@ -180,10 +197,17 @@ export function findFirstCharacterMismatched(
 	return [index, charA, charB];
 }
 
-function withoutLocalOpMetadata(message: IPendingMessage): IPendingMessage {
+/**
+ * Returns a shallow copy of the given message with the non-serializable properties removed.
+ * Note that the runtimeOp's data has already been serialized in the content property.
+ */
+function toSerializableForm(
+	message: IPendingMessage,
+): IPendingMessage & { runtimeOp: undefined; localOpMetadata: undefined } {
 	return {
 		...message,
 		localOpMetadata: undefined,
+		runtimeOp: undefined,
 	};
 }
 
@@ -273,7 +297,7 @@ export class PendingStateManager implements IDisposable {
 		return {
 			pendingStates: [
 				...newSavedOps,
-				...this.pendingMessages.toArray().map((message) => withoutLocalOpMetadata(message)),
+				...this.pendingMessages.toArray().map((message) => toSerializableForm(message)),
 			],
 		};
 	}
@@ -296,17 +320,36 @@ export class PendingStateManager implements IDisposable {
 	}
 	public readonly dispose = (): void => this.disposeOnce.value;
 
+	/**
+	 * We've flushed an empty batch, and need to track it locally until the corresponding
+	 * ack is processed, to properly track batch IDs
+	 */
+	public onFlushEmptyBatch(
+		placeholder: LocalEmptyBatchPlaceholder,
+		clientSequenceNumber: number | undefined,
+		staged: boolean,
+	): void {
+		// We have to cast because runtimeOp doesn't apply for empty batches and is missing on LocalEmptyBatchPlaceholder
+		this.onFlushBatch(
+			[placeholder satisfies Omit<LocalBatchMessage, "runtimeOp"> as LocalBatchMessage],
+			clientSequenceNumber,
+			staged,
+		);
+	}
+
 	/**
 	 * The given batch has been flushed, and needs to be tracked locally until the corresponding
 	 * acks are processed, to ensure it is successfully sent.
 	 * @param batch - The batch that was flushed
 	 * @param clientSequenceNumber - The CSN of the first message in the batch,
 	 * or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
+	 * @param staged - Indicates whether batch is staged (not to be submitted while runtime is in Staging Mode)
 	 * @param ignoreBatchId - Whether to ignore the batchId in the batchStartInfo
 	 */
 	public onFlushBatch(
 		batch: LocalBatchMessage[],
 		clientSequenceNumber: number | undefined,
+		staged: boolean,
 		ignoreBatchId?: boolean,
 	): void {
 		// clientId and batchStartCsn are used for generating the batchId so we can detect container forks
@@ -315,6 +358,9 @@ export class PendingStateManager implements IDisposable {
 		// In the case where the batch was not sent, use a random uuid for clientId, and -1 for clientSequenceNumber to indicate this case.
 		// This will guarantee uniqueness of the batchId, and is a suitable fallback since clientId/CSN is only needed if the batch was actually sent/sequenced.
 		const batchWasSent = clientSequenceNumber !== undefined;
+		if (batchWasSent) {
+			assert(!staged, 0xb84 /* Staged batches should not have been submitted */);
+		}
 		const [clientId, batchStartCsn] = batchWasSent
 			? [this.stateHandler.clientId(), clientSequenceNumber]
 			: [uuid(), -1]; // -1 will indicate not a real clientId/CSN pair
@@ -325,7 +371,7 @@ export class PendingStateManager implements IDisposable {
 
 		for (const message of batch) {
 			const {
-				serializedOp: content,
+				runtimeOp,
 				referenceSequenceNumber,
 				localOpMetadata,
 				metadata: opMetadata,
@@ -333,11 +379,12 @@ export class PendingStateManager implements IDisposable {
 			const pendingMessage: IPendingMessage = {
 				type: "message",
 				referenceSequenceNumber,
-				content,
+				content: serializeOp(runtimeOp),
+				runtimeOp,
 				localOpMetadata,
 				opMetadata,
 				// Note: We only will read this off the first message, but put it on all for simplicity
-				batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId },
+				batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId, staged },
 			};
 			this.pendingMessages.push(pendingMessage);
 		}
@@ -375,6 +422,11 @@ export class PendingStateManager implements IDisposable {
 				const localOpMetadata = await this.stateHandler.applyStashedOp(nextMessage.content);
 				if (this.stateHandler.isAttached()) {
 					nextMessage.localOpMetadata = localOpMetadata;
+					// NOTE: This runtimeOp has been roundtripped through string, which is technically lossy.
+					// e.g. At this point, handles are in their encoded form.
+					nextMessage.runtimeOp = JSON.parse(
+						nextMessage.content,
+					) as LocalContainerRuntimeMessage;
 					// then we push onto pendingMessages which will cause PendingStateManager to resubmit when we connect
 					patchbatchInfo(nextMessage); // Back compat
 					this.pendingMessages.push(nextMessage);
@@ -508,7 +560,7 @@ export class PendingStateManager implements IDisposable {
 		);
 
 		pendingMessage.sequenceNumber = sequenceNumber;
-		this.savedOps.push(withoutLocalOpMetadata(pendingMessage));
+		this.savedOps.push(toSerializableForm(pendingMessage));
 
 		this.pendingMessages.shift();
 
@@ -579,6 +631,7 @@ export class PendingStateManager implements IDisposable {
 			pendingMessage !== undefined,
 			0xa21 /* No pending message found as we start processing this remote batch */,
 		);
+		assert(!pendingMessage.batchInfo.staged, 0xb85 /* Can't get an ack from a staged batch */);
 
 		// If this batch became empty on resubmit, batch.messages will be empty (but keyMessage is always set)
 		// and the next pending message should be an empty batch marker.
@@ -630,18 +683,22 @@ export class PendingStateManager implements IDisposable {
 	 * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
 	 * states in its queue. This includes triggering resubmission of unacked ops.
 	 * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
+	 * @param onlyStagedBatches - If true, only replay staged batches. This is used when we are exiting staging mode and want to rebase and submit the staged batches.
 	 */
-	public replayPendingStates(): void {
+	public replayPendingStates(onlyStagedBatches?: boolean): void {
 		assert(
 			this.stateHandler.connected(),
 			0x172 /* "The connection state is not consistent with the runtime" */,
 		);
 
-		// This assert suggests we are about to send same ops twice, which will result in data loss.
-		assert(
-			this.clientIdFromLastReplay !== this.stateHandler.clientId(),
-			0x173 /* "replayPendingStates called twice for same clientId!" */,
-		);
+		// Staged batches have not yet been submitted so check doesn't apply
+		if (!onlyStagedBatches) {
+			// This assert suggests we are about to send same ops twice, which will result in data loss.
+			assert(
+				this.clientIdFromLastReplay !== this.stateHandler.clientId(),
+				0x173 /* "replayPendingStates called twice for same clientId!" */,
+			);
+		}
 		this.clientIdFromLastReplay = this.stateHandler.clientId();
 
 		assert(
@@ -652,6 +709,8 @@ export class PendingStateManager implements IDisposable {
 		const initialPendingMessagesCount = this.pendingMessages.length;
 		let remainingPendingMessagesCount = this.pendingMessages.length;
 
+		let seenStagedBatch = false;
+
 		// Process exactly `pendingMessagesCount` items in the queue as it represents the number of messages that were
 		// pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
 		// which must not be replayed.
@@ -660,18 +719,37 @@ export class PendingStateManager implements IDisposable {
 			let pendingMessage = this.pendingMessages.shift()!;
 			remainingPendingMessagesCount--;
 
+			// Re-queue pre-staging messages if we are only processing staged batches
+			if (onlyStagedBatches) {
+				if (!pendingMessage.batchInfo.staged) {
+					assert(!seenStagedBatch, 0xb86 /* Staged batch was followed by non-staged batch */);
+					this.pendingMessages.push(pendingMessage);
+					continue;
+				}
+
+				seenStagedBatch = true;
+				pendingMessage.batchInfo.staged = false; // Clear staged flag so we can submit
+			}
+
 			const batchMetadataFlag = asBatchMetadata(pendingMessage.opMetadata)?.batch;
 			assert(batchMetadataFlag !== false, 0x41b /* We cannot process batches in chunks */);
 
 			// The next message starts a batch (possibly single-message), and we'll need its batchId.
 			const batchId = getEffectiveBatchId(pendingMessage);
 
+			const staged = pendingMessage.batchInfo.staged;
+
 			if (asEmptyBatchLocalOpMetadata(pendingMessage.localOpMetadata)?.emptyBatch === true) {
 				// Resubmit no messages, with the batchId. Will result in another empty batch marker.
-				this.stateHandler.reSubmitBatch([], batchId);
+				this.stateHandler.reSubmitBatch([], batchId, staged);
 				continue;
 			}
 
+			assert(
+				pendingMessage.runtimeOp !== undefined,
+				0xb87 /* viableOp is only undefined for empty batches */,
+			);
+
 			/**
 			 * We must preserve the distinct batches on resubmit.
 			 * Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will
@@ -683,12 +761,13 @@ export class PendingStateManager implements IDisposable {
 				this.stateHandler.reSubmitBatch(
 					[
 						{
-							content: pendingMessage.content,
+							runtimeOp: pendingMessage.runtimeOp,
 							localOpMetadata: pendingMessage.localOpMetadata,
 							opMetadata: pendingMessage.opMetadata,
 						},
 					],
 					batchId,
+					staged,
 				);
 				continue;
 			}
@@ -703,12 +782,17 @@ export class PendingStateManager implements IDisposable {
 
 			// check is >= because batch end may be last pending message
 			while (remainingPendingMessagesCount >= 0) {
+				assert(
+					pendingMessage.runtimeOp !== undefined,
+					0xb88 /* viableOp is only undefined for empty batches */,
+				);
 				batch.push({
-					content: pendingMessage.content,
+					runtimeOp: pendingMessage.runtimeOp,
 					localOpMetadata: pendingMessage.localOpMetadata,
 					opMetadata: pendingMessage.opMetadata,
 				});
 
+				// End of the batch
 				if (pendingMessage.opMetadata?.batch === false) {
 					break;
 				}
@@ -723,7 +807,7 @@ export class PendingStateManager implements IDisposable {
 				);
 			}
 
-			this.stateHandler.reSubmitBatch(batch, batchId);
+			this.stateHandler.reSubmitBatch(batch, batchId, staged);
 		}
 
 		// pending ops should no longer depend on previous sequenced local ops after resubmit
@@ -740,6 +824,36 @@ export class PendingStateManager implements IDisposable {
 			});
 		}
 	}
+
+	/**
+	 * Clears the 'staged' flag off all pending messages.
+	 */
+	public clearStagingFlags(): void {
+		for (const message of this.pendingMessages.toArray()) {
+			if (message.batchInfo.staged) {
+				message.batchInfo.staged = false;
+			}
+		}
+	}
+
+	/**
+	 * Pops all staged batches, invoking the callback on each one in order (LIFO)
+	 */
+	public popStagedBatches(callback: (stagedMessage: IPendingMessage) => void): void {
+		while (!this.pendingMessages.isEmpty()) {
+			const stagedMessage = this.pendingMessages.peekBack();
+			if (stagedMessage?.batchInfo.staged === true) {
+				callback(stagedMessage);
+				this.pendingMessages.pop();
+			} else {
+				break; // no more staged messages
+			}
+		}
+		assert(
+			this.pendingMessages.toArray().every((m) => m.batchInfo.staged !== true),
+			0xb89 /* Shouldn't be any more staged messages */,
+		);
+	}
 }
 
 /**
@@ -751,6 +865,6 @@ function patchbatchInfo(
 	const batchInfo: IPendingMessageFromStash["batchInfo"] = message.batchInfo;
 	if (batchInfo === undefined) {
 		// Using uuid guarantees uniqueness, retaining existing behavior
-		message.batchInfo = { clientId: uuid(), batchStartCsn: -1, length: -1 };
+		message.batchInfo = { clientId: uuid(), batchStartCsn: -1, length: -1, staged: false };
 	}
 }

package/src/runtimeLayerCompatState.ts

@@ -9,7 +9,10 @@ import {
 	type ILayerCompatSupportRequirements,
 } from "@fluid-internal/client-utils";
 import type { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { encodeHandlesInContainerRuntime } from "@fluidframework/runtime-definitions/internal";
+import {
+	encodeHandlesInContainerRuntime,
+	notifiesReadOnlyState,
+} from "@fluidframework/runtime-definitions/internal";
 import { UsageError } from "@fluidframework/telemetry-utils/internal";
 
 import { pkgVersion } from "./packageVersion.js";
@@ -66,7 +69,7 @@ export const runtimeCompatDetailsForDataStore: ILayerCompatDetails = {
 	/**
 	 * The features supported by the Runtime layer across the Runtime / DataStore boundary.
 	 */
-	supportedFeatures: new Set<string>([encodeHandlesInContainerRuntime]),
+	supportedFeatures: new Set<string>([encodeHandlesInContainerRuntime, notifiesReadOnlyState]),
 };
 
 /**

package/src/summary/documentSchema.ts

@@ -96,6 +96,7 @@ export interface IDocumentSchemaFeatures {
 	compressionLz4: boolean;
 	idCompressorMode: IdCompressorMode;
 	opGroupingEnabled: boolean;
+	createBlobPayloadPending: true | undefined;
 
 	/**
 	 * List of disallowed versions of the runtime.
@@ -227,6 +228,7 @@ const documentSchemaSupportedConfigs = {
 	idCompressorMode: new IdCompressorProperty(["delayed", "on"]),
 	opGroupingEnabled: new TrueOrUndefined(),
 	compressionLz4: new TrueOrUndefined(),
+	createBlobPayloadPending: new TrueOrUndefined(),
 	disallowedVersions: new CheckVersions(),
 };
 
@@ -482,6 +484,7 @@ export class DocumentsSchemaController {
 				compressionLz4: boolToProp(features.compressionLz4),
 				idCompressorMode: features.idCompressorMode,
 				opGroupingEnabled: boolToProp(features.opGroupingEnabled),
+				createBlobPayloadPending: features.createBlobPayloadPending,
 				disallowedVersions: arrayToProp(features.disallowedVersions),
 			},
 		};