@fluidframework/container-runtime 2.92.0 → 2.93.0

package/src/containerRuntime.ts

@@ -2069,6 +2069,24 @@ export class ContainerRuntime
 			}),
 			reSubmit: this.reSubmit.bind(this),
 			opReentrancy: () => this.dataModelChangeRunner.running,
+			generateIdAllocationOp: (): LocalBatchMessage | undefined => {
+				if (this._idCompressor === undefined) {
+					return undefined;
+				}
+				const idRange = this._idCompressor.takeNextCreationRange();
+				if (idRange.ids === undefined) {
+					return undefined;
+				}
+				const idAllocationMessage: ContainerRuntimeIdAllocationMessage = {
+					type: ContainerMessageType.IdAllocation,
+					contents: idRange,
+				};
+				return {
+					runtimeOp: idAllocationMessage,
+					referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+					staged: false,
+				};
+			},
 		});
 
 		this._quorum = quorum;
@@ -3687,9 +3705,6 @@ export class ContainerRuntime
 
 						this.stageControls = undefined;
 
-						// During Staging Mode, we avoid submitting any ID Allocation ops (apart from resubmitting pre-staging ops).
-						// Now that we've exited, we need to submit an ID Allocation op for any IDs that were generated while in Staging Mode.
-						this.submitIdAllocationOpIfNeeded({ staged: false });
 						const batchInfos = discardOrCommit();
 						event.reportProgress({
 							details: {
@@ -4272,7 +4287,6 @@ export class ContainerRuntime
 					outboxLength: this.outbox.messageCount,
 					mainBatchLength: this.outbox.mainBatchMessageCount,
 					blobAttachBatchLength: this.outbox.blobAttachBatchMessageCount,
-					idAllocationBatchLength: this.outbox.idAllocationBatchMessageCount,
 				},
 			);
 		}
@@ -4605,7 +4619,8 @@ export class ContainerRuntime
 
 	/**
 	 * This helper is called during summarization. If the container is dirty, it will return a failed summarize result
-	 * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
+	 * (IBaseSummarizeResult) unless this is the final summarize attempt, in which case the summary is allowed to
+	 * proceed to make progress in documents where there are consistently pending ops in the summarizer.
 	 * @param logger - The logger to be used for sending telemetry.
 	 * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
 	 * @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
@@ -4624,13 +4639,9 @@ export class ContainerRuntime
 			return;
 		}
 
-		// If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
-		// This is a fallback to make progress in documents where there are consistently pending ops in
-		// the summarizer.
-		if (
-			finalAttempt &&
-			this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary") === true
-		) {
+		// Don't fail the summary in the last attempt. This is a fallback to make progress in
+		// documents where there are consistently pending ops in the summarizer.
+		if (finalAttempt) {
 			const error = DataProcessingError.create(
 				"Pending ops during summarization",
 				"submitSummary",
@@ -4639,7 +4650,7 @@ export class ContainerRuntime
 			);
 			logger.sendErrorEvent(
 				{
-					eventName: "SkipFailingIncorrectSummary",
+					eventName: "PendingOpsDuringSummaryFinalAttempt",
 					referenceSequenceNumber,
 					minimumSequenceNumber,
 					beforeGenerate: beforeSummaryGeneration,
@@ -4733,33 +4744,6 @@ export class ContainerRuntime
 		return this.blobManager.lookupTemporaryBlobStorageId(localId);
 	}
 
-	private submitIdAllocationOpIfNeeded({
-		resubmitOutstandingRanges = false,
-		staged,
-	}: {
-		resubmitOutstandingRanges?: boolean;
-		staged: boolean;
-	}): void {
-		if (this._idCompressor) {
-			const idRange = resubmitOutstandingRanges
-				? this._idCompressor.takeUnfinalizedCreationRange()
-				: this._idCompressor.takeNextCreationRange();
-			// Don't include the idRange if there weren't any Ids allocated
-			if (idRange.ids !== undefined) {
-				const idAllocationMessage: ContainerRuntimeIdAllocationMessage = {
-					type: ContainerMessageType.IdAllocation,
-					contents: idRange,
-				};
-				const idAllocationBatchMessage: LocalBatchMessage = {
-					runtimeOp: idAllocationMessage,
-					referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
-					staged,
-				};
-				this.outbox.submitIdAllocation(idAllocationBatchMessage);
-			}
-		}
-	}
-
 	private submit(
 		containerRuntimeMessage: LocalContainerRuntimeMessage,
 		localOpMetadata: unknown = undefined,
@@ -4803,11 +4787,6 @@ export class ContainerRuntime
 				0xbba /* Unexpected message type submitted in Staging Mode */,
 			);
 
-			// Before submitting any non-staged change, submit the ID Allocation op to cover any compressed IDs included in the op.
-			if (!staged) {
-				this.submitIdAllocationOpIfNeeded({ staged: false });
-			}
-
 			// Allow document schema controller to send a message if it needs to propose change in document schema.
 			// If it needs to send a message, it will call provided callback with payload of such message and rely
 			// on this callback to do actual sending.
@@ -4867,7 +4846,7 @@ export class ContainerRuntime
 		// Incoming ops still break the batch via direct this.flush() calls elsewhere
 		// (deltaManager "op" handler, process(), connection changes, getPendingLocalState,
 		// exitStagingMode). Those all bypass scheduleFlush(), so they're unaffected by this check.
-		// Additionally, outbox.maybeFlushPartialBatch() (called on every submit) detects
+		// Additionally, outbox.outboxSequenceNumberCoherencyCheck() (called on every submit) detects
 		// sequence number changes and throws if unexpected changes are detected.
 		if (
 			this.inStagingMode &&

package/src/opLifecycle/batchManager.ts

@@ -20,17 +20,8 @@ import { serializeOp } from "./opSerialization.js";
 import type { BatchStartInfo } from "./remoteMessageProcessor.js";
 
 export interface IBatchManagerOptions {
+	readonly disableGroupedBatching: boolean;
 	readonly compressionOptions?: ICompressionRuntimeOptions;
-
-	/**
-	 * If true, the outbox is allowed to rebase the batch during flushing.
-	 */
-	readonly canRebase: boolean;
-
-	/**
-	 * If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored
-	 */
-	readonly ignoreBatchId?: boolean;
 }
 
 export interface BatchSequenceNumbers {
@@ -127,8 +118,9 @@ export class BatchManager {
 
 	/**
 	 * Gets the pending batch and clears state for the next batch.
+	 * The caller is responsible for calling {@link addBatchMetadata} after any modifications (e.g. prepending messages).
 	 */
-	public popBatch(batchId?: BatchId): LocalBatch {
+	public popBatch(): LocalBatch {
 		assert(this.pendingBatch[0] !== undefined, 0xb8a /* expected non-empty batch */);
 		const batch: LocalBatch = {
 			messages: this.pendingBatch,
@@ -141,7 +133,7 @@ export class BatchManager {
 		this.clientSequenceNumber = undefined;
 		this.hasReentrantOps = false;
 
-		return addBatchMetadata(batch, batchId);
+		return batch;
 	}
 
 	/**

package/src/opLifecycle/outbox.ts

@@ -28,6 +28,7 @@ import {
 	type BatchSequenceNumbers,
 	sequenceNumbersMatch,
 	type BatchId,
+	addBatchMetadata,
 } from "./batchManager.js";
 import type {
 	LocalBatchMessage,
@@ -65,6 +66,13 @@ export interface IOutboxParameters {
 	readonly getCurrentSequenceNumbers: () => BatchSequenceNumbers;
 	readonly reSubmit: (message: PendingMessageResubmitData, squash: boolean) => void;
 	readonly opReentrancy: () => boolean;
+	/**
+	 * JIT callback to generate an ID allocation op at flush time.
+	 * Called after rebase (if any), so the returned message has the correct refSeq.
+	 *
+	 * @returns A LocalBatchMessage for the ID allocation op, or undefined if no IDs need allocating.
+	 */
+	readonly generateIdAllocationOp: () => LocalBatchMessage | undefined;
 }
 
 /**
@@ -189,7 +197,6 @@ export class Outbox {
 	private readonly logger: ITelemetryLoggerExt;
 	private readonly mainBatch: BatchManager;
 	private readonly blobAttachBatch: BatchManager;
-	private readonly idAllocationBatch: BatchManager;
 	private batchRebasesToReport = 5;
 	private rebasing = false;
 
@@ -205,16 +212,12 @@ export class Outbox {
 	constructor(private readonly params: IOutboxParameters) {
 		this.logger = createChildLogger({ logger: params.logger, namespace: "Outbox" });
 
-		this.mainBatch = new BatchManager({ canRebase: true });
-		this.blobAttachBatch = new BatchManager({ canRebase: true });
-		this.idAllocationBatch = new BatchManager({
-			canRebase: false,
-			ignoreBatchId: true,
-		});
+		this.mainBatch = new BatchManager({ disableGroupedBatching: false });
+		this.blobAttachBatch = new BatchManager({ disableGroupedBatching: true });
 	}
 
 	public get messageCount(): number {
-		return this.mainBatch.length + this.blobAttachBatch.length + this.idAllocationBatch.length;
+		return this.mainBatch.length + this.blobAttachBatch.length;
 	}
 
 	public get mainBatchMessageCount(): number {
@@ -225,19 +228,12 @@ export class Outbox {
 		return this.blobAttachBatch.length;
 	}
 
-	public get idAllocationBatchMessageCount(): number {
-		return this.idAllocationBatch.length;
-	}
-
 	public get isEmpty(): boolean {
 		return this.messageCount === 0;
 	}
 
 	public containsUserChanges(): boolean {
-		return (
-			this.mainBatch.containsUserChanges() || this.blobAttachBatch.containsUserChanges()
-			// ID Allocation ops are not user changes
-		);
+		return this.mainBatch.containsUserChanges() || this.blobAttachBatch.containsUserChanges();
 	}
 
 	/**
@@ -251,13 +247,11 @@ export class Outbox {
 	 * last message processed by the ContainerRuntime. In the absence of op reentrancy, this
 	 * pair will remain stable during a single JS turn during which the batch is being built up.
 	 */
-	private maybeFlushPartialBatch(): void {
+	private outboxSequenceNumberCoherencyCheck(): void {
 		const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
 		const blobAttachSeqNums = this.blobAttachBatch.sequenceNumbers;
-		const idAllocSeqNums = this.idAllocationBatch.sequenceNumbers;
 		assert(
-			sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums) &&
-				sequenceNumbersMatch(mainBatchSeqNums, idAllocSeqNums),
+			sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums),
 			0x58d /* Reference sequence numbers from both batches must be in sync */,
 		);
 
@@ -265,8 +259,7 @@ export class Outbox {
 
 		if (
 			sequenceNumbersMatch(mainBatchSeqNums, currentSequenceNumbers) &&
-			sequenceNumbersMatch(blobAttachSeqNums, currentSequenceNumbers) &&
-			sequenceNumbersMatch(idAllocSeqNums, currentSequenceNumbers)
+			sequenceNumbersMatch(blobAttachSeqNums, currentSequenceNumbers)
 		) {
 			// The reference sequence numbers are stable, there is nothing to do
 			return;
@@ -314,23 +307,17 @@ export class Outbox {
 	}
 
 	public submit(message: LocalBatchMessage): void {
-		this.maybeFlushPartialBatch();
+		this.outboxSequenceNumberCoherencyCheck();
 
 		this.addMessageToBatchManager(this.mainBatch, message);
 	}
 
 	public submitBlobAttach(message: LocalBatchMessage): void {
-		this.maybeFlushPartialBatch();
+		this.outboxSequenceNumberCoherencyCheck();
 
 		this.addMessageToBatchManager(this.blobAttachBatch, message);
 	}
 
-	public submitIdAllocation(message: LocalBatchMessage): void {
-		this.maybeFlushPartialBatch();
-
-		this.addMessageToBatchManager(this.idAllocationBatch, message);
-	}
-
 	private addMessageToBatchManager(
 		batchManager: BatchManager,
 		message: LocalBatchMessage,
@@ -352,11 +339,12 @@ export class Outbox {
 	public flush(resubmitInfo?: BatchResubmitInfo): void {
 		// We have nothing to flush if all batchManagers are empty, and we we're not needing to resubmit an empty batch placeholder
 		if (
-			this.idAllocationBatch.empty &&
 			this.blobAttachBatch.empty &&
 			this.mainBatch.empty &&
 			resubmitInfo?.batchId === undefined
 		) {
+			// Note that it's possible that there are unfinalized ranges in the ID Compressor,
+			// but there's no urgency to flush those if they're not referenced in any messages.
 			return;
 		}
 
@@ -368,8 +356,7 @@ export class Outbox {
 	}
 
 	private flushAll(resubmitInfo?: BatchResubmitInfo): void {
-		const allBatchesEmpty =
-			this.idAllocationBatch.empty && this.blobAttachBatch.empty && this.mainBatch.empty;
+		const allBatchesEmpty = this.blobAttachBatch.empty && this.mainBatch.empty;
 		if (allBatchesEmpty) {
 			// If we're resubmitting with a batchId and all batches are empty, we need to flush an empty batch.
 			// Note that we currently resubmit one batch at a time, so on resubmit, 1 of the 2 batches will *always* be empty.
@@ -382,22 +369,8 @@ export class Outbox {
 			return;
 		}
 
-		// Don't use resubmittingBatchId for idAllocationBatch.
-		// ID Allocation messages are not directly resubmitted so don't pass the resubmitInfo
-		this.flushInternal({
-			batchManager: this.idAllocationBatch,
-			// Note: For now, we will never stage ID Allocation messages.
-			// They won't contain personal info and no harm in extra allocations in case of discarding the staged changes
-		});
-		this.flushInternal({
-			batchManager: this.blobAttachBatch,
-			disableGroupedBatching: true,
-			resubmitInfo,
-		});
-		this.flushInternal({
-			batchManager: this.mainBatch,
-			resubmitInfo,
-		});
+		this.flushInternal(this.blobAttachBatch, resubmitInfo);
+		this.flushInternal(this.mainBatch, resubmitInfo);
 	}
 
 	private flushEmptyBatch(
@@ -429,17 +402,15 @@ export class Outbox {
 		return;
 	}
 
-	private flushInternal(params: {
-		batchManager: BatchManager;
-		disableGroupedBatching?: boolean;
-		resubmitInfo?: BatchResubmitInfo; // undefined if not resubmitting
-	}): void {
-		const { batchManager, disableGroupedBatching = false, resubmitInfo } = params;
+	private flushInternal(
+		batchManager: BatchManager,
+		resubmitInfo?: BatchResubmitInfo, // undefined if not resubmitting
+	): void {
 		if (batchManager.empty) {
 			return;
 		}
 
-		const rawBatch = batchManager.popBatch(resubmitInfo?.batchId);
+		let rawBatch = batchManager.popBatch();
 
 		// On resubmit we use the original batch's staged state, so these should match as well.
 		const staged = rawBatch.staged === true;
@@ -449,13 +420,15 @@ export class Outbox {
 		);
 
 		const groupingEnabled =
-			!disableGroupedBatching && this.params.groupingManager.groupedBatchingEnabled();
-		if (
-			batchManager.options.canRebase &&
-			rawBatch.hasReentrantOps === true &&
-			groupingEnabled
-		) {
+			!batchManager.options.disableGroupedBatching &&
+			this.params.groupingManager.groupedBatchingEnabled();
+		if (rawBatch.hasReentrantOps === true) {
+			assert(
+				resubmitInfo === undefined,
+				0xcf2 /* Re-submitting a batch with reentrant ops is not supported */,
+			);
 			assert(!this.rebasing, 0x6fa /* A rebased batch should never have reentrant ops */);
+			// Rebase the current batch (resubmit the ops one-by-one) and then reinvoke flushInternal.
 			// If a batch contains reentrant ops (ops created as a result from processing another op)
 			// it needs to be rebased so that we can ensure consistent reference sequence numbers
 			// and eventual consistency at the DDS level.
@@ -466,11 +439,22 @@ export class Outbox {
 			return;
 		}
 
-		let clientSequenceNumber: number | undefined;
 		// Did we disconnect? (i.e. is shouldSend false?)
 		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
 		// Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
-		if (this.params.shouldSend() && !staged) {
+		const shouldSendNow = this.params.shouldSend() && !staged;
+		let clientSequenceNumber: number | undefined;
+		if (shouldSendNow) {
+			// Generate ID Allocation op just-in-time, after rebase (if any), and before addBatchMetadata,
+			// so that the prepended idAllocMsg is correctly marked as the first op in the batch.
+			// This ensures the refSeq is correct (matching the rest of the batch) and that
+			// ID ranges aren't lost during rebase (since reSubmit drops IdAllocation ops).
+			// Only generate for non-staged batches — ID alloc ops are always non-staged.
+			const idAllocMsg = this.params.generateIdAllocationOp();
+			if (idAllocMsg !== undefined) {
+				rawBatch = { ...rawBatch, messages: [idAllocMsg, ...rawBatch.messages] };
+			}
+			addBatchMetadata(rawBatch, resubmitInfo?.batchId);
 			const virtualizedBatch = this.virtualizeBatch(rawBatch, groupingEnabled);
 
 			clientSequenceNumber = this.sendBatch(virtualizedBatch);
@@ -478,13 +462,14 @@ export class Outbox {
 				clientSequenceNumber === undefined || clientSequenceNumber >= 0,
 				0x9d2 /* unexpected negative clientSequenceNumber (empty batch should yield undefined) */,
 			);
+		} else {
+			addBatchMetadata(rawBatch, resubmitInfo?.batchId);
 		}
 
 		this.params.pendingStateManager.onFlushBatch(
 			rawBatch.messages,
 			clientSequenceNumber,
 			staged,
-			batchManager.options.ignoreBatchId,
 		);
 	}
 
@@ -496,7 +481,6 @@ export class Outbox {
 	 */
 	private rebase(rawBatch: LocalBatch, batchManager: BatchManager): void {
 		assert(!this.rebasing, 0x6fb /* Reentrancy */);
-		assert(batchManager.options.canRebase, 0x9a7 /* BatchManager does not support rebase */);
 
 		this.rebasing = true;
 		const squash = false;
@@ -523,7 +507,7 @@ export class Outbox {
 			this.batchRebasesToReport--;
 		}
 
-		this.flushInternal({ batchManager });
+		this.flushInternal(batchManager);
 		this.rebasing = false;
 	}
 
@@ -664,7 +648,6 @@ export class Outbox {
 	 */
 	public getBatchCheckpoints(): {
 		mainBatch: IBatchCheckpoint;
-		idAllocationBatch: IBatchCheckpoint;
 		blobAttachBatch: IBatchCheckpoint;
 	} {
 		// This variable is declared with a specific type so that we have a standard import of the IBatchCheckpoint type.
@@ -672,7 +655,6 @@ export class Outbox {
 		const mainBatch: IBatchCheckpoint = this.mainBatch.checkpoint();
 		return {
 			mainBatch,
-			idAllocationBatch: this.idAllocationBatch.checkpoint(),
 			blobAttachBatch: this.blobAttachBatch.checkpoint(),
 		};
 	}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.92.0";
+export const pkgVersion = "2.93.0";

package/src/pendingStateManager.ts

@@ -84,10 +84,6 @@ export interface IPendingMessage {
 		 * length of the batch (how many runtime messages here)
 		 */
 		length: number;
-		/**
-		 * If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored
-		 */
-		ignoreBatchId?: boolean;
 		/**
 		 * If true, this batch is staged and should not actually be submitted on replayPendingStates.
 		 */
@@ -407,13 +403,11 @@ export class PendingStateManager implements IDisposable {
 	 * @param clientSequenceNumber - The CSN of the first message in the batch,
 	 * or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
 	 * @param staged - Indicates whether batch is staged (not to be submitted while runtime is in Staging Mode)
-	 * @param ignoreBatchId - Whether to ignore the batchId in the batchStartInfo
 	 */
 	public onFlushBatch(
 		batch: LocalBatchMessage[] | [LocalEmptyBatchPlaceholder],
 		clientSequenceNumber: number | undefined,
 		staged: boolean,
-		ignoreBatchId?: boolean,
 	): void {
 		// clientId and batchStartCsn are used for generating the batchId so we can detect container forks
 		// where this batch was submitted by two different clients rehydrating from the same local state.
@@ -431,7 +425,7 @@ export class PendingStateManager implements IDisposable {
 			clientId !== undefined,
 			0xa33 /* clientId (from stateHandler) could only be undefined if we've never connected, but we have a CSN so we know that's not the case */,
 		);
-		const batchInfo = { clientId, batchStartCsn, length: batch.length, ignoreBatchId, staged };
+		const batchInfo = { clientId, batchStartCsn, length: batch.length, staged };
 		for (const message of batch) {
 			const {
 				runtimeOp,
@@ -512,23 +506,14 @@ export class PendingStateManager implements IDisposable {
 	 * @returns whether the batch IDs match
 	 */
 	private remoteBatchMatchesPendingBatch(remoteBatchStart: BatchStartInfo): boolean {
-		// Find the first pending message that uses Batch ID, to compare to the incoming remote batch.
-		// If there is no such message, then the incoming remote batch doesn't have a match here and we can return.
-		const firstIndexUsingBatchId = Array.from({
-			length: this.pendingMessages.length,
-		}).findIndex((_, i) => this.pendingMessages.get(i)?.batchInfo.ignoreBatchId !== true);
-		const pendingMessageUsingBatchId =
-			firstIndexUsingBatchId === -1
-				? undefined
-				: this.pendingMessages.get(firstIndexUsingBatchId);
-
-		if (pendingMessageUsingBatchId === undefined) {
+		const pendingMessage = this.pendingMessages.peekFront();
+		if (pendingMessage === undefined) {
 			return false;
 		}
 
 		// We must compare the effective batch IDs, since one of these ops
 		// may have been the original, not resubmitted, so wouldn't have its batch ID stamped yet.
-		const pendingBatchId = getEffectiveBatchId(pendingMessageUsingBatchId);
+		const pendingBatchId = getEffectiveBatchId(pendingMessage);
 		const inboundBatchId = getEffectiveBatchId(remoteBatchStart);
 
 		return pendingBatchId === inboundBatchId;
@@ -810,10 +795,7 @@ export class PendingStateManager implements IDisposable {
 			assert(batchMetadataFlag !== false, 0x41b /* We cannot process batches in chunks */);
 
 			// The next message starts a batch (possibly single-message), and we'll need its batchId.
-			const batchId =
-				pendingMessage.batchInfo.ignoreBatchId === true
-					? undefined
-					: getEffectiveBatchId(pendingMessage);
+			const batchId = getEffectiveBatchId(pendingMessage);
 
 			const staged = pendingMessage.batchInfo.staged;