@fluidframework/container-runtime 2.0.0-dev-rc.3.0.0.254674 → 2.0.0-dev-rc.4.0.0.261659

package/src/gc/gcTelemetry.ts

@@ -5,15 +5,14 @@
 
 import { IGarbageCollectionData } from "@fluidframework/runtime-definitions";
 import {
-	type ITelemetryGenericEventExt,
 	ITelemetryLoggerExt,
-} from "@fluidframework/telemetry-utils";
-import {
 	MonitoringContext,
 	generateStack,
 	tagCodeArtifacts,
+	type ITelemetryGenericEventExt,
 } from "@fluidframework/telemetry-utils/internal";
 
+import type { Tagged } from "@fluidframework/core-interfaces";
 import { RuntimeHeaderData } from "../containerRuntime.js";
 import { ICreateContainerMetadata } from "../summary/index.js";
 
@@ -43,11 +42,12 @@ interface ICommonProps {
 
 /** The event that is logged when unreferenced node is used after a certain time. */
 interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps {
+	/** The id that GC uses to track the node. May or may not match id */
+	trackedId: string;
 	state: UnreferencedState;
-	id: {
-		value: string;
-		tag: string;
-	};
+	/** The full path (in GC Path format) to the node in question */
+	id: Tagged<string>;
+	fromId?: Tagged<string>;
 	type: GCNodeType;
 	unrefTime: number;
 	age: number;
@@ -56,19 +56,24 @@ interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps
 		[K in keyof GCFeatureMatrix]: GCFeatureMatrix[K];
 	};
 	timeout?: number;
-	fromId?: {
-		value: string;
-		tag: string;
-	};
 }
 
 /** Properties passed to nodeUsed function when a node is used. */
 interface INodeUsageProps extends ICommonProps {
+	/** The full path (in GC Path format) to the node in question */
 	id: string;
+	/** Latest timestamp received from the server, as a baseline for computing GC state/age */
 	currentReferenceTimestampMs: number | undefined;
+	/** The package path of the node. This may not be available if the node hasn't been loaded yet */
 	packagePath: readonly string[] | undefined;
+	/** In case of Revived - what node added the reference? */
 	fromId?: string;
+	/** In case of Revived - was it revived due to autorecovery? */
 	autorecovery?: true;
+	/** URL (including query string) if this usage came from a request */
+	requestUrl?: string;
+	/** Original request headers if this usage came from a request or handle.get */
+	requestHeaders?: string;
 }
 
 /**
@@ -143,18 +148,34 @@ export class GCTelemetryTracker {
 	}
 
 	/**
-	 * Called when a node is used. If the node is not active or tombstoned, log telemetry indicating object is used
+	 * Called when a node is used. If the node is inactive or tombstoned, log telemetry indicating object is used
 	 * when it should not have been.
+	 * @param trackedId - The id that GC uses to track the node. For SubDataStore nodes, this should be the DataStore ID.
+	 * @param INodeUsageProps - All kind of details about this event to be logged
 	 */
-	public nodeUsed(nodeUsageProps: INodeUsageProps) {
+	public nodeUsed(
+		trackedId: string,
+		{
+			usageType,
+			currentReferenceTimestampMs,
+			packagePath,
+			id: untaggedId,
+			fromId: untaggedFromId,
+			isTombstoned,
+			...otherNodeUsageProps
+		}: INodeUsageProps,
+	) {
 		// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
 		// logging as nothing interesting would have happened worth logging.
-		if (nodeUsageProps.currentReferenceTimestampMs === undefined) {
+		if (currentReferenceTimestampMs === undefined) {
 			return;
 		}
 
-		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
-		const nodeType = this.getNodeType(nodeUsageProps.id);
+		// Note: For SubDataStore Load usage, trackedId will be the DataStore's id, not the full path in question.
+		// This is necessary because the SubDataStore path may be unrecognized by GC (if suited for a custom request handler)
+		const nodeStateTracker = this.getNodeStateTracker(trackedId);
+		const nodeType = this.getNodeType(untaggedId);
+
 		const timeout = (() => {
 			switch (nodeStateTracker?.state) {
 				case UnreferencedState.Inactive:
@@ -170,33 +191,27 @@ export class GCTelemetryTracker {
 					return undefined;
 			}
 		})();
-		const {
-			usageType,
-			currentReferenceTimestampMs,
-			packagePath,
-			id: untaggedId,
-			fromId: untaggedFromId,
-			...propsToLog
-		} = nodeUsageProps;
 		const { persistedGcFeatureMatrix, ...configs } = this.configs;
-		const unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
+		const unrefEventProps = {
+			trackedId,
 			type: nodeType,
 			unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
 			age:
 				nodeStateTracker !== undefined
-					? nodeUsageProps.currentReferenceTimestampMs -
-					  nodeStateTracker.unreferencedTimestampMs
+					? currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs
 					: -1,
 			timeout,
+			isTombstoned,
 			...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
-			...propsToLog,
+			...otherNodeUsageProps,
 			...this.createContainerMetadata,
 			gcConfigs: { ...configs, ...persistedGcFeatureMatrix },
-		};
+		} satisfies Omit<IUnreferencedEventProps, "state" | "usageType"> &
+			typeof otherNodeUsageProps;
 
 		// If the node that is used is tombstoned, log a tombstone telemetry.
-		if (nodeUsageProps.isTombstoned) {
-			this.logTombstoneUsageTelemetry(nodeUsageProps, unrefEventProps, nodeType, usageType);
+		if (isTombstoned) {
+			this.logTombstoneUsageTelemetry(unrefEventProps, nodeType, usageType, packagePath);
 		}
 
 		// After logging tombstone telemetry, if the node's unreferenced state is not tracked, there is nothing
@@ -206,16 +221,9 @@ export class GCTelemetryTracker {
 		}
 
 		const state = nodeStateTracker.state;
-		const uniqueEventId = `${state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+		const uniqueEventId = `${state}-${untaggedId}-${usageType}`;
 
-		if (
-			!this.shouldLogNonActiveEvent(
-				nodeType,
-				nodeUsageProps.usageType,
-				nodeStateTracker,
-				uniqueEventId,
-			)
-		) {
+		if (!this.shouldLogNonActiveEvent(nodeType, usageType, nodeStateTracker, uniqueEventId)) {
 			return;
 		}
 
@@ -229,8 +237,8 @@ export class GCTelemetryTracker {
 		// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
 		if (this.isSummarizerClient) {
 			this.pendingEventsQueue.push({
-				...unrefEventProps,
-				usageType: nodeUsageProps.usageType,
+				...unrefEventProps, // Note: Contains some properties from INodeUsageProps as well
+				usageType,
 				state,
 			});
 		} else {
@@ -238,11 +246,11 @@ export class GCTelemetryTracker {
 			// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
 			// Events generated:
 			// InactiveObject_Loaded, SweepReadyObject_Loaded
-			if (nodeUsageProps.usageType === "Loaded") {
+			if (usageType === "Loaded") {
 				const { id, fromId, headers, gcConfigs, ...detailedProps } = unrefEventProps;
 				const event = {
-					eventName: `${state}Object_${nodeUsageProps.usageType}`,
-					...tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }),
+					eventName: `${state}Object_${usageType}`,
+					...tagCodeArtifacts({ pkg: packagePath?.join("/") }),
 					stack: generateStack(),
 					id,
 					fromId,
@@ -262,10 +270,10 @@ export class GCTelemetryTracker {
 	 * Logs telemetry when a tombstoned object is changed, revived or loaded.
 	 */
 	private logTombstoneUsageTelemetry(
-		nodeUsageProps: INodeUsageProps,
 		unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
+		packagePath?: readonly string[],
 	) {
 		// This will log the following events:
 		// GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
@@ -275,12 +283,12 @@ export class GCTelemetryTracker {
 		const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
 		const event = {
 			eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
-			pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+			...tagCodeArtifacts({ pkg: packagePath?.join("/") }),
 			stack: generateStack(),
 			id,
 			fromId,
 			headers: { ...headers },
-			details: detailedProps,
+			details: detailedProps, // Also includes some properties from INodeUsageProps type
 			gcConfigs,
 			tombstoneFlags: {
 				DisableTombstone: this.mc.config.getBoolean(disableTombstoneKey),
@@ -372,7 +380,6 @@ export class GCTelemetryTracker {
 		// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
 		// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
 		for (const eventProps of this.pendingEventsQueue) {
-			// const { usageType, state, id, fromId, ...propsToLog } = eventProps;
 			const { usageType, state, id, fromId, headers, gcConfigs, ...detailedProps } =
 				eventProps;
 			/**
@@ -381,7 +388,7 @@ export class GCTelemetryTracker {
 			 * Loaded and Changed events are logged only if the node is not active. If the node is active, it was
 			 * revived and a Revived event will be logged for it.
 			 */
-			const nodeStateTracker = this.getNodeStateTracker(eventProps.id.value);
+			const nodeStateTracker = this.getNodeStateTracker(detailedProps.trackedId); // Note: This is never SubDataStore path
 			const active =
 				nodeStateTracker === undefined ||
 				nodeStateTracker.state === UnreferencedState.Active;

package/src/gc/index.ts

@@ -23,6 +23,7 @@ export {
 	IGarbageCollectorCreateParams,
 	IGCMetadata,
 	IGCMetadata_Deprecated,
+	IGCNodeUpdatedProps,
 	IGCResult,
 	IGCRuntimeOptions,
 	IMarkPhaseStats,
@@ -46,8 +47,8 @@ export {
 	cloneGCData,
 	concatGarbageCollectionStates,
 	getGCVersionInEffect,
-	trimLeadingAndTrailingSlashes,
 	unpackChildNodesGCDetails,
+	urlToGCNodePath,
 } from "./gcHelpers.js";
 export { runGarbageCollection } from "./gcReferenceGraphAlgorithm.js";
 export {

package/src/index.ts

@@ -13,6 +13,7 @@ export {
 	isRuntimeMessage,
 	agentSchedulerId,
 	ContainerRuntime,
+	DeletedResponseHeaderKey,
 	TombstoneResponseHeaderKey,
 	InactiveResponseHeaderKey,
 	ISummaryConfiguration,
@@ -47,6 +48,7 @@ export {
 	IGCRuntimeOptions,
 	IMarkPhaseStats,
 	ISweepPhaseStats,
+	IGCNodeUpdatedProps,
 	IGCStats,
 } from "./gc/index.js";
 export {

package/src/metadata.ts

@@ -19,8 +19,8 @@ export interface IBlobMetadata {
 }
 
 /**
- * The IdCompressor needs to know if this is a replayed savedOp as those need to be skipped in stashed ops scenarios.
+ * ContainerRuntime needs to know if this is a replayed savedOp as those need to be skipped in stashed ops scenarios.
  */
-export interface IIdAllocationMetadata {
+export interface ISavedOpMetadata {
 	savedOp?: boolean;
 }

package/src/opLifecycle/README.md

@@ -339,19 +339,19 @@ stateDiagram-v2
 	state "Store original (uncompressed, unchunked, ungrouped) batch locally" as store
     state if_compression <<choice>>
 	[*] --> ContainerRuntime.submit
-	ContainerRuntime.submit --> outbox.submitAttach
+	ContainerRuntime.submit --> outbox.submitIdAllocation
 	ContainerRuntime.submit --> outbox.submitBlobAttach
 	ContainerRuntime.submit --> outbox.submit
 	outbox.submit --> scheduleFlush
-	outbox.submitAttach --> scheduleFlush
+	outbox.submitIdAllocation --> scheduleFlush
 	outbox.submitBlobAttach --> scheduleFlush
 	scheduleFlush --> jsTurn
 	jsTurn --> flush
 	flush --> outbox.flushInternalMain
-	flush --> outbox.flushInternalAttach
+	flush --> outbox.flushInternalIdAllocation
 	flush --> outbox.flushInternalBlobAttach
 	outbox.flushInternalMain --> flushInternal
-	outbox.flushInternalAttach --> flushInternal
+	outbox.flushInternalIdAllocation --> flushInternal
 	outbox.flushInternalBlobAttach --> flushInternal
 	flushInternal --> ContainerRuntime.reSubmit: if batch has reentrant ops and should group
 	ContainerRuntime.reSubmit --> flushInternal

package/src/opLifecycle/batchManager.ts

@@ -9,7 +9,6 @@ import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.js";
 
 export interface IBatchManagerOptions {
 	readonly hardLimit: number;
-	readonly softLimit?: number;
 	readonly compressionOptions?: ICompressionRuntimeOptions;
 }
 
@@ -72,19 +71,6 @@ export class BatchManager {
 		// initially stored as base64, and that requires only 2 extra escape characters.
 		const socketMessageSize = contentSize + opOverhead * opCount;
 
-		// If we were provided soft limit, check for exceeding it.
-		// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
-		// and start over. That's not an option if we have no ops.
-		// If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
-		// Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
-		if (
-			this.options.softLimit !== undefined &&
-			this.length > 0 &&
-			socketMessageSize >= this.options.softLimit
-		) {
-			return false;
-		}
-
 		if (socketMessageSize >= this.options.hardLimit) {
 			return false;
 		}

package/src/opLifecycle/opDecompressor.ts

@@ -98,7 +98,10 @@ export class OpDecompressor {
 			message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
 			0x511 /* Only lz4 compression is supported */,
 		);
-		assert(this.isCompressedMessage(message), "provided message should be compressed");
+		assert(
+			this.isCompressedMessage(message),
+			0x940 /* provided message should be compressed */,
+		);
 
 		assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
 		this.activeBatch = true;
@@ -107,7 +110,7 @@ export class OpDecompressor {
 		if (batchMetadata === undefined) {
 			this.isSingleMessageBatch = true;
 		} else {
-			assert(batchMetadata === true, "invalid batch metadata");
+			assert(batchMetadata === true, 0x941 /* invalid batch metadata */);
 		}
 
 		const contents = IsoBuffer.from(
@@ -125,9 +128,12 @@ export class OpDecompressor {
 	 * @returns the unrolled `ISequencedDocumentMessage`
 	 */
 	public unroll(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
-		assert(this.currentlyUnrolling, "not currently unrolling");
-		assert(this.rootMessageContents !== undefined, "missing rootMessageContents");
-		assert(this.rootMessageContents.length > this.processedCount, "no more content to unroll");
+		assert(this.currentlyUnrolling, 0x942 /* not currently unrolling */);
+		assert(this.rootMessageContents !== undefined, 0x943 /* missing rootMessageContents */);
+		assert(
+			this.rootMessageContents.length > this.processedCount,
+			0x944 /* no more content to unroll */,
+		);
 
 		const batchMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;
 
@@ -149,7 +155,7 @@ export class OpDecompressor {
 			return newMessage(message, this.rootMessageContents[this.processedCount++]);
 		}
 
-		assert(batchMetadata === undefined, "invalid batch metadata");
+		assert(batchMetadata === undefined, 0x945 /* invalid batch metadata */);
 		assert(message.contents === undefined, 0x512 /* Expecting empty message */);
 
 		// Continuation of compressed batch

package/src/opLifecycle/opGroupingManager.ts

@@ -50,7 +50,7 @@ export class OpGroupingManager {
 	}
 
 	public groupBatch(batch: IBatch): IBatch {
-		assert(this.shouldGroup(batch), "cannot group the provided batch");
+		assert(this.shouldGroup(batch), 0x946 /* cannot group the provided batch */);
 
 		if (batch.content.length >= 1000) {
 			this.logger.sendTelemetryEvent({
@@ -96,7 +96,7 @@ export class OpGroupingManager {
 	}
 
 	public ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
-		assert(isGroupContents(op.contents), "can only ungroup a grouped batch");
+		assert(isGroupContents(op.contents), 0x947 /* can only ungroup a grouped batch */);
 		const contents: IGroupedBatchMessageContents = op.contents;
 
 		let fakeCsn = 1;

package/src/opLifecycle/opSplitter.ts

@@ -182,7 +182,7 @@ export class OpSplitter {
 	}
 
 	public processChunk(message: ISequencedDocumentMessage): ProcessChunkResult {
-		assert(isChunkedContents(message.contents), "message not of type ChunkedOp");
+		assert(isChunkedContents(message.contents), 0x948 /* message not of type ChunkedOp */);
 		const contents: IChunkedContents = message.contents;
 
 		// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)

package/src/opLifecycle/outbox.ts

@@ -89,11 +89,9 @@ export function getLongStack<T>(action: () => T, length: number = 50): T {
 
 export class Outbox {
 	private readonly mc: MonitoringContext;
-	private readonly attachFlowBatch: BatchManager;
 	private readonly mainBatch: BatchManager;
 	private readonly blobAttachBatch: BatchManager;
 	private readonly idAllocationBatch: BatchManager;
-	private readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;
 	private batchRebasesToReport = 5;
 	private rebasing = false;
 
@@ -113,21 +111,14 @@ export class Outbox {
 			Number.POSITIVE_INFINITY;
 		// We need to allow infinite size batches if we enable compression
 		const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
-		const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
 
-		this.attachFlowBatch = new BatchManager({ hardLimit, softLimit });
 		this.mainBatch = new BatchManager({ hardLimit });
 		this.blobAttachBatch = new BatchManager({ hardLimit });
 		this.idAllocationBatch = new BatchManager({ hardLimit });
 	}
 
 	public get messageCount(): number {
-		return (
-			this.attachFlowBatch.length +
-			this.mainBatch.length +
-			this.blobAttachBatch.length +
-			this.idAllocationBatch.length
-		);
+		return this.mainBatch.length + this.blobAttachBatch.length + this.idAllocationBatch.length;
 	}
 
 	public get isEmpty(): boolean {
@@ -142,13 +133,11 @@ export class Outbox {
 	 */
 	private maybeFlushPartialBatch() {
 		const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
-		const attachFlowBatchSeqNums = this.attachFlowBatch.sequenceNumbers;
 		const blobAttachSeqNums = this.blobAttachBatch.sequenceNumbers;
 		const idAllocSeqNums = this.idAllocationBatch.sequenceNumbers;
 		assert(
 			this.params.config.disablePartialFlush ||
-				(sequenceNumbersMatch(mainBatchSeqNums, attachFlowBatchSeqNums) &&
-					sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums) &&
+				(sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums) &&
 					sequenceNumbersMatch(mainBatchSeqNums, idAllocSeqNums)),
 			0x58d /* Reference sequence numbers from both batches must be in sync */,
 		);
@@ -157,7 +146,6 @@ export class Outbox {
 
 		if (
 			sequenceNumbersMatch(mainBatchSeqNums, currentSequenceNumbers) &&
-			sequenceNumbersMatch(attachFlowBatchSeqNums, currentSequenceNumbers) &&
 			sequenceNumbersMatch(blobAttachSeqNums, currentSequenceNumbers) &&
 			sequenceNumbersMatch(idAllocSeqNums, currentSequenceNumbers)
 		) {
@@ -172,8 +160,6 @@ export class Outbox {
 					eventName: "ReferenceSequenceNumberMismatch",
 					mainReferenceSequenceNumber: mainBatchSeqNums.referenceSequenceNumber,
 					mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
-					attachReferenceSequenceNumber: attachFlowBatchSeqNums.referenceSequenceNumber,
-					attachClientSequenceNumber: attachFlowBatchSeqNums.clientSequenceNumber,
 					blobAttachReferenceSequenceNumber: blobAttachSeqNums.referenceSequenceNumber,
 					blobAttachClientSequenceNumber: blobAttachSeqNums.clientSequenceNumber,
 					currentReferenceSequenceNumber: currentSequenceNumbers.referenceSequenceNumber,
@@ -194,37 +180,6 @@ export class Outbox {
 		this.addMessageToBatchManager(this.mainBatch, message);
 	}
 
-	public submitAttach(message: BatchMessage) {
-		this.maybeFlushPartialBatch();
-
-		if (
-			!this.attachFlowBatch.push(
-				message,
-				this.isContextReentrant(),
-				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
-			)
-		) {
-			// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
-			// when queue is not empty.
-			// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
-			this.flushInternal(this.attachFlowBatch);
-
-			this.addMessageToBatchManager(this.attachFlowBatch, message);
-		}
-
-		// If compression is enabled, we will always successfully receive
-		// attach ops and compress then send them at the next JS turn, regardless
-		// of the overall size of the accumulated ops in the batch.
-		// However, it is more efficient to flush these ops faster, preferably
-		// after they reach a size which would benefit from compression.
-		if (
-			this.attachFlowBatch.contentSizeInBytes >=
-			this.params.config.compressionOptions.minimumBatchSizeInBytes
-		) {
-			this.flushInternal(this.attachFlowBatch);
-		}
-	}
-
 	public submitBlobAttach(message: BatchMessage) {
 		this.maybeFlushPartialBatch();
 
@@ -303,7 +258,6 @@ export class Outbox {
 
 	private flushAll() {
 		this.flushInternal(this.idAllocationBatch);
-		this.flushInternal(this.attachFlowBatch);
 		this.flushInternal(this.blobAttachBatch, true /* disableGroupedBatching */);
 		this.flushInternal(this.mainBatch);
 	}
@@ -479,7 +433,6 @@ export class Outbox {
 		const mainBatch: IBatchCheckpoint = this.mainBatch.checkpoint();
 		return {
 			mainBatch,
-			attachFlowBatch: this.attachFlowBatch.checkpoint(),
 			blobAttachBatch: this.blobAttachBatch.checkpoint(),
 		};
 	}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev-rc.3.0.0.254674";
+export const pkgVersion = "2.0.0-dev-rc.4.0.0.261659";

package/src/pendingStateManager.ts

@@ -7,8 +7,11 @@ import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import { IDisposable } from "@fluidframework/core-interfaces";
 import { assert, Lazy } from "@fluidframework/core-utils/internal";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { DataProcessingError } from "@fluidframework/telemetry-utils/internal";
+import {
+	ITelemetryLoggerExt,
+	DataProcessingError,
+	LoggingError,
+} from "@fluidframework/telemetry-utils/internal";
 import Deque from "double-ended-queue";
 
 import { InboundSequencedContainerRuntimeMessage } from "./messageTypes.js";
@@ -25,6 +28,7 @@ export interface IPendingMessage {
 	content: string;
 	localOpMetadata: unknown;
 	opMetadata: Record<string, unknown> | undefined;
+	sequenceNumber?: number;
 }
 
 export interface IPendingLocalState {
@@ -119,22 +123,31 @@ export class PendingStateManager implements IDisposable {
 		return this.pendingMessagesCount !== 0;
 	}
 
-	public getLocalState(): IPendingLocalState | undefined {
+	public getLocalState(snapshotSequenceNumber?: number): IPendingLocalState {
 		assert(
 			this.initialMessages.isEmpty(),
 			0x2e9 /* "Must call getLocalState() after applying initial states" */,
 		);
-		if (!this.pendingMessages.isEmpty()) {
-			return {
-				pendingStates: [...this.savedOps, ...this.pendingMessages.toArray()].map(
-					(message) => {
-						// delete localOpMetadata since it may not be serializable
-						// and will be regenerated by applyStashedOp()
-						return { ...message, localOpMetadata: undefined };
-					},
-				),
-			};
-		}
+		const newSavedOps = [...this.savedOps].filter((message) => {
+			assert(
+				message.sequenceNumber !== undefined,
+				"saved op should already have a sequence number",
+			);
+			return message.sequenceNumber >= (snapshotSequenceNumber ?? 0);
+		});
+		this.pendingMessages.toArray().forEach((message) => {
+			if (
+				snapshotSequenceNumber !== undefined &&
+				message.referenceSequenceNumber < snapshotSequenceNumber
+			) {
+				throw new LoggingError("trying to stash ops older than our latest snapshot");
+			}
+		});
+		return {
+			pendingStates: [...newSavedOps, ...this.pendingMessages.toArray()].map((message) => {
+				return { ...message, localOpMetadata: undefined };
+			}),
+		};
 	}
 
 	constructor(
@@ -221,13 +234,13 @@ export class PendingStateManager implements IDisposable {
 	public processPendingLocalMessage(message: InboundSequencedContainerRuntimeMessage): unknown {
 		// Pre-processing part - This may be the start of a batch.
 		this.maybeProcessBatchBegin(message);
-
 		// Get the next message from the pending queue. Verify a message exists.
 		const pendingMessage = this.pendingMessages.peekFront();
 		assert(
 			pendingMessage !== undefined,
 			0x169 /* "No pending message found for this remote message" */,
 		);
+		pendingMessage.sequenceNumber = message.sequenceNumber;
 		this.savedOps.push(pendingMessage);
 
 		this.pendingMessages.shift();

package/src/scheduleManager.ts

@@ -9,8 +9,8 @@ import { IDeltaManager } from "@fluidframework/container-definitions";
 import { assert } from "@fluidframework/core-utils/internal";
 import { isRuntimeMessage } from "@fluidframework/driver-utils/internal";
 import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import {
+	ITelemetryLoggerExt,
 	DataCorruptionError,
 	DataProcessingError,
 	createChildLogger,