@fluidframework/container-runtime 2.1.0-281041 → 2.2.0

package/src/containerRuntime.ts

@@ -162,7 +162,7 @@ import {
 	ContainerRuntimeGCMessage,
 	type ContainerRuntimeIdAllocationMessage,
 	type InboundSequencedContainerRuntimeMessage,
-	type InboundSequencedContainerRuntimeMessageOrSystemMessage,
+	type InboundSequencedNonContainerRuntimeMessage,
 	type LocalContainerRuntimeMessage,
 	type OutboundContainerRuntimeMessage,
 	type UnknownContainerRuntimeMessage,
@@ -171,6 +171,7 @@ import { IBatchMetadata, ISavedOpMetadata } from "./metadata.js";
 import {
 	BatchId,
 	BatchMessage,
+	ensureContentsDeserialized,
 	IBatch,
 	IBatchCheckpoint,
 	OpCompressor,
@@ -179,6 +180,7 @@ import {
 	OpSplitter,
 	Outbox,
 	RemoteMessageProcessor,
+	type InboundBatch,
 } from "./opLifecycle/index.js";
 import { pkgVersion } from "./packageVersion.js";
 import {
@@ -521,6 +523,9 @@ export const TombstoneResponseHeaderKey = "isTombstoned";
  * Inactive error responses will have this header set to true
  * @legacy
  * @alpha
+ *
+ * @deprecated this header is deprecated and will be removed in the future. The functionality corresponding
+ * to this was experimental and is no longer supported.
  */
 export const InactiveResponseHeaderKey = "isInactive";
 
@@ -532,7 +537,6 @@ export interface RuntimeHeaderData {
 	wait?: boolean;
 	viaHandle?: boolean;
 	allowTombstone?: boolean;
-	allowInactive?: boolean;
 }
 
 /** Default values for Runtime Headers */
@@ -540,7 +544,6 @@ export const defaultRuntimeHeaderData: Required<RuntimeHeaderData> = {
 	wait: true,
 	viaHandle: false,
 	allowTombstone: false,
-	allowInactive: false,
 };
 
 /**
@@ -628,6 +631,7 @@ export const defaultPendingOpsRetryDelayMs = 1000;
 const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
 
 /**
+ * Checks whether a message.type is one of the values in ContainerMessageType
  * @deprecated please use version in driver-utils
  * @internal
  */
@@ -687,9 +691,7 @@ export const makeLegacySendBatchFn =
 /** Helper type for type constraints passed through several functions.
  * local - Did this client send the op?
  * savedOp - Is this op being replayed after being serialized (having been sequenced previously)
- * batchStartCsn - The clientSequenceNumber given on submit to the start of this batch
- * message - The unpacked message. Likely a TypedContainerRuntimeMessage, but could also be a system op
- * modernRuntimeMessage - Does this appear like a current TypedContainerRuntimeMessage?
+ * localOpMetadata - Metadata maintained locally for a local op.
  */
 type MessageWithContext = {
 	local: boolean;
@@ -698,14 +700,13 @@ type MessageWithContext = {
 } & (
 	| {
 			message: InboundSequencedContainerRuntimeMessage;
-			modernRuntimeMessage: true;
+			isRuntimeMessage: true;
 	  }
 	| {
-			message: InboundSequencedContainerRuntimeMessageOrSystemMessage;
-			modernRuntimeMessage: false;
+			message: InboundSequencedNonContainerRuntimeMessage;
+			isRuntimeMessage: false;
 	  }
 );
-
 const summarizerRequestUrl = "_summarizer";
 
 /**
@@ -760,6 +761,23 @@ function lastMessageFromMetadata(metadata: IContainerRuntimeMetadata | undefined
 		: metadata?.message;
 }
 
+/**
+ * There is some ancient back-compat code that we'd like to instrument
+ * to understand if/when it is hit.
+ * We only want to log this once, to avoid spamming telemetry if we are wrong and these cases are hit commonly.
+ */
+let getSingleUseLegacyLogCallback = (logger: ITelemetryLoggerExt, type: string) => {
+	// We only want to log this once per ContainerRuntime instance, to avoid spamming telemetry.
+	getSingleUseLegacyLogCallback = () => () => {};
+
+	return (codePath: string) => {
+		logger.sendTelemetryEvent({
+			eventName: "LegacyMessageFormat",
+			details: { codePath, type },
+		});
+	};
+};
+
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
@@ -1218,6 +1236,7 @@ export class ContainerRuntime
 
 	private _orderSequentiallyCalls: number = 0;
 	private readonly _flushMode: FlushMode;
+	private readonly offlineEnabled: boolean;
 	private flushTaskExists = false;
 
 	private _connected: boolean;
@@ -1329,14 +1348,22 @@ export class ContainerRuntime
 	 */
 	private nextSummaryNumber: number;
 
-	/** If false, loading or using a Tombstoned object should merely log, not fail */
+	/**
+	 * If false, loading or using a Tombstoned object should merely log, not fail.
+	 * @deprecated NOT SUPPORTED - hardcoded to return false since it's deprecated.
+	 */
+	// eslint-disable-next-line @typescript-eslint/class-literal-property-style
 	public get gcTombstoneEnforcementAllowed(): boolean {
-		return this.garbageCollector.tombstoneEnforcementAllowed;
+		return false;
 	}
 
-	/** If true, throw an error when a tombstone data store is used. */
+	/**
+	 * If true, throw an error when a tombstone data store is used.
+	 * @deprecated NOT SUPPORTED - hardcoded to return false since it's deprecated.
+	 */
+	// eslint-disable-next-line @typescript-eslint/class-literal-property-style
 	public get gcThrowOnTombstoneUsage(): boolean {
-		return this.garbageCollector.throwOnTombstoneUsage;
+		return false;
 	}
 
 	/**
@@ -1546,7 +1573,6 @@ export class ContainerRuntime
 			{
 				applyStashedOp: this.applyStashedOp.bind(this),
 				clientId: () => this.clientId,
-				close: this.closeFn,
 				connected: () => this.connected,
 				reSubmitBatch: this.reSubmitBatch.bind(this),
 				isActiveConnection: () => this.innerDeltaManager.active,
@@ -1601,6 +1627,14 @@ export class ContainerRuntime
 		} else {
 			this._flushMode = runtimeOptions.flushMode;
 		}
+		this.offlineEnabled =
+			this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad") ?? false;
+
+		if (this.offlineEnabled && this._flushMode !== FlushMode.TurnBased) {
+			const error = new UsageError("Offline mode is only supported in turn-based mode");
+			this.closeFn(error);
+			throw error;
+		}
 
 		if (context.attachState === AttachState.Attached) {
 			const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
@@ -2608,62 +2642,92 @@ export class ContainerRuntime
 		await this.pendingStateManager.applyStashedOpsAt(message.sequenceNumber);
 	}
 
-	public process(messageArg: ISequencedDocumentMessage, local: boolean) {
+	/**
+	 * Processes the op.
+	 * @param messageCopy - Sequenced message for a distributed document.
+	 * @param local - true if the message was originally generated by the client receiving it.
+	 */
+	public process({ ...messageCopy }: ISequencedDocumentMessage, local: boolean) {
+		// spread operator above ensure we make a shallow copy of message, as the processing flow will modify it.
+		// There might be multiple container instances receiving the same message.
+
 		this.verifyNotClosed();
 
 		// Whether or not the message appears to be a runtime message from an up-to-date client.
 		// It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
 		// or something different, like a system message.
-		const modernRuntimeMessage = messageArg.type === MessageType.Operation;
-
-		// Do shallow copy of message, as the processing flow will modify it.
-		// There might be multiple container instances receiving the same message.
-		// We do not need to make a deep copy. Each layer will just replace message.contents itself,
-		// but will not modify the contents object (likely it will replace it on the message).
-		const messageCopy = { ...messageArg };
+		const hasModernRuntimeMessageEnvelope = messageCopy.type === MessageType.Operation;
 		const savedOp = (messageCopy.metadata as ISavedOpMetadata)?.savedOp;
-		if (modernRuntimeMessage) {
-			const processResult = this.remoteMessageProcessor.process(messageCopy);
-			if (processResult === undefined) {
+		const logLegacyCase = getSingleUseLegacyLogCallback(this.logger, messageCopy.type);
+
+		// We expect runtime messages to have JSON contents - deserialize it in place.
+		ensureContentsDeserialized(messageCopy, hasModernRuntimeMessageEnvelope, logLegacyCase);
+		if (hasModernRuntimeMessageEnvelope) {
+			// If the message has the modern message envelope, then process it here.
+			// Here we unpack the message (decompress, unchunk, and/or ungroup) into a batch of messages with ContainerMessageType
+			const inboundBatch = this.remoteMessageProcessor.process(messageCopy, logLegacyCase);
+			if (inboundBatch === undefined) {
 				// This means the incoming message is an incomplete part of a message or batch
 				// and we need to process more messages before the rest of the system can understand it.
 				return;
 			}
-			const batchStartCsn = processResult.batchStartCsn;
-			const batch = processResult.messages;
-			const messages: {
-				message: InboundSequencedContainerRuntimeMessage;
-				localOpMetadata: unknown;
-			}[] = local
-				? this.pendingStateManager.processPendingLocalBatch(batch, batchStartCsn)
-				: batch.map((message) => ({ message, localOpMetadata: undefined }));
-			messages.forEach(({ message, localOpMetadata }) => {
-				const msg: MessageWithContext = {
-					message,
-					local,
-					modernRuntimeMessage,
-					savedOp,
-					localOpMetadata,
-				};
-				this.ensureNoDataModelChanges(() => this.processCore(msg));
-			});
-		} else {
-			const msg: MessageWithContext = {
-				message: messageCopy as InboundSequencedContainerRuntimeMessageOrSystemMessage,
+
+			// Reach out to PendingStateManager to zip localOpMetadata into the message list if it's a local batch
+			const messagesWithPendingState = this.pendingStateManager.processInboundBatch(
+				inboundBatch,
 				local,
-				modernRuntimeMessage,
-				savedOp,
-			};
-			this.ensureNoDataModelChanges(() => this.processCore(msg));
+			);
+			if (messagesWithPendingState.length > 0) {
+				messagesWithPendingState.forEach(({ message, localOpMetadata }) => {
+					const msg: MessageWithContext = {
+						message,
+						local,
+						isRuntimeMessage: true,
+						savedOp,
+						localOpMetadata,
+					};
+					this.ensureNoDataModelChanges(() => this.processRuntimeMessage(msg));
+				});
+			} else {
+				this.ensureNoDataModelChanges(() => this.processEmptyBatch(inboundBatch, local));
+			}
+		} else {
+			// Check if message.type is one of values in ContainerMessageType
+			// eslint-disable-next-line import/no-deprecated
+			if (isRuntimeMessage(messageCopy)) {
+				// Legacy op received
+				this.ensureNoDataModelChanges(() =>
+					this.processRuntimeMessage({
+						message: messageCopy as InboundSequencedContainerRuntimeMessage,
+						local,
+						isRuntimeMessage: true,
+						savedOp,
+					}),
+				);
+			} else {
+				// A non container runtime message (like other system ops - join, ack, leave, nack etc.)
+				this.ensureNoDataModelChanges(() =>
+					this.observeNonRuntimeMessage({
+						message: messageCopy as InboundSequencedNonContainerRuntimeMessage,
+						local,
+						isRuntimeMessage: false,
+						savedOp,
+					}),
+				);
+			}
 		}
 	}
 
 	private _processedClientSequenceNumber: number | undefined;
 
 	/**
-	 * Direct the message to the correct subsystem for processing, and implement other side effects
+	 * Processes messages that are intended for the runtime layer to process.
+	 * It redirects the message to the correct subsystem for processing, and implement other side effects
+	 * @param messageWithContext - message to process with additional context and isRuntimeMessage prop as true
 	 */
-	private processCore(messageWithContext: MessageWithContext) {
+	private processRuntimeMessage(
+		messageWithContext: MessageWithContext & { isRuntimeMessage: true },
+	) {
 		const { message, local, localOpMetadata } = messageWithContext;
 
 		// Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
@@ -2698,7 +2762,75 @@ export class ContainerRuntime
 
 			this.validateAndProcessRuntimeMessage(messageWithContext, localOpMetadata);
 
-			this.emit("op", message, messageWithContext.modernRuntimeMessage);
+			this.emit("op", message, messageWithContext.isRuntimeMessage);
+
+			this.scheduleManager.afterOpProcessing(undefined, message);
+
+			if (local) {
+				// If we have processed a local op, this means that the container is
+				// making progress and we can reset the counter for how many times
+				// we have consecutively replayed the pending states
+				this.resetReconnectCount();
+			}
+		} catch (e) {
+			this.scheduleManager.afterOpProcessing(e, message);
+			throw e;
+		}
+	}
+
+	/**
+	 * Process an empty batch, which will execute expected actions while processing even if there are no messages.
+	 * This is a separate function because the processCore function expects at least one message to process.
+	 * It is expected to happen only when the outbox produces an empty batch due to a resubmit flow.
+	 */
+	private processEmptyBatch(emptyBatch: InboundBatch, local: boolean) {
+		const { emptyBatchSequenceNumber: sequenceNumber, batchStartCsn } = emptyBatch;
+		assert(sequenceNumber !== undefined, 0x9fa /* emptyBatchSequenceNumber must be defined */);
+		this.emit("batchBegin", { sequenceNumber });
+		this._processedClientSequenceNumber = batchStartCsn;
+		if (!this.hasPendingMessages()) {
+			this.updateDocumentDirtyState(false);
+		}
+		this.emit("batchEnd", undefined, { sequenceNumber });
+		if (local) {
+			this.resetReconnectCount();
+		}
+	}
+
+	/**
+	 * Observes messages that are not intended for the runtime layer, updating/notifying Runtime systems as needed.
+	 * @param messageWithContext - non-runtime messages to process with additional context and isRuntimeMessage prop as false
+	 */
+	private observeNonRuntimeMessage(
+		messageWithContext: MessageWithContext & { isRuntimeMessage: false },
+	) {
+		const { message, local } = messageWithContext;
+
+		// Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
+		// Sequence numbers are not guaranteed to follow any sort of order. Re-entrancy is one of those situations
+		if (
+			this.deltaManager.minimumSequenceNumber <
+			messageWithContext.message.minimumSequenceNumber
+		) {
+			messageWithContext.message.minimumSequenceNumber =
+				this.deltaManager.minimumSequenceNumber;
+		}
+
+		// Surround the actual processing of the operation with messages to the schedule manager indicating
+		// the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
+		// messages once a batch has been fully processed.
+		this.scheduleManager.beforeOpProcessing(message);
+
+		this._processedClientSequenceNumber = message.clientSequenceNumber;
+
+		try {
+			// If there are no more pending messages after processing a local message,
+			// the document is no longer dirty.
+			if (!this.hasPendingMessages()) {
+				this.updateDocumentDirtyState(false);
+			}
+
+			this.emit("op", message, messageWithContext.isRuntimeMessage);
 
 			this.scheduleManager.afterOpProcessing(undefined, message);
 
@@ -2713,33 +2845,33 @@ export class ContainerRuntime
 			throw e;
 		}
 	}
+
 	/**
 	 * Assuming the given message is also a TypedContainerRuntimeMessage,
 	 * checks its type and dispatches the message to the appropriate handler in the runtime.
 	 * Throws a DataProcessingError if the message looks like but doesn't conform to a known TypedContainerRuntimeMessage type.
 	 */
 	private validateAndProcessRuntimeMessage(
-		messageWithContext: MessageWithContext,
+		messageWithContext: MessageWithContext & { isRuntimeMessage: true },
 		localOpMetadata: unknown,
 	): void {
-		// TODO: destructure message and modernRuntimeMessage once using typescript 5.2.2+
-		const { local } = messageWithContext;
-		switch (messageWithContext.message.type) {
+		const { local, message, savedOp } = messageWithContext;
+		switch (message.type) {
 			case ContainerMessageType.Attach:
 			case ContainerMessageType.Alias:
 			case ContainerMessageType.FluidDataStoreOp:
-				this.channelCollection.process(messageWithContext.message, local, localOpMetadata);
+				this.channelCollection.process(message, local, localOpMetadata);
 				break;
 			case ContainerMessageType.BlobAttach:
-				this.blobManager.processBlobAttachOp(messageWithContext.message, local);
+				this.blobManager.processBlobAttachOp(message, local);
 				break;
 			case ContainerMessageType.IdAllocation:
 				// Don't re-finalize the range if we're processing a "savedOp" in
 				// stashed ops flow. The compressor is stashed with these ops already processed.
 				// That said, in idCompressorMode === "delayed", we might not serialize ID compressor, and
 				// thus we need to process all the ops.
-				if (!(this.skipSavedCompressorOps && messageWithContext.savedOp === true)) {
-					const range = messageWithContext.message.contents;
+				if (!(this.skipSavedCompressorOps && savedOp === true)) {
+					const range = message.contents;
 					// Some other client turned on the id compressor. If we have not turned it on,
 					// put it in a pending queue and delay finalization.
 					if (this._idCompressor === undefined) {
@@ -2758,11 +2890,7 @@ export class ContainerRuntime
 				}
 				break;
 			case ContainerMessageType.GC:
-				this.garbageCollector.processMessage(
-					messageWithContext.message,
-					messageWithContext.message.timestamp,
-					local,
-				);
+				this.garbageCollector.processMessage(message, message.timestamp, local);
 				break;
 			case ContainerMessageType.ChunkedOp:
 				// From observability POV, we should not exppse the rest of the system (including "op" events on object) to these messages.
@@ -2772,23 +2900,14 @@ export class ContainerRuntime
 				break;
 			case ContainerMessageType.DocumentSchemaChange:
 				this.documentsSchemaController.processDocumentSchemaOp(
-					messageWithContext.message.contents,
-					messageWithContext.local,
-					messageWithContext.message.sequenceNumber,
+					message.contents,
+					local,
+					message.sequenceNumber,
 				);
 				break;
 			default: {
-				// If we didn't necessarily expect a runtime message type, then no worries - just return
-				// e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
-				if (!messageWithContext.modernRuntimeMessage) {
-					return;
-				}
-
-				const compatBehavior = messageWithContext.message.compatDetails?.behavior;
-				if (
-					!compatBehaviorAllowsMessageType(messageWithContext.message.type, compatBehavior)
-				) {
-					const { message } = messageWithContext;
+				const compatBehavior = message.compatDetails?.behavior;
+				if (!compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
 					const error = DataProcessingError.create(
 						// Former assert 0x3ce
 						"Runtime message of unknown type",
@@ -3970,8 +4089,6 @@ export class ContainerRuntime
 			0x93f /* metadata */,
 		);
 
-		const serializedContent = JSON.stringify(containerRuntimeMessage);
-
 		// Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
 		// container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
 		if (this.innerDeltaManager.readOnlyInfo.readonly) {
@@ -3986,12 +4103,6 @@ export class ContainerRuntime
 			type !== ContainerMessageType.IdAllocation,
 			0x9a5 /* IdAllocation should be submitted directly to outbox. */,
 		);
-		const message: BatchMessage = {
-			contents: serializedContent,
-			metadata,
-			localOpMetadata,
-			referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
-		};
 
 		try {
 			this.submitIdAllocationOpIfNeeded(false);
@@ -3999,19 +4110,19 @@ export class ContainerRuntime
 			// Allow document schema controller to send a message if it needs to propose change in document schema.
 			// If it needs to send a message, it will call provided callback with payload of such message and rely
 			// on this callback to do actual sending.
-			const contents = this.documentsSchemaController.maybeSendSchemaMessage();
-			if (contents) {
+			const schemaChangeMessage = this.documentsSchemaController.maybeSendSchemaMessage();
+			if (schemaChangeMessage) {
 				this.logger.sendTelemetryEvent({
 					eventName: "SchemaChangeProposal",
-					refSeq: contents.refSeq,
-					version: contents.version,
-					newRuntimeSchema: JSON.stringify(contents.runtime),
+					refSeq: schemaChangeMessage.refSeq,
+					version: schemaChangeMessage.version,
+					newRuntimeSchema: JSON.stringify(schemaChangeMessage.runtime),
 					sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
 					oldRuntimeSchema: JSON.stringify(this.metadata?.documentSchema?.runtime),
 				});
 				const msg: ContainerRuntimeDocumentSchemaMessage = {
 					type: ContainerMessageType.DocumentSchemaChange,
-					contents,
+					contents: schemaChangeMessage,
 				};
 				this.outbox.submit({
 					contents: JSON.stringify(msg),
@@ -4019,6 +4130,12 @@ export class ContainerRuntime
 				});
 			}
 
+			const message: BatchMessage = {
+				contents: JSON.stringify(containerRuntimeMessage) /* serialized content */,
+				metadata,
+				localOpMetadata,
+				referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+			};
 			if (type === ContainerMessageType.BlobAttach) {
 				// BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
 				this.outbox.submitBlobAttach(message);
@@ -4118,10 +4235,7 @@ export class ContainerRuntime
 
 		// Only include Batch ID if "Offline Load" feature is enabled
 		// It's only needed to identify batches across container forks arising from misuse of offline load.
-		const includeBatchId =
-			this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad") ?? false;
-
-		this.flush(includeBatchId ? batchId : undefined);
+		this.flush(this.offlineEnabled ? batchId : undefined);
 	}
 
 	private reSubmit(message: PendingMessageResubmitData) {
@@ -4324,7 +4438,7 @@ export class ContainerRuntime
 						fetchSource: FetchSource.noCache,
 					});
 					const id = snapshot.snapshotTree.id;
-					assert(id !== undefined, "id of the fetched snapshot should be defined");
+					assert(id !== undefined, 0x9d0 /* id of the fetched snapshot should be defined */);
 					props.snapshotVersion = id;
 					snapshotTree = snapshot.snapshotTree;
 				} else {