@fluidframework/container-runtime 2.0.0-dev.4.4.0.162574 → 2.0.0-dev.5.3.2.178189

package/src/gc/gcDefinitions.ts

@@ -13,7 +13,7 @@ import {
 	ITelemetryContext,
 } from "@fluidframework/runtime-definitions";
 import { ReadAndParseBlob } from "@fluidframework/runtime-utils";
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import {
 	IContainerRuntimeMetadata,
 	ICreateContainerMetadata,
@@ -211,7 +211,7 @@ export interface IGarbageCollector {
 	/** Run garbage collection and update the reference / used state of the system. */
 	collectGarbage(
 		options: {
-			logger?: ITelemetryLogger;
+			logger?: ITelemetryLoggerExt;
 			runSweep?: boolean;
 			fullGC?: boolean;
 		},
@@ -253,7 +253,7 @@ export interface IGarbageCollector {
 export interface IGarbageCollectorCreateParams {
 	readonly runtime: IGarbageCollectionRuntime;
 	readonly gcOptions: IGCRuntimeOptions;
-	readonly baseLogger: ITelemetryLogger;
+	readonly baseLogger: ITelemetryLoggerExt;
 	readonly existing: boolean;
 	readonly metadata: IContainerRuntimeMetadata | undefined;
 	readonly createContainerMetadata: ICreateContainerMetadata;
@@ -263,7 +263,6 @@ export interface IGarbageCollectorCreateParams {
 	readonly getLastSummaryTimestampMs: () => number | undefined;
 	readonly readAndParseBlob: ReadAndParseBlob;
 	readonly activeConnection: () => boolean;
-	readonly getContainerDiagnosticId: () => string;
 }
 
 export interface IGCRuntimeOptions {

package/src/gc/gcTelemetry.ts

@@ -3,10 +3,14 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryGenericEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryGenericEvent } from "@fluidframework/core-interfaces";
 import { IGarbageCollectionData } from "@fluidframework/runtime-definitions";
 import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
-import { generateStack, MonitoringContext } from "@fluidframework/telemetry-utils";
+import {
+	generateStack,
+	ITelemetryLoggerExt,
+	MonitoringContext,
+} from "@fluidframework/telemetry-utils";
 import { ICreateContainerMetadata } from "../summary";
 import {
 	disableSweepLogKey,
@@ -243,7 +247,7 @@ export class GCTelemetryTracker {
 		currentGCData: IGarbageCollectionData,
 		previousGCData: IGarbageCollectionData,
 		explicitReferences: Map<string, string[]>,
-		logger: ITelemetryLogger,
+		logger: ITelemetryLoggerExt,
 	) {
 		for (const [nodeId, currentOutboundRoutes] of Object.entries(currentGCData.gcNodes)) {
 			const previousRoutes = previousGCData.gcNodes[nodeId] ?? [];
@@ -284,7 +288,7 @@ export class GCTelemetryTracker {
 	 * Log events that are pending in pendingEventsQueue. This is called after GC runs in the summarizer client
 	 * so that the state of an unreferenced node is updated.
 	 */
-	public async logPendingEvents(logger: ITelemetryLogger) {
+	public async logPendingEvents(logger: ITelemetryLoggerExt) {
 		// Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
 		// summary time they are then logged.
 		// Events generated:
@@ -333,7 +337,7 @@ export class GCTelemetryTracker {
 	 * this will give us a view into how much deleted content a container has.
 	 */
 	public logSweepEvents(
-		logger: ITelemetryLogger,
+		logger: ITelemetryLoggerExt,
 		currentReferenceTimestampMs: number,
 		unreferencedNodesState: Map<string, UnreferencedStateTracker>,
 		completedGCRuns: number,

package/src/id-compressor/idCompressor.ts

@@ -5,7 +5,7 @@
 
 /* eslint-disable @typescript-eslint/restrict-plus-operands */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import { assert } from "@fluidframework/common-utils";
 import {
 	IIdCompressor,
@@ -311,7 +311,7 @@ export class IdCompressor implements IIdCompressorCore, IIdCompressor {
 	 */
 	public constructor(
 		public readonly localSessionId: SessionId,
-		private readonly logger?: ITelemetryLogger,
+		private readonly logger?: ITelemetryLoggerExt,
 	) {
 		this.localSession = this.createSession(localSessionId);
 	}

package/src/id-compressor/uuidUtilities.ts

@@ -5,7 +5,7 @@
 
 import { assert } from "@fluidframework/common-utils";
 import { StableId, UuidString } from "@fluidframework/runtime-definitions";
-import { v4, NIL } from "uuid";
+import { v4 } from "uuid";
 
 const hexadecimalCharCodes = Array.from("09afAF").map((c) => c.charCodeAt(0)) as [
 	zero: number,
@@ -24,9 +24,6 @@ function isHexadecimalCharacter(charCode: number): boolean {
 	);
 }
 
-/** The null (lowest/all-zeros) UUID */
-export const nilUuid = assertIsUuidString(NIL);
-
 /**
  * Asserts that the given string is a UUID
  */

package/src/index.ts

@@ -67,6 +67,8 @@ export {
 	OpActionEventListener,
 	OpActionEventName,
 	ICancellableSummarizerController,
+	SubmitSummaryFailureData,
+	SummaryStage,
 } from "./summary";
 export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";
 export { generateStableId, isStableId, assertIsStableId } from "./id-compressor";

package/src/metadata.ts

@@ -0,0 +1,19 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * Batching makes assumptions about what might be on the metadata. This interface codifies those assumptions, but does not validate them.
+ */
+export interface IBatchMetadata {
+	batch?: boolean;
+}
+
+/**
+ * Blob handling makes assumptions about what might be on the metadata. This interface codifies those assumptions, but does not validate them.
+ */
+export interface IBlobMetadata {
+	blobId?: string;
+	localId?: string;
+}

package/src/opLifecycle/README.md

@@ -51,6 +51,26 @@ and verifying that the following expectation changes won't have any effects:
 -   client sequence numbers on batch messages can only be used to order messages with the same sequenceNumber
 -   requires all ops to be processed by runtime layer (version "2.0.0-internal.1.2.0" or later https://github.com/microsoft/FluidFramework/pull/11832)
 
+Grouped batching may become problematic for batches which contain reentrant ops. This is the case when changes are made to a DDS inside a DDS 'onChanged' event handler. This means that the reentrant op will have a different reference sequence number than the rest of the ops in the batch, resulting in a different view of the state of the data model.
+
+Therefore, when grouped batching is enabled, all batches with reentrant ops are rebased to the current reference sequence number and resubmitted to the data stores so that all ops are in agreement about the state of the data model and ensure eventual consistency.
+
+### How to enable
+
+**This feature is disabled by default, currently considered experimental and not ready for production usage.**
+
+If all prerequisites in the previous section are met, enabling the feature can be done via the `IContainerRuntimeOptions` as following:
+
+```
+    const runtimeOptions: IContainerRuntimeOptions = {
+        (...)
+        enableGroupedBatching: true,
+        (...)
+    }
+```
+
+In case of emergency grouped batching can be disabled at runtime, using feature gates. If `"Fluid.ContainerRuntime.DisableGroupedBatching"` is set to `true`, it will disable grouped batching if enabled from `IContainerRuntimeOptions` in the code.
+
 ## Chunking for compression
 
 **Op chunking for compression targets payloads which exceed the max batch size after compression.** So, only payloads which are already compressed. By default, the feature is enabled.

package/src/opLifecycle/batchManager.ts

@@ -29,6 +29,7 @@ const opOverhead = 200;
 export class BatchManager {
 	private pendingBatch: BatchMessage[] = [];
 	private batchContentSize = 0;
+	private hasReentrantOps = false;
 
 	public get length() {
 		return this.pendingBatch.length;
@@ -54,9 +55,14 @@ export class BatchManager {
 
 	constructor(public readonly options: IBatchManagerOptions) {}
 
-	public push(message: BatchMessage, currentClientSequenceNumber?: number): boolean {
+	public push(
+		message: BatchMessage,
+		reentrant: boolean,
+		currentClientSequenceNumber?: number,
+	): boolean {
 		const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
 		const opCount = this.pendingBatch.length;
+		this.hasReentrantOps = this.hasReentrantOps || reentrant;
 
 		// Attempt to estimate batch size, aka socket message size.
 		// Each op has pretty large envelope, estimating to be 200 bytes.
@@ -100,11 +106,13 @@ export class BatchManager {
 			content: this.pendingBatch,
 			contentSizeInBytes: this.batchContentSize,
 			referenceSequenceNumber: this.referenceSequenceNumber,
+			hasReentrantOps: this.hasReentrantOps,
 		};
 
 		this.pendingBatch = [];
 		this.batchContentSize = 0;
 		this.clientSequenceNumber = undefined;
+		this.hasReentrantOps = false;
 
 		return addBatchMetadata(batch);
 	}

package/src/opLifecycle/definitions.ts

@@ -5,14 +5,14 @@
 
 import { IBatchMessage } from "@fluidframework/container-definitions";
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
-import { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from "..";
+import { CompressionAlgorithms, ContainerMessageType } from "..";
 
 /**
  * Batch message type used internally by the runtime
  */
 export type BatchMessage = IBatchMessage & {
 	localOpMetadata: unknown;
-	deserializedContent: ContainerRuntimeMessage;
+	type: ContainerMessageType;
 	referenceSequenceNumber: number;
 	compression?: CompressionAlgorithms;
 };
@@ -34,6 +34,17 @@ export interface IBatch {
 	 * The reference sequence number for the batch
 	 */
 	readonly referenceSequenceNumber: number | undefined;
+	/**
+	 * Wether or not the batch contains at least one op which was produced as the result
+	 * of processing another op. This means that the batch must be rebased before
+	 * submitted, to ensure that all ops have the same reference sequence numbers and a
+	 * consistent view of the data model. This happens when the op is created within a
+	 * 'changed' event handler of a DDS and will have a different reference sequence number
+	 * than the rest of the ops in the batch, meaning that it has a different view of the
+	 * state of the data model, therefore all ops must be resubmitted and rebased to the current
+	 * reference sequence number to be in agreement about the data model state.
+	 */
+	readonly hasReentrantOps?: boolean;
 }
 
 export interface IBatchCheckpoint {

package/src/opLifecycle/index.ts

@@ -11,7 +11,7 @@ export {
 	IChunkedOp,
 	IMessageProcessingResult,
 } from "./definitions";
-export { Outbox } from "./outbox";
+export { Outbox, getLongStack } from "./outbox";
 export { OpCompressor } from "./opCompressor";
 export { OpDecompressor } from "./opDecompressor";
 export { OpSplitter, splitOp } from "./opSplitter";

package/src/opLifecycle/opCompressor.ts

@@ -3,10 +3,9 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
 import { assert, IsoBuffer } from "@fluidframework/common-utils";
 import { UsageError } from "@fluidframework/container-utils";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { compress } from "lz4js";
 import { CompressionAlgorithms } from "../containerRuntime";
 import { estimateSocketSize } from "./batchManager";
@@ -20,7 +19,7 @@ import { IBatch, BatchMessage } from "./definitions";
 export class OpCompressor {
 	private readonly logger;
 
-	constructor(logger: ITelemetryLogger) {
+	constructor(logger: ITelemetryLoggerExt) {
 		this.logger = ChildLogger.create(logger, "OpCompressor");
 	}
 
@@ -47,10 +46,7 @@ export class OpCompressor {
 		// Add empty placeholder messages to reserve the sequence numbers
 		for (const message of batch.content.slice(1)) {
 			messages.push({
-				deserializedContent: {
-					contents: undefined,
-					type: message.deserializedContent.type,
-				},
+				type: message.type,
 				localOpMetadata: message.localOpMetadata,
 				metadata: message.metadata,
 				referenceSequenceNumber: message.referenceSequenceNumber,
@@ -79,7 +75,7 @@ export class OpCompressor {
 
 	private serializeBatch(batch: IBatch): string {
 		try {
-			return JSON.stringify(batch.content.map((message) => message.deserializedContent));
+			return `[${batch.content.map((message) => message.contents).join(",")}]`;
 		} catch (e: any) {
 			if (e.message === "Invalid string length") {
 				// This is how JSON.stringify signals that

package/src/opLifecycle/opDecompressor.ts

@@ -6,11 +6,18 @@
 import { decompress } from "lz4js";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ChildLogger, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import { CompressionAlgorithms } from "../containerRuntime";
+import { IBatchMetadata } from "../metadata";
 import { IMessageProcessingResult } from "./definitions";
 
+/**
+ * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
+ */
+interface IPackedContentsContents {
+	packedContents: string;
+}
+
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
  * This class relies on some implicit contracts defined below:
@@ -25,7 +32,7 @@ export class OpDecompressor {
 	private processedCount = 0;
 	private readonly logger;
 
-	constructor(logger: ITelemetryLogger) {
+	constructor(logger: ITelemetryLoggerExt) {
 		this.logger = ChildLogger.create(logger, "OpDecompressor");
 	}
 
@@ -35,7 +42,10 @@ export class OpDecompressor {
 			0x511 /* Only lz4 compression is supported */,
 		);
 
-		if (message.metadata?.batch === true && this.isCompressed(message)) {
+		if (
+			(message.metadata as IBatchMetadata | undefined)?.batch === true &&
+			this.isCompressed(message)
+		) {
 			// Beginning of a compressed batch
 			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
 			if (message.compression) {
@@ -48,7 +58,10 @@ export class OpDecompressor {
 
 			this.activeBatch = true;
 
-			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const contents = IsoBuffer.from(
+				(message.contents as IPackedContentsContents).packedContents,
+				"base64",
+			);
 			const decompressedMessage = decompress(contents);
 			const intoString = Uint8ArrayToString(decompressedMessage);
 			const asObj = JSON.parse(intoString);
@@ -62,7 +75,7 @@ export class OpDecompressor {
 
 		if (
 			this.rootMessageContents !== undefined &&
-			message.metadata?.batch === undefined &&
+			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
 			this.activeBatch
 		) {
 			assert(message.contents === undefined, 0x512 /* Expecting empty message */);
@@ -74,7 +87,10 @@ export class OpDecompressor {
 			};
 		}
 
-		if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
+		if (
+			this.rootMessageContents !== undefined &&
+			(message.metadata as IBatchMetadata | undefined)?.batch === false
+		) {
 			// End of compressed batch
 			const returnMessage = newMessage(
 				message,
@@ -91,14 +107,20 @@ export class OpDecompressor {
 			};
 		}
 
-		if (message.metadata?.batch === undefined && this.isCompressed(message)) {
+		if (
+			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
+			this.isCompressed(message)
+		) {
 			// Single compressed message
 			assert(
 				this.activeBatch === false,
 				0x4ba /* shouldn't receive compressed message in middle of a batch */,
 			);
 
-			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const contents = IsoBuffer.from(
+				(message.contents as IPackedContentsContents).packedContents,
+				"base64",
+			);
 			const decompressedMessage = decompress(contents);
 			const intoString = new TextDecoder().decode(decompressedMessage);
 			const asObj = JSON.parse(intoString);
@@ -136,16 +158,19 @@ export class OpDecompressor {
 				message.contents !== null &&
 				typeof message.contents === "object" &&
 				Object.keys(message.contents).length === 1 &&
-				message.contents?.packedContents !== undefined &&
-				typeof message.contents?.packedContents === "string" &&
-				message.contents.packedContents.length > 0 &&
-				IsoBuffer.from(message.contents.packedContents, "base64").toString("base64") ===
-					message.contents.packedContents
+				typeof (message.contents as { packedContents?: unknown }).packedContents ===
+					"string" &&
+				(message.contents as IPackedContentsContents).packedContents.length > 0 &&
+				IsoBuffer.from(
+					(message.contents as IPackedContentsContents).packedContents,
+					"base64",
+				).toString("base64") ===
+					(message.contents as IPackedContentsContents).packedContents
 			) {
 				this.logger.sendTelemetryEvent({
 					eventName: "LegacyCompression",
 					type: message.type,
-					batch: message.metadata?.batch,
+					batch: (message.metadata as IBatchMetadata | undefined)?.batch,
 				});
 				return true;
 			}
@@ -165,5 +190,7 @@ const newMessage = (
 	...originalMessage,
 	contents,
 	compression: undefined,
-	metadata: { ...originalMessage.metadata },
+	// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+	metadata: { ...(originalMessage.metadata as any) },
 });

package/src/opLifecycle/opGroupingManager.ts

@@ -5,9 +5,17 @@
 
 import { assert } from "@fluidframework/common-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ContainerMessageType, ContainerRuntimeMessage } from "..";
+import { ContainerMessageType } from "..";
 import { IBatch } from "./definitions";
 
+/**
+ * Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
+ */
+interface IGroupedBatchMessageContents {
+	type: typeof OpGroupingManager.groupedBatchOp;
+	contents: IGroupedMessage[];
+}
+
 interface IGroupedMessage {
 	contents?: unknown;
 	metadata?: Record<string, unknown>;
@@ -15,7 +23,7 @@ interface IGroupedMessage {
 }
 
 export class OpGroupingManager {
-	static groupedBatchOp = "groupedBatch";
+	static readonly groupedBatchOp = "groupedBatch";
 
 	constructor(private readonly groupedBatchingEnabled: boolean) {}
 
@@ -25,10 +33,6 @@ export class OpGroupingManager {
 		}
 
 		for (const message of batch.content) {
-			// Blob attaches cannot be grouped (grouped batching would hide metadata)
-			if (message.deserializedContent.type === ContainerMessageType.BlobAttach) {
-				return batch;
-			}
 			if (message.metadata) {
 				const keys = Object.keys(message.metadata);
 				assert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
@@ -39,14 +43,14 @@ export class OpGroupingManager {
 			}
 		}
 
-		const deserializedContent = {
+		const serializedContent = JSON.stringify({
 			type: OpGroupingManager.groupedBatchOp,
 			contents: batch.content.map<IGroupedMessage>((message) => ({
 				contents: message.contents === undefined ? undefined : JSON.parse(message.contents),
 				metadata: message.metadata,
 				compression: message.compression,
 			})),
-		};
+		});
 
 		const groupedBatch: IBatch = {
 			...batch,
@@ -55,9 +59,8 @@ export class OpGroupingManager {
 					localOpMetadata: undefined,
 					metadata: undefined,
 					referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
-					// Need deserializedContent for back-compat
-					deserializedContent: deserializedContent as ContainerRuntimeMessage,
-					contents: JSON.stringify(deserializedContent),
+					contents: serializedContent,
+					type: OpGroupingManager.groupedBatchOp as ContainerMessageType,
 				},
 			],
 		};
@@ -65,11 +68,14 @@ export class OpGroupingManager {
 	}
 
 	public ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
-		if (op.contents?.type !== OpGroupingManager.groupedBatchOp) {
+		if (
+			(op.contents as { type?: unknown } | undefined)?.type !==
+			OpGroupingManager.groupedBatchOp
+		) {
 			return [op];
 		}
 
-		const messages = op.contents.contents as IGroupedMessage[];
+		const messages = (op.contents as IGroupedBatchMessageContents).contents;
 		let fakeCsn = 1;
 		return messages.map((subMessage) => ({
 			...op,

package/src/opLifecycle/opSplitter.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
 import { assert } from "@fluidframework/common-utils";
 import { IBatchMessage } from "@fluidframework/container-definitions";
 import {
@@ -11,7 +11,6 @@ import {
 	extractSafePropertiesFromMessage,
 } from "@fluidframework/container-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
 import { estimateSocketSize } from "./batchManager";
 import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
@@ -31,7 +30,7 @@ export class OpSplitter {
 			| undefined,
 		public readonly chunkSizeInBytes: number,
 		private readonly maxBatchSizeInBytes: number,
-		logger: ITelemetryLogger,
+		logger: ITelemetryLoggerExt,
 	) {
 		this.chunkMap = new Map<string, string[]>(chunks);
 		this.logger = ChildLogger.create(logger, "OpSplitter");
@@ -53,7 +52,9 @@ export class OpSplitter {
 			};
 		}
 
-		const clientId = message.clientId;
+		// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
+		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+		const clientId = message.clientId as string;
 		const chunkedContent = message.contents as IChunkedOp;
 		this.addChunk(clientId, chunkedContent, message);
 
@@ -214,7 +215,7 @@ const chunkToBatchMessage = (
 	};
 	return {
 		contents: JSON.stringify(payload),
-		deserializedContent: payload,
+		type: payload.type,
 		metadata,
 		localOpMetadata: undefined,
 		referenceSequenceNumber,
@@ -251,7 +252,7 @@ export const splitOp = (
 		const chunk: IChunkedOp = {
 			chunkId,
 			contents: op.contents.substr(offset, chunkSizeInBytes),
-			originalType: op.deserializedContent.type,
+			originalType: op.type,
 			totalChunks: chunkCount,
 		};