@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258

package/src/id-compressor/utilities.ts

@@ -0,0 +1,187 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/* eslint-disable no-bitwise */
+import { assert } from "@fluidframework/core-utils";
+import { SessionId, StableId } from "@fluidframework/runtime-definitions";
+import { v4 } from "uuid";
+import { LocalCompressedId, NumericUuid } from "./identifiers";
+
+const hexadecimalCharCodes = Array.from("09afAF").map((c) => c.charCodeAt(0)) as [
+	zero: number,
+	nine: number,
+	a: number,
+	f: number,
+	A: number,
+	F: number,
+];
+
+function isHexadecimalCharacter(charCode: number): boolean {
+	return (
+		(charCode >= hexadecimalCharCodes[0] && charCode <= hexadecimalCharCodes[1]) ||
+		(charCode >= hexadecimalCharCodes[2] && charCode <= hexadecimalCharCodes[3]) ||
+		(charCode >= hexadecimalCharCodes[4] && charCode <= hexadecimalCharCodes[5])
+	);
+}
+
+/**
+ * Generate a random session ID
+ */
+export function createSessionId(): SessionId {
+	return assertIsStableId(v4()) as SessionId;
+}
+
+/**
+ * Asserts that the given string is a stable ID.
+ */
+export function assertIsStableId(stableId: string): StableId {
+	assert(isStableId(stableId), 0x4a3 /* Expected a StableId */);
+	return stableId;
+}
+
+/**
+ * Asserts that the given string is a stable ID.
+ */
+export function assertIsSessionId(stableId: string): SessionId {
+	assertIsStableId(stableId);
+	return stableId as SessionId;
+}
+
+/**
+ * Generate a random stable ID
+ */
+export function generateStableId(): StableId {
+	return assertIsStableId(v4());
+}
+
+/**
+ * Returns true iff the given string is a valid Version 4, variant 2 UUID
+ * 'xxxxxxxx-xxxx-4xxx-vxxx-xxxxxxxxxxxx'
+ */
+export function isStableId(str: string): str is StableId {
+	if (str.length !== 36) {
+		return false;
+	}
+
+	for (let i = 0; i < str.length; i++) {
+		switch (i) {
+			case 8:
+			case 13:
+			case 18:
+			case 23:
+				if (str.charAt(i) !== "-") {
+					return false;
+				}
+				break;
+
+			case 14:
+				if (str.charAt(i) !== "4") {
+					return false;
+				}
+				break;
+
+			case 19: {
+				const char = str.charAt(i);
+				if (char !== "8" && char !== "9" && char !== "a" && char !== "b") {
+					return false;
+				}
+				break;
+			}
+
+			default:
+				if (!isHexadecimalCharacter(str.charCodeAt(i))) {
+					return false;
+				}
+				break;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * A numeric comparator used for sorting in ascending order.
+ *
+ * Handles +/-0 like Map: -0 is equal to +0.
+ */
+export function compareFiniteNumbers<T extends number>(a: T, b: T): number {
+	return a - b;
+}
+
+/**
+ * Compares strings lexically to form a strict partial ordering.
+ */
+export function compareStrings<T extends string>(a: T, b: T): number {
+	return a > b ? 1 : a === b ? 0 : -1;
+}
+
+/**
+ * Compares bigints to form a strict partial ordering.
+ */
+export function compareBigints<T extends bigint>(a: T, b: T): number {
+	return a > b ? 1 : a === b ? 0 : -1;
+}
+
+export function genCountFromLocalId(localId: LocalCompressedId): number {
+	return -localId;
+}
+
+export function localIdFromGenCount(genCount: number): LocalCompressedId {
+	return -genCount as LocalCompressedId;
+}
+
+// xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx
+const versionMask = 0x4n << (19n * 4n); // Version 4
+const variantMask = 0x8n << (15n * 4n); // Variant RFC4122 (1 0 x x)
+const upperMask = 0xffffffffffffn << (20n * 4n);
+// Upper mask when version/variant bits are removed
+const strippedUpperMask = upperMask >> 6n;
+const middieBittiesMask = 0xfffn << (16n * 4n);
+// Middie mask when version/variant bits are removed
+const strippedMiddieBittiesMask = middieBittiesMask >> 2n;
+// Note: leading character should be 3 to mask at 0011
+// The more-significant half of the N nibble is used to denote the variant (10xx)
+const lowerMask = 0x3fffffffffffffffn;
+
+export function numericUuidFromStableId(stableId: StableId): NumericUuid {
+	const uuidU128 = BigInt(`0x${stableId.replace(/-/g, "")}`);
+	const upperMasked = uuidU128 & upperMask;
+	const middieBittiesMasked = uuidU128 & middieBittiesMask;
+	const lowerMasked = uuidU128 & lowerMask;
+
+	const upperMaskedPlaced = upperMasked >> 6n;
+	const middieBittiesMaskedPlaced = middieBittiesMasked >> 2n;
+
+	const id = upperMaskedPlaced | middieBittiesMaskedPlaced | lowerMasked;
+	return id as NumericUuid;
+}
+
+export function stableIdFromNumericUuid(numericUuid: NumericUuid): StableId {
+	// bitwise reverse transform
+	const upperMasked = (numericUuid & strippedUpperMask) << 6n;
+	const middieBittiesMasked = (numericUuid & strippedMiddieBittiesMask) << 2n;
+	const lowerMasked = numericUuid & lowerMask;
+	const uuidU128 = upperMasked | versionMask | middieBittiesMasked | variantMask | lowerMasked;
+	// Pad to 32 characters, inserting leading zeroes if needed
+	const uuidString = uuidU128.toString(16).padStart(32, "0");
+	return `${uuidString.substring(0, 8)}-${uuidString.substring(8, 12)}-${uuidString.substring(
+		12,
+		16,
+	)}-${uuidString.substring(16, 20)}-${uuidString.substring(20, 32)}` as StableId;
+}
+
+export function offsetNumericUuid(numericUuid: NumericUuid, offset: number): NumericUuid {
+	// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+	return (numericUuid + BigInt(offset)) as NumericUuid;
+}
+
+export function subtractNumericUuids(a: NumericUuid, b: NumericUuid): NumericUuid {
+	return (a - b) as NumericUuid;
+}
+
+export function addNumericUuids(a: NumericUuid, b: NumericUuid): NumericUuid {
+	// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+	return (a + b) as NumericUuid;
+}

package/src/index.ts

@@ -6,20 +6,23 @@
 export {
 	ContainerMessageType,
 	ContainerRuntimeMessage,
+	IContainerRuntimeMessageCompatDetails,
+	CompatModeBehavior,
 	ISummaryRuntimeOptions,
 	ISummaryBaseConfiguration,
 	ISummaryConfigurationHeuristics,
 	ISummaryConfigurationDisableSummarizer,
 	ISummaryConfigurationDisableHeuristics,
 	IContainerRuntimeOptions,
-	IRootSummaryTreeWithStats,
 	isRuntimeMessage,
 	RuntimeMessage,
 	agentSchedulerId,
 	ContainerRuntime,
 	RuntimeHeaders,
 	AllowTombstoneRequestHeaderKey,
+	AllowInactiveRequestHeaderKey,
 	TombstoneResponseHeaderKey,
+	InactiveResponseHeaderKey,
 	ISummaryConfiguration,
 	DefaultSummaryConfiguration,
 	ICompressionRuntimeOptions,
@@ -67,6 +70,10 @@ export {
 	OpActionEventListener,
 	OpActionEventName,
 	ICancellableSummarizerController,
+	SubmitSummaryFailureData,
+	SummaryStage,
+	IRetriableFailureResult,
+	ISummarizeEventProps,
 } from "./summary";
+export { isStableId, generateStableId, assertIsStableId } from "./id-compressor";
 export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";
-export { generateStableId, isStableId, assertIsStableId } from "./id-compressor";

package/src/metadata.ts

@@ -0,0 +1,19 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * Batching makes assumptions about what might be on the metadata. This interface codifies those assumptions, but does not validate them.
+ */
+export interface IBatchMetadata {
+	batch?: boolean;
+}
+
+/**
+ * Blob handling makes assumptions about what might be on the metadata. This interface codifies those assumptions, but does not validate them.
+ */
+export interface IBlobMetadata {
+	blobId?: string;
+	localId?: string;
+}

package/src/opLifecycle/README.md

@@ -51,6 +51,26 @@ and verifying that the following expectation changes won't have any effects:
 -   client sequence numbers on batch messages can only be used to order messages with the same sequenceNumber
 -   requires all ops to be processed by runtime layer (version "2.0.0-internal.1.2.0" or later https://github.com/microsoft/FluidFramework/pull/11832)
 
+Grouped batching may become problematic for batches which contain reentrant ops. This is the case when changes are made to a DDS inside a DDS 'onChanged' event handler. This means that the reentrant op will have a different reference sequence number than the rest of the ops in the batch, resulting in a different view of the state of the data model.
+
+Therefore, when grouped batching is enabled, all batches with reentrant ops are rebased to the current reference sequence number and resubmitted to the data stores so that all ops are in agreement about the state of the data model and ensure eventual consistency.
+
+### How to enable
+
+**This feature is disabled by default, currently considered experimental and not ready for production usage.**
+
+If all prerequisites in the previous section are met, enabling the feature can be done via the `IContainerRuntimeOptions` as following:
+
+```
+    const runtimeOptions: IContainerRuntimeOptions = {
+        (...)
+        enableGroupedBatching: true,
+        (...)
+    }
+```
+
+In case of emergency grouped batching can be disabled at runtime, using feature gates. If `"Fluid.ContainerRuntime.DisableGroupedBatching"` is set to `true`, it will disable grouped batching if enabled from `IContainerRuntimeOptions` in the code.
+
 ## Chunking for compression
 
 **Op chunking for compression targets payloads which exceed the max batch size after compression.** So, only payloads which are already compressed. By default, the feature is enabled.

package/src/opLifecycle/batchManager.ts

@@ -29,6 +29,7 @@ const opOverhead = 200;
 export class BatchManager {
 	private pendingBatch: BatchMessage[] = [];
 	private batchContentSize = 0;
+	private hasReentrantOps = false;
 
 	public get length() {
 		return this.pendingBatch.length;
@@ -54,9 +55,14 @@ export class BatchManager {
 
 	constructor(public readonly options: IBatchManagerOptions) {}
 
-	public push(message: BatchMessage, currentClientSequenceNumber?: number): boolean {
+	public push(
+		message: BatchMessage,
+		reentrant: boolean,
+		currentClientSequenceNumber?: number,
+	): boolean {
 		const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
 		const opCount = this.pendingBatch.length;
+		this.hasReentrantOps = this.hasReentrantOps || reentrant;
 
 		// Attempt to estimate batch size, aka socket message size.
 		// Each op has pretty large envelope, estimating to be 200 bytes.
@@ -100,11 +106,13 @@ export class BatchManager {
 			content: this.pendingBatch,
 			contentSizeInBytes: this.batchContentSize,
 			referenceSequenceNumber: this.referenceSequenceNumber,
+			hasReentrantOps: this.hasReentrantOps,
 		};
 
 		this.pendingBatch = [];
 		this.batchContentSize = 0;
 		this.clientSequenceNumber = undefined;
+		this.hasReentrantOps = false;
 
 		return addBatchMetadata(batch);
 	}

package/src/opLifecycle/definitions.ts

@@ -34,6 +34,17 @@ export interface IBatch {
 	 * The reference sequence number for the batch
 	 */
 	readonly referenceSequenceNumber: number | undefined;
+	/**
+	 * Wether or not the batch contains at least one op which was produced as the result
+	 * of processing another op. This means that the batch must be rebased before
+	 * submitted, to ensure that all ops have the same reference sequence numbers and a
+	 * consistent view of the data model. This happens when the op is created within a
+	 * 'changed' event handler of a DDS and will have a different reference sequence number
+	 * than the rest of the ops in the batch, meaning that it has a different view of the
+	 * state of the data model, therefore all ops must be resubmitted and rebased to the current
+	 * reference sequence number to be in agreement about the data model state.
+	 */
+	readonly hasReentrantOps?: boolean;
 }
 
 export interface IBatchCheckpoint {

package/src/opLifecycle/index.ts

@@ -11,7 +11,7 @@ export {
 	IChunkedOp,
 	IMessageProcessingResult,
 } from "./definitions";
-export { Outbox } from "./outbox";
+export { Outbox, getLongStack } from "./outbox";
 export { OpCompressor } from "./opCompressor";
 export { OpDecompressor } from "./opDecompressor";
 export { OpSplitter, splitOp } from "./opSplitter";

package/src/opLifecycle/opCompressor.ts

@@ -3,10 +3,11 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, IsoBuffer } from "@fluidframework/common-utils";
-import { UsageError } from "@fluidframework/container-utils";
+import { createChildLogger, UsageError } from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer } from "@fluid-internal/client-utils";
 import { compress } from "lz4js";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { CompressionAlgorithms } from "../containerRuntime";
 import { estimateSocketSize } from "./batchManager";
 import { IBatch, BatchMessage } from "./definitions";
@@ -19,8 +20,8 @@ import { IBatch, BatchMessage } from "./definitions";
 export class OpCompressor {
 	private readonly logger;
 
-	constructor(logger: ITelemetryLoggerExt) {
-		this.logger = ChildLogger.create(logger, "OpCompressor");
+	constructor(logger: ITelemetryBaseLogger) {
+		this.logger = createChildLogger({ logger, namespace: "OpCompressor" });
 	}
 
 	public compressBatch(batch: IBatch): IBatch {

package/src/opLifecycle/opDecompressor.ts

@@ -5,11 +5,21 @@
 
 import { decompress } from "lz4js";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
-import { ChildLogger, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer, Uint8ArrayToString } from "@fluid-internal/client-utils";
+import { createChildLogger } from "@fluidframework/telemetry-utils";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { CompressionAlgorithms } from "../containerRuntime";
+import { IBatchMetadata } from "../metadata";
 import { IMessageProcessingResult } from "./definitions";
 
+/**
+ * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
+ */
+interface IPackedContentsContents {
+	packedContents: string;
+}
+
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
  * This class relies on some implicit contracts defined below:
@@ -24,8 +34,8 @@ export class OpDecompressor {
 	private processedCount = 0;
 	private readonly logger;
 
-	constructor(logger: ITelemetryLoggerExt) {
-		this.logger = ChildLogger.create(logger, "OpDecompressor");
+	constructor(logger: ITelemetryBaseLogger) {
+		this.logger = createChildLogger({ logger, namespace: "OpDecompressor" });
 	}
 
 	public processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
@@ -34,7 +44,10 @@ export class OpDecompressor {
 			0x511 /* Only lz4 compression is supported */,
 		);
 
-		if (message.metadata?.batch === true && this.isCompressed(message)) {
+		if (
+			(message.metadata as IBatchMetadata | undefined)?.batch === true &&
+			this.isCompressed(message)
+		) {
 			// Beginning of a compressed batch
 			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
 			if (message.compression) {
@@ -47,7 +60,10 @@ export class OpDecompressor {
 
 			this.activeBatch = true;
 
-			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const contents = IsoBuffer.from(
+				(message.contents as IPackedContentsContents).packedContents,
+				"base64",
+			);
 			const decompressedMessage = decompress(contents);
 			const intoString = Uint8ArrayToString(decompressedMessage);
 			const asObj = JSON.parse(intoString);
@@ -61,7 +77,7 @@ export class OpDecompressor {
 
 		if (
 			this.rootMessageContents !== undefined &&
-			message.metadata?.batch === undefined &&
+			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
 			this.activeBatch
 		) {
 			assert(message.contents === undefined, 0x512 /* Expecting empty message */);
@@ -73,7 +89,10 @@ export class OpDecompressor {
 			};
 		}
 
-		if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
+		if (
+			this.rootMessageContents !== undefined &&
+			(message.metadata as IBatchMetadata | undefined)?.batch === false
+		) {
 			// End of compressed batch
 			const returnMessage = newMessage(
 				message,
@@ -90,14 +109,20 @@ export class OpDecompressor {
 			};
 		}
 
-		if (message.metadata?.batch === undefined && this.isCompressed(message)) {
+		if (
+			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
+			this.isCompressed(message)
+		) {
 			// Single compressed message
 			assert(
 				this.activeBatch === false,
 				0x4ba /* shouldn't receive compressed message in middle of a batch */,
 			);
 
-			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const contents = IsoBuffer.from(
+				(message.contents as IPackedContentsContents).packedContents,
+				"base64",
+			);
 			const decompressedMessage = decompress(contents);
 			const intoString = new TextDecoder().decode(decompressedMessage);
 			const asObj = JSON.parse(intoString);
@@ -135,16 +160,19 @@ export class OpDecompressor {
 				message.contents !== null &&
 				typeof message.contents === "object" &&
 				Object.keys(message.contents).length === 1 &&
-				message.contents?.packedContents !== undefined &&
-				typeof message.contents?.packedContents === "string" &&
-				message.contents.packedContents.length > 0 &&
-				IsoBuffer.from(message.contents.packedContents, "base64").toString("base64") ===
-					message.contents.packedContents
+				typeof (message.contents as { packedContents?: unknown }).packedContents ===
+					"string" &&
+				(message.contents as IPackedContentsContents).packedContents.length > 0 &&
+				IsoBuffer.from(
+					(message.contents as IPackedContentsContents).packedContents,
+					"base64",
+				).toString("base64") ===
+					(message.contents as IPackedContentsContents).packedContents
 			) {
 				this.logger.sendTelemetryEvent({
 					eventName: "LegacyCompression",
 					type: message.type,
-					batch: message.metadata?.batch,
+					batch: (message.metadata as IBatchMetadata | undefined)?.batch,
 				});
 				return true;
 			}
@@ -164,5 +192,7 @@ const newMessage = (
 	...originalMessage,
 	contents,
 	compression: undefined,
-	metadata: { ...originalMessage.metadata },
+	// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+	metadata: { ...(originalMessage.metadata as any) },
 });

package/src/opLifecycle/opGroupingManager.ts

@@ -3,19 +3,33 @@
  * Licensed under the MIT License.
  */
 
-import { assert } from "@fluidframework/common-utils";
+import { assert } from "@fluidframework/core-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { ContainerMessageType } from "..";
 import { IBatch } from "./definitions";
 
+/**
+ * Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
+ */
+interface IGroupedBatchMessageContents {
+	type: typeof OpGroupingManager.groupedBatchOp;
+	contents: IGroupedMessage[];
+}
+
 interface IGroupedMessage {
 	contents?: unknown;
 	metadata?: Record<string, unknown>;
 	compression?: string;
 }
 
+function isGroupContents(
+	opContents: IGroupedBatchMessageContents | { type?: unknown } | undefined,
+): opContents is IGroupedBatchMessageContents {
+	return opContents?.type === OpGroupingManager.groupedBatchOp;
+}
+
 export class OpGroupingManager {
-	static groupedBatchOp = "groupedBatch";
+	static readonly groupedBatchOp = "groupedBatch";
 
 	constructor(private readonly groupedBatchingEnabled: boolean) {}
 
@@ -25,10 +39,6 @@ export class OpGroupingManager {
 		}
 
 		for (const message of batch.content) {
-			// Blob attaches cannot be grouped (grouped batching would hide metadata)
-			if (message.type === ContainerMessageType.BlobAttach) {
-				return batch;
-			}
 			if (message.metadata) {
 				const keys = Object.keys(message.metadata);
 				assert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
@@ -64,11 +74,11 @@ export class OpGroupingManager {
 	}
 
 	public ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
-		if (op.contents?.type !== OpGroupingManager.groupedBatchOp) {
+		if (!isGroupContents(op.contents)) {
 			return [op];
 		}
 
-		const messages = op.contents.contents as IGroupedMessage[];
+		const messages = op.contents.contents;
 		let fakeCsn = 1;
 		return messages.map((subMessage) => ({
 			...op,

package/src/opLifecycle/opSplitter.ts

@@ -3,14 +3,15 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
-import { assert } from "@fluidframework/common-utils";
-import { IBatchMessage } from "@fluidframework/container-definitions";
 import {
+	createChildLogger,
 	DataCorruptionError,
 	extractSafePropertiesFromMessage,
-} from "@fluidframework/container-utils";
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IBatchMessage } from "@fluidframework/container-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
 import { estimateSocketSize } from "./batchManager";
 import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
@@ -30,10 +31,10 @@ export class OpSplitter {
 			| undefined,
 		public readonly chunkSizeInBytes: number,
 		private readonly maxBatchSizeInBytes: number,
-		logger: ITelemetryLoggerExt,
+		logger: ITelemetryBaseLogger,
 	) {
 		this.chunkMap = new Map<string, string[]>(chunks);
-		this.logger = ChildLogger.create(logger, "OpSplitter");
+		this.logger = createChildLogger({ logger, namespace: "OpSplitter" });
 	}
 
 	public get isBatchChunkingEnabled(): boolean {
@@ -52,7 +53,9 @@ export class OpSplitter {
 			};
 		}
 
-		const clientId = message.clientId;
+		// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
+		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+		const clientId = message.clientId as string;
 		const chunkedContent = message.contents as IChunkedOp;
 		this.addChunk(clientId, chunkedContent, message);