@fluidframework/container-runtime 2.0.0-internal.6.1.1 → 2.0.0-internal.6.3.0

package/src/id-compressor/utilities.ts

@@ -0,0 +1,187 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/* eslint-disable no-bitwise */
+import { assert } from "@fluidframework/core-utils";
+import { SessionId, StableId } from "@fluidframework/runtime-definitions";
+import { v4 } from "uuid";
+import { LocalCompressedId, NumericUuid } from "./identifiers";
+
+const hexadecimalCharCodes = Array.from("09afAF").map((c) => c.charCodeAt(0)) as [
+	zero: number,
+	nine: number,
+	a: number,
+	f: number,
+	A: number,
+	F: number,
+];
+
+function isHexadecimalCharacter(charCode: number): boolean {
+	return (
+		(charCode >= hexadecimalCharCodes[0] && charCode <= hexadecimalCharCodes[1]) ||
+		(charCode >= hexadecimalCharCodes[2] && charCode <= hexadecimalCharCodes[3]) ||
+		(charCode >= hexadecimalCharCodes[4] && charCode <= hexadecimalCharCodes[5])
+	);
+}
+
+/**
+ * Generate a random session ID
+ */
+export function createSessionId(): SessionId {
+	return assertIsStableId(v4()) as SessionId;
+}
+
+/**
+ * Asserts that the given string is a stable ID.
+ */
+export function assertIsStableId(stableId: string): StableId {
+	assert(isStableId(stableId), 0x4a3 /* Expected a StableId */);
+	return stableId;
+}
+
+/**
+ * Asserts that the given string is a stable ID.
+ */
+export function assertIsSessionId(stableId: string): SessionId {
+	assertIsStableId(stableId);
+	return stableId as SessionId;
+}
+
+/**
+ * Generate a random stable ID
+ */
+export function generateStableId(): StableId {
+	return assertIsStableId(v4());
+}
+
+/**
+ * Returns true iff the given string is a valid Version 4, variant 2 UUID
+ * 'xxxxxxxx-xxxx-4xxx-vxxx-xxxxxxxxxxxx'
+ */
+export function isStableId(str: string): str is StableId {
+	if (str.length !== 36) {
+		return false;
+	}
+
+	for (let i = 0; i < str.length; i++) {
+		switch (i) {
+			case 8:
+			case 13:
+			case 18:
+			case 23:
+				if (str.charAt(i) !== "-") {
+					return false;
+				}
+				break;
+
+			case 14:
+				if (str.charAt(i) !== "4") {
+					return false;
+				}
+				break;
+
+			case 19: {
+				const char = str.charAt(i);
+				if (char !== "8" && char !== "9" && char !== "a" && char !== "b") {
+					return false;
+				}
+				break;
+			}
+
+			default:
+				if (!isHexadecimalCharacter(str.charCodeAt(i))) {
+					return false;
+				}
+				break;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * A numeric comparator used for sorting in ascending order.
+ *
+ * Handles +/-0 like Map: -0 is equal to +0.
+ */
+export function compareFiniteNumbers<T extends number>(a: T, b: T): number {
+	return a - b;
+}
+
+/**
+ * Compares strings lexically to form a strict partial ordering.
+ */
+export function compareStrings<T extends string>(a: T, b: T): number {
+	return a > b ? 1 : a === b ? 0 : -1;
+}
+
+/**
+ * Compares bigints to form a strict partial ordering.
+ */
+export function compareBigints<T extends bigint>(a: T, b: T): number {
+	return a > b ? 1 : a === b ? 0 : -1;
+}
+
+export function genCountFromLocalId(localId: LocalCompressedId): number {
+	return -localId;
+}
+
+export function localIdFromGenCount(genCount: number): LocalCompressedId {
+	return -genCount as LocalCompressedId;
+}
+
+// xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx
+const versionMask = 0x4n << (19n * 4n); // Version 4
+const variantMask = 0x8n << (15n * 4n); // Variant RFC4122 (1 0 x x)
+const upperMask = 0xffffffffffffn << (20n * 4n);
+// Upper mask when version/variant bits are removed
+const strippedUpperMask = upperMask >> 6n;
+const middieBittiesMask = 0xfffn << (16n * 4n);
+// Middie mask when version/variant bits are removed
+const strippedMiddieBittiesMask = middieBittiesMask >> 2n;
+// Note: leading character should be 3 to mask at 0011
+// The more-significant half of the N nibble is used to denote the variant (10xx)
+const lowerMask = 0x3fffffffffffffffn;
+
+export function numericUuidFromStableId(stableId: StableId): NumericUuid {
+	const uuidU128 = BigInt(`0x${stableId.replace(/-/g, "")}`);
+	const upperMasked = uuidU128 & upperMask;
+	const middieBittiesMasked = uuidU128 & middieBittiesMask;
+	const lowerMasked = uuidU128 & lowerMask;
+
+	const upperMaskedPlaced = upperMasked >> 6n;
+	const middieBittiesMaskedPlaced = middieBittiesMasked >> 2n;
+
+	const id = upperMaskedPlaced | middieBittiesMaskedPlaced | lowerMasked;
+	return id as NumericUuid;
+}
+
+export function stableIdFromNumericUuid(numericUuid: NumericUuid): StableId {
+	// bitwise reverse transform
+	const upperMasked = (numericUuid & strippedUpperMask) << 6n;
+	const middieBittiesMasked = (numericUuid & strippedMiddieBittiesMask) << 2n;
+	const lowerMasked = numericUuid & lowerMask;
+	const uuidU128 = upperMasked | versionMask | middieBittiesMasked | variantMask | lowerMasked;
+	// Pad to 32 characters, inserting leading zeroes if needed
+	const uuidString = uuidU128.toString(16).padStart(32, "0");
+	return `${uuidString.substring(0, 8)}-${uuidString.substring(8, 12)}-${uuidString.substring(
+		12,
+		16,
+	)}-${uuidString.substring(16, 20)}-${uuidString.substring(20, 32)}` as StableId;
+}
+
+export function offsetNumericUuid(numericUuid: NumericUuid, offset: number): NumericUuid {
+	// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+	return (numericUuid + BigInt(offset)) as NumericUuid;
+}
+
+export function subtractNumericUuids(a: NumericUuid, b: NumericUuid): NumericUuid {
+	return (a - b) as NumericUuid;
+}
+
+export function addNumericUuids(a: NumericUuid, b: NumericUuid): NumericUuid {
+	// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+	return (a + b) as NumericUuid;
+}

package/src/index.ts

@@ -6,6 +6,8 @@
 export {
 	ContainerMessageType,
 	ContainerRuntimeMessage,
+	IContainerRuntimeMessageCompatDetails,
+	CompatModeBehavior,
 	ISummaryRuntimeOptions,
 	ISummaryBaseConfiguration,
 	ISummaryConfigurationHeuristics,
@@ -18,7 +20,9 @@ export {
 	ContainerRuntime,
 	RuntimeHeaders,
 	AllowTombstoneRequestHeaderKey,
+	AllowInactiveRequestHeaderKey,
 	TombstoneResponseHeaderKey,
+	InactiveResponseHeaderKey,
 	ISummaryConfiguration,
 	DefaultSummaryConfiguration,
 	ICompressionRuntimeOptions,
@@ -68,6 +72,8 @@ export {
 	ICancellableSummarizerController,
 	SubmitSummaryFailureData,
 	SummaryStage,
+	IRetriableFailureResult,
+	ISummarizeEventProps,
 } from "./summary";
+export { isStableId, generateStableId, assertIsStableId } from "./id-compressor";
 export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";
-export { generateStableId, isStableId, assertIsStableId } from "./id-compressor";

package/src/opLifecycle/opCompressor.ts

@@ -3,9 +3,9 @@
  * Licensed under the MIT License.
  */
 
-import { createChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, IsoBuffer } from "@fluidframework/common-utils";
-import { UsageError } from "@fluidframework/container-utils";
+import { createChildLogger, UsageError } from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer } from "@fluid-internal/client-utils";
 import { compress } from "lz4js";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { CompressionAlgorithms } from "../containerRuntime";

package/src/opLifecycle/opDecompressor.ts

@@ -5,7 +5,8 @@
 
 import { decompress } from "lz4js";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer, Uint8ArrayToString } from "@fluid-internal/client-utils";
 import { createChildLogger } from "@fluidframework/telemetry-utils";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { CompressionAlgorithms } from "../containerRuntime";

package/src/opLifecycle/opGroupingManager.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { assert } from "@fluidframework/common-utils";
+import { assert } from "@fluidframework/core-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { ContainerMessageType } from "..";
 import { IBatch } from "./definitions";

package/src/opLifecycle/opSplitter.ts

@@ -3,13 +3,13 @@
  * Licensed under the MIT License.
  */
 
-import { createChildLogger } from "@fluidframework/telemetry-utils";
-import { assert } from "@fluidframework/common-utils";
-import { IBatchMessage } from "@fluidframework/container-definitions";
 import {
+	createChildLogger,
 	DataCorruptionError,
 	extractSafePropertiesFromMessage,
-} from "@fluidframework/container-utils";
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IBatchMessage } from "@fluidframework/container-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";

package/src/opLifecycle/outbox.ts

@@ -3,10 +3,14 @@
  * Licensed under the MIT License.
  */
 
-import { createChildMonitoringContext, MonitoringContext } from "@fluidframework/telemetry-utils";
-import { assert } from "@fluidframework/common-utils";
+import {
+	createChildMonitoringContext,
+	GenericError,
+	MonitoringContext,
+	UsageError,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
 import { IBatchMessage, ICriticalContainerError } from "@fluidframework/container-definitions";
-import { GenericError, UsageError } from "@fluidframework/container-utils";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { ICompressionRuntimeOptions } from "../containerRuntime";
 import { IPendingBatchMessage, PendingStateManager } from "../pendingStateManager";
@@ -63,9 +67,8 @@ export function getLongStack<T>(action: () => T, length: number = 50): T {
 	if (
 		(
 			Object.getOwnPropertyDescriptor(errorObj, "stackTraceLimit") ||
-			Object.getOwnPropertyDescriptor(Object.getPrototypeOf(errorObj), "stackTraceLimit") ||
-			{}
-		).writable !== true
+			Object.getOwnPropertyDescriptor(Object.getPrototypeOf(errorObj), "stackTraceLimit")
+		)?.writable !== true
 	) {
 		return action();
 	}
@@ -111,12 +114,12 @@ export class Outbox {
 		this.blobAttachBatch = new BatchManager({ hardLimit });
 	}
 
+	public get messageCount(): number {
+		return this.attachFlowBatch.length + this.mainBatch.length + this.blobAttachBatch.length;
+	}
+
 	public get isEmpty(): boolean {
-		return (
-			this.attachFlowBatch.length === 0 &&
-			this.mainBatch.length === 0 &&
-			this.blobAttachBatch.length === 0
-		);
+		return this.messageCount === 0;
 	}
 
 	/**

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -4,7 +4,11 @@
  */
 
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
-import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
+import {
+	ContainerMessageType,
+	ContainerRuntimeMessage,
+	SequencedContainerRuntimeMessage,
+} from "../containerRuntime";
 import { OpDecompressor } from "./opDecompressor";
 import { OpGroupingManager } from "./opGroupingManager";
 import { OpSplitter } from "./opSplitter";
@@ -98,14 +102,23 @@ const copy = (remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessa
 };
 
 /**
- * For a given message, it moves the nested contents and type on level up.
+ * For a given message, it moves the nested ContainerRuntimeMessage props one level up.
  *
+ * The return type illustrates the assumption that the message param
+ * becomes a ContainerRuntimeMessage by the time the function returns
+ * (but there is no runtime validation of the 'type' or 'compatDetails' values)
  */
-const unpack = (message: ISequencedDocumentMessage) => {
+function unpack(
+	message: ISequencedDocumentMessage,
+): asserts message is SequencedContainerRuntimeMessage {
 	const innerContents = message.contents as ContainerRuntimeMessage;
-	message.type = innerContents.type;
-	message.contents = innerContents.contents;
-};
+
+	// We're going to turn message into a SequencedContainerRuntimeMessage in-place
+	const sequencedContainerRuntimeMessage = message as SequencedContainerRuntimeMessage;
+	sequencedContainerRuntimeMessage.type = innerContents.type;
+	sequencedContainerRuntimeMessage.contents = innerContents.contents;
+	sequencedContainerRuntimeMessage.compatDetails = innerContents.compatDetails;
+}
 
 /**
  * Unpacks runtime messages.

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.6.1.1";
+export const pkgVersion = "2.0.0-internal.6.3.0";

package/src/pendingStateManager.ts

@@ -3,15 +3,15 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable } from "@fluidframework/common-definitions";
-import { assert } from "@fluidframework/common-utils";
+import Deque from "double-ended-queue";
+
+import { IDisposable } from "@fluidframework/core-interfaces";
+import { assert, Lazy } from "@fluidframework/core-utils";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { DataProcessingError } from "@fluidframework/container-utils";
-import { Lazy } from "@fluidframework/core-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import Deque from "double-ended-queue";
-import { ContainerMessageType } from "./containerRuntime";
+import { DataProcessingError, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+
+import { ContainerMessageType, SequencedContainerRuntimeMessage } from "./containerRuntime";
 import { pkgVersion } from "./packageVersion";
 import { IBatchMetadata } from "./metadata";
 
@@ -81,15 +81,17 @@ export interface IRuntimeStateHandler {
 export class PendingStateManager implements IDisposable {
 	private readonly pendingMessages = new Deque<IPendingMessageNew>();
 	private readonly initialMessages = new Deque<IPendingMessageNew>();
+
+	/**
+	 * Sequenced local ops that are saved when stashing since pending ops may depend on them
+	 */
+	private savedOps: IPendingMessageNew[] = [];
+
 	private readonly disposeOnce = new Lazy<void>(() => {
 		this.initialMessages.clear();
 		this.pendingMessages.clear();
 	});
 
-	public get pendingMessagesCount(): number {
-		return this.pendingMessages.length;
-	}
-
 	// Indicates whether we are processing a batch.
 	private isProcessingBatch: boolean = false;
 
@@ -99,12 +101,20 @@ export class PendingStateManager implements IDisposable {
 
 	private clientId: string | undefined;
 
+	/**
+	 * The pending messages count. Includes `pendingMessages` and `initialMessages` to keep in sync with
+	 * 'hasPendingMessages'.
+	 */
+	public get pendingMessagesCount(): number {
+		return this.pendingMessages.length + this.initialMessages.length;
+	}
+
 	/**
 	 * Called to check if there are any pending messages in the pending message queue.
 	 * @returns A boolean indicating whether there are messages or not.
 	 */
 	public hasPendingMessages(): boolean {
-		return !this.pendingMessages.isEmpty() || !this.initialMessages.isEmpty();
+		return this.pendingMessagesCount !== 0;
 	}
 
 	public getLocalState(): IPendingLocalState | undefined {
@@ -114,20 +124,22 @@ export class PendingStateManager implements IDisposable {
 		);
 		if (!this.pendingMessages.isEmpty()) {
 			return {
-				pendingStates: this.pendingMessages.toArray().map((message) => {
-					let content = message.content;
-					const parsedContent = JSON.parse(content);
-					// IdAllocations need their localOpMetadata stashed in the contents
-					// of the op to correctly resume the session when processing stashed ops
-					if (parsedContent.type === ContainerMessageType.IdAllocation) {
-						parsedContent.contents.stashedState = message.localOpMetadata;
-						content = JSON.stringify(parsedContent);
-					}
-
-					// delete localOpMetadata since it may not be serializable
-					// and will be regenerated by applyStashedOp()
-					return { ...message, content, localOpMetadata: undefined };
-				}),
+				pendingStates: [...this.savedOps, ...this.pendingMessages.toArray()].map(
+					(message) => {
+						let content = message.content;
+						const parsedContent = JSON.parse(content);
+						// IdAllocations need their localOpMetadata stashed in the contents
+						// of the op to correctly resume the session when processing stashed ops
+						if (parsedContent.type === ContainerMessageType.IdAllocation) {
+							parsedContent.contents.stashedState = message.localOpMetadata;
+							content = JSON.stringify(parsedContent);
+						}
+
+						// delete localOpMetadata since it may not be serializable
+						// and will be regenerated by applyStashedOp()
+						return { ...message, content, localOpMetadata: undefined };
+					},
+				),
 			};
 		}
 	}
@@ -230,7 +242,7 @@ export class PendingStateManager implements IDisposable {
 	 * the batch information was preserved for batch messages.
 	 * @param message - The message that got ack'd and needs to be processed.
 	 */
-	public processPendingLocalMessage(message: ISequencedDocumentMessage): unknown {
+	public processPendingLocalMessage(message: SequencedContainerRuntimeMessage): unknown {
 		// Pre-processing part - This may be the start of a batch.
 		this.maybeProcessBatchBegin(message);
 
@@ -240,10 +252,16 @@ export class PendingStateManager implements IDisposable {
 			pendingMessage !== undefined,
 			0x169 /* "No pending message found for this remote message" */,
 		);
+		this.savedOps.push(pendingMessage);
+
 		this.pendingMessages.shift();
 
-		const messageContent = JSON.stringify({ type: message.type, contents: message.contents });
-		// Stringified content does not match
+		// IMPORTANT: Order matters here, this must match the order of the properties used
+		// when submitting the message.
+		const { type, contents, compatDetails } = message;
+		const messageContent = JSON.stringify({ type, contents, compatDetails });
+
+		// Stringified content should match
 		if (pendingMessage.content !== messageContent) {
 			this.stateHandler.close(
 				DataProcessingError.create(
@@ -424,6 +442,9 @@ export class PendingStateManager implements IDisposable {
 			}
 		}
 
+		// pending ops should no longer depend on previous sequenced local ops after resubmit
+		this.savedOps = [];
+
 		// We replayPendingStates on read connections too - we expect these to get nack'd though, and to then reconnect
 		// on a write connection and replay again. This filters out the replay that happens on the read connection so
 		// we only see the replays on write connections (that have a chance to go through).

package/src/scheduleManager.ts

@@ -5,14 +5,16 @@
 import { EventEmitter } from "events";
 import { IDeltaManager } from "@fluidframework/container-definitions";
 import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt, createChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, performance } from "@fluidframework/common-utils";
-import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import {
+	createChildLogger,
 	DataCorruptionError,
 	DataProcessingError,
 	extractSafePropertiesFromMessage,
-} from "@fluidframework/container-utils";
+	ITelemetryLoggerExt,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { performance } from "@fluid-internal/client-utils";
+import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import { DeltaScheduler } from "./deltaScheduler";
 import { pkgVersion } from "./packageVersion";
 import { IBatchMetadata } from "./metadata";

package/src/summary/index.ts

@@ -11,7 +11,7 @@ export {
 	OrderedClientCollection,
 	OrderedClientElection,
 } from "./orderedClientElection";
-export { RunningSummarizer } from "./runningSummarizer";
+export { defaultMaxAttemptsForSubmitFailures, RunningSummarizer } from "./runningSummarizer";
 export {
 	ICancellableSummarizerController,
 	neverCancelledSummaryToken,
@@ -28,10 +28,9 @@ export { SummarizeHeuristicData, SummarizeHeuristicRunner } from "./summarizerHe
 export {
 	createRootSummarizerNode,
 	createRootSummarizerNodeWithGC,
-	IFetchSnapshotResult,
+	IRefreshSummaryResult,
 	IRootSummarizerNode,
 	IRootSummarizerNodeWithGC,
-	RefreshSummaryResult,
 } from "./summarizerNode";
 export {
 	IConnectableRuntime,
@@ -64,6 +63,8 @@ export {
 	SummarizeResultPart,
 	SubmitSummaryFailureData,
 	SummaryStage,
+	IRetriableFailureResult,
+	ISummarizeEventProps,
 } from "./summarizerTypes";
 export {
 	IAckedSummary,

package/src/summary/orderedClientElection.ts

@@ -3,13 +3,16 @@
  * Licensed under the MIT License.
  */
 /* eslint-disable @rushstack/no-new-null */
-import { IEvent, IEventProvider } from "@fluidframework/common-definitions";
-import { ITelemetryLoggerExt, createChildLogger } from "@fluidframework/telemetry-utils";
-import { assert, TypedEventEmitter } from "@fluidframework/common-utils";
+import { IEvent, IEventProvider, ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import {
+	ITelemetryLoggerExt,
+	createChildLogger,
+	UsageError,
+} from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { TypedEventEmitter } from "@fluid-internal/client-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";
-import { UsageError } from "@fluidframework/container-utils";
 import { IClient, IQuorumClients, ISequencedClient } from "@fluidframework/protocol-definitions";
-import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { summarizerClientType } from "./summarizerClientElection";
 
 // helper types for recursive readonly.

package/src/summary/runWhileConnectedCoordinator.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { assert, Deferred } from "@fluidframework/common-utils";
+import { assert, Deferred } from "@fluidframework/core-utils";
 import {
 	SummarizerStopReason,
 	IConnectableRuntime,