@fluidframework/container-runtime 2.11.0 → 2.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.11.0",
+  "version": "2.13.0",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -129,39 +129,39 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "~2.11.0",
-    "@fluidframework/container-definitions": "~2.11.0",
-    "@fluidframework/container-runtime-definitions": "~2.11.0",
-    "@fluidframework/core-interfaces": "~2.11.0",
-    "@fluidframework/core-utils": "~2.11.0",
-    "@fluidframework/datastore": "~2.11.0",
-    "@fluidframework/driver-definitions": "~2.11.0",
-    "@fluidframework/driver-utils": "~2.11.0",
-    "@fluidframework/id-compressor": "~2.11.0",
-    "@fluidframework/runtime-definitions": "~2.11.0",
-    "@fluidframework/runtime-utils": "~2.11.0",
-    "@fluidframework/telemetry-utils": "~2.11.0",
+    "@fluid-internal/client-utils": "~2.13.0",
+    "@fluidframework/container-definitions": "~2.13.0",
+    "@fluidframework/container-runtime-definitions": "~2.13.0",
+    "@fluidframework/core-interfaces": "~2.13.0",
+    "@fluidframework/core-utils": "~2.13.0",
+    "@fluidframework/datastore": "~2.13.0",
+    "@fluidframework/driver-definitions": "~2.13.0",
+    "@fluidframework/driver-utils": "~2.13.0",
+    "@fluidframework/id-compressor": "~2.13.0",
+    "@fluidframework/runtime-definitions": "~2.13.0",
+    "@fluidframework/runtime-utils": "~2.13.0",
+    "@fluidframework/telemetry-utils": "~2.13.0",
     "@tylerbu/sorted-btree-es6": "^1.8.0",
     "double-ended-queue": "^2.1.0-0",
     "lz4js": "^0.2.0",
     "uuid": "^9.0.0"
   },
   "devDependencies": {
-    "@arethetypeswrong/cli": "^0.16.4",
+    "@arethetypeswrong/cli": "^0.17.1",
     "@biomejs/biome": "~1.9.3",
-    "@fluid-internal/mocha-test-setup": "~2.11.0",
-    "@fluid-private/stochastic-test-utils": "~2.11.0",
-    "@fluid-private/test-pairwise-generator": "~2.11.0",
+    "@fluid-internal/mocha-test-setup": "~2.13.0",
+    "@fluid-private/stochastic-test-utils": "~2.13.0",
+    "@fluid-private/test-pairwise-generator": "~2.13.0",
     "@fluid-tools/benchmark": "^0.50.0",
     "@fluid-tools/build-cli": "^0.51.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.51.0",
-    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.10.0",
+    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.12.0",
     "@fluidframework/eslint-config-fluid": "^5.6.0",
-    "@fluidframework/test-runtime-utils": "~2.11.0",
+    "@fluidframework/test-runtime-utils": "~2.13.0",
     "@microsoft/api-extractor": "7.47.8",
     "@types/double-ended-queue": "^2.1.0",
-    "@types/mocha": "^9.1.1",
+    "@types/mocha": "^10.0.10",
     "@types/node": "^18.19.0",
     "@types/sinon": "^17.0.3",
     "@types/uuid": "^9.0.2",

package/src/blobManager/blobManagerSnapSum.ts

@@ -47,7 +47,7 @@ const loadV1 = async (
 		return {};
 	}
 	let redirectTableEntries: [string, string][] = [];
-	const tableId = blobsTree.blobs[redirectTableBlobName];
+	const tableId: string | undefined = blobsTree.blobs[redirectTableBlobName];
 	if (tableId) {
 		redirectTableEntries = await readAndParse(context.storage, tableId);
 	}

package/src/channelCollection.ts

@@ -1551,7 +1551,7 @@ export function getSummaryForDatastores(
 	}
 
 	if (rootHasIsolatedChannels(metadata)) {
-		const datastoresSnapshot = snapshot.trees[channelsTreeName];
+		const datastoresSnapshot: ISnapshotTree | undefined = snapshot.trees[channelsTreeName];
 		assert(!!datastoresSnapshot, 0x168 /* Expected tree in snapshot not found */);
 		return datastoresSnapshot;
 	} else {

package/src/containerRuntime.ts

@@ -64,6 +64,7 @@ import {
 	ISequencedDocumentMessage,
 	ISignalMessage,
 	type ISummaryContext,
+	type SummaryObject,
 } from "@fluidframework/driver-definitions/internal";
 import { readAndParse } from "@fluidframework/driver-utils/internal";
 import type { IIdCompressor } from "@fluidframework/id-compressor";
@@ -218,7 +219,6 @@ import {
 	ISubmitSummaryOptions,
 	ISummarizeResults,
 	ISummarizer,
-	ISummarizerEvents,
 	ISummarizerInternalsProvider,
 	ISummarizerRuntime,
 	ISummaryMetadataMessage,
@@ -475,6 +475,8 @@ export interface IContainerRuntimeOptions {
 	 * send all operations to the driver layer, while in TurnBased the operations will be buffered
 	 * and then sent them as a single batch at the end of the turn.
 	 * By default, flush mode is TurnBased.
+	 *
+	 * @deprecated Only the default value TurnBased is supported. This option will be removed in the future.
 	 */
 	readonly flushMode?: FlushMode;
 	/**
@@ -516,9 +518,10 @@ export interface IContainerRuntimeOptions {
 	/**
 	 * If enabled, the runtime will group messages within a batch into a single
 	 * message to be sent to the service.
-	 * The grouping an ungrouping of such messages is handled by the "OpGroupingManager".
+	 * The grouping and ungrouping of such messages is handled by the "OpGroupingManager".
 	 *
 	 * By default, the feature is enabled.
+	 * @deprecated  The ability to disable Grouped Batching is deprecated and will be removed in v2.20.0. This feature is required for the proper functioning of the Fluid Framework.
 	 */
 	readonly enableGroupedBatching?: boolean;
 
@@ -532,6 +535,31 @@ export interface IContainerRuntimeOptions {
 	readonly explicitSchemaControl?: boolean;
 }
 
+/**
+ * Internal extension of @see IContainerRuntimeOptions
+ *
+ * These options are not available to consumers when creating a new container runtime,
+ * but we do need to expose them for internal use, e.g. when configuring the container runtime
+ * to ensure compability with older versions.
+ *
+ * @internal
+ */
+export interface IContainerRuntimeOptionsInternal extends IContainerRuntimeOptions {
+	/**
+	 * Sets the flush mode for the runtime. In Immediate flush mode the runtime will immediately
+	 * send all operations to the driver layer, while in TurnBased the operations will be buffered
+	 * and then sent them as a single batch at the end of the turn.
+	 * By default, flush mode is TurnBased.
+	 */
+	readonly flushMode?: FlushMode;
+
+	/**
+	 * Allows Grouped Batching to be disabled by setting to false (default is true).
+	 * In that case, batched messages will be sent individually (but still all at the same time).
+	 */
+	readonly enableGroupedBatching?: boolean;
+}
+
 /**
  * Error responses when requesting a deleted object will have this header set to true
  * @legacy
@@ -836,11 +864,15 @@ export async function loadContainerRuntime(
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
+ *
+ * @deprecated To be removed from the Legacy-Alpha API in version 2.20.0.
+ * Use the loadContainerRuntime function and interfaces IContainerRuntime / IRuntime instead.
+ *
  * @legacy
  * @alpha
  */
 export class ContainerRuntime
-	extends TypedEventEmitter<IContainerRuntimeEvents & ISummarizerEvents>
+	extends TypedEventEmitter<IContainerRuntimeEvents>
 	implements
 		IContainerRuntime,
 		IRuntime,
@@ -867,7 +899,7 @@ export class ContainerRuntime
 		context: IContainerContext;
 		registryEntries: NamedFluidDataStoreRegistryEntries;
 		existing: boolean;
-		runtimeOptions?: IContainerRuntimeOptions;
+		runtimeOptions?: IContainerRuntimeOptions; // May also include options from IContainerRuntimeOptionsInternal
 		containerScope?: FluidObject;
 		containerRuntimeCtor?: typeof ContainerRuntime;
 		/** @deprecated Will be removed once Loader LTS version is "2.0.0-internal.7.0.0". Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md */
@@ -914,7 +946,7 @@ export class ContainerRuntime
 			chunkSizeInBytes = defaultChunkSizeInBytes,
 			enableGroupedBatching = true,
 			explicitSchemaControl = false,
-		} = runtimeOptions;
+		}: IContainerRuntimeOptionsInternal = runtimeOptions;
 
 		const registry = new FluidDataStoreRegistry(registryEntries);
 
@@ -1093,6 +1125,21 @@ export class ContainerRuntime
 
 		const featureGatesForTelemetry: Record<string, boolean | number | undefined> = {};
 
+		// Make sure we've got all the options including internal ones
+		const internalRuntimeOptions: Readonly<Required<IContainerRuntimeOptionsInternal>> = {
+			summaryOptions,
+			gcOptions,
+			loadSequenceNumberVerification,
+			flushMode,
+			compressionOptions,
+			maxBatchSizeInBytes,
+			chunkSizeInBytes,
+			// Requires<> drops undefined from IdCompressorType
+			enableRuntimeIdCompressor: enableRuntimeIdCompressor as "on" | "delayed",
+			enableGroupedBatching,
+			explicitSchemaControl,
+		};
+
 		const runtime = new containerRuntimeCtor(
 			context,
 			registry,
@@ -1100,19 +1147,7 @@ export class ContainerRuntime
 			electedSummarizerData,
 			chunks ?? [],
 			aliases ?? [],
-			{
-				summaryOptions,
-				gcOptions,
-				loadSequenceNumberVerification,
-				flushMode,
-				compressionOptions,
-				maxBatchSizeInBytes,
-				chunkSizeInBytes,
-				// Requires<> drops undefined from IdCompressorType
-				enableRuntimeIdCompressor: enableRuntimeIdCompressor as "on" | "delayed",
-				enableGroupedBatching,
-				explicitSchemaControl,
-			},
+			internalRuntimeOptions,
 			containerScope,
 			logger,
 			existing,
@@ -1474,6 +1509,11 @@ export class ContainerRuntime
 		expiry: { policy: "absolute", durationMs: 60000 },
 	});
 
+	/**
+	 * The options to apply to this ContainerRuntime instance (including internal options hidden from the public API)
+	 */
+	private readonly runtimeOptions: Readonly<Required<IContainerRuntimeOptionsInternal>>;
+
 	/***/
 	protected constructor(
 		context: IContainerContext,
@@ -1482,7 +1522,10 @@ export class ContainerRuntime
 		electedSummarizerData: ISerializedElection | undefined,
 		chunks: [string, string[]][],
 		dataStoreAliasMap: [string, string][],
-		private readonly runtimeOptions: Readonly<Required<IContainerRuntimeOptions>>,
+		runtimeOptions: Readonly<
+			Required<Omit<IContainerRuntimeOptions, "flushMode" | "enableGroupedBatching">> &
+				IContainerRuntimeOptions // Let flushMode and enabledGroupedBatching be optional now since they're soon to be removed
+		>,
 		private readonly containerScope: FluidObject,
 		// Create a custom ITelemetryBaseLogger to output telemetry events.
 		public readonly baseLogger: ITelemetryBaseLogger,
@@ -1527,6 +1570,12 @@ export class ContainerRuntime
 			snapshotWithContents,
 		} = context;
 
+		// Backfill in defaults for the internal runtimeOptions, since they may not be present on the provided runtimeOptions object
+		this.runtimeOptions = {
+			flushMode: defaultFlushMode,
+			enableGroupedBatching: true,
+			...runtimeOptions,
+		};
 		this.logger = createChildLogger({ logger: this.baseLogger });
 		this.mc = createChildMonitoringContext({
 			logger: this.logger,
@@ -1697,14 +1746,15 @@ export class ContainerRuntime
 			this.defaultMaxConsecutiveReconnects;
 
 		if (
-			runtimeOptions.flushMode === (FlushModeExperimental.Async as unknown as FlushMode) &&
+			this.runtimeOptions.flushMode ===
+				(FlushModeExperimental.Async as unknown as FlushMode) &&
 			supportedFeatures?.get("referenceSequenceNumbers") !== true
 		) {
 			// The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
 			this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
 			this._flushMode = FlushMode.TurnBased;
 		} else {
-			this._flushMode = runtimeOptions.flushMode;
+			this._flushMode = this.runtimeOptions.flushMode;
 		}
 		this.offlineEnabled =
 			this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad") ?? false;
@@ -2015,9 +2065,19 @@ export class ContainerRuntime
 						initialDelayMs: this.initialSummarizerDelayMs,
 					},
 				);
-				this.summaryManager.on("summarize", (eventProps) => {
-					this.emit("summarize", eventProps);
+				// Forward events from SummaryManager
+				[
+					"summarize",
+					"summarizeAllAttemptsFailed",
+					"summarizerStop",
+					"summarizerStart",
+					"summarizerStartupFailed",
+				].forEach((eventName) => {
+					this.summaryManager?.on(eventName, (...args: any[]) => {
+						this.emit(eventName, ...args);
+					});
 				});
+
 				this.summaryManager.start();
 			}
 		}
@@ -4111,7 +4171,7 @@ export class ContainerRuntime
 			// Counting dataStores and handles
 			// Because handles are unchanged dataStores in the current logic,
 			// summarized dataStore count is total dataStore count minus handle count
-			const dataStoreTree = summaryTree.tree[channelsTreeName];
+			const dataStoreTree: SummaryObject | undefined = summaryTree.tree[channelsTreeName];
 
 			assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
 			const handleCount = Object.values(dataStoreTree.tree).filter(

package/src/gc/garbageCollection.ts

@@ -5,6 +5,7 @@
 
 import { IRequest } from "@fluidframework/core-interfaces";
 import { assert, LazyPromise, Timer } from "@fluidframework/core-utils/internal";
+import type { ISnapshotTree } from "@fluidframework/driver-definitions/internal";
 import {
 	IGarbageCollectionDetailsBase,
 	ISummarizeResult,
@@ -226,7 +227,7 @@ export class GarbageCollector implements IGarbageCollector {
 
 				try {
 					// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
-					const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
+					const gcSnapshotTree: ISnapshotTree | undefined = baseSnapshot.trees[gcTreeKey];
 					if (gcSnapshotTree === undefined) {
 						// back-compat - Older documents get their gc data reset for simplicity as there are few of them
 						// incremental gc summary will not work with older gc data as well

package/src/gc/gcConfigs.ts

@@ -87,7 +87,7 @@ export function generateGCConfigs(
 		tombstoneTimeoutMs =
 			testOverrideTombstoneTimeoutMs ?? computeTombstoneTimeout(sessionExpiryTimeoutMs);
 
-		const gcGeneration = createParams.gcOptions[gcGenerationOptionName];
+		const gcGeneration = createParams.gcOptions?.[gcGenerationOptionName];
 		if (gcGeneration !== undefined) {
 			persistedGcFeatureMatrix = { gcGeneration };
 		}

package/src/gc/gcHelpers.ts

@@ -109,7 +109,7 @@ export function concatGarbageCollectionStates(
 	}
 
 	for (const [nodeId, nodeData] of Object.entries(gcState2.gcNodes)) {
-		let combineNodeData = combinedGCNodes[nodeId];
+		let combineNodeData: IGarbageCollectionNodeData | undefined = combinedGCNodes[nodeId];
 		if (combineNodeData === undefined) {
 			combineNodeData = {
 				outboundRoutes: Array.from(nodeData.outboundRoutes),
@@ -202,7 +202,7 @@ export async function getGCDataFromSnapshot(
 			continue;
 		}
 
-		const blobId = gcSnapshotTree.blobs[key];
+		const blobId: string | undefined = gcSnapshotTree.blobs[key];
 		if (blobId === undefined) {
 			continue;
 		}

package/src/gc/gcReferenceGraphAlgorithm.ts

@@ -31,7 +31,7 @@ export function runGarbageCollection(
 
 		// Get the node for the referenced id and add its outbound routes to referencedIds since they are
 		// also referenced.
-		const routes = referenceGraph[id];
+		const routes: string[] | undefined = referenceGraph[id];
 		if (routes !== undefined) {
 			referencedIds.push(...routes);
 		}

package/src/index.ts

@@ -10,6 +10,7 @@ export {
 	ISummaryConfigurationDisableSummarizer,
 	ISummaryConfigurationDisableHeuristics,
 	IContainerRuntimeOptions,
+	IContainerRuntimeOptionsInternal,
 	loadContainerRuntime,
 	LoadContainerRuntimeParams,
 	agentSchedulerId,

package/src/opLifecycle/opDecompressor.ts

@@ -168,7 +168,7 @@ const newMessage = (
 	contents,
 	compression: undefined,
 	// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
-	// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+
 	metadata:
 		originalMessage.metadata === undefined
 			? undefined

package/src/opLifecycle/opSplitter.ts

@@ -190,7 +190,7 @@ export class OpSplitter {
 		const contents: IChunkedContents = message.contents;
 
 		// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
-		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+
 		const clientId = message.clientId as string;
 		const chunkedContent = contents.contents;
 		this.addChunk(clientId, chunkedContent, message);

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.11.0";
+export const pkgVersion = "2.13.0";

package/src/pendingStateManager.ts

@@ -131,6 +131,28 @@ function scrubAndStringify(
 	return JSON.stringify(scrubbed);
 }
 
+/**
+ * @returns The index where the strings diverge, and the character at that index in each string (or undefined if not applicable)
+ */
+export function findFirstCharacterMismatched(
+	a: string,
+	b: string,
+): [index: number, charA?: string, charB?: string] {
+	const minLength = Math.min(a.length, b.length);
+	for (let i = 0; i < minLength; i++) {
+		if (a[i] !== b[i]) {
+			return [i, a[i], b[i]];
+		}
+	}
+
+	// Since we didn't return in the loop, the shorter string must be a prefix of the other.
+	// If they're the same length, return -1 to indicate they're identical.
+	// Otherwise, the next character of the longer one is where they differ. No need to return that next character.
+	return a.length === b.length
+		? [-1, undefined, undefined]
+		: [minLength, a[minLength], b[minLength]];
+}
+
 function withoutLocalOpMetadata(message: IPendingMessage): IPendingMessage {
 	return {
 		...message,
@@ -459,7 +481,17 @@ export class PendingStateManager implements IDisposable {
 			const messageContent = buildPendingMessageContent(message);
 
 			// Stringified content should match
+			// If it doesn't, collect as much info about the difference as possible (privacy-wise) and log it
 			if (pendingMessage.content !== messageContent) {
+				const [pendingLength, incomingLength] = [
+					pendingMessage.content.length,
+					messageContent.length,
+				];
+				const [mismatchStartIndex, pendingChar, incomingChar] = findFirstCharacterMismatched(
+					pendingMessage.content,
+					messageContent,
+				);
+
 				const pendingContentObj = JSON.parse(
 					pendingMessage.content,
 				) as LocalContainerRuntimeMessage;
@@ -467,6 +499,7 @@ export class PendingStateManager implements IDisposable {
 					messageContent,
 				) as InboundContainerRuntimeMessage;
 
+				// Compare inner contents object, since that both should be { type, contents }
 				const contentsMatch =
 					pendingContentObj.contents === incomingContentObj.contents ||
 					(pendingContentObj.contents !== undefined &&
@@ -480,6 +513,11 @@ export class PendingStateManager implements IDisposable {
 						pendingContentScrubbed: scrubAndStringify(pendingContentObj),
 						incomingContentScrubbed: scrubAndStringify(incomingContentObj),
 						contentsMatch,
+						pendingLength,
+						incomingLength,
+						mismatchStartIndex,
+						pendingChar,
+						incomingChar,
 					},
 				});
 

package/src/scheduleManager.ts

@@ -103,7 +103,7 @@ class ScheduleManagerCore {
 			// Set the batch flag to false on the last message to indicate the end of the send batch
 			const lastMessage = messages[messages.length - 1];
 			// TODO: It's not clear if this shallow clone is required, as opposed to just setting "batch" to false.
-			// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+
 			lastMessage.metadata = { ...(lastMessage.metadata as any), batch: false };
 		});
 
@@ -303,7 +303,7 @@ class ScheduleManagerCore {
 			);
 			this.pauseSequenceNumber = message.sequenceNumber;
 			// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
-			// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+
 			this.currentBatchClientId = message.clientId as string;
 			// Start of the batch
 			// Only pause processing if queue has no other ops!

package/src/summary/runWhileConnectedCoordinator.ts

@@ -3,13 +3,10 @@
  * Licensed under the MIT License.
  */
 
+import type { SummarizerStopReason } from "@fluidframework/container-runtime-definitions/internal";
 import { assert, Deferred } from "@fluidframework/core-utils/internal";
 
-import {
-	IConnectableRuntime,
-	ISummaryCancellationToken,
-	SummarizerStopReason,
-} from "./summarizerTypes.js";
+import { IConnectableRuntime, ISummaryCancellationToken } from "./summarizerTypes.js";
 
 /**
  * Similar to AbortController, but using promise instead of events