@fluidframework/container-runtime 2.32.0 → 2.33.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.32.0",
+  "version": "2.33.0",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -119,41 +119,43 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "~2.32.0",
-    "@fluidframework/container-definitions": "~2.32.0",
-    "@fluidframework/container-runtime-definitions": "~2.32.0",
-    "@fluidframework/core-interfaces": "~2.32.0",
-    "@fluidframework/core-utils": "~2.32.0",
-    "@fluidframework/datastore": "~2.32.0",
-    "@fluidframework/driver-definitions": "~2.32.0",
-    "@fluidframework/driver-utils": "~2.32.0",
-    "@fluidframework/id-compressor": "~2.32.0",
-    "@fluidframework/runtime-definitions": "~2.32.0",
-    "@fluidframework/runtime-utils": "~2.32.0",
-    "@fluidframework/telemetry-utils": "~2.32.0",
+    "@fluid-internal/client-utils": "~2.33.0",
+    "@fluidframework/container-definitions": "~2.33.0",
+    "@fluidframework/container-runtime-definitions": "~2.33.0",
+    "@fluidframework/core-interfaces": "~2.33.0",
+    "@fluidframework/core-utils": "~2.33.0",
+    "@fluidframework/datastore": "~2.33.0",
+    "@fluidframework/driver-definitions": "~2.33.0",
+    "@fluidframework/driver-utils": "~2.33.0",
+    "@fluidframework/id-compressor": "~2.33.0",
+    "@fluidframework/runtime-definitions": "~2.33.0",
+    "@fluidframework/runtime-utils": "~2.33.0",
+    "@fluidframework/telemetry-utils": "~2.33.0",
     "@tylerbu/sorted-btree-es6": "^1.8.0",
     "double-ended-queue": "^2.1.0-0",
     "lz4js": "^0.2.0",
+    "semver": "^7.7.1",
     "uuid": "^9.0.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.17.1",
     "@biomejs/biome": "~1.9.3",
-    "@fluid-internal/mocha-test-setup": "~2.32.0",
-    "@fluid-private/stochastic-test-utils": "~2.32.0",
-    "@fluid-private/test-pairwise-generator": "~2.32.0",
+    "@fluid-internal/mocha-test-setup": "~2.33.0",
+    "@fluid-private/stochastic-test-utils": "~2.33.0",
+    "@fluid-private/test-pairwise-generator": "~2.33.0",
     "@fluid-tools/benchmark": "^0.50.0",
     "@fluid-tools/build-cli": "^0.55.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.55.0",
-    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.31.0",
+    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.32.0",
     "@fluidframework/eslint-config-fluid": "^5.7.3",
-    "@fluidframework/test-runtime-utils": "~2.32.0",
-    "@microsoft/api-extractor": "7.50.1",
+    "@fluidframework/test-runtime-utils": "~2.33.0",
+    "@microsoft/api-extractor": "7.52.5",
     "@types/double-ended-queue": "^2.1.0",
     "@types/lz4js": "^0.2.0",
     "@types/mocha": "^10.0.10",
     "@types/node": "^18.19.0",
+    "@types/semver": "^7.7.0",
     "@types/sinon": "^17.0.3",
     "@types/uuid": "^9.0.2",
     "c8": "^8.0.1",
@@ -163,7 +165,6 @@
     "eslint": "~8.55.0",
     "mocha": "^10.8.2",
     "mocha-multi-reporters": "^1.5.1",
-    "moment": "^2.21.0",
     "rimraf": "^4.4.0",
     "sinon": "^18.0.1",
     "typescript": "~5.4.5"

package/src/blobManager/blobManager.ts

@@ -18,6 +18,7 @@ import type {
 	IEventProvider,
 	IFluidHandleContext,
 	IFluidHandleInternal,
+	IFluidHandleInternalPayloadPending,
 } from "@fluidframework/core-interfaces/internal";
 import { assert, Deferred } from "@fluidframework/core-utils/internal";
 import {
@@ -62,7 +63,10 @@ import {
  * DataObject.request() recognizes requests in the form of `/blobs/<id>`
  * and loads blob.
  */
-export class BlobHandle extends FluidHandleBase<ArrayBufferLike> {
+export class BlobHandle
+	extends FluidHandleBase<ArrayBufferLike>
+	implements IFluidHandleInternalPayloadPending<ArrayBufferLike>
+{
 	private attached: boolean = false;
 
 	public get isAttached(): boolean {
@@ -75,6 +79,7 @@ export class BlobHandle extends FluidHandleBase<ArrayBufferLike> {
 		public readonly path: string,
 		public readonly routeContext: IFluidHandleContext,
 		public get: () => Promise<ArrayBufferLike>,
+		public readonly payloadPending: boolean,
 		private readonly onAttachGraph?: () => void,
 	) {
 		super();
@@ -133,7 +138,8 @@ export interface IBlobManagerEvents extends IEvent {
 }
 
 interface IBlobManagerInternalEvents extends IEvent {
-	(event: "blobAttached", listener: (pending: PendingBlob) => void);
+	(event: "handleAttached", listener: (pending: PendingBlob) => void);
+	(event: "processedBlobAttach", listener: (localId: string, storageId: string) => void);
 }
 
 const stashedPendingBlobOverrides: Pick<
@@ -176,7 +182,7 @@ export class BlobManager {
 	 * we can resolve all pending blobs with the same storage ID even though they may have different local IDs. That's
 	 * because we know that the server will not delete the blob corresponding to that storage ID.
 	 */
-	private readonly opsInFlight: Map<string, string[]> = new Map();
+	private readonly opsInFlight: Map<string, Set<string>> = new Map();
 
 	private readonly sendBlobAttachOp: (localId: string, storageId?: string) => void;
 	private stopAttaching: boolean = false;
@@ -195,7 +201,7 @@ export class BlobManager {
 		new Map();
 	public readonly stashedBlobsUploadP: Promise<(void | ICreateBlobResponse)[]>;
 
-	constructor(props: {
+	public constructor(props: {
 		readonly routeContext: IFluidHandleContext;
 
 		blobManagerLoadInfo: IBlobManagerLoadInfo;
@@ -220,6 +226,7 @@ export class BlobManager {
 		readonly runtime: IBlobManagerRuntime;
 		stashedBlobs: IPendingBlobs | undefined;
 		readonly localBlobIdGenerator?: (() => string) | undefined;
+		readonly createBlobPayloadPending: boolean;
 	}) {
 		const {
 			routeContext,
@@ -351,7 +358,11 @@ export class BlobManager {
 		return [...this.pendingBlobs.values()].some((e) => e.stashedUpload === true);
 	}
 
-	public async getBlob(blobId: string): Promise<ArrayBufferLike> {
+	public hasBlob(blobId: string): boolean {
+		return this.redirectTable.get(blobId) !== undefined;
+	}
+
+	public async getBlob(blobId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
 		// Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
 		// an error, failing the call.
 		this.verifyBlobNotDeleted(blobId);
@@ -374,8 +385,24 @@ export class BlobManager {
 			storageId = blobId;
 		} else {
 			const attachedStorageId = this.redirectTable.get(blobId);
-			assert(!!attachedStorageId, 0x11f /* "requesting unknown blobs" */);
-			storageId = attachedStorageId;
+			if (!payloadPending) {
+				assert(!!attachedStorageId, 0x11f /* "requesting unknown blobs" */);
+			}
+			// If we didn't find it in the redirectTable, assume the attach op is coming eventually and wait.
+			// We do this even if the local client doesn't have the blob payloadPending flag enabled, in case a
+			// remote client does have it enabled. This wait may be infinite if the uploading client failed
+			// the upload and doesn't exist anymore.
+			storageId =
+				attachedStorageId ??
+				(await new Promise<string>((resolve) => {
+					const onProcessBlobAttach = (localId: string, _storageId: string): void => {
+						if (localId === blobId) {
+							this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
+							resolve(_storageId);
+						}
+					};
+					this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
+				}));
 		}
 
 		return PerformanceEvent.timedExecAsync(
@@ -406,14 +433,15 @@ export class BlobManager {
 			? () => {
 					pending.attached = true;
 					// Notify listeners (e.g. serialization process) that blob has been attached
-					this.internalEvents.emit("blobAttached", pending);
+					this.internalEvents.emit("handleAttached", pending);
 					this.deletePendingBlobMaybe(localId);
 				}
 			: undefined;
 		return new BlobHandle(
 			getGCNodePathFromBlobId(localId),
 			this.routeContext,
-			async () => this.getBlob(localId),
+			async () => this.getBlob(localId, false),
+			false, // payloadPending
 			callback,
 		);
 	}
@@ -597,10 +625,14 @@ export class BlobManager {
 			// If there is already an op for this storage ID, append the local ID to the list. Once any op for
 			// this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
 			// blob alive in storage.
-			this.opsInFlight.set(response.id, [
-				...(this.opsInFlight.get(response.id) ?? []),
-				localId,
-			]);
+			let setForRemoteId = this.opsInFlight.get(response.id);
+			if (setForRemoteId === undefined) {
+				setForRemoteId = new Set();
+				this.opsInFlight.set(response.id, setForRemoteId);
+			}
+			// seeing the same localId twice can happen if a blob is being reuploaded and stashed.
+			// TODO: review stashing logic and see if we can avoid this, as well in tests.
+			setForRemoteId.add(localId);
 		}
 		return response;
 	}
@@ -677,6 +709,7 @@ export class BlobManager {
 				this.deletePendingBlobMaybe(localId);
 			}
 		}
+		this.internalEvents.emit("processedBlobAttach", localId, blobId);
 	}
 
 	public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
@@ -870,14 +903,14 @@ export class BlobManager {
 									);
 									const onBlobAttached = (attachedEntry: PendingBlob): void => {
 										if (attachedEntry === entry) {
-											this.internalEvents.off("blobAttached", onBlobAttached);
+											this.internalEvents.off("handleAttached", onBlobAttached);
 											resolve();
 										}
 									};
 									if (entry.attached) {
 										resolve();
 									} else {
-										this.internalEvents.on("blobAttached", onBlobAttached);
+										this.internalEvents.on("handleAttached", onBlobAttached);
 									}
 								}),
 							);

package/src/channelCollection.ts

@@ -124,6 +124,7 @@ interface FluidDataStoreMessage {
  * Creates a shallow wrapper of {@link IFluidParentContext}. The wrapper can then have its methods overwritten as needed
  */
 export function wrapContext(context: IFluidParentContext): IFluidParentContext {
+	const isReadOnly = context.isReadOnly;
 	return {
 		get IFluidDataStoreRegistry() {
 			return context.IFluidDataStoreRegistry;
@@ -149,6 +150,7 @@ export function wrapContext(context: IFluidParentContext): IFluidParentContext {
 		get attachState() {
 			return context.attachState;
 		},
+		isReadOnly: isReadOnly === undefined ? undefined : () => isReadOnly(),
 		containerRuntime: context.containerRuntime,
 		scope: context.scope,
 		gcThrowOnTombstoneUsage: context.gcThrowOnTombstoneUsage,
@@ -1116,6 +1118,31 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 		}
 	}
 
+	/**
+	 * Enumerates the contexts and calls notifyReadOnlyState on them.
+	 */
+	public notifyReadOnlyState(readonly: boolean): void {
+		for (const [fluidDataStoreId, context] of this.contexts) {
+			try {
+				context.notifyReadOnlyState(readonly);
+			} catch (error) {
+				this.mc.logger.sendErrorEvent(
+					{
+						eventName: "SetReadOnlyStateError",
+						...tagCodeArtifacts({
+							fluidDataStoreId,
+						}),
+						details: {
+							runtimeReadonly: this.parentContext.isReadOnly?.(),
+							readonly,
+						},
+					},
+					error,
+				);
+			}
+		}
+	}
+
 	public setAttachState(attachState: AttachState.Attaching | AttachState.Attached): void {
 		for (const [, context] of this.contexts) {
 			// Fire only for bounded stores.

package/src/compatUtils.ts

@@ -0,0 +1,211 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { FlushMode } from "@fluidframework/runtime-definitions/internal";
+// The semver package documents and encourages these imports for users that only need some of the semver functionality.
+// eslint-disable-next-line import/no-internal-modules
+import semverGte from "semver/functions/gte.js";
+// eslint-disable-next-line import/no-internal-modules
+import semverLte from "semver/functions/lte.js";
+// eslint-disable-next-line import/no-internal-modules
+import semverValid from "semver/functions/valid.js";
+
+import {
+	disabledCompressionConfig,
+	enabledCompressionConfig,
+} from "./compressionDefinitions.js";
+import type { ContainerRuntimeOptionsInternal } from "./containerRuntime.js";
+import { pkgVersion } from "./packageVersion.js";
+
+/**
+ * Our policy is to support N/N-1 compatibility by default, where N is the most
+ * recent public major release of the runtime.
+ * Therefore, if the customer does not provide a compatibility mode, we will
+ * default to use N-1.
+ *
+ * However, this is not consistent with today's behavior. Some options (i.e.
+ * batching, compression) are enabled by default despite not being compatible
+ * with 1.x clients. Since the policy was introduced during 2.x's lifespan,
+ * N/N-1 compatibility by **default** will be in effect starting with 3.0.
+ * Importantly though, N/N-2 compatibility is still guaranteed with the proper
+ * configurations set.
+ *
+ * Further to distinguish unspecified `compatibilityVersion` from a specified
+ * version and allow `enableExplicitSchemaControl` to default to `true` for
+ * any 2.0.0+ version, we will use a special value of `2.0.0-defaults`, which
+ * is semantically less than 2.0.0.
+ */
+export const defaultCompatibilityVersion = "2.0.0-defaults" as const;
+
+/**
+ * String in a valid semver format specifying bottom of a minor version
+ * or special "defaults" prerelease of a major.
+ * @remarks Only 2.0.0-defaults is expected, but index signatures cannot be a
+ * literal; so, just allow any major -defaults prerelease.
+ */
+export type MinimumMinorSemanticVersion = `${bigint}.${bigint}.0` | `${bigint}.0.0-defaults`;
+
+/**
+ * String in a valid semver format of a specific version at least specifying minor.
+ */
+export type SemanticVersion =
+	| `${bigint}.${bigint}.${bigint}`
+	| `${bigint}.${bigint}.${bigint}-${string}`;
+
+/**
+ * Generic type for runtimeOptionsAffectingDocSchemaConfigMap
+ */
+export type ConfigMap<T extends Record<string, unknown>> = {
+	[K in keyof T]-?: {
+		[version: MinimumMinorSemanticVersion]: T[K];
+	};
+};
+
+/**
+ * Subset of the {@link ContainerRuntimeOptionsInternal} properties which
+ * affect {@link IDocumentSchemaFeatures}.
+ *
+ * @remarks
+ * When a new option is added to {@link ContainerRuntimeOptionsInternal}, we
+ * must consider if it changes the DocumentSchema. If so, then a corresponding
+ * entry must be added to {@link runtimeOptionsAffectingDocSchemaConfigMap}
+ * below. If not, then it must be omitted from this type.
+ *
+ * Note: `Omit` is used instead of `Pick` to ensure that all new options are
+ * included in this type by default. If any new properties are added to
+ * {@link ContainerRuntimeOptionsInternal}, they will be included in this
+ * type unless explicitly omitted. This will prevent us from forgetting to
+ * account for any new properties in the future.
+ */
+export type RuntimeOptionsAffectingDocSchema = Omit<
+	ContainerRuntimeOptionsInternal,
+	| "chunkSizeInBytes"
+	| "maxBatchSizeInBytes"
+	| "loadSequenceNumberVerification"
+	| "summaryOptions"
+>;
+
+/**
+ * Mapping of RuntimeOptionsAffectingDocSchema to their compatibility related configs.
+ *
+ * Each key in this map corresponds to a property in RuntimeOptionsAffectingDocSchema. The value is an object that maps SemanticVersions
+ * to the appropriate default value for that property to supporting that SemanticVersion. If clients running SemanticVersion X are able to understand
+ * the format changes introduced by the property, then the default value for that SemanticVersion will enable the feature associated with the property.
+ * Otherwise, the feature will be disabled.
+ *
+ * For example if the compatibilityVersion is a 1.x version (i.e. "1.5.0"), then the default value for `enableGroupedBatching` will be false since 1.x
+ * clients do not understand the document format when batching is enabled. If the compatibilityVersion is a 2.x client (i.e. "2.0.0" or later), then the
+ * default value for `enableGroupedBatching` will be true because clients running 2.0 or later will be able to understand the format changes associated
+ * with the batching feature.
+ */
+const runtimeOptionsAffectingDocSchemaConfigMap = {
+	enableGroupedBatching: {
+		"1.0.0": false,
+		"2.0.0-defaults": true,
+	} as const,
+	compressionOptions: {
+		"1.0.0": disabledCompressionConfig,
+		"2.0.0-defaults": enabledCompressionConfig,
+	} as const,
+	enableRuntimeIdCompressor: {
+		// For IdCompressorMode, `undefined` represents a logical state (off).
+		// However, to satisfy the Required<> constraint while
+		// `exactOptionalPropertyTypes` is `false` (TODO: AB#8215), we need
+		// to have it defined, so we trick the type checker here.
+		"1.0.0": undefined,
+		// We do not yet want to enable idCompressor by default since it will
+		// increase bundle sizes, and not all customers will benefit from it.
+		// Therefore, we will require customers to explicitly enable it. We
+		// are keeping it as a DocSchema affecting option for now as this may
+		// change in the future.
+	} as const,
+	explicitSchemaControl: {
+		"1.0.0": false,
+		// This option's intention is to prevent 1.x clients from joining sessions
+		// when enabled. This is set to true when the compatibility version is set
+		// to >=2.0.0 (explicitly). This is different than other 2.0 defaults
+		// because it was not enabled by default prior to the implementation of
+		// `compatibilityVersion`.
+		// `defaultCompatibilityVersion` is set to "2.0.0-defaults" which "2.0.0"
+		// does not satisfy to avoiding enabling this option by default as of
+		// `compatibilityVersion` introduction, which could be unexpected.
+		// Only enable as a default when `compatibilityVersion` is specified at
+		// 2.0.0+.
+		"2.0.0": true,
+	} as const,
+	flushMode: {
+		// Note: 1.x clients are compatible with TurnBased flushing, but here we elect to remain on Immediate flush mode
+		// as a work-around for inability to send batches larger than 1Mb. Immediate flushing keeps batches smaller as
+		// fewer messages will be included per flush.
+		"1.0.0": FlushMode.Immediate,
+		"2.0.0-defaults": FlushMode.TurnBased,
+	} as const,
+	gcOptions: {
+		"1.0.0": {},
+		// Although sweep is supported in 2.x, it is disabled by default until compatibilityVersion>=3.0.0 to be extra safe.
+		"3.0.0": { enableGCSweep: true },
+	} as const,
+	createBlobPayloadPending: {
+		// This feature is new and disabled by default. In the future we will enable it by default, but we have not
+		// closed on the version where that will happen yet.  Probably a .10 release since blob functionality is not
+		// exposed on the `@public` API surface.
+		"1.0.0": undefined,
+	} as const,
+} as const satisfies ConfigMap<RuntimeOptionsAffectingDocSchema>;
+
+/**
+ * Returns the default RuntimeOptionsAffectingDocSchema configuration for a given compatibility version.
+ */
+export function getCompatibilityVersionDefaults(
+	compatibilityVersion: SemanticVersion,
+): RuntimeOptionsAffectingDocSchema {
+	return getConfigsForCompatMode(
+		compatibilityVersion,
+		runtimeOptionsAffectingDocSchemaConfigMap,
+		// This is a bad cast away from Partial that getConfigsForCompatMode provides.
+		// ConfigMap should be restructured to provide RuntimeOptionsAffectingDocSchema guarantee.
+	) as RuntimeOptionsAffectingDocSchema;
+}
+
+/**
+ * Returns a default configuration given compatibility version and configuration version map.
+ */
+export function getConfigsForCompatMode<T extends Record<SemanticVersion, unknown>>(
+	compatibilityVersion: SemanticVersion,
+	configMap: ConfigMap<T>,
+): Partial<T> {
+	const defaultConfigs: Partial<T> = {};
+	// Iterate over configMap to get default values for each option.
+	for (const key of Object.keys(configMap)) {
+		const config = configMap[key as keyof T];
+		// Sort the versions in ascending order so we can short circuit the loop.
+		const versions = Object.keys(config).sort((a, b) => (semverGte(b, a) ? -1 : 1));
+		// For each config, we iterate over the keys and check if compatibilityVersion is greater than or equal to the version.
+		// If so, we set it as the default value for the option. At the end of the loop we should have the most recent default
+		// value that is compatible with the version specified as the compatibilityVersion.
+		for (const version of versions) {
+			if (semverGte(compatibilityVersion, version)) {
+				defaultConfigs[key] = config[version as MinimumMinorSemanticVersion];
+			} else {
+				// If the compatibility mode is less than the version, we break out of the loop since we don't need to check
+				// any later versions.
+				break;
+			}
+		}
+	}
+	return defaultConfigs;
+}
+
+/**
+ * Checks if the compatibility version is valid.
+ * A valid compatibility version is a string that is a valid semver version and is less than or equal to the current package version.
+ */
+export function isValidCompatVersion(compatibilityVersion: SemanticVersion): boolean {
+	return (
+		compatibilityVersion !== undefined &&
+		semverValid(compatibilityVersion) !== null &&
+		semverLte(compatibilityVersion, pkgVersion)
+	);
+}

package/src/compressionDefinitions.ts

@@ -0,0 +1,47 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * Available compression algorithms for op compression.
+ * @legacy
+ * @alpha
+ */
+export enum CompressionAlgorithms {
+	lz4 = "lz4",
+}
+
+/**
+ * Options for op compression.
+ * @legacy
+ * @alpha
+ */
+export interface ICompressionRuntimeOptions {
+	/**
+	 * The value the batch's content size must exceed for the batch to be compressed.
+	 * By default the value is 600 * 1024 = 614400 bytes. If the value is set to `Infinity`, compression will be disabled.
+	 */
+	readonly minimumBatchSizeInBytes: number;
+
+	/**
+	 * The compression algorithm that will be used to compress the op.
+	 * By default the value is `lz4` which is the only compression algorithm currently supported.
+	 */
+	readonly compressionAlgorithm: CompressionAlgorithms;
+}
+
+/**
+ * @legacy
+ * @alpha
+ */
+export const disabledCompressionConfig: ICompressionRuntimeOptions = {
+	minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
+	compressionAlgorithm: CompressionAlgorithms.lz4,
+};
+
+export const enabledCompressionConfig = {
+	// Batches with content size exceeding this value will be compressed
+	minimumBatchSizeInBytes: 614400,
+	compressionAlgorithm: CompressionAlgorithms.lz4,
+} as const satisfies ICompressionRuntimeOptions;