@fluidframework/container-runtime 2.0.0-dev.7.4.0.217212 → 2.0.0-dev.7.4.0.221926

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.0.0-dev.7.4.0.217212",
+  "version": "2.0.0-dev.7.4.0.221926",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -35,18 +35,19 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/container-definitions": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/container-runtime-definitions": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/core-interfaces": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/core-utils": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/datastore": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/driver-definitions": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/driver-utils": "2.0.0-dev.7.4.0.217212",
+    "@fluid-internal/client-utils": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/container-definitions": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/container-runtime-definitions": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/core-interfaces": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/core-utils": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/datastore": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/driver-definitions": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/driver-utils": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/id-compressor": "2.0.0-dev.7.4.0.221926",
     "@fluidframework/protocol-definitions": "^3.0.0",
-    "@fluidframework/runtime-definitions": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/runtime-utils": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/telemetry-utils": "2.0.0-dev.7.4.0.217212",
+    "@fluidframework/runtime-definitions": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/runtime-utils": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/telemetry-utils": "2.0.0-dev.7.4.0.221926",
     "double-ended-queue": "^2.1.0-0",
     "events": "^3.1.0",
     "lz4js": "^0.2.0",
@@ -54,15 +55,16 @@
     "uuid": "^9.0.0"
   },
   "devDependencies": {
-    "@fluid-private/stochastic-test-utils": "2.0.0-dev.7.4.0.217212",
+    "@arethetypeswrong/cli": "^0.13.3",
+    "@fluid-private/stochastic-test-utils": "2.0.0-dev.7.4.0.221926",
     "@fluid-tools/benchmark": "^0.48.0",
     "@fluid-tools/build-cli": "^0.28.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.28.0",
     "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.7.2.0",
     "@fluidframework/eslint-config-fluid": "^3.1.0",
-    "@fluidframework/mocha-test-setup": "2.0.0-dev.7.4.0.217212",
-    "@fluidframework/test-runtime-utils": "2.0.0-dev.7.4.0.217212",
+    "@fluidframework/mocha-test-setup": "2.0.0-dev.7.4.0.221926",
+    "@fluidframework/test-runtime-utils": "2.0.0-dev.7.4.0.221926",
     "@microsoft/api-extractor": "^7.38.3",
     "@types/double-ended-queue": "^2.1.0",
     "@types/events": "^3.0.0",
@@ -115,6 +117,7 @@
     "build:esnext": "tsc --project ./tsconfig.esnext.json",
     "build:genver": "gen-version",
     "build:test": "tsc --project ./src/test/tsconfig.json",
+    "check:are-the-types-wrong": "attw --pack",
     "check:release-tags": "api-extractor run --local --config ./api-extractor-lint.json",
     "ci:build:docs": "api-extractor run",
     "clean": "rimraf --glob dist lib \"**/*.tsbuildinfo\" \"**/*.build.log\" _api-extractor-temp nyc",

package/src/blobManager.ts

@@ -85,7 +85,7 @@ export class BlobHandle implements IFluidHandle<ArrayBufferLike> {
 
 /**
  * Information from a snapshot needed to load BlobManager
- * @internal
+ * @alpha
  */
 export interface IBlobManagerLoadInfo {
 	ids?: string[];
@@ -729,7 +729,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	 * This is called to update blobs whose routes are unused. The unused blobs are deleted.
 	 * @param unusedRoutes - The routes of the blob nodes that are unused. These routes will be based off of local ids.
 	 */
-	public updateUnusedRoutes(unusedRoutes: string[]): void {
+	public updateUnusedRoutes(unusedRoutes: readonly string[]): void {
 		this.deleteBlobsFromRedirectTable(unusedRoutes);
 	}
 
@@ -739,7 +739,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	 * be based off of local ids.
 	 * @returns The routes of blobs that were deleted.
 	 */
-	public deleteSweepReadyNodes(sweepReadyBlobRoutes: string[]): string[] {
+	public deleteSweepReadyNodes(sweepReadyBlobRoutes: readonly string[]): readonly string[] {
 		// If sweep for attachment blobs is not enabled, return empty list indicating nothing is deleted.
 		if (this.mc.config.getBoolean(disableAttachmentBlobSweepKey) === true) {
 			return [];
@@ -760,7 +760,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	 * Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
 	 * will remove them the next summary. The service would them delete them some time in the future.
 	 */
-	private deleteBlobsFromRedirectTable(blobRoutes: string[]) {
+	private deleteBlobsFromRedirectTable(blobRoutes: readonly string[]) {
 		if (blobRoutes.length === 0) {
 			return;
 		}
@@ -805,7 +805,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	 * accessing deleted content without actually deleting content from summaries.
 	 * @param tombstonedRoutes - The routes of blob nodes that are tombstones.
 	 */
-	public updateTombstonedRoutes(tombstonedRoutes: string[]) {
+	public updateTombstonedRoutes(tombstonedRoutes: readonly string[]) {
 		const tombstonedBlobsSet: Set<string> = new Set();
 		// The routes or blob node paths are in the same format as returned in getGCData -
 		// `/<BlobManager.basePath>/<blobId>`.

package/src/containerRuntime.ts

@@ -47,6 +47,7 @@ import {
 	wrapError,
 	ITelemetryLoggerExt,
 	UsageError,
+	LoggingError,
 } from "@fluidframework/telemetry-utils";
 import {
 	DriverHeader,
@@ -87,12 +88,14 @@ import {
 	channelsTreeName,
 	IDataStore,
 	ITelemetryContext,
+} from "@fluidframework/runtime-definitions";
+import type {
 	SerializedIdCompressorWithNoSession,
 	IIdCompressor,
 	IIdCompressorCore,
 	IdCreationRange,
 	SerializedIdCompressorWithOngoingSession,
-} from "@fluidframework/runtime-definitions";
+} from "@fluidframework/id-compressor";
 import {
 	addBlobToSummary,
 	addSummarizeResultToSummary,
@@ -163,7 +166,7 @@ import { formExponentialFn, Throttler } from "./throttler";
 import {
 	GarbageCollector,
 	GCNodeType,
-	gcTombstoneGenerationOptionName,
+	gcGenerationOptionName,
 	IGarbageCollector,
 	IGCRuntimeOptions,
 	IGCStats,
@@ -194,6 +197,7 @@ import {
 	type LocalContainerRuntimeMessage,
 	type OutboundContainerRuntimeMessage,
 	type UnknownContainerRuntimeMessage,
+	ContainerRuntimeGCMessage,
 } from "./messageTypes";
 
 /**
@@ -213,7 +217,7 @@ function compatBehaviorAllowsMessageType(
 }
 
 /**
- * @internal
+ * @alpha
  */
 export interface ISummaryBaseConfiguration {
 	/**
@@ -235,7 +239,7 @@ export interface ISummaryBaseConfiguration {
 }
 
 /**
- * @internal
+ * @alpha
  */
 export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfiguration {
 	state: "enabled";
@@ -298,21 +302,21 @@ export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfigurati
 }
 
 /**
- * @internal
+ * @alpha
  */
 export interface ISummaryConfigurationDisableSummarizer {
 	state: "disabled";
 }
 
 /**
- * @internal
+ * @alpha
  */
 export interface ISummaryConfigurationDisableHeuristics extends ISummaryBaseConfiguration {
 	state: "disableHeuristics";
 }
 
 /**
- * @internal
+ * @alpha
  */
 export type ISummaryConfiguration =
 	| ISummaryConfigurationDisableSummarizer
@@ -349,7 +353,7 @@ export const DefaultSummaryConfiguration: ISummaryConfiguration = {
 };
 
 /**
- * @internal
+ * @alpha
  */
 export interface ISummaryRuntimeOptions {
 	/** Override summary configurations set by the server. */
@@ -366,7 +370,7 @@ export interface ISummaryRuntimeOptions {
 
 /**
  * Options for op compression.
- * @internal
+ * @alpha
  */
 export interface ICompressionRuntimeOptions {
 	/**
@@ -384,7 +388,7 @@ export interface ICompressionRuntimeOptions {
 
 /**
  * Options for container runtime.
- * @internal
+ * @alpha
  */
 export interface IContainerRuntimeOptions {
 	readonly summaryOptions?: ISummaryRuntimeOptions;
@@ -516,7 +520,7 @@ export const defaultRuntimeHeaderData: Required<RuntimeHeaderData> = {
 
 /**
  * Available compression algorithms for op compression.
- * @internal
+ * @alpha
  */
 export enum CompressionAlgorithms {
 	lz4 = "lz4",
@@ -718,7 +722,7 @@ export async function TEST_requestSummarizer(loader: ILoader, url: string): Prom
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
- * @internal
+ * @alpha
  */
 export class ContainerRuntime
 	extends TypedEventEmitter<IContainerRuntimeEvents & ISummarizerEvents>
@@ -909,16 +913,18 @@ export class ContainerRuntime
 			metadata?.idCompressorEnabled ?? runtimeOptions.enableRuntimeIdCompressor ?? false;
 		let idCompressor: (IIdCompressor & IIdCompressorCore) | undefined;
 		if (idCompressorEnabled) {
-			const { IdCompressor, createSessionId } = await import("./id-compressor");
+			const { createIdCompressor, deserializeIdCompressor, createSessionId } = await import(
+				"@fluidframework/id-compressor"
+			);
 
 			const pendingLocalState = context.pendingLocalState as IPendingRuntimeState;
 
 			if (pendingLocalState?.pendingIdCompressorState !== undefined) {
-				idCompressor = IdCompressor.deserialize(pendingLocalState.pendingIdCompressorState);
+				idCompressor = deserializeIdCompressor(pendingLocalState.pendingIdCompressorState);
 			} else if (serializedIdCompressor !== undefined) {
-				idCompressor = IdCompressor.deserialize(serializedIdCompressor, createSessionId());
+				idCompressor = deserializeIdCompressor(serializedIdCompressor, createSessionId());
 			} else {
-				idCompressor = IdCompressor.create(logger);
+				idCompressor = createIdCompressor(logger);
 			}
 		}
 
@@ -1341,8 +1347,7 @@ export class ContainerRuntime
 			eventName: "GCFeatureMatrix",
 			metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
 			inputs: JSON.stringify({
-				gcOptions_gcTombstoneGeneration:
-					this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName],
+				gcOptions_gcGeneration: this.runtimeOptions.gcOptions[gcGenerationOptionName],
 			}),
 		});
 
@@ -1447,6 +1452,7 @@ export class ContainerRuntime
 			// GC runs in summarizer client and needs access to the real (non-proxy) active information. The proxy
 			// delta manager would always return false for summarizer client.
 			activeConnection: () => this.innerDeltaManager.active,
+			submitMessage: (message: ContainerRuntimeGCMessage) => this.submit(message),
 		});
 
 		const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
@@ -2082,6 +2088,9 @@ export class ContainerRuntime
 				throw new Error("chunkedOp not expected here");
 			case ContainerMessageType.Rejoin:
 				throw new Error("rejoin not expected here");
+			case ContainerMessageType.GC:
+				// GC op is only sent in summarizer which should never have stashed ops.
+				throw new LoggingError("GC op not expected to be stashed in summarizer");
 			default: {
 				// This should be extremely rare for stashed ops.
 				// It would require a newer runtime stashing ops and then an older one applying them,
@@ -2335,6 +2344,9 @@ export class ContainerRuntime
 					this.idCompressor.finalizeCreationRange(messageWithContext.message.contents);
 				}
 				break;
+			case ContainerMessageType.GC:
+				this.garbageCollector.processMessage(messageWithContext.message, local);
+				break;
 			case ContainerMessageType.ChunkedOp:
 			case ContainerMessageType.Rejoin:
 				break;
@@ -2632,18 +2644,28 @@ export class ContainerRuntime
 	}
 
 	private isContainerMessageDirtyable({ type, contents }: OutboundContainerRuntimeMessage) {
-		// For legacy purposes, exclude the old built-in AgentScheduler from dirty consideration as a special-case.
-		// Ultimately we should have no special-cases from the ContainerRuntime's perspective.
-		if (type === ContainerMessageType.Attach) {
-			const attachMessage = contents as InboundAttachMessage;
-			if (attachMessage.id === agentSchedulerId) {
-				return false;
+		// Certain container runtime messages should not mark the container dirty such as the old built-in
+		// AgentScheduler and Garbage collector messages.
+		switch (type) {
+			case ContainerMessageType.Attach: {
+				const attachMessage = contents as InboundAttachMessage;
+				if (attachMessage.id === agentSchedulerId) {
+					return false;
+				}
+				break;
 			}
-		} else if (type === ContainerMessageType.FluidDataStoreOp) {
-			const envelope = contents;
-			if (envelope.address === agentSchedulerId) {
+			case ContainerMessageType.FluidDataStoreOp: {
+				const envelope = contents;
+				if (envelope.address === agentSchedulerId) {
+					return false;
+				}
+				break;
+			}
+			case ContainerMessageType.GC: {
 				return false;
 			}
+			default:
+				break;
 		}
 		return true;
 	}
@@ -2874,7 +2896,7 @@ export class ContainerRuntime
 	 * @param usedRoutes - The routes that are used in all nodes in this Container.
 	 * @see IGarbageCollectionRuntime.updateUsedRoutes
 	 */
-	public updateUsedRoutes(usedRoutes: string[]) {
+	public updateUsedRoutes(usedRoutes: readonly string[]) {
 		// Update our summarizer node's used routes. Updating used routes in summarizer node before
 		// summarizing is required and asserted by the the summarizer node. We are the root and are
 		// always referenced, so the used routes is only self-route (empty string).
@@ -2888,7 +2910,7 @@ export class ContainerRuntime
 	 * This is called to update objects whose routes are unused.
 	 * @param unusedRoutes - Data store and attachment blob routes that are unused in this Container.
 	 */
-	public updateUnusedRoutes(unusedRoutes: string[]) {
+	public updateUnusedRoutes(unusedRoutes: readonly string[]) {
 		const { blobManagerRoutes, dataStoreRoutes } =
 			this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
 		this.blobManager.updateUnusedRoutes(blobManagerRoutes);
@@ -2898,7 +2920,7 @@ export class ContainerRuntime
 	/**
 	 * @deprecated Replaced by deleteSweepReadyNodes.
 	 */
-	public deleteUnusedNodes(unusedRoutes: string[]): string[] {
+	public deleteUnusedNodes(unusedRoutes: readonly string[]): string[] {
 		throw new Error("deleteUnusedRoutes should not be called");
 	}
 
@@ -2907,7 +2929,7 @@ export class ContainerRuntime
 	 * @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
 	 * @returns The routes of nodes that were deleted.
 	 */
-	public deleteSweepReadyNodes(sweepReadyRoutes: string[]): string[] {
+	public deleteSweepReadyNodes(sweepReadyRoutes: readonly string[]): readonly string[] {
 		const { dataStoreRoutes, blobManagerRoutes } =
 			this.getDataStoreAndBlobManagerRoutes(sweepReadyRoutes);
 
@@ -2919,7 +2941,7 @@ export class ContainerRuntime
 	 * This is called to update objects that are tombstones.
 	 * @param tombstonedRoutes - Data store and attachment blob routes that are tombstones in this Container.
 	 */
-	public updateTombstonedRoutes(tombstonedRoutes: string[]) {
+	public updateTombstonedRoutes(tombstonedRoutes: readonly string[]) {
 		const { blobManagerRoutes, dataStoreRoutes } =
 			this.getDataStoreAndBlobManagerRoutes(tombstonedRoutes);
 		this.blobManager.updateTombstonedRoutes(blobManagerRoutes);
@@ -2979,7 +3001,7 @@ export class ContainerRuntime
 	 * @returns Two route lists - One that contains routes for blob manager and another one that contains routes
 	 * for data stores.
 	 */
-	private getDataStoreAndBlobManagerRoutes(routes: string[]) {
+	private getDataStoreAndBlobManagerRoutes(routes: readonly string[]) {
 		const blobManagerRoutes: string[] = [];
 		const dataStoreRoutes: string[] = [];
 		for (const route of routes) {
@@ -3056,10 +3078,10 @@ export class ContainerRuntime
 			);
 		}
 
-		// If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
-		// incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
-		// within the timeout, check if it should be failed or can continue.
-		if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
+		// If the container is dirty, i.e., there are pending unacked ops, the summary will not be eventual consistent
+		// and it may even be incorrect. So, wait for the container to be saved with a timeout. If the container is not
+		// saved within the timeout, check if it should be failed or can continue.
+		if (this.validateSummaryBeforeUpload && this.isDirty) {
 			const countBefore = this.pendingMessagesCount;
 			// The timeout for waiting for pending ops can be overridden via configurations.
 			const pendingOpsTimeout =
@@ -3081,7 +3103,7 @@ export class ContainerRuntime
 			// happens, whether we attempted to wait for these ops to be acked and what was the result.
 			summaryNumberLogger.sendTelemetryEvent({
 				eventName: "PendingOpsWhileSummarizing",
-				saved: this.hasPendingMessages() ? false : true,
+				saved: !this.isDirty,
 				timeout: pendingOpsTimeout,
 				countBefore,
 				countAfter: this.pendingMessagesCount,
@@ -3360,7 +3382,7 @@ export class ContainerRuntime
 	}
 
 	/**
-	 * This helper is called during summarization. If there are pending ops, it will return a failed summarize result
+	 * This helper is called during summarization. If the container is dirty, it will return a failed summarize result
 	 * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
 	 * @param logger - The logger to be used for sending telemetry.
 	 * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
@@ -3376,7 +3398,7 @@ export class ContainerRuntime
 		finalAttempt: boolean,
 		beforeSummaryGeneration: boolean,
 	): Promise<IBaseSummarizeResult | undefined> {
-		if (!this.hasPendingMessages()) {
+		if (!this.isDirty) {
 			return;
 		}
 
@@ -3724,6 +3746,7 @@ export class ContainerRuntime
 	/**
 	 * Finds the right store and asks it to resubmit the message. This typically happens when we
 	 * reconnect and there are pending messages.
+	 * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
 	 * @param message - The original LocalContainerRuntimeMessage.
 	 * @param localOpMetadata - The local metadata associated with the original message.
 	 */
@@ -3752,6 +3775,9 @@ export class ContainerRuntime
 			case ContainerMessageType.Rejoin:
 				this.submit(message);
 				break;
+			case ContainerMessageType.GC:
+				// GC op is only sent in summarizer which should never reconnect.
+				throw new LoggingError("GC op not expected to be resubmitted in summarizer");
 			default: {
 				// This case should be very rare - it would imply an op was stashed from a
 				// future version of runtime code and now is being applied on an older version