@fluidframework/container-runtime 2.0.0-internal.3.2.2 → 2.0.0-internal.3.3.1

package/src/gc/gcConfigs.ts

@@ -0,0 +1,177 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { UsageError } from "@fluidframework/driver-utils";
+import { MonitoringContext } from "@fluidframework/telemetry-utils";
+import { IContainerRuntimeMetadata } from "../summary";
+import {
+	defaultInactiveTimeoutMs,
+	defaultSessionExpiryDurationMs,
+	disableTombstoneKey,
+	GCFeatureMatrix,
+	gcTestModeKey,
+	gcTombstoneGenerationOptionName,
+	GCVersion,
+	IGarbageCollectorConfigs,
+	IGCRuntimeOptions,
+	maxSnapshotCacheExpiryMs,
+	oneDayMs,
+	runGCKey,
+	runSessionExpiryKey,
+	runSweepKey,
+} from "./gcDefinitions";
+import { getGCVersion } from "./gcHelpers";
+
+/**
+ * Generates configurations for the Garbage Collector that it uses to determine what to run and how.
+ * @param mc - The monitoring context for reading configs from the config provider.
+ * @param createParams - The creation params:
+ * gcOptions - The garbage collector runtime options.
+ * metadata - The container runtime's createParams.metadata.
+ * existing - Whether the container is new or an existing one.
+ * @returns The garbage collector configurations.
+ */
+export function generateGCConfigs(
+	mc: MonitoringContext,
+	createParams: {
+		gcOptions: IGCRuntimeOptions;
+		metadata: IContainerRuntimeMetadata | undefined;
+		existing: boolean;
+	},
+): IGarbageCollectorConfigs {
+	let gcEnabled: boolean;
+	let sweepEnabled: boolean;
+	let sessionExpiryTimeoutMs: number | undefined;
+	let sweepTimeoutMs: number | undefined;
+	let persistedGcFeatureMatrix: GCFeatureMatrix | undefined;
+	let gcVersionInBaseSnapshot: GCVersion | undefined;
+
+	/**
+	 * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
+	 * 1. Whether running GC mark phase is allowed or not.
+	 * 2. Whether running GC sweep phase is allowed or not.
+	 * 3. Whether GC session expiry is enabled or not.
+	 * For existing containers, we get this information from the createParams.metadata blob of its summary.
+	 */
+	if (createParams.existing) {
+		gcVersionInBaseSnapshot = getGCVersion(createParams.metadata);
+		// Existing documents which did not have createParams.metadata blob or had GC disabled have version as 0. For all
+		// other existing documents, GC is enabled.
+		gcEnabled = gcVersionInBaseSnapshot > 0;
+		sweepEnabled = createParams.metadata?.sweepEnabled ?? false;
+		sessionExpiryTimeoutMs = createParams.metadata?.sessionExpiryTimeoutMs;
+		sweepTimeoutMs =
+			createParams.metadata?.sweepTimeoutMs ?? computeSweepTimeout(sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
+		persistedGcFeatureMatrix = createParams.metadata?.gcFeatureMatrix;
+	} else {
+		// Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
+		// scenario but explicitly failing makes it clearer and promotes correct usage.
+		if (createParams.gcOptions.sweepAllowed && createParams.gcOptions.gcAllowed === false) {
+			throw new UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
+		}
+
+		// This Test Override only applies for new containers
+		const testOverrideSweepTimeoutMs = mc.config.getNumber(
+			"Fluid.GarbageCollection.TestOverride.SweepTimeoutMs",
+		);
+
+		// For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
+		// flag in GC options to false.
+		gcEnabled = createParams.gcOptions.gcAllowed !== false;
+		// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
+		sweepEnabled = createParams.gcOptions.sweepAllowed === true;
+
+		// Set the Session Expiry if GC is enabled and session expiry flag isn't explicitly set to false.
+		if (gcEnabled && mc.config.getBoolean(runSessionExpiryKey) !== false) {
+			sessionExpiryTimeoutMs =
+				createParams.gcOptions.sessionExpiryTimeoutMs ?? defaultSessionExpiryDurationMs;
+		}
+		sweepTimeoutMs = testOverrideSweepTimeoutMs ?? computeSweepTimeout(sessionExpiryTimeoutMs);
+
+		if (createParams.gcOptions[gcTombstoneGenerationOptionName] !== undefined) {
+			persistedGcFeatureMatrix = {
+				tombstoneGeneration: createParams.gcOptions[gcTombstoneGenerationOptionName],
+			};
+		}
+	}
+
+	/**
+	 * Whether GC should run or not. The following conditions have to be met to run sweep:
+	 *
+	 * 1. GC should be enabled for this container.
+	 *
+	 * 2. GC should not be disabled via disableGC GC option.
+	 *
+	 * These conditions can be overridden via runGCKey feature flag.
+	 */
+	const shouldRunGC =
+		mc.config.getBoolean(runGCKey) ??
+		// GC must be enabled for the document.
+		(gcEnabled &&
+			// GC must not be disabled via GC options.
+			!createParams.gcOptions.disableGC);
+
+	/**
+	 * Whether sweep should run or not. The following conditions have to be met to run sweep:
+	 *
+	 * 1. Overall GC or mark phase must be enabled (this.configs.shouldRunGC).
+	 * 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
+	 * 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
+	 * the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
+	 * 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
+	 * feature flag.
+	 */
+	const shouldRunSweep =
+		shouldRunGC &&
+		sweepTimeoutMs !== undefined &&
+		(mc.config.getBoolean(runSweepKey) ?? sweepEnabled);
+
+	// Override inactive timeout if test config or gc options to override it is set.
+	const inactiveTimeoutMs =
+		mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs") ??
+		createParams.gcOptions.inactiveTimeoutMs ??
+		defaultInactiveTimeoutMs;
+
+	// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
+	if (sweepTimeoutMs !== undefined && inactiveTimeoutMs > sweepTimeoutMs) {
+		throw new UsageError("inactive timeout should not be greater than the sweep timeout");
+	}
+
+	// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
+	const testMode =
+		mc.config.getBoolean(gcTestModeKey) ?? createParams.gcOptions.runGCInTestMode === true;
+	// Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
+	// via feature flags.
+	const tombstoneMode = !shouldRunSweep && mc.config.getBoolean(disableTombstoneKey) !== true;
+	const runFullGC = createParams.gcOptions.runFullGC;
+
+	return {
+		gcEnabled,
+		sweepEnabled,
+		shouldRunGC,
+		shouldRunSweep,
+		runFullGC,
+		testMode,
+		tombstoneMode,
+		sessionExpiryTimeoutMs,
+		sweepTimeoutMs,
+		inactiveTimeoutMs,
+		persistedGcFeatureMatrix,
+		gcVersionInBaseSnapshot,
+	};
+}
+
+/**
+ * Sweep timeout is the time after which unreferenced content can be swept.
+ * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
+ *
+ * The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
+ * The buffer is added to account for any clock skew or other edge cases.
+ * We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
+ */
+function computeSweepTimeout(sessionExpiryTimeoutMs: number | undefined): number | undefined {
+	const bufferMs = oneDayMs;
+	return sessionExpiryTimeoutMs && sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs;
+}

package/src/gc/gcDefinitions.ts

@@ -14,7 +14,6 @@ import {
 } from "@fluidframework/runtime-definitions";
 import { ReadAndParseBlob, RefreshSummaryResult } from "@fluidframework/runtime-utils";
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IGCRuntimeOptions } from "../containerRuntime";
 import { IContainerRuntimeMetadata, ICreateContainerMetadata } from "../summary";
 
 export type GCVersion = number;
@@ -35,8 +34,6 @@ export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
 // Feature gate key to expire a session after a set period of time.
 export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
-// Feature gate key to write the gc blob as a handle if the data is the same.
-export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
 // Feature gate key to turn GC sweep log off.
 export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
 // Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
@@ -56,6 +53,14 @@ export const sweepAttachmentBlobsKey = "Fluid.GarbageCollection.Test.SweepAttach
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
 
+/**
+ * The maximum snapshot cache expiry in the driver. This is used to calculate the sweep timeout.
+ * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + a buffer.
+ * The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days, i.e., any snapshot
+ * in cache will be invalidated before 5 days.
+ */
+export const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
+
 export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
 export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days
 
@@ -138,7 +143,11 @@ export const GCNodeType = {
 };
 export type GCNodeType = typeof GCNodeType[keyof typeof GCNodeType];
 
-/** Defines the APIs for the runtime object to be passed to the garbage collector. */
+// NOTE: Once this is removed from the package exports in the next major, the deprecation tag can be removed as well
+/**
+ * @deprecated - Was only to be used internally anyway, no replacement provided.
+ * Defines the APIs for the runtime object to be passed to the garbage collector.
+ */
 export interface IGarbageCollectionRuntime {
 	/** Before GC runs, called to notify the runtime to update any pending GC state. */
 	updateStateBeforeGC(): Promise<void>;
@@ -232,6 +241,103 @@ export interface IGarbageCollectorCreateParams {
 	readonly getContainerDiagnosticId: () => string;
 }
 
+export interface IGCRuntimeOptions {
+	/**
+	 * Flag that if true, will enable running garbage collection (GC) for a new container.
+	 *
+	 * GC has mark phase and sweep phase. In mark phase, unreferenced objects are identified
+	 * and marked as such in the summary. This option enables the mark phase.
+	 * In sweep phase, unreferenced objects are eventually deleted from the container if they meet certain conditions.
+	 * Sweep phase can be enabled via the "sweepAllowed" option.
+	 *
+	 * Note: This setting is persisted in the container's summary and cannot be changed.
+	 */
+	gcAllowed?: boolean;
+
+	/**
+	 * Flag that if true, enables GC's sweep phase for a new container.
+	 *
+	 * This will allow GC to eventually delete unreferenced objects from the container.
+	 * This flag should only be set to true if "gcAllowed" is true.
+	 *
+	 * Note: This setting is persisted in the container's summary and cannot be changed.
+	 */
+	sweepAllowed?: boolean;
+
+	/**
+	 * Flag that if true, will disable garbage collection for the session.
+	 * Can be used to disable running GC on containers where it is allowed via the gcAllowed option.
+	 */
+	disableGC?: boolean;
+
+	/**
+	 * Flag that will bypass optimizations and generate GC data for all nodes irrespective of whether a node
+	 * changed or not.
+	 */
+	runFullGC?: boolean;
+
+	/**
+	 * Maximum session duration for a new container. If not present, a default value will be used.
+	 *
+	 * Note: This setting is persisted in the container's summary and cannot be changed.
+	 */
+	sessionExpiryTimeoutMs?: number;
+
+	/**
+	 * Allows additional GC options to be passed.
+	 */
+	[key: string]: any;
+}
+
+/**
+ * The configurations for Garbage Collector that determines what runs and how.
+ */
+export interface IGarbageCollectorConfigs {
+	/**
+	 * Tracks if GC is enabled for this document. This is specified during document creation and doesn't change
+	 * throughout its lifetime.
+	 */
+	readonly gcEnabled: boolean;
+	/**
+	 * Tracks if sweep phase is enabled for this document. This is specified during document creation and doesn't change
+	 * throughout its lifetime.
+	 */
+	readonly sweepEnabled: boolean;
+	/**
+	 * Tracks if GC should run or not. Even if GC is enabled for a document (see gcEnabled), it can be explicitly
+	 * disabled via runtime options or feature flags.
+	 */
+	readonly shouldRunGC: boolean;
+	/**
+	 * Tracks if sweep phase should run or not. Even if the sweep phase is enabled for a document (see sweepEnabled), it
+	 * can be explicitly disabled via feature flags. It also won't run if session expiry is not enabled.
+	 */
+	readonly shouldRunSweep: boolean;
+	/**
+	 * If true, bypass optimizations and generate GC data for all nodes irrespective of whether a node changed or not.
+	 */
+	readonly runFullGC: boolean | undefined;
+	/** The time in ms to expire a session for a client for gc. */
+	readonly sessionExpiryTimeoutMs: number | undefined;
+	/** The time after which an unreferenced node is ready to be swept. */
+	readonly sweepTimeoutMs: number | undefined;
+	/** The time after which an unreferenced node is inactive. */
+	readonly inactiveTimeoutMs: number;
+	/** Tracks whether GC should run in test mode. In this mode, unreferenced objects are deleted immediately. */
+	readonly testMode: boolean;
+	/**
+	 * Tracks whether GC should run in tombstone mode. In this mode, sweep ready objects are marked as tombstones.
+	 * In interactive (non-summarizer) clients, tombstone objects behave as if they are deleted, i.e., access to them
+	 * is not allowed. However, these objects can be accessed after referencing them first. It is used as a staging
+	 * step for sweep where accidental sweep ready objects can be recovered.
+	 */
+	readonly tombstoneMode: boolean;
+	/** @see GCFeatureMatrix. */
+	readonly persistedGcFeatureMatrix: GCFeatureMatrix | undefined;
+	/** The version of GC in the base snapshot. */
+	readonly gcVersionInBaseSnapshot: GCVersion | undefined;
+}
+
 /** The state of node that is unreferenced. */
 export const UnreferencedState = {
 	/** The node is active, i.e., it can become referenced again. */

package/src/gc/gcHelpers.ts

@@ -4,12 +4,23 @@
  */
 
 import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
+import { assert } from "@fluidframework/common-utils";
+import { ISnapshotTree } from "@fluidframework/protocol-definitions";
 import {
+	gcTreeKey,
 	IGarbageCollectionNodeData,
+	IGarbageCollectionSnapshotData,
 	IGarbageCollectionState,
+	IGarbageCollectionSummaryDetailsLegacy,
 } from "@fluidframework/runtime-definitions";
-import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
+import { packagePathToTelemetryProperty, ReadAndParseBlob } from "@fluidframework/runtime-utils";
 import { MonitoringContext } from "@fluidframework/telemetry-utils";
+import { getSummaryForDatastores } from "../dataStores";
+import {
+	dataStoreAttributesBlobName,
+	IContainerRuntimeMetadata,
+	ReadFluidDataStoreAttributes,
+} from "../summary";
 import {
 	disableTombstoneKey,
 	GCVersion,
@@ -74,6 +85,9 @@ export function shouldAllowGcTombstoneEnforcement(
 	return persistedGeneration === currentGeneration;
 }
 
+/**
+ * Sorts the given GC state as per the id of the GC nodes. It also sorts the outbound routes array of each node.
+ */
 export function generateSortedGCState(gcState: IGarbageCollectionState): IGarbageCollectionState {
 	const sortableArray: [string, IGarbageCollectionNodeData][] = Object.entries(gcState.gcNodes);
 	sortableArray.sort(([a], [b]) => a.localeCompare(b));
@@ -84,3 +98,66 @@ export function generateSortedGCState(gcState: IGarbageCollectionState): IGarbag
 	}
 	return sortedGCState;
 }
+
+/**
+ * This is for back-compat only - Before GC data was written at the root of the summary tree, individual GC blobs were
+ * written at data store's snapshot tree. This function consolidates them into the new IGarbageCollectionState format.
+ */
+export async function getSnapshotDataFromOldSnapshotFormat(
+	oldSnapshot: ISnapshotTree,
+	metadata: IContainerRuntimeMetadata | undefined,
+	readAndParseBlob: ReadAndParseBlob,
+): Promise<IGarbageCollectionSnapshotData | undefined> {
+	// Add a node for the root node that is not present in older snapshot format.
+	const gcState: IGarbageCollectionState = {
+		gcNodes: { "/": { outboundRoutes: [] } },
+	};
+	const dataStoreSnapshotTree = getSummaryForDatastores(oldSnapshot, metadata);
+	assert(
+		dataStoreSnapshotTree !== undefined,
+		0x2a8 /* "Expected data store snapshot tree in base snapshot" */,
+	);
+	for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
+		const blobId = dsSnapshotTree.blobs[gcTreeKey];
+		if (blobId === undefined) {
+			continue;
+		}
+
+		const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionSummaryDetailsLegacy>(
+			blobId,
+		);
+		// If there are no nodes for this data store, skip it.
+		if (gcSummaryDetails.gcData?.gcNodes === undefined) {
+			continue;
+		}
+
+		const dsRootId = `/${dsId}`;
+		// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
+		// Construct that entry by adding root data store ids to its outbound routes.
+		const initialSnapshotDetails = await readAndParseBlob<ReadFluidDataStoreAttributes>(
+			dsSnapshotTree.blobs[dataStoreAttributesBlobName],
+		);
+		if (initialSnapshotDetails.isRootDataStore) {
+			gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
+		}
+
+		for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
+			// Prefix the data store id to the GC node ids to make them relative to the root from being
+			// relative to the data store. Similar to how its done in DataStore::getGCData.
+			const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
+			gcState.gcNodes[rootId] = {
+				outboundRoutes: Array.from(outboundRoutes),
+			};
+		}
+		assert(
+			gcState.gcNodes[dsRootId] !== undefined,
+			0x2a9 /* GC nodes for data store not in GC blob */,
+		);
+		gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
+	}
+	// If there is only one node (root node just added above), either GC is disabled or we are loading from
+	// the first summary generated by detached container. In both cases, GC was not run - return undefined.
+	return Object.keys(gcState.gcNodes).length === 1
+		? undefined
+		: { gcState, tombstones: undefined, deletedNodes: undefined };
+}

package/src/gc/gcSummaryStateTracker.ts

@@ -63,8 +63,6 @@ export class GCSummaryStateTracker {
 	constructor(
 		// Tells whether GC should run or not.
 		private readonly shouldRunGC: boolean,
-		// Tells whether GC state should be tracked across summaries or not.
-		private readonly trackGCState: boolean,
 		// Tells whether tombstone mode is enabled or not.
 		private readonly tombstoneMode: boolean,
 		private readonly mc: MonitoringContext,
@@ -159,38 +157,37 @@ export class GCSummaryStateTracker {
 		 * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
 		 * for each of these that did not change, write a summary handle.
 		 */
-		if (this.trackGCState) {
-			this.pendingSummaryData = {
+		this.pendingSummaryData = {
+			serializedGCState,
+			serializedTombstones,
+			serializedDeletedNodes,
+		};
+
+		if (trackState && !fullTree && this.latestSummaryData !== undefined) {
+			// If nothing changed since last summary, send a summary handle for the entire GC data.
+			if (
+				this.latestSummaryData.serializedGCState === serializedGCState &&
+				this.latestSummaryData.serializedTombstones === serializedTombstones
+			) {
+				const stats = mergeStats();
+				stats.handleNodeCount++;
+				return {
+					summary: {
+						type: SummaryType.Handle,
+						handle: `/${gcTreeKey}`,
+						handleType: SummaryType.Tree,
+					},
+					stats,
+				};
+			}
+
+			// If some state changed, build a GC summary tree.
+			return this.buildGCSummaryTree(
 				serializedGCState,
 				serializedTombstones,
 				serializedDeletedNodes,
-			};
-			if (trackState && !fullTree && this.latestSummaryData !== undefined) {
-				// If nothing changed since last summary, send a summary handle for the entire GC data.
-				if (
-					this.latestSummaryData.serializedGCState === serializedGCState &&
-					this.latestSummaryData.serializedTombstones === serializedTombstones
-				) {
-					const stats = mergeStats();
-					stats.handleNodeCount++;
-					return {
-						summary: {
-							type: SummaryType.Handle,
-							handle: `/${gcTreeKey}`,
-							handleType: SummaryType.Tree,
-						},
-						stats,
-					};
-				}
-
-				// If some state changed, build a GC summary tree.
-				return this.buildGCSummaryTree(
-					serializedGCState,
-					serializedTombstones,
-					serializedDeletedNodes,
-					true /* trackState */,
-				);
-			}
+				true /* trackState */,
+			);
 		}
 		// If not tracking GC state, build a GC summary tree without any summary handles.
 		return this.buildGCSummaryTree(
@@ -290,10 +287,8 @@ export class GCSummaryStateTracker {
 		// Update latest state from pending.
 		if (result.wasSummaryTracked) {
 			this.latestSummaryGCVersion = this.currentGCVersion;
-			if (this.trackGCState) {
-				this.latestSummaryData = this.pendingSummaryData;
-				this.pendingSummaryData = undefined;
-			}
+			this.latestSummaryData = this.pendingSummaryData;
+			this.pendingSummaryData = undefined;
 			return undefined;
 		}
 
@@ -328,12 +323,10 @@ export class GCSummaryStateTracker {
 		}
 
 		// If tracking state across summaries, update latest summary data from the snapshot's GC data.
-		if (this.trackGCState) {
-			this.latestSummaryData = {
-				serializedGCState: JSON.stringify(generateSortedGCState(gcSnapshotData.gcState)),
-				serializedTombstones: JSON.stringify(gcSnapshotData.tombstones),
-				serializedDeletedNodes: JSON.stringify(gcSnapshotData.deletedNodes),
-			};
-		}
+		this.latestSummaryData = {
+			serializedGCState: JSON.stringify(generateSortedGCState(gcSnapshotData.gcState)),
+			serializedTombstones: JSON.stringify(gcSnapshotData.tombstones),
+			serializedDeletedNodes: JSON.stringify(gcSnapshotData.deletedNodes),
+		};
 	}
 }

package/src/gc/index.ts

@@ -14,10 +14,12 @@ export {
 	gcTombstoneGenerationOptionName,
 	GCVersion,
 	gcVersionUpgradeToV2Key,
-	IGarbageCollectionRuntime,
+	IGarbageCollectionRuntime, // Deprecated
 	IGarbageCollector,
+	IGarbageCollectorConfigs,
 	IGarbageCollectorCreateParams,
 	IGCMetadata,
+	IGCRuntimeOptions,
 	IGCStats,
 	oneDayMs,
 	runGCKey,
@@ -30,7 +32,11 @@ export {
 	throwOnTombstoneUsageKey,
 	UnreferencedState,
 } from "./gcDefinitions";
-export { sendGCUnexpectedUsageEvent, shouldAllowGcTombstoneEnforcement } from "./gcHelpers";
+export {
+	getSnapshotDataFromOldSnapshotFormat,
+	sendGCUnexpectedUsageEvent,
+	shouldAllowGcTombstoneEnforcement,
+} from "./gcHelpers";
 export { GCSummaryStateTracker } from "./gcSummaryStateTracker";
 export {
 	skipClosureForXDaysKey,

package/src/index.ts

@@ -6,7 +6,6 @@
 export {
 	ContainerMessageType,
 	ContainerRuntimeMessage,
-	IGCRuntimeOptions,
 	ISummaryRuntimeOptions,
 	ISummaryBaseConfiguration,
 	ISummaryConfigurationHeuristics,
@@ -27,7 +26,7 @@ export {
 	CompressionAlgorithms,
 } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
-export { IGCStats } from "./gc";
+export { IGCRuntimeOptions, IGCStats } from "./gc";
 export {
 	IPendingFlush,
 	IPendingLocalState,

package/src/opLifecycle/README.md

@@ -30,9 +30,9 @@ There are two features which can be used to work around this size limit, batch c
 
 **Op chunking for compression targets payloads which exceed the max batch size (1MB) after compression.** So, only payloads which are already compressed. By default, the feature is disabled.
 
-To enable, use the `IContainerRuntimeOptions.chunkSizeInBytes` property, which represents the size of the chunked ops, when chunking is necessary. When chunking is performed, the large op is split into smaller ops (chunks). This config represents the size of the chunks. The value enables a trade-off between large chunks / few ops and small chunks / many ops. A good value for this would be at around 614400.
+To enable, use the `IContainerRuntimeOptions.chunkSizeInBytes` property, which represents the size of the chunked ops, when chunking is necessary. When chunking is performed, the large op is split into smaller ops (chunks). This config represents both the size of the chunks and the threshold for the feature to activate. The value enables a trade-off between large chunks / few ops and small chunks / many ops. A good value for this would be at around 204800.
 
-This config would govern chunking compressed batches only. We will not be enabling chunking across all types of ops/batches but **only when compression is enabled and when the batch is compressed**, and its payload size is more than `maxBatchSizeInBytes`. Therefore, for this feature to be working, it is required that compression is enabled using `IContainerRuntimeOptions.compressionOptions`.
+This config would govern chunking compressed batches only. We will not be enabling chunking across all types of ops/batches but **only when compression is enabled and when the batch is compressed**, and its payload size is more than `IContainerRuntimeOptions.chunkSizeInBytes`. Therefore, for this feature to be working, it is required that compression is enabled using `IContainerRuntimeOptions.compressionOptions`.
 
 It is recommended to also change the `maxBatchSizeInBytes` property in `IContainerRuntimeOptions`. By default, if unspecified it is 972800. We recommend a lower value such as `716800`, to account for any overhead on the server side. The reason for this is that chunking will only kick in after this configuration limit is exceeded. If the limit is too high, we might allow batches which are under 1MB but can increase in size due to overhead after they reach the server.
 

package/src/opLifecycle/batchManager.ts

@@ -12,6 +12,12 @@ export interface IBatchManagerOptions {
 	readonly compressionOptions?: ICompressionRuntimeOptions;
 }
 
+/**
+ * Estimated size of the stringification overhead for an op accumulated
+ * from runtime to loader to the service.
+ */
+const opOverhead = 200;
+
 /**
  * Helper class that manages partial batch & rollback.
  */
@@ -43,7 +49,7 @@ export class BatchManager {
 		// Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
 		// Not taking it into account, as compression work should help there - compressed payload will be
 		// initially stored as base64, and that requires only 2 extra escape characters.
-		const socketMessageSize = contentSize + 200 * opCount;
+		const socketMessageSize = contentSize + opOverhead * opCount;
 
 		// If we were provided soft limit, check for exceeding it.
 		// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
@@ -118,3 +124,15 @@ const addBatchMetadata = (batch: IBatch): IBatch => {
 
 	return batch;
 };
+
+/**
+ * Estimates the real size in bytes on the socket for a given batch. It assumes that
+ * the envelope size (and the size of an empty op) is 200 bytes, taking into account
+ * extra overhead from stringification.
+ *
+ * @param batch - the batch to inspect
+ * @returns An estimate of the payload size in bytes which will be produced when the batch is sent over the wire
+ */
+export const estimateSocketSize = (batch: IBatch): number => {
+	return batch.contentSizeInBytes + opOverhead * batch.content.length;
+};

package/src/opLifecycle/index.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-export { BatchManager } from "./batchManager";
+export { BatchManager, estimateSocketSize } from "./batchManager";
 export {
 	BatchMessage,
 	IBatch,