@fluidframework/container-runtime 2.1.0-281041 → 2.2.0

package/src/gc/gcHelpers.ts

@@ -240,10 +240,11 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
 			continue;
 		}
 
-		assert(id.startsWith("/"), 0x5d9 /* node id should always be an absolute route */);
-		// TODO Why are we non null asserting here
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		const childId = id.split("/")[1]!;
+		const childId = id.split("/")[1];
+		assert(
+			childId !== undefined,
+			0x9fe /* node id should be an absolute route with child id part */,
+		);
 		let childGCNodeId = id.slice(childId.length + 1);
 		// GC node id always begins with "/". Handle the special case where a child's id in the parent's GC nodes is
 		// of format `/root`. In this case, the childId is root and childGCNodeId is "". Make childGCNodeId = "/".
@@ -271,10 +272,11 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
 	// Remove the node's self used route, if any, and generate the children used routes.
 	const usedRoutes = gcDetails.usedRoutes.filter((route) => route !== "" && route !== "/");
 	for (const route of usedRoutes) {
-		assert(route.startsWith("/"), 0x5db /* Used route should always be an absolute route */);
-		// TODO Why are we non null asserting here
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		const childId = route.split("/")[1]!;
+		const childId = route.split("/")[1];
+		assert(
+			childId !== undefined,
+			0x9ff /* used route should be an absolute route with child id part */,
+		);
 		const childUsedRoute = route.slice(childId.length + 1);
 
 		const childGCDetails = childGCDetailsMap.get(childId);

package/src/gc/gcSummaryStateTracker.ts

@@ -66,7 +66,7 @@ export class GCSummaryStateTracker {
 		// Tells whether GC should run or not.
 		private readonly configs: Pick<
 			IGarbageCollectorConfigs,
-			"gcEnabled" | "tombstoneMode" | "gcVersionInBaseSnapshot" | "gcVersionInEffect"
+			"gcAllowed" | "gcVersionInBaseSnapshot" | "gcVersionInEffect"
 		>,
 	) {}
 
@@ -100,7 +100,7 @@ export class GCSummaryStateTracker {
 		deletedNodes: Set<string>,
 		tombstones: string[],
 	): ISummarizeResult | undefined {
-		if (!this.configs.gcEnabled) {
+		if (!this.configs.gcAllowed) {
 			return;
 		}
 
@@ -109,12 +109,9 @@ export class GCSummaryStateTracker {
 		// to identify deleted nodes' usage.
 		const serializedDeletedNodes =
 			deletedNodes.size > 0 ? JSON.stringify(Array.from(deletedNodes).sort()) : undefined;
-		// If running in tombstone mode, serialize and write tombstones, if any.
-		const serializedTombstones = this.configs.tombstoneMode
-			? tombstones.length > 0
-				? JSON.stringify(tombstones.sort())
-				: undefined
-			: undefined;
+		// Serialize and write tombstones, if any.
+		const serializedTombstones =
+			tombstones.length > 0 ? JSON.stringify(tombstones.sort()) : undefined;
 
 		/**
 		 * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
@@ -231,7 +228,7 @@ export class GCSummaryStateTracker {
 	 * Called to refresh the latest summary state. This happens when a pending summary is acked.
 	 */
 	public async refreshLatestSummary(result: IRefreshSummaryResult): Promise<void> {
-		if (!this.configs.gcEnabled || !result.isSummaryTracked) {
+		if (!this.configs.gcAllowed || !result.isSummaryTracked) {
 			return;
 		}
 

package/src/gc/gcTelemetry.ts

@@ -10,7 +10,6 @@ import {
 	MonitoringContext,
 	generateStack,
 	tagCodeArtifacts,
-	type ITelemetryGenericEventExt,
 	type ITelemetryPropertiesExt,
 } from "@fluidframework/telemetry-utils/internal";
 
@@ -22,11 +21,7 @@ import {
 	GCNodeType,
 	IGarbageCollectorConfigs,
 	UnreferencedState,
-	disableTombstoneKey,
-	throwOnTombstoneLoadOverrideKey,
-	throwOnTombstoneUsageKey,
 } from "./gcDefinitions.js";
-import { getGCVersionInEffect } from "./gcHelpers.js";
 import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker.js";
 
 type NodeUsageType = "Changed" | "Loaded" | "Revived" | "Realized";
@@ -287,18 +282,12 @@ export class GCTelemetryTracker {
 			headers: { ...headers },
 			details: { ...detailedProps, ...additionalProps }, // Also includes some properties from INodeUsageProps type
 			gcConfigs,
-			tombstoneFlags: {
-				DisableTombstone: this.mc.config.getBoolean(disableTombstoneKey),
-				ThrowOnTombstoneUsage: this.mc.config.getBoolean(throwOnTombstoneUsageKey),
-				ThrowOnTombstoneLoad: this.mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
-			},
 		};
 
 		if (
-			(usageType === "Loaded" &&
-				this.configs.throwOnTombstoneLoad &&
-				!headers?.allowTombstone) ||
-			(usageType === "Changed" && this.configs.throwOnTombstoneUsage)
+			usageType === "Loaded" &&
+			this.configs.throwOnTombstoneLoad &&
+			!headers?.allowTombstone
 		) {
 			this.mc.logger.sendErrorEvent(event);
 		} else {
@@ -419,27 +408,3 @@ export class GCTelemetryTracker {
 		this.pendingEventsQueue = [];
 	}
 }
-
-/**
- * Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
- * tombstoned or deleted object is loaded.
- */
-export function sendGCUnexpectedUsageEvent(
-	mc: MonitoringContext,
-	event: ITelemetryGenericEventExt & {
-		category: "error" | "generic";
-		gcTombstoneEnforcementAllowed: boolean | undefined;
-	},
-	packagePath: readonly string[] | undefined,
-	error?: unknown,
-) {
-	event.pkg = tagCodeArtifacts({ pkg: packagePath?.join("/") })?.pkg;
-	event.tombstoneFlags = JSON.stringify({
-		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
-		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
-		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
-	});
-	event.gcVersion = getGCVersionInEffect(mc.config);
-
-	mc.logger.sendTelemetryEvent(event, error);
-}

package/src/gc/index.ts

@@ -11,8 +11,6 @@ export {
 	defaultSessionExpiryDurationMs,
 	GCNodeType,
 	gcTestModeKey,
-	gcDisableDataStoreSweepOptionName,
-	gcDisableThrowOnTombstoneLoadOptionName,
 	gcGenerationOptionName,
 	GCFeatureMatrix,
 	GCVersion,
@@ -32,10 +30,8 @@ export {
 	oneDayMs,
 	runSessionExpiryKey,
 	stableGCVersion,
-	disableAutoRecoveryKey,
-	disableDatastoreSweepKey,
 	UnreferencedState,
-	throwOnTombstoneLoadOverrideKey,
+	disableThrowOnTombstoneLoadKey,
 	GarbageCollectionMessage,
 	GarbageCollectionMessageType,
 	ISweepMessage,
@@ -59,5 +55,5 @@ export {
 	GCSummaryStateTracker,
 	IGCSummaryTrackingData,
 } from "./gcSummaryStateTracker.js";
-export { GCTelemetryTracker, sendGCUnexpectedUsageEvent } from "./gcTelemetry.js";
+export { GCTelemetryTracker } from "./gcTelemetry.js";
 export { UnreferencedStateTracker } from "./gcUnreferencedStateTracker.js";

package/src/index.ts

@@ -37,7 +37,6 @@ export {
 	RuntimeHeaders,
 	ChannelCollectionFactory,
 	AllowTombstoneRequestHeaderKey,
-	AllowInactiveRequestHeaderKey,
 } from "./channelCollection.js";
 export {
 	GCNodeType,

package/src/messageTypes.ts

@@ -88,12 +88,18 @@ export interface IContainerRuntimeMessageCompatDetails {
  * IMPORTANT: when creating one to be serialized, set the properties in the order they appear here.
  * This way stringified values can be compared.
  */
-interface TypedContainerRuntimeMessage<TType extends ContainerMessageType, TContents> {
+type TypedContainerRuntimeMessage<
+	TType extends ContainerMessageType,
+	TContents,
+	TUSedCompatDetails extends boolean = false,
+> = {
 	/** Type of the op, within the ContainerRuntime's domain */
 	type: TType;
 	/** Domain-specific contents, interpreted according to the type */
 	contents: TContents;
-}
+} & (TUSedCompatDetails extends true
+	? Partial<RecentlyAddedContainerRuntimeMessageDetails>
+	: { compatDetails?: undefined });
 
 /**
  * Additional details expected for any recently added message.
@@ -139,10 +145,9 @@ export type ContainerRuntimeIdAllocationMessage = TypedContainerRuntimeMessage<
 >;
 export type ContainerRuntimeGCMessage = TypedContainerRuntimeMessage<
 	ContainerMessageType.GC,
-	GarbageCollectionMessage
-> &
-	// While deprecating: GC messages may still contain compat details for now
-	Partial<RecentlyAddedContainerRuntimeMessageDetails>;
+	GarbageCollectionMessage,
+	true // TUsedCompatDetails
+>;
 export type ContainerRuntimeDocumentSchemaMessage = TypedContainerRuntimeMessage<
 	ContainerMessageType.DocumentSchemaChange,
 	IDocumentSchemaChangeMessage
@@ -223,15 +228,11 @@ export type InboundSequencedContainerRuntimeMessage = Omit<
  * There should never be a runtime value of "__not_a_...".
  * Currently additionally replaces `contents` type until protocol-definitions update is taken with `unknown` instead of `any`.
  */
-type InboundSequencedNonContainerRuntimeMessage = Omit<
+export type InboundSequencedNonContainerRuntimeMessage = Omit<
 	ISequencedDocumentMessage,
 	"type" | "contents"
 > & { type: "__not_a_container_runtime_message_type__"; contents: unknown };
 
-export type InboundSequencedContainerRuntimeMessageOrSystemMessage =
-	| InboundSequencedContainerRuntimeMessage
-	| InboundSequencedNonContainerRuntimeMessage;
-
 /** A [loose] InboundSequencedContainerRuntimeMessage that is recent and may contain compat details.
  * It exists solely to to provide access to those details.
  */

package/src/metadata.ts

@@ -6,10 +6,24 @@
 import type { BatchId } from "./opLifecycle/index.js";
 
 /** Syntactic sugar for casting */
-export function asBatchMetadata(metadata: unknown): IBatchMetadata | undefined {
-	return metadata as IBatchMetadata | undefined;
+export function asBatchMetadata(metadata: unknown): Partial<IBatchMetadata> | undefined {
+	return metadata as Partial<IBatchMetadata> | undefined;
 }
 
+/** Syntactic sugar for casting */
+export function asEmptyBatchLocalOpMetadata(
+	localOpMetadata: unknown,
+): IEmptyBatchMetadata | undefined {
+	return localOpMetadata as IEmptyBatchMetadata | undefined;
+}
+
+/**
+ * Properties put on the localOpMetadata object for empty batches
+ */
+export interface IEmptyBatchMetadata {
+	// Set to true on localOpMetadata for empty batches
+	emptyBatch?: boolean;
+}
 /**
  * Properties put on the op metadata object for batch tracking
  */

package/src/opLifecycle/batchManager.ts

@@ -156,7 +156,10 @@ const addBatchMetadata = (batch: IBatch, batchId?: BatchId): IBatch => {
 
 	const firstMsg = batch.messages[0];
 	const lastMsg = batch.messages[batchEnd];
-	assert(firstMsg !== undefined && lastMsg !== undefined, "expected non-empty batch");
+	assert(
+		firstMsg !== undefined && lastMsg !== undefined,
+		0x9d1 /* expected non-empty batch */,
+	);
 
 	const firstMetadata: Partial<IBatchMetadata> = firstMsg.metadata ?? {};
 	const lastMetadata: Partial<IBatchMetadata> = lastMsg.metadata ?? {};

package/src/opLifecycle/index.ts

@@ -16,7 +16,12 @@ export { Outbox, getLongStack } from "./outbox.js";
 export { OpCompressor } from "./opCompressor.js";
 export { OpDecompressor } from "./opDecompressor.js";
 export { OpSplitter, splitOp, isChunkedMessage } from "./opSplitter.js";
-export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor.js";
+export {
+	ensureContentsDeserialized,
+	InboundBatch,
+	RemoteMessageProcessor,
+	unpackRuntimeMessage,
+} from "./remoteMessageProcessor.js";
 export {
 	OpGroupingManager,
 	OpGroupingManagerConfig,

package/src/opLifecycle/opGroupingManager.ts

@@ -49,6 +49,40 @@ export class OpGroupingManager {
 		this.logger = createChildLogger({ logger, namespace: "OpGroupingManager" });
 	}
 
+	/**
+	 * Creates a new batch with a single message of type "groupedBatch" and empty contents.
+	 * This is needed as a placeholder if a batch becomes empty on resubmit, but we are tracking batch IDs.
+	 * @param resubmittingBatchId - batch ID of the resubmitting batch
+	 * @param referenceSequenceNumber - reference sequence number
+	 * @returns - IBatch containing a single empty Grouped Batch op
+	 */
+	public createEmptyGroupedBatch(
+		resubmittingBatchId: string,
+		referenceSequenceNumber: number,
+	): IBatch<[BatchMessage]> {
+		assert(
+			this.config.groupedBatchingEnabled,
+			0xa00 /* cannot create empty grouped batch when grouped batching is disabled */,
+		);
+		const serializedContent = JSON.stringify({
+			type: OpGroupingManager.groupedBatchOp,
+			contents: [],
+		});
+
+		return {
+			contentSizeInBytes: 0,
+			messages: [
+				{
+					metadata: { batchId: resubmittingBatchId },
+					localOpMetadata: { emptyBatch: true },
+					referenceSequenceNumber,
+					contents: serializedContent,
+				},
+			],
+			referenceSequenceNumber,
+		};
+	}
+
 	/**
 	 * Converts the given batch into a "grouped batch" - a batch with a single message of type "groupedBatch",
 	 * with contents being an array of the original batch's messages.
@@ -70,10 +104,14 @@ export class OpGroupingManager {
 				referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
 			});
 		}
-
+		// We expect this will be on the first message, if present at all.
+		let groupedBatchId;
 		for (const message of batch.messages) {
 			if (message.metadata) {
 				const { batch: _batch, batchId, ...rest } = message.metadata;
+				if (batchId) {
+					groupedBatchId = batchId;
+				}
 				assert(Object.keys(rest).length === 0, 0x5dd /* cannot group ops with metadata */);
 			}
 		}
@@ -91,7 +129,7 @@ export class OpGroupingManager {
 			...batch,
 			messages: [
 				{
-					metadata: undefined,
+					metadata: { batchId: groupedBatchId },
 					// TODO why are we non null asserting here?
 					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 					referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
@@ -121,7 +159,8 @@ export class OpGroupingManager {
 			// Grouped batching must be enabled
 			this.config.groupedBatchingEnabled &&
 			// The number of ops in the batch must surpass the configured threshold
-			batch.messages.length >= this.config.opCountThreshold &&
+			// or be empty (to allow for empty batches to be grouped)
+			(batch.messages.length === 0 || batch.messages.length >= this.config.opCountThreshold) &&
 			// Support for reentrant batches must be explicitly enabled
 			(this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true)
 		);

package/src/opLifecycle/outbox.ts

@@ -254,6 +254,14 @@ export class Outbox {
 		// Don't use resubmittingBatchId for idAllocationBatch.
 		// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
 		this.flushInternal(this.idAllocationBatch);
+		// We need to flush an empty batch if the main batch *becomes* empty on resubmission.
+		// When resubmitting the main batch, the blobAttach batch will always be empty since we don't resubmit them simultaneously.
+		// And conversely, the blobAttach will never *become* empty on resubmit.
+		// So if both blobAttachBatch and mainBatch are empty, we must submit an empty main batch.
+		if (resubmittingBatchId && this.blobAttachBatch.empty && this.mainBatch.empty) {
+			this.flushEmptyBatch(resubmittingBatchId);
+			return;
+		}
 		this.flushInternal(
 			this.blobAttachBatch,
 			true /* disableGroupedBatching */,
@@ -266,6 +274,28 @@ export class Outbox {
 		);
 	}
 
+	private flushEmptyBatch(resubmittingBatchId: BatchId) {
+		const referenceSequenceNumber =
+			this.params.getCurrentSequenceNumbers().referenceSequenceNumber;
+		assert(
+			referenceSequenceNumber !== undefined,
+			0xa01 /* reference sequence number should be defined */,
+		);
+		const emptyGroupedBatch = this.params.groupingManager.createEmptyGroupedBatch(
+			resubmittingBatchId,
+			referenceSequenceNumber,
+		);
+		let clientSequenceNumber: number | undefined;
+		if (this.params.shouldSend()) {
+			clientSequenceNumber = this.sendBatch(emptyGroupedBatch);
+		}
+		this.params.pendingStateManager.onFlushBatch(
+			emptyGroupedBatch.messages, // This is the single empty Grouped Batch message
+			clientSequenceNumber,
+		);
+		return;
+	}
+
 	private flushInternal(
 		batchManager: BatchManager,
 		disableGroupedBatching: boolean = false,
@@ -298,7 +328,7 @@ export class Outbox {
 			clientSequenceNumber = this.sendBatch(processedBatch);
 			assert(
 				clientSequenceNumber === undefined || clientSequenceNumber >= 0,
-				"unexpected negative clientSequenceNumber (empty batch should yield undefined)",
+				0x9d2 /* unexpected negative clientSequenceNumber (empty batch should yield undefined) */,
 			);
 		}