@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.3.1.0.125672

package/src/garbageCollectionConstants.ts

@@ -0,0 +1,38 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { GCVersion } from "./summaryFormat";
+
+/** The stable version of garbage collection in production. */
+export const stableGCVersion: GCVersion = 1;
+/** The current version of garbage collection. */
+export const currentGCVersion: GCVersion = 2;
+
+// Feature gate key to turn GC on / off.
+export const runGCKey = "Fluid.GarbageCollection.RunGC";
+// Feature gate key to turn GC sweep on / off.
+export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
+// Feature gate key to turn GC test mode on / off.
+export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
+// Feature gate key to expire a session after a set period of time.
+export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
+// Feature gate key to write the gc blob as a handle if the data is the same.
+export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
+// Feature gate key to turn GC sweep log off.
+export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
+// Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
+export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
+// Feature gate to enable throwing an error when tombstone object is loaded (requested).
+export const throwOnTombstoneLoadKey = "Fluid.GarbageCollection.ThrowOnTombstoneLoad";
+// Feature gate to enable throwing an error when tombstone object is used (e.g. outgoing or incoming ops).
+export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
+// Feature gate to enable GC version upgrade.
+export const gcVersionUpgradeToV2Key = "Fluid.GarbageCollection.GCVersionUpgradeToV2";
+
+// One day in milliseconds.
+export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
+
+export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
+export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days

package/src/garbageCollectionHelpers.ts

@@ -0,0 +1,37 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
+import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
+import { MonitoringContext } from "@fluidframework/telemetry-utils";
+import {
+	disableTombstoneKey,
+	runSweepKey,
+	throwOnTombstoneLoadKey,
+	throwOnTombstoneUsageKey,
+} from "./garbageCollectionConstants";
+
+/**
+ * Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
+ * tombstoned or deleted object is loaded.
+ */
+export function sendGCUnexpectedUsageEvent(
+	mc: MonitoringContext,
+	event: ITelemetryGenericEvent & { category: "error" | "generic"; isSummarizerClient: boolean },
+	packagePath: readonly string[] | undefined,
+	error?: unknown,
+) {
+	event.pkg = packagePathToTelemetryProperty(packagePath);
+	event.tombstoneFlags = JSON.stringify({
+		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
+		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
+		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
+	});
+	event.sweepFlags = JSON.stringify({
+		EnableSweepFlag: mc.config.getBoolean(runSweepKey),
+	});
+
+	mc.logger.sendTelemetryEvent(event, error);
+}

package/src/gcSweepReadyUsageDetection.ts

@@ -6,41 +6,44 @@
 import { ITelemetryProperties } from "@fluidframework/common-definitions";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import {
-    IConfigProvider,
-    IFluidErrorBase,
-    LoggingError,
-    MonitoringContext,
+	IConfigProvider,
+	IFluidErrorBase,
+	LoggingError,
+	MonitoringContext,
 } from "@fluidframework/telemetry-utils";
-import { oneDayMs } from "./garbageCollection";
+import { oneDayMs } from "./garbageCollectionConstants";
 
 /**
  * Feature Gate Key -
  * How many days between closing the container from this error (avoids locking user out of their file altogether)
  */
-export const skipClosureForXDaysKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
+export const skipClosureForXDaysKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
 
 /**
  * LocalStorage key (NOT via feature gate / monitoring context)
  * A map from docId to info about the last time we closed due to this error
  */
-export const closuresMapLocalStorageKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
+export const closuresMapLocalStorageKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
 
 /**
  * Feature gate key to enable closing the container if SweepReady objects are used.
  * Value should contain keywords "interactiveClient" and/or "summarizer" to enable detection in each container type
  */
 const sweepReadyUsageDetectionSetting = {
-    read(config: IConfigProvider) {
-        const sweepReadyUsageDetectionKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
-        const value = config.getString(sweepReadyUsageDetectionKey);
-        if (value === undefined) {
-            return { interactiveClient: false, summarizer: false };
-        }
-        return {
-            interactiveClient: value.includes("interactiveClient"),
-            summarizer: value.includes("summarizer"),
-        };
-    },
+	read(config: IConfigProvider) {
+		const sweepReadyUsageDetectionKey =
+			"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
+		const value = config.getString(sweepReadyUsageDetectionKey);
+		if (value === undefined) {
+			return { interactiveClient: false, summarizer: false };
+		}
+		return {
+			interactiveClient: value.includes("interactiveClient"),
+			summarizer: value.includes("summarizer"),
+		};
+	},
 };
 
 /**
@@ -52,8 +55,8 @@ const sweepReadyUsageDetectionSetting = {
  * since only the Summarizer has the latest truth about unreferenced node tracking
  */
 export class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {
-    /** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
-    public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
+	/** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
+	public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
 }
 
 /**
@@ -65,75 +68,78 @@ export class SweepReadyUsageError extends LoggingError implements IFluidErrorBas
  * (via skipClosureForXDaysKey above.  Uses localStorage and closuresMapLocalStorageKey to implement this behavior)
  */
 export class SweepReadyUsageDetectionHandler {
-    private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
+	private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
 
-    constructor(
-        private readonly uniqueContainerKey: string,
-        private readonly mc: MonitoringContext,
-        private readonly closeFn: (error?: ICriticalContainerError) => void,
-        localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
-    ) {
-        const noopStorage = { getItem: () => null, setItem: () => {} };
-        // localStorage is not defined in Node environment, so fall back to noopStorage if needed.
-        this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
+	constructor(
+		private readonly uniqueContainerKey: string,
+		private readonly mc: MonitoringContext,
+		private readonly closeFn: (error?: ICriticalContainerError) => void,
+		localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
+	) {
+		const noopStorage = { getItem: () => null, setItem: () => {} };
+		// localStorage is not defined in Node environment, so fall back to noopStorage if needed.
+		this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
 
-        if (this.localStorage === noopStorage) {
-            // This means the Skip Closure Period logic will not work.
-            this.mc.logger.sendTelemetryEvent({ eventName: "SweepReadyUsageDetectionHandlerNoopStorage" });
-        }
-    }
+		if (this.localStorage === noopStorage) {
+			// This means the Skip Closure Period logic will not work.
+			this.mc.logger.sendTelemetryEvent({
+				eventName: "SweepReadyUsageDetectionHandlerNoopStorage",
+			});
+		}
+	}
 
-    /**
-      * If SweepReady Usage Detection is enabled, close the interactive container.
-      * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
-      *
-      * Once Sweep is fully implemented, this will be removed since the objects will be gone
-      * and errors will arise elsewhere in the runtime
-     */
-    public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
-        if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
-            return;
-        }
+	/**
+	 * If SweepReady Usage Detection is enabled, close the interactive container.
+	 * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
+	 *
+	 * Once Sweep is fully implemented, this will be removed since the objects will be gone
+	 * and errors will arise elsewhere in the runtime
+	 */
+	public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
+		if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
+			return;
+		}
 
-        // Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
-        // However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
-        // which will only allow the container to close once during that period, to avoid locking users out.
-        let shouldClose: boolean = true;
-        let pastClosuresMap: Record<string, { lastCloseTime: number; } | undefined> = {};
-        let lastCloseTime: number | undefined;
-        const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
-        if (skipClosureForXDays !== undefined) {
-            // Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
-            try {
-                const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
-                const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
-                if (typeof parsedValue === "object") {
-                    pastClosuresMap = parsedValue;
-                }
-            } catch (e) {
-            }
-            lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
+		// Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
+		// However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
+		// which will only allow the container to close once during that period, to avoid locking users out.
+		let shouldClose: boolean = true;
+		let pastClosuresMap: Record<string, { lastCloseTime: number } | undefined> = {};
+		let lastCloseTime: number | undefined;
+		const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
+		if (skipClosureForXDays !== undefined) {
+			// Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
+			try {
+				const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
+				const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
+				if (typeof parsedValue === "object") {
+					pastClosuresMap = parsedValue;
+				}
+			} catch (e) {}
+			lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
 
-            // Don't close if we did already within the Skip Closure Period
-            if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs) {
-                shouldClose = false;
-            }
-        }
+			// Don't close if we did already within the Skip Closure Period
+			if (
+				lastCloseTime !== undefined &&
+				Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs
+			) {
+				shouldClose = false;
+			}
+		}
 
-        const error = new SweepReadyUsageError(
-            "SweepReady object used in Non-Summarizer Client",
-            { errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }) },
-        );
-        if (shouldClose) {
-            // Update closures map in localStorage before closing
-            // Note there is a race condition between different tabs updating localStorage and overwriting
-            // each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
-            pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
-            this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
+		const error = new SweepReadyUsageError("SweepReady object used in Non-Summarizer Client", {
+			errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }),
+		});
+		if (shouldClose) {
+			// Update closures map in localStorage before closing
+			// Note there is a race condition between different tabs updating localStorage and overwriting
+			// each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
+			pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
+			this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
 
-            this.closeFn(error);
-        } else {
-            this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
-        }
-    }
+			this.closeFn(error);
+		} else {
+			this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
+		}
+	}
 }

package/src/index.ts

@@ -4,81 +4,80 @@
  */
 
 export {
-    ContainerMessageType,
-    IChunkedOp,
-    ContainerRuntimeMessage,
-    IGCRuntimeOptions,
-    ISummaryRuntimeOptions,
-    ISummaryBaseConfiguration,
-    ISummaryConfigurationHeuristics,
-    ISummaryConfigurationDisableSummarizer,
-    ISummaryConfigurationDisableHeuristics,
-    IContainerRuntimeOptions,
-    IRootSummaryTreeWithStats,
-    isRuntimeMessage,
-    RuntimeMessage,
-    unpackRuntimeMessage,
-    agentSchedulerId,
-    ContainerRuntime,
-    RuntimeHeaders,
-    ISummaryConfiguration,
-    DefaultSummaryConfiguration,
-    ICompressionRuntimeOptions,
-    CompressionAlgorithms,
+	ContainerMessageType,
+	ContainerRuntimeMessage,
+	IGCRuntimeOptions,
+	ISummaryRuntimeOptions,
+	ISummaryBaseConfiguration,
+	ISummaryConfigurationHeuristics,
+	ISummaryConfigurationDisableSummarizer,
+	ISummaryConfigurationDisableHeuristics,
+	IContainerRuntimeOptions,
+	IRootSummaryTreeWithStats,
+	isRuntimeMessage,
+	RuntimeMessage,
+	agentSchedulerId,
+	ContainerRuntime,
+	RuntimeHeaders,
+	AllowTombstoneRequestHeaderKey,
+	TombstoneResponseHeaderKey,
+	ISummaryConfiguration,
+	DefaultSummaryConfiguration,
+	ICompressionRuntimeOptions,
+	CompressionAlgorithms,
 } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
+export { IGCStats } from "./garbageCollection";
 export {
-    gcBlobPrefix,
-    gcTombstoneBlobKey,
-    gcTreeKey,
-    IGCStats,
-} from "./garbageCollection";
-export {
-    IPendingFlush,
-    IPendingLocalState,
-    IPendingMessage,
-    IPendingState,
+	IPendingFlush,
+	IPendingLocalState,
+	IPendingMessage,
+	IPendingState,
 } from "./pendingStateManager";
 export { Summarizer } from "./summarizer";
 export {
-    EnqueueSummarizeResult,
-    IAckSummaryResult,
-    IBaseSummarizeResult,
-    IBroadcastSummaryResult,
-    ICancellationToken,
-    IConnectableRuntime,
-    IEnqueueSummarizeOptions,
-    IGenerateSummaryTreeResult,
-    IGeneratedSummaryStats,
-    INackSummaryResult,
-    IOnDemandSummarizeOptions,
-    IProvideSummarizer,
-    IRefreshSummaryAckOptions,
-    ISubmitSummaryOpResult,
-    ISubmitSummaryOptions,
-    ISummarizeOptions,
-    ISummarizeResults,
-    ISummarizer,
-    ISummarizerEvents,
-    ISummarizerInternalsProvider,
-    ISummarizerRuntime,
-    ISummarizingWarning,
-    ISummaryCancellationToken,
-    IUploadSummaryResult,
-    SubmitSummaryResult,
-    SummarizeResultPart,
-    SummarizerStopReason,
+	EnqueueSummarizeResult,
+	IAckSummaryResult,
+	IBaseSummarizeResult,
+	IBroadcastSummaryResult,
+	ICancellationToken,
+	IConnectableRuntime,
+	IEnqueueSummarizeOptions,
+	IGenerateSummaryTreeResult,
+	IGeneratedSummaryStats,
+	INackSummaryResult,
+	IOnDemandSummarizeOptions,
+	IProvideSummarizer,
+	IRefreshSummaryAckOptions,
+	ISubmitSummaryOpResult,
+	ISubmitSummaryOptions,
+	ISummarizeOptions,
+	ISummarizeResults,
+	ISummarizer,
+	ISummarizerEvents,
+	ISummarizerInternalsProvider,
+	ISummarizerRuntime,
+	ISummarizingWarning,
+	ISummaryCancellationToken,
+	IUploadSummaryResult,
+	SubmitSummaryResult,
+	SummarizeResultPart,
+	SummarizerStopReason,
 } from "./summarizerTypes";
 export {
-    IAckedSummary,
-    IClientSummaryWatcher,
-    ISummary,
-    ISummaryCollectionOpEvents,
-    ISummaryAckMessage,
-    ISummaryNackMessage,
-    ISummaryOpMessage,
-    OpActionEventListener,
-    OpActionEventName,
-    SummaryCollection,
+	IAckedSummary,
+	IClientSummaryWatcher,
+	ISummary,
+	ISummaryCollectionOpEvents,
+	ISummaryAckMessage,
+	ISummaryNackMessage,
+	ISummaryOpMessage,
+	OpActionEventListener,
+	OpActionEventName,
+	SummaryCollection,
 } from "./summaryCollection";
-export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
+export {
+	ICancellableSummarizerController,
+	neverCancelledSummaryToken,
+} from "./runWhileConnectedCoordinator";
+export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";

package/src/opLifecycle/batchManager.ts

@@ -0,0 +1,167 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { UsageError } from "@fluidframework/driver-utils";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
+import { ICompressionRuntimeOptions } from "../containerRuntime";
+import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions";
+
+export interface IBatchManagerOptions {
+	readonly enableOpReentryCheck?: boolean;
+	readonly hardLimit: number;
+	readonly softLimit?: number;
+	readonly compressionOptions?: ICompressionRuntimeOptions;
+}
+
+/**
+ * Helper class that manages partial batch & rollback.
+ */
+export class BatchManager {
+	private readonly logger;
+	private pendingBatch: BatchMessage[] = [];
+	private batchContentSize = 0;
+	/**
+	 * Track the number of ops which were detected to have a mismatched
+	 * reference sequence number, in order to self-throttle the telemetry events.
+	 *
+	 * This should be removed as part of ADO:2322
+	 */
+	private readonly maxMismatchedOpsToReport = 5;
+	private mismatchedOpsReported = 0;
+
+	public get length() {
+		return this.pendingBatch.length;
+	}
+	public get contentSizeInBytes() {
+		return this.batchContentSize;
+	}
+
+	constructor(public readonly options: IBatchManagerOptions, logger: ITelemetryLogger) {
+		this.logger = ChildLogger.create(logger, "BatchManager");
+	}
+
+	public push(message: BatchMessage): boolean {
+		this.checkReferenceSequenceNumber(message);
+
+		const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
+		const opCount = this.pendingBatch.length;
+
+		// Attempt to estimate batch size, aka socket message size.
+		// Each op has pretty large envelope, estimating to be 200 bytes.
+		// Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
+		// Not taking it into account, as compression work should help there - compressed payload will be
+		// initially stored as base64, and that requires only 2 extra escape characters.
+		const socketMessageSize = contentSize + 200 * opCount;
+
+		// If we were provided soft limit, check for exceeding it.
+		// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
+		// and start over. That's not an option if we have no ops.
+		// If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
+		// Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
+		if (
+			this.options.softLimit !== undefined &&
+			this.length > 0 &&
+			socketMessageSize >= this.options.softLimit
+		) {
+			return false;
+		}
+
+		if (socketMessageSize >= this.options.hardLimit) {
+			return false;
+		}
+
+		this.batchContentSize = contentSize;
+		this.pendingBatch.push(message);
+		return true;
+	}
+
+	public get empty() {
+		return this.pendingBatch.length === 0;
+	}
+
+	public popBatch(): IBatch {
+		const batch: IBatch = {
+			content: this.pendingBatch,
+			contentSizeInBytes: this.batchContentSize,
+		};
+
+		this.pendingBatch = [];
+		this.batchContentSize = 0;
+
+		return addBatchMetadata(batch);
+	}
+
+	/**
+	 * Capture the pending state at this point
+	 */
+	public checkpoint(): IBatchCheckpoint {
+		const startPoint = this.pendingBatch.length;
+		return {
+			rollback: (process: (message: BatchMessage) => void) => {
+				for (let i = this.pendingBatch.length; i > startPoint; ) {
+					i--;
+					const message = this.pendingBatch[i];
+					this.batchContentSize -= message.contents?.length ?? 0;
+					process(message);
+				}
+
+				this.pendingBatch.length = startPoint;
+			},
+		};
+	}
+
+	private checkReferenceSequenceNumber(message: BatchMessage) {
+		if (
+			this.pendingBatch.length === 0 ||
+			message.referenceSequenceNumber === this.pendingBatch[0].referenceSequenceNumber
+		) {
+			// The reference sequence numbers are stable
+			return;
+		}
+
+		const telemetryProperties = {
+			referenceSequenceNumber: this.pendingBatch[0].referenceSequenceNumber,
+			messageReferenceSequenceNumber: message.referenceSequenceNumber,
+			type: message.deserializedContent.type,
+			length: this.pendingBatch.length,
+			enableOpReentryCheck: this.options.enableOpReentryCheck === true,
+		};
+		const error = new UsageError("Submission of an out of order message");
+		const eventName = "ReferenceSequenceNumberMismatch";
+
+		if (this.options.enableOpReentryCheck === true) {
+			this.logger.sendErrorEvent({ eventName, ...telemetryProperties }, error);
+			throw error;
+		}
+
+		if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
+			this.logger.sendErrorEvent(
+				{
+					eventName,
+					...telemetryProperties,
+					ops: this.mismatchedOpsReported,
+					maxOps: this.maxMismatchedOpsToReport,
+				},
+				error,
+			);
+		}
+	}
+}
+
+const addBatchMetadata = (batch: IBatch): IBatch => {
+	if (batch.content.length > 1) {
+		batch.content[0].metadata = {
+			...batch.content[0].metadata,
+			batch: true,
+		};
+		batch.content[batch.content.length - 1].metadata = {
+			...batch.content[batch.content.length - 1].metadata,
+			batch: false,
+		};
+	}
+
+	return batch;
+};