@fluidframework/container-runtime 2.0.0-internal.7.0.0 → 2.0.0-internal.7.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.0.0-internal.7.0.0",
+  "version": "2.0.0-internal.7.2.0",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -35,18 +35,18 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/container-definitions": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/container-runtime-definitions": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/core-interfaces": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/core-utils": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/datastore": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/driver-definitions": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/driver-utils": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
+    "@fluid-internal/client-utils": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/container-definitions": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/container-runtime-definitions": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/core-interfaces": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/core-utils": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/datastore": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/driver-definitions": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/driver-utils": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
     "@fluidframework/protocol-definitions": "^3.0.0",
-    "@fluidframework/runtime-definitions": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/runtime-utils": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/telemetry-utils": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
+    "@fluidframework/runtime-definitions": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/runtime-utils": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/telemetry-utils": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
     "double-ended-queue": "^2.1.0-0",
     "events": "^3.1.0",
     "lz4js": "^0.2.0",
@@ -54,15 +54,15 @@
     "uuid": "^9.0.0"
   },
   "devDependencies": {
-    "@fluid-internal/stochastic-test-utils": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
+    "@fluid-internal/stochastic-test-utils": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
     "@fluid-tools/benchmark": "^0.48.0",
-    "@fluid-tools/build-cli": "^0.24.0",
-    "@fluidframework/build-common": "^2.0.0",
-    "@fluidframework/build-tools": "^0.24.0",
-    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.6.3.0",
-    "@fluidframework/eslint-config-fluid": "^2.1.0",
-    "@fluidframework/mocha-test-setup": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
-    "@fluidframework/test-runtime-utils": ">=2.0.0-internal.7.0.0 <2.0.0-internal.7.1.0",
+    "@fluid-tools/build-cli": "^0.26.1",
+    "@fluidframework/build-common": "^2.0.3",
+    "@fluidframework/build-tools": "^0.26.1",
+    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.7.1.0",
+    "@fluidframework/eslint-config-fluid": "^3.0.0",
+    "@fluidframework/mocha-test-setup": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
+    "@fluidframework/test-runtime-utils": ">=2.0.0-internal.7.2.0 <2.0.0-internal.7.3.0",
     "@microsoft/api-extractor": "^7.37.0",
     "@types/double-ended-queue": "^2.1.0",
     "@types/events": "^3.0.0",
@@ -70,14 +70,13 @@
     "@types/sinon": "^7.0.13",
     "@types/uuid": "^9.0.2",
     "c8": "^7.7.1",
-    "copyfiles": "^2.4.1",
     "cross-env": "^7.0.3",
-    "eslint": "~8.6.0",
+    "eslint": "~8.50.0",
     "mocha": "^10.2.0",
     "mocha-json-output-reporter": "^2.0.1",
     "mocha-multi-reporters": "^1.5.1",
     "moment": "^2.21.0",
-    "prettier": "~2.6.2",
+    "prettier": "~3.0.3",
     "rimraf": "^4.4.0",
     "sinon": "^7.4.2",
     "typescript": "~5.1.6"
@@ -85,38 +84,7 @@
   "typeValidation": {
     "broken": {
       "ClassDeclaration_ContainerRuntime": {
-        "forwardCompat": false,
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IAckedSummary": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IAckSummaryResult": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IBroadcastSummaryResult": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_ISummaryOpMessage": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_ISummaryNackMessage": {
-        "backCompat": false
-      },
-      "ClassDeclaration_Summarizer": {
-        "backCompat": false
-      },
-      "ClassDeclaration_SummaryCollection": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_ISummarizerRuntime": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_INackSummaryResult": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_ISummaryAckMessage": {
-        "backCompat": false
+        "forwardCompat": false
       }
     }
   },
@@ -124,12 +92,12 @@
     "build": "fluid-build . --task build",
     "build:commonjs": "fluid-build . --task commonjs",
     "build:compile": "fluid-build . --task compile",
-    "build:docs": "api-extractor run --local --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
+    "build:docs": "api-extractor run --local",
     "build:esnext": "tsc --project ./tsconfig.esnext.json",
     "build:genver": "gen-version",
     "build:test": "tsc --project ./src/test/tsconfig.json",
-    "ci:build:docs": "api-extractor run --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
-    "clean": "rimraf --glob 'dist' 'lib' '*.tsbuildinfo' '*.build.log' '_api-extractor-temp' 'nyc'",
+    "ci:build:docs": "api-extractor run",
+    "clean": "rimraf --glob dist lib \"*.tsbuildinfo\" \"*.build.log\" _api-extractor-temp nyc",
     "eslint": "eslint --format stylish src",
     "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
     "format": "npm run prettier:fix",

package/src/blobManager.ts

@@ -30,6 +30,7 @@ import {
 	LoggingError,
 	MonitoringContext,
 	PerformanceEvent,
+	wrapError,
 } from "@fluidframework/telemetry-utils";
 import {
 	IGarbageCollectionData,
@@ -37,14 +38,9 @@ import {
 	ITelemetryContext,
 } from "@fluidframework/runtime-definitions";
 
+import { canRetryOnError, runWithRetry } from "@fluidframework/driver-utils";
 import { ContainerRuntime, TombstoneResponseHeaderKey } from "./containerRuntime";
-import {
-	sendGCUnexpectedUsageEvent,
-	disableAttachmentBlobSweepKey,
-	throwOnTombstoneLoadKey,
-} from "./gc";
-import { Throttler, formExponentialFn, IThrottler } from "./throttler";
-import { summarizerClientType } from "./summary";
+import { sendGCUnexpectedUsageEvent, disableAttachmentBlobSweepKey } from "./gc";
 import { IBlobMetadata } from "./metadata";
 
 /**
@@ -88,25 +84,9 @@ export class BlobHandle implements IFluidHandle<ArrayBufferLike> {
 	}
 }
 
-class CancellableThrottler {
-	constructor(private readonly throttler: IThrottler) {}
-	private cancelP = new Deferred<void>();
-
-	public async getDelay(): Promise<void> {
-		return Promise.race([
-			this.cancelP.promise,
-			new Promise<void>((resolve) => setTimeout(resolve, this.throttler.getDelay())),
-		]);
-	}
-
-	public cancel() {
-		this.cancelP.resolve();
-		this.cancelP = new Deferred<void>();
-	}
-}
-
 /**
  * Information from a snapshot needed to load BlobManager
+ * @public
  */
 export interface IBlobManagerLoadInfo {
 	ids?: string[];
@@ -119,7 +99,7 @@ export type IBlobManagerRuntime = Pick<
 	IContainerRuntime,
 	"attachState" | "connected" | "logger" | "clientDetails"
 > &
-	Pick<ContainerRuntime, "gcTombstoneEnforcementAllowed"> &
+	Pick<ContainerRuntime, "gcTombstoneEnforcementAllowed" | "gcThrowOnTombstoneLoad"> &
 	TypedEventEmitter<IContainerRuntimeEvents>;
 
 type ICreateBlobResponseWithTTL = ICreateBlobResponse & Partial<Record<"minTTLInSeconds", number>>;
@@ -180,17 +160,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	 */
 	private readonly opsInFlight: Map<string, string[]> = new Map();
 
-	private readonly retryThrottler = new CancellableThrottler(
-		new Throttler(
-			60 * 1000, // 60 sec delay window
-			30 * 1000, // 30 sec max delay
-			// throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
-			formExponentialFn({ coefficient: 20, initialDelay: 0 }),
-		),
-	);
-
-	/** If true, throw an error when a tombstone attachment blob is retrieved. */
-	private readonly throwOnTombstoneLoad: boolean;
 	/**
 	 * This stores IDs of tombstoned blobs.
 	 * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
@@ -198,6 +167,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	private readonly tombstonedBlobs: Set<string> = new Set();
 
 	private readonly sendBlobAttachOp: (localId: string, storageId?: string) => void;
+	private stopAttaching: boolean = false;
 
 	constructor(
 		private readonly routeContext: IFluidHandleContext,
@@ -229,11 +199,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 			logger: this.runtime.logger,
 			namespace: "BlobManager",
 		});
-		// Read the feature flag that tells whether to throw when a tombstone blob is requested.
-		this.throwOnTombstoneLoad =
-			this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
-			this.runtime.gcTombstoneEnforcementAllowed &&
-			this.runtime.clientDetails.type !== summarizerClientType;
 
 		this.redirectTable = this.load(snapshot);
 
@@ -339,7 +304,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	 * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
 	 */
 	public async processStashedChanges() {
-		this.retryThrottler.cancel();
 		const pendingUploads = Array.from(this.pendingBlobs.values())
 			.filter((e) => e.pendingStashed === true)
 			.map(async (e) => e.uploadP);
@@ -493,14 +457,39 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		localId: string,
 		blob: ArrayBufferLike,
 	): Promise<ICreateBlobResponse | void> {
-		return PerformanceEvent.timedExecAsync(
+		return runWithRetry(
+			async () => {
+				try {
+					return await this.getStorage().createBlob(blob);
+				} catch (error) {
+					const entry = this.pendingBlobs.get(localId);
+					assert(
+						!!entry,
+						0x387 /* Must have pending blob entry for blob which failed to upload */,
+					);
+					if (entry.opsent && !canRetryOnError(error)) {
+						throw wrapError(
+							error,
+							() => new LoggingError(`uploadBlob error`, { canRetry: true }),
+						);
+					}
+					throw error;
+				}
+			},
+			"createBlob",
 			this.mc.logger,
-			{ eventName: "createBlob" },
-			async () => this.getStorage().createBlob(blob),
-			{ end: true, cancel: this.runtime.connected ? "error" : "generic" },
+			{
+				cancel: this.pendingBlobs.get(localId)?.abortSignal,
+			},
 		).then(
 			(response) => this.onUploadResolve(localId, response),
-			async (err) => this.onUploadReject(localId, err),
+			(error) => {
+				// it will only reject if we haven't sent an op
+				// and is a non-retriable error. It will only reject
+				// the promise but not throw any error outside.
+				this.pendingBlobs.get(localId)?.handleP.reject(error);
+				this.deletePendingBlob(localId);
+			},
 		);
 	}
 
@@ -530,7 +519,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	private onUploadResolve(localId: string, response: ICreateBlobResponseWithTTL) {
 		const entry = this.pendingBlobs.get(localId);
 		assert(entry !== undefined, 0x6c8 /* pending blob entry not found for uploaded blob */);
-		if (entry.abortSignal?.aborted === true && !entry.opsent) {
+		if ((entry.abortSignal?.aborted === true && !entry.opsent) || this.stopAttaching) {
 			this.deletePendingBlob(localId);
 			return;
 		}
@@ -568,25 +557,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		return response;
 	}
 
-	private async onUploadReject(localId: string, error: any) {
-		const entry = this.pendingBlobs.get(localId);
-		assert(!!entry, 0x387 /* Must have pending blob entry for blob which failed to upload */);
-		if (entry.abortSignal?.aborted === true && !entry.opsent) {
-			this.deletePendingBlob(localId);
-			return;
-		}
-		if (!this.runtime.connected) {
-			// we are probably not connected to storage but start another upload request in case we are
-			entry.uploadP = this.retryThrottler
-				.getDelay()
-				.then(async () => this.uploadBlob(localId, entry.blob));
-			return entry.uploadP;
-		} else {
-			entry.handleP.reject(error);
-			throw error;
-		}
-	}
-
 	/**
 	 * Resubmit a BlobAttach op. Used to add storage IDs to ops that were
 	 * submitted to runtime while disconnected.
@@ -881,7 +851,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 
 		// If the blob is deleted or throw on tombstone load is enabled, throw an error which will fail any attempt
 		// to load the blob.
-		const shouldFail = state === "deleted" || this.throwOnTombstoneLoad;
+		const shouldFail = state === "deleted" || this.runtime.gcThrowOnTombstoneLoad;
 		const request = { url: blobId };
 		const error = responseToException(
 			createResponseError(
@@ -930,7 +900,9 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		}
 	}
 
-	public async attachAndGetPendingBlobs(): Promise<IPendingBlobs | undefined> {
+	public async attachAndGetPendingBlobs(
+		stopBlobAttachingSignal?: AbortSignal,
+	): Promise<IPendingBlobs | undefined> {
 		return PerformanceEvent.timedExecAsync(
 			this.mc.logger,
 			{ eventName: "GetPendingBlobs" },
@@ -945,12 +917,17 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 					for (const [id, entry] of this.pendingBlobs) {
 						if (!localBlobs.has(entry)) {
 							localBlobs.add(entry);
-							if (!entry.opsent) {
-								this.sendBlobAttachOp(id, entry.storageId);
-							}
 							entry.handleP.resolve(this.getBlobHandle(id));
 							attachBlobsP.push(
-								new Promise<void>((resolve) => {
+								new Promise<void>((resolve, reject) => {
+									stopBlobAttachingSignal?.addEventListener(
+										"abort",
+										() => {
+											this.stopAttaching = true;
+											reject(new Error("Operation aborted"));
+										},
+										{ once: true },
+									);
 									const onBlobAttached = (attachedEntry) => {
 										if (attachedEntry === entry) {
 											this.off("blobAttached", onBlobAttached);
@@ -966,11 +943,21 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 							);
 						}
 					}
-					await Promise.all(attachBlobsP);
+					await Promise.allSettled(attachBlobsP).catch(() => {});
 				}
 
 				for (const [id, entry] of this.pendingBlobs) {
+					if (stopBlobAttachingSignal?.aborted && !entry.attached) {
+						this.mc.logger.sendTelemetryEvent({
+							eventName: "UnableToStashBlob",
+							id,
+						});
+						continue;
+					}
 					assert(entry.attached === true, 0x790 /* stashed blob should be attached */);
+					if (!entry.opsent) {
+						this.sendBlobAttachOp(id, entry.storageId);
+					}
 					blobs[id] = {
 						blob: bufferToString(entry.blob, "base64"),
 						storageId: entry.storageId,

package/src/connectionTelemetry.ts

@@ -4,8 +4,11 @@
  */
 
 import {
+	IEventSampler,
+	ISampledTelemetryLogger,
 	ITelemetryLoggerExt,
 	createChildLogger,
+	createSampledLogger,
 	formatTick,
 } from "@fluidframework/telemetry-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";
@@ -53,7 +56,6 @@ interface IOpPerfTimings {
 }
 
 class OpPerfTelemetry {
-	private pongCount: number = 0;
 	private pingLatency: number | undefined;
 
 	// Collab window tracking. This is timestamp of %1000 message.
@@ -61,11 +63,14 @@ class OpPerfTelemetry {
 	private msnTrackingTimestamp: number = 0;
 	// To track round trip time for every %500 client message.
 	private clientSequenceNumberForLatencyStatistics: number | undefined;
-
-	private opProcessingTimes: Partial<IOpPerfTimings> = {};
-
 	// Performance Data to be reported for ops round trips and processing.
-	private opPerfData: Partial<IOpPerfTelemetryProperties> = {};
+	private readonly latencyStatistics = new Map<
+		number,
+		{
+			opProcessingTimes: Partial<IOpPerfTimings>;
+			opPerfData: Partial<IOpPerfTelemetryProperties>;
+		}
+	>();
 
 	private firstConnection = true;
 	private connectionOpSeqNumber: number | undefined;
@@ -75,6 +80,12 @@ class OpPerfTelemetry {
 
 	private readonly logger: ITelemetryLoggerExt;
 
+	private static readonly OP_LATENCY_SAMPLE_RATE = 500;
+	private readonly opLatencyLogger: ISampledTelemetryLogger;
+
+	private static readonly DELTA_LATENCY_SAMPLE_RATE = 100;
+	private readonly deltaLatencyLogger: ISampledTelemetryLogger;
+
 	public constructor(
 		private clientId: string | undefined,
 		private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
@@ -82,6 +93,28 @@ class OpPerfTelemetry {
 	) {
 		this.logger = createChildLogger({ logger, namespace: "OpPerf" });
 
+		const deltaLatencyEventSampler: IEventSampler = (() => {
+			let eventCount = -1;
+			return {
+				sample: () => {
+					eventCount++;
+					const shouldSample =
+						eventCount % OpPerfTelemetry.DELTA_LATENCY_SAMPLE_RATE === 0;
+					if (shouldSample) {
+						eventCount = 0;
+					}
+					return shouldSample;
+				},
+			};
+		})();
+
+		this.deltaLatencyLogger = createSampledLogger(logger, deltaLatencyEventSampler);
+
+		// The SampledLogger here is used get access to the isSamplingDisabled property dervied from
+		// telemetry config properties. The actual sampling logic for op messages happens outside this SampledLogger
+		// due to complexity of the different asynchronus scenarios of the op message lifecycle.
+		this.opLatencyLogger = createSampledLogger(logger);
+
 		this.deltaManager.on("pong", (latency) => this.recordPingTime(latency));
 		this.deltaManager.on("submitOp", (message) => this.beforeOpSubmit(message));
 
@@ -103,42 +136,47 @@ class OpPerfTelemetry {
 		this.deltaManager.on("disconnect", () => {
 			this.sequenceNumberForMsnTracking = undefined;
 			this.clientSequenceNumberForLatencyStatistics = undefined;
-			this.opProcessingTimes = {};
-			this.opPerfData = {};
 			this.connectionOpSeqNumber = undefined;
 			this.firstConnection = false;
-			this.pongCount = 0;
+			this.latencyStatistics.clear();
 		});
 
 		this.deltaManager.outbound.on("push", (messages) => {
 			for (const msg of messages) {
 				if (
 					msg.type === MessageType.Operation &&
-					this.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber
+					(this.opLatencyLogger.isSamplingDisabled ||
+						this.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber)
 				) {
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					const latencyStats = this.latencyStatistics.get(msg.clientSequenceNumber)!;
+					assert(
+						latencyStats !== undefined,
+						0x7c2 /* Latency stats for op should exist */,
+					);
 					assert(
-						this.opProcessingTimes.outboundPushEventTime === undefined,
+						latencyStats.opProcessingTimes.outboundPushEventTime === undefined,
 						0x2c8 /* "outboundPushEventTime should be undefined" */,
 					);
 					assert(
-						this.opPerfData.durationNetwork === undefined,
+						latencyStats.opPerfData.durationNetwork === undefined,
 						0x2c9 /* "durationNetwork should be undefined" */,
 					);
-					this.opProcessingTimes.outboundPushEventTime = Date.now();
+					latencyStats.opProcessingTimes.outboundPushEventTime = Date.now();
 
 					assert(
-						this.opPerfData.durationOutboundBatching === undefined,
+						latencyStats.opPerfData.durationOutboundBatching === undefined,
 						0x2ca /* "durationOutboundBatching should be undefined" */,
 					);
 
 					assert(
-						this.opProcessingTimes.submitOpEventTime !== undefined,
+						latencyStats.opProcessingTimes.submitOpEventTime !== undefined,
 						0x2cb /* "submitOpEventTime should be undefined" */,
 					);
 
-					this.opPerfData.durationOutboundBatching =
-						this.opProcessingTimes.outboundPushEventTime -
-						this.opProcessingTimes.submitOpEventTime;
+					latencyStats.opPerfData.durationOutboundBatching =
+						latencyStats.opProcessingTimes.outboundPushEventTime -
+						latencyStats.opProcessingTimes.submitOpEventTime;
 				}
 			}
 		});
@@ -147,15 +185,20 @@ class OpPerfTelemetry {
 			if (
 				this.clientId === message.clientId &&
 				message.type === MessageType.Operation &&
-				this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber &&
-				this.opProcessingTimes.outboundPushEventTime !== undefined
+				(this.opLatencyLogger.isSamplingDisabled ||
+					this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber)
 			) {
-				this.opProcessingTimes.inboundPushEventTime = Date.now();
-				this.opPerfData.durationNetwork =
-					this.opProcessingTimes.inboundPushEventTime -
-					this.opProcessingTimes.outboundPushEventTime;
-				this.opProcessingTimes.outboundPushEventTime = undefined;
-				this.opPerfData.lengthInboundQueue = this.deltaManager.inbound.length;
+				// We do an explicit check for undefined right after this
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				const latencyStats = this.latencyStatistics.get(message.clientSequenceNumber)!;
+				assert(latencyStats !== undefined, 0x7c3 /* Latency stats for op should exist */);
+				if (latencyStats.opProcessingTimes.outboundPushEventTime !== undefined) {
+					latencyStats.opProcessingTimes.inboundPushEventTime = Date.now();
+					latencyStats.opPerfData.durationNetwork =
+						latencyStats.opProcessingTimes.inboundPushEventTime -
+						latencyStats.opProcessingTimes.outboundPushEventTime;
+					latencyStats.opPerfData.lengthInboundQueue = this.deltaManager.inbound.length;
+				}
 			}
 		});
 
@@ -201,32 +244,33 @@ class OpPerfTelemetry {
 			});
 		}
 
-		// logging one in every 100 pongs, including the first time, if it is a "write" client.
-		if (this.pongCount % 100 === 0 && this.deltaManager.active) {
-			this.logger.sendPerformanceEvent({
+		// logging one in every DELTA_LATENCY_SAMPLE_RATE pongs, including the first time, if it is a "write" client.
+		if (this.deltaManager.active) {
+			this.deltaLatencyLogger.sendPerformanceEvent({
 				eventName: "DeltaLatency",
 				duration: latency,
 			});
 		}
-		this.pongCount++;
 	}
 
 	private beforeOpSubmit(message: IDocumentMessage) {
 		// start with first client op and measure latency every 500 client ops
 		if (
-			this.clientSequenceNumberForLatencyStatistics === undefined &&
-			message.clientSequenceNumber % 500 === 1
+			this.opLatencyLogger.isSamplingDisabled ||
+			(this.clientSequenceNumberForLatencyStatistics === undefined &&
+				message.clientSequenceNumber % OpPerfTelemetry.OP_LATENCY_SAMPLE_RATE === 1)
 		) {
 			assert(
-				this.opProcessingTimes.outboundPushEventTime === undefined,
-				0x2cc /* "OpTimeSittingInboundQueue should be undefined" */,
-			);
-			assert(
-				this.opPerfData.durationNetwork === undefined,
-				0x2cd /* "durationNetwork should be undefined" */,
+				this.latencyStatistics.get(message.clientSequenceNumber) === undefined,
+				0x7c4 /* Existing op perf data for client sequence number */,
 			);
-			this.opProcessingTimes.submitOpEventTime = Date.now();
 			this.clientSequenceNumberForLatencyStatistics = message.clientSequenceNumber;
+			this.latencyStatistics.set(message.clientSequenceNumber, {
+				opProcessingTimes: {
+					submitOpEventTime: Date.now(),
+				},
+				opPerfData: {},
+			});
 		}
 	}
 
@@ -261,20 +305,23 @@ class OpPerfTelemetry {
 
 		if (
 			this.clientId === message.clientId &&
-			this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber
+			(this.opLatencyLogger.isSamplingDisabled ||
+				this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber)
 		) {
+			// We do an explicit check for undefined right after this
+			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+			const latencyData = this.latencyStatistics.get(message.clientSequenceNumber)!;
+			assert(latencyData !== undefined, 0x7c5 /* Undefined latency statistics for op */);
 			assert(
-				this.opProcessingTimes.submitOpEventTime !== undefined,
-				0x120 /* "Undefined latency statistics (op send time)" */,
+				latencyData.opProcessingTimes.submitOpEventTime !== undefined,
+				0x120 /* "Undefined latency statistics for op (op send time)" */,
 			);
 			const currentTime = Date.now();
-
-			if (this.opProcessingTimes.inboundPushEventTime !== undefined) {
-				this.opPerfData.durationInboundToProcessing =
-					currentTime - this.opProcessingTimes.inboundPushEventTime;
+			if (latencyData.opProcessingTimes.inboundPushEventTime !== undefined) {
+				latencyData.opPerfData.durationInboundToProcessing =
+					currentTime - latencyData.opProcessingTimes.inboundPushEventTime;
 			}
-
-			const duration = currentTime - this.opProcessingTimes.submitOpEventTime;
+			const duration = currentTime - latencyData.opProcessingTimes.submitOpEventTime;
 
 			// One of the core expectations for Fluid service is to be fast.
 			// When it's not the case, we want to learn about it and be able to investigate, so
@@ -284,8 +331,7 @@ class OpPerfTelemetry {
 			// The threshold could be adjusted, but ideally it stays  workload-agnostic, as service
 			// performance impacts all workloads relying on service.
 			const category = duration > latencyThreshold ? "error" : "performance";
-
-			this.logger.sendPerformanceEvent({
+			this.opLatencyLogger.sendPerformanceEvent({
 				eventName: "OpRoundtripTime",
 				sequenceNumber,
 				referenceSequenceNumber: message.referenceSequenceNumber,
@@ -294,11 +340,10 @@ class OpPerfTelemetry {
 				pingLatency: this.pingLatency,
 				msnDistance:
 					this.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,
-				...this.opPerfData,
+				...latencyData.opPerfData,
 			});
 			this.clientSequenceNumberForLatencyStatistics = undefined;
-			this.opPerfData = {};
-			this.opProcessingTimes = {};
+			this.latencyStatistics.delete(message.clientSequenceNumber);
 		}
 	}
 }