@fluidframework/container-runtime 2.0.0-internal.7.0.0 → 2.0.0-internal.7.2.0

package/dist/connectionTelemetry.js

@@ -17,16 +17,32 @@ class OpPerfTelemetry {
     constructor(clientId, deltaManager, logger) {
         this.clientId = clientId;
         this.deltaManager = deltaManager;
-        this.pongCount = 0;
         this.msnTrackingTimestamp = 0;
-        this.opProcessingTimes = {};
         // Performance Data to be reported for ops round trips and processing.
-        this.opPerfData = {};
+        this.latencyStatistics = new Map();
         this.firstConnection = true;
         this.bootTime = client_utils_1.performance.now();
         this.connectionStartTime = 0;
         this.gap = 0;
         this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpPerf" });
+        const deltaLatencyEventSampler = (() => {
+            let eventCount = -1;
+            return {
+                sample: () => {
+                    eventCount++;
+                    const shouldSample = eventCount % OpPerfTelemetry.DELTA_LATENCY_SAMPLE_RATE === 0;
+                    if (shouldSample) {
+                        eventCount = 0;
+                    }
+                    return shouldSample;
+                },
+            };
+        })();
+        this.deltaLatencyLogger = (0, telemetry_utils_1.createSampledLogger)(logger, deltaLatencyEventSampler);
+        // The SampledLogger here is used get access to the isSamplingDisabled property dervied from
+        // telemetry config properties. The actual sampling logic for op messages happens outside this SampledLogger
+        // due to complexity of the different asynchronus scenarios of the op message lifecycle.
+        this.opLatencyLogger = (0, telemetry_utils_1.createSampledLogger)(logger);
         this.deltaManager.on("pong", (latency) => this.recordPingTime(latency));
         this.deltaManager.on("submitOp", (message) => this.beforeOpSubmit(message));
         this.deltaManager.on("op", (message) => this.afterProcessingOp(message));
@@ -45,38 +61,45 @@ class OpPerfTelemetry {
         this.deltaManager.on("disconnect", () => {
             this.sequenceNumberForMsnTracking = undefined;
             this.clientSequenceNumberForLatencyStatistics = undefined;
-            this.opProcessingTimes = {};
-            this.opPerfData = {};
             this.connectionOpSeqNumber = undefined;
             this.firstConnection = false;
-            this.pongCount = 0;
+            this.latencyStatistics.clear();
         });
         this.deltaManager.outbound.on("push", (messages) => {
             for (const msg of messages) {
                 if (msg.type === protocol_definitions_1.MessageType.Operation &&
-                    this.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber) {
-                    (0, core_utils_1.assert)(this.opProcessingTimes.outboundPushEventTime === undefined, 0x2c8 /* "outboundPushEventTime should be undefined" */);
-                    (0, core_utils_1.assert)(this.opPerfData.durationNetwork === undefined, 0x2c9 /* "durationNetwork should be undefined" */);
-                    this.opProcessingTimes.outboundPushEventTime = Date.now();
-                    (0, core_utils_1.assert)(this.opPerfData.durationOutboundBatching === undefined, 0x2ca /* "durationOutboundBatching should be undefined" */);
-                    (0, core_utils_1.assert)(this.opProcessingTimes.submitOpEventTime !== undefined, 0x2cb /* "submitOpEventTime should be undefined" */);
-                    this.opPerfData.durationOutboundBatching =
-                        this.opProcessingTimes.outboundPushEventTime -
-                            this.opProcessingTimes.submitOpEventTime;
+                    (this.opLatencyLogger.isSamplingDisabled ||
+                        this.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber)) {
+                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                    const latencyStats = this.latencyStatistics.get(msg.clientSequenceNumber);
+                    (0, core_utils_1.assert)(latencyStats !== undefined, 0x7c2 /* Latency stats for op should exist */);
+                    (0, core_utils_1.assert)(latencyStats.opProcessingTimes.outboundPushEventTime === undefined, 0x2c8 /* "outboundPushEventTime should be undefined" */);
+                    (0, core_utils_1.assert)(latencyStats.opPerfData.durationNetwork === undefined, 0x2c9 /* "durationNetwork should be undefined" */);
+                    latencyStats.opProcessingTimes.outboundPushEventTime = Date.now();
+                    (0, core_utils_1.assert)(latencyStats.opPerfData.durationOutboundBatching === undefined, 0x2ca /* "durationOutboundBatching should be undefined" */);
+                    (0, core_utils_1.assert)(latencyStats.opProcessingTimes.submitOpEventTime !== undefined, 0x2cb /* "submitOpEventTime should be undefined" */);
+                    latencyStats.opPerfData.durationOutboundBatching =
+                        latencyStats.opProcessingTimes.outboundPushEventTime -
+                            latencyStats.opProcessingTimes.submitOpEventTime;
                 }
             }
         });
         this.deltaManager.inbound.on("push", (message) => {
             if (this.clientId === message.clientId &&
                 message.type === protocol_definitions_1.MessageType.Operation &&
-                this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber &&
-                this.opProcessingTimes.outboundPushEventTime !== undefined) {
-                this.opProcessingTimes.inboundPushEventTime = Date.now();
-                this.opPerfData.durationNetwork =
-                    this.opProcessingTimes.inboundPushEventTime -
-                        this.opProcessingTimes.outboundPushEventTime;
-                this.opProcessingTimes.outboundPushEventTime = undefined;
-                this.opPerfData.lengthInboundQueue = this.deltaManager.inbound.length;
+                (this.opLatencyLogger.isSamplingDisabled ||
+                    this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber)) {
+                // We do an explicit check for undefined right after this
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                const latencyStats = this.latencyStatistics.get(message.clientSequenceNumber);
+                (0, core_utils_1.assert)(latencyStats !== undefined, 0x7c3 /* Latency stats for op should exist */);
+                if (latencyStats.opProcessingTimes.outboundPushEventTime !== undefined) {
+                    latencyStats.opProcessingTimes.inboundPushEventTime = Date.now();
+                    latencyStats.opPerfData.durationNetwork =
+                        latencyStats.opProcessingTimes.inboundPushEventTime -
+                            latencyStats.opProcessingTimes.outboundPushEventTime;
+                    latencyStats.opPerfData.lengthInboundQueue = this.deltaManager.inbound.length;
+                }
             }
         });
         this.deltaManager.inbound.on("idle", (count, duration) => {
@@ -117,23 +140,27 @@ class OpPerfTelemetry {
                 duration: latency,
             });
         }
-        // logging one in every 100 pongs, including the first time, if it is a "write" client.
-        if (this.pongCount % 100 === 0 && this.deltaManager.active) {
-            this.logger.sendPerformanceEvent({
+        // logging one in every DELTA_LATENCY_SAMPLE_RATE pongs, including the first time, if it is a "write" client.
+        if (this.deltaManager.active) {
+            this.deltaLatencyLogger.sendPerformanceEvent({
                 eventName: "DeltaLatency",
                 duration: latency,
             });
         }
-        this.pongCount++;
     }
     beforeOpSubmit(message) {
         // start with first client op and measure latency every 500 client ops
-        if (this.clientSequenceNumberForLatencyStatistics === undefined &&
-            message.clientSequenceNumber % 500 === 1) {
-            (0, core_utils_1.assert)(this.opProcessingTimes.outboundPushEventTime === undefined, 0x2cc /* "OpTimeSittingInboundQueue should be undefined" */);
-            (0, core_utils_1.assert)(this.opPerfData.durationNetwork === undefined, 0x2cd /* "durationNetwork should be undefined" */);
-            this.opProcessingTimes.submitOpEventTime = Date.now();
+        if (this.opLatencyLogger.isSamplingDisabled ||
+            (this.clientSequenceNumberForLatencyStatistics === undefined &&
+                message.clientSequenceNumber % OpPerfTelemetry.OP_LATENCY_SAMPLE_RATE === 1)) {
+            (0, core_utils_1.assert)(this.latencyStatistics.get(message.clientSequenceNumber) === undefined, 0x7c4 /* Existing op perf data for client sequence number */);
             this.clientSequenceNumberForLatencyStatistics = message.clientSequenceNumber;
+            this.latencyStatistics.set(message.clientSequenceNumber, {
+                opProcessingTimes: {
+                    submitOpEventTime: Date.now(),
+                },
+                opPerfData: {},
+            });
         }
     }
     afterProcessingOp(message) {
@@ -158,14 +185,19 @@ class OpPerfTelemetry {
             this.sequenceNumberForMsnTracking = undefined;
         }
         if (this.clientId === message.clientId &&
-            this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber) {
-            (0, core_utils_1.assert)(this.opProcessingTimes.submitOpEventTime !== undefined, 0x120 /* "Undefined latency statistics (op send time)" */);
+            (this.opLatencyLogger.isSamplingDisabled ||
+                this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber)) {
+            // We do an explicit check for undefined right after this
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            const latencyData = this.latencyStatistics.get(message.clientSequenceNumber);
+            (0, core_utils_1.assert)(latencyData !== undefined, 0x7c5 /* Undefined latency statistics for op */);
+            (0, core_utils_1.assert)(latencyData.opProcessingTimes.submitOpEventTime !== undefined, 0x120 /* "Undefined latency statistics for op (op send time)" */);
             const currentTime = Date.now();
-            if (this.opProcessingTimes.inboundPushEventTime !== undefined) {
-                this.opPerfData.durationInboundToProcessing =
-                    currentTime - this.opProcessingTimes.inboundPushEventTime;
+            if (latencyData.opProcessingTimes.inboundPushEventTime !== undefined) {
+                latencyData.opPerfData.durationInboundToProcessing =
+                    currentTime - latencyData.opProcessingTimes.inboundPushEventTime;
             }
-            const duration = currentTime - this.opProcessingTimes.submitOpEventTime;
+            const duration = currentTime - latencyData.opProcessingTimes.submitOpEventTime;
             // One of the core expectations for Fluid service is to be fast.
             // When it's not the case, we want to learn about it and be able to investigate, so
             // raise awareness.
@@ -174,7 +206,7 @@ class OpPerfTelemetry {
             // The threshold could be adjusted, but ideally it stays  workload-agnostic, as service
             // performance impacts all workloads relying on service.
             const category = duration > exports.latencyThreshold ? "error" : "performance";
-            this.logger.sendPerformanceEvent({
+            this.opLatencyLogger.sendPerformanceEvent({
                 eventName: "OpRoundtripTime",
                 sequenceNumber,
                 referenceSequenceNumber: message.referenceSequenceNumber,
@@ -182,14 +214,15 @@ class OpPerfTelemetry {
                 category,
                 pingLatency: this.pingLatency,
                 msnDistance: this.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,
-                ...this.opPerfData,
+                ...latencyData.opPerfData,
             });
             this.clientSequenceNumberForLatencyStatistics = undefined;
-            this.opPerfData = {};
-            this.opProcessingTimes = {};
+            this.latencyStatistics.delete(message.clientSequenceNumber);
         }
     }
 }
+OpPerfTelemetry.OP_LATENCY_SAMPLE_RATE = 500;
+OpPerfTelemetry.DELTA_LATENCY_SAMPLE_RATE = 100;
 function ReportOpPerfTelemetry(clientId, deltaManager, logger) {
     new OpPerfTelemetry(clientId, deltaManager, logger);
 }

package/dist/connectionTelemetry.js.map

@@ -1 +1 @@
-{"version":3,"file":"connectionTelemetry.js","sourceRoot":"","sources":["../src/connectionTelemetry.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,qEAIyC;AAEzC,+EAI8C;AAC9C,2DAAoD;AACpD,+DAA2D;AAE3D;;GAEG;AACU,QAAA,gBAAgB,GAAG,IAAI,CAAC;AAgCrC,MAAM,eAAe;IAuBpB,YACS,QAA4B,EACnB,YAAwE,EACzF,MAA2B;QAFnB,aAAQ,GAAR,QAAQ,CAAoB;QACnB,iBAAY,GAAZ,YAAY,CAA4D;QAxBlF,cAAS,GAAW,CAAC,CAAC;QAKtB,yBAAoB,GAAW,CAAC,CAAC;QAIjC,sBAAiB,GAA4B,EAAE,CAAC;QAExD,sEAAsE;QAC9D,eAAU,GAAwC,EAAE,CAAC;QAErD,oBAAe,GAAG,IAAI,CAAC;QAEd,aAAQ,GAAG,0BAAW,CAAC,GAAG,EAAE,CAAC;QACtC,wBAAmB,GAAG,CAAC,CAAC;QACxB,QAAG,GAAG,CAAC,CAAC;QASf,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEjE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5E,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAEzE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE;YACtD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;YACjC,IAAI,SAAS,KAAK,SAAS,EAAE;gBAC5B,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBAClE,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;gBACrB,IAAI,CAAC,mBAAmB,GAAG,0BAAW,CAAC,GAAG,EAAE,CAAC;gBAE7C,6DAA6D;gBAC7D,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE;oBAClB,IAAI,CAAC,qBAAqB,EAAE,CAAC;iBAC7B;aACD;QACF,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE;YACvC,IAAI,CAAC,4BAA4B,GAAG,SAAS,CAAC;YAC9C,IAAI,CAAC,wCAAwC,GAAG,SAAS,CAAC;YAC1D,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;YAC5B,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;YACvC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC7B,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACpB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE;YAClD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE;gBAC3B,IACC,GAAG,CAAC,IAAI,KAAK,kCAAW,CAAC,SAAS;oBAClC,IAAI,CAAC,wCAAwC,KAAK,GAAG,CAAC,oBAAoB,EACzE;oBACD,IAAA,mBAAM,EACL,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,KAAK,SAAS,EAC1D,KAAK,CAAC,iDAAiD,CACvD,CAAC;oBACF,IAAA,mBAAM,EACL,IAAI,CAAC,UAAU,CAAC,eAAe,KAAK,SAAS,EAC7C,KAAK,CAAC,2CAA2C,CACjD,CAAC;oBACF,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE1D,IAAA,mBAAM,EACL,IAAI,CAAC,UAAU,CAAC,wBAAwB,KAAK,SAAS,EACtD,KAAK,CAAC,oDAAoD,CAC1D,CAAC;oBAEF,IAAA,mBAAM,EACL,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,KAAK,SAAS,EACtD,KAAK,CAAC,6CAA6C,CACnD,CAAC;oBAEF,IAAI,CAAC,UAAU,CAAC,wBAAwB;wBACvC,IAAI,CAAC,iBAAiB,CAAC,qBAAqB;4BAC5C,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC;iBAC1C;aACD;QACF,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAkC,EAAE,EAAE;YAC3E,IACC,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ;gBAClC,OAAO,CAAC,IAAI,KAAK,kCAAW,CAAC,SAAS;gBACtC,IAAI,CAAC,wCAAwC,KAAK,OAAO,CAAC,oBAAoB;gBAC9E,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,KAAK,SAAS,EACzD;gBACD,IAAI,CAAC,iBAAiB,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzD,IAAI,CAAC,UAAU,CAAC,eAAe;oBAC9B,IAAI,CAAC,iBAAiB,CAAC,oBAAoB;wBAC3C,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,CAAC;gBAC9C,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,GAAG,SAAS,CAAC;gBACzD,IAAI,CAAC,UAAU,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC;aACtE;QACF,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,QAAgB,EAAE,EAAE;YACxE,oCAAoC;YACpC,gGAAgG;YAChG,0GAA0G;YAC1G,uBAAuB;YACvB,4FAA4F;YAC5F,8BAA8B;YAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,IAAI,GAAG,EAAE;gBAC9C,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAChC,SAAS,EAAE,wBAAwB;oBACnC,KAAK;oBACL,QAAQ;iBACR,CAAC,CAAC;aACH;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,qBAAqB;QAC5B,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,SAAS,EAAE,iBAAiB;YAC5B,QAAQ,EAAE,0BAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,mBAAmB;YACtD,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,mDAAmD;YACnD,aAAa,EAAE,IAAI,CAAC,eAAe;gBAClC,CAAC,CAAC,IAAA,4BAAU,EAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,QAAQ,CAAC;gBACtD,CAAC,CAAC,SAAS;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;SACrC,CAAC,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,OAAe;QACrC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;QAE3B,sCAAsC;QACtC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAE,EAAE;YACxB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBAC1B,SAAS,EAAE,gBAAgB;gBAC3B,QAAQ,EAAE,OAAO;aACjB,CAAC,CAAC;SACH;QAED,uFAAuF;QACvF,IAAI,IAAI,CAAC,SAAS,GAAG,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;YAC3D,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,cAAc;gBACzB,QAAQ,EAAE,OAAO;aACjB,CAAC,CAAC;SACH;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,OAAyB;QAC/C,sEAAsE;QACtE,IACC,IAAI,CAAC,wCAAwC,KAAK,SAAS;YAC3D,OAAO,CAAC,oBAAoB,GAAG,GAAG,KAAK,CAAC,EACvC;YACD,IAAA,mBAAM,EACL,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,KAAK,SAAS,EAC1D,KAAK,CAAC,qDAAqD,CAC3D,CAAC;YACF,IAAA,mBAAM,EACL,IAAI,CAAC,UAAU,CAAC,eAAe,KAAK,SAAS,EAC7C,KAAK,CAAC,2CAA2C,CACjD,CAAC;YACF,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtD,IAAI,CAAC,wCAAwC,GAAG,OAAO,CAAC,oBAAoB,CAAC;SAC7E;IACF,CAAC;IAEO,iBAAiB,CAAC,OAAkC;QAC3D,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAE9C,IAAI,cAAc,KAAK,IAAI,CAAC,qBAAqB,EAAE;YAClD,IAAI,CAAC,qBAAqB,EAAE,CAAC;SAC7B;QAED,uDAAuD;QACvD,IAAI,IAAI,CAAC,4BAA4B,KAAK,SAAS,IAAI,cAAc,GAAG,IAAI,KAAK,CAAC,EAAE;YACnF,IAAI,CAAC,4BAA4B,GAAG,cAAc,CAAC;YACnD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,SAAS,CAAC;SAC9C;QACD,IACC,IAAI,CAAC,4BAA4B,KAAK,SAAS;YAC/C,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,4BAA4B,EACjE;YACD,IAAA,mBAAM,EACL,IAAI,CAAC,oBAAoB,KAAK,SAAS,EACvC,KAAK,CAAC,oDAAoD,CAC1D,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,eAAe;gBAC1B,cAAc;gBACd,WAAW,EAAE,cAAc,GAAG,IAAI,CAAC,4BAA4B;gBAC/D,QAAQ,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,oBAAoB;aACvD,CAAC,CAAC;YACH,IAAI,CAAC,4BAA4B,GAAG,SAAS,CAAC;SAC9C;QAED,IACC,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ;YAClC,IAAI,CAAC,wCAAwC,KAAK,OAAO,CAAC,oBAAoB,EAC7E;YACD,IAAA,mBAAM,EACL,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,KAAK,SAAS,EACtD,KAAK,CAAC,mDAAmD,CACzD,CAAC;YACF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE/B,IAAI,IAAI,CAAC,iBAAiB,CAAC,oBAAoB,KAAK,SAAS,EAAE;gBAC9D,IAAI,CAAC,UAAU,CAAC,2BAA2B;oBAC1C,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,oBAAoB,CAAC;aAC3D;YAED,MAAM,QAAQ,GAAG,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC;YAExE,gEAAgE;YAChE,mFAAmF;YACnF,mBAAmB;YACnB,0FAA0F;YAC1F,yFAAyF;YACzF,uFAAuF;YACvF,wDAAwD;YACxD,MAAM,QAAQ,GAAG,QAAQ,GAAG,wBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC;YAEvE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,iBAAiB;gBAC5B,cAAc;gBACd,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,QAAQ;gBACR,QAAQ;gBACR,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EACV,IAAI,CAAC,YAAY,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,qBAAqB;gBAC/E,GAAG,IAAI,CAAC,UAAU;aAClB,CAAC,CAAC;YACH,IAAI,CAAC,wCAAwC,GAAG,SAAS,CAAC;YAC1D,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;SAC5B;IACF,CAAC;CACD;AAuBD,SAAgB,qBAAqB,CACpC,QAA4B,EAC5B,YAAwE,EACxE,MAA2B;IAE3B,IAAI,eAAe,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AAND,sDAMC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport {\n\tITelemetryLoggerExt,\n\tcreateChildLogger,\n\tformatTick,\n} from \"@fluidframework/telemetry-utils\";\nimport { IDeltaManager } from \"@fluidframework/container-definitions\";\nimport {\n\tIDocumentMessage,\n\tISequencedDocumentMessage,\n\tMessageType,\n} from \"@fluidframework/protocol-definitions\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { performance } from \"@fluid-internal/client-utils\";\n\n/**\n * We report various latency-related errors when waiting for op roundtrip takes longer than that amout of time.\n */\nexport const latencyThreshold = 5000;\n\n// Phases in OpPerfTelemetry:\n// 1.\tOp is added to DeltaManager (DM) buffer.\n// 2.\tOp is sent to service (op leaves outbound queue).\n// \t - Note: We do not know for sure when op is sent, we only track when it is added to outbound queue.\n//     If outbound queue is paused, time queue is paused is counted as network time.\n// 3.\tOp received from service back (pushed to inbound queue).\n// 4.\tOp is processed.\ninterface IOpPerfTelemetryProperties {\n\t/** Measure time between (1) and (2) - Measure time outbound op is sitting in queue due to active batch */\n\tdurationOutboundBatching: number; // was durationOutboundQueue in previous versions\n\t/** Measure time between (2) and (3) - Track how long it took for op to be acked by service */\n\tdurationNetwork: number; // was durationInboundQueue\n\t/** Measure time between (3) and (4) - Time between DM's inbound \"push\" event until DM's \"op\" event */\n\tdurationInboundToProcessing: number;\n\t/** Length of the DeltaManager's inbound queue at the time of the DM's inbound \"push\" event (3) */\n\tlengthInboundQueue: number;\n}\n\n/**\n * Timings collected at various moments during the op processing.\n */\ninterface IOpPerfTimings {\n\t/** Starting time for (1) */\n\tsubmitOpEventTime: number;\n\t/** Starting time for (2) */\n\toutboundPushEventTime: number;\n\t/** Starting time for (3) */\n\tinboundPushEventTime: number;\n}\n\nclass OpPerfTelemetry {\n\tprivate pongCount: number = 0;\n\tprivate pingLatency: number | undefined;\n\n\t// Collab window tracking. This is timestamp of %1000 message.\n\tprivate sequenceNumberForMsnTracking: number | undefined;\n\tprivate msnTrackingTimestamp: number = 0;\n\t// To track round trip time for every %500 client message.\n\tprivate clientSequenceNumberForLatencyStatistics: number | undefined;\n\n\tprivate opProcessingTimes: Partial<IOpPerfTimings> = {};\n\n\t// Performance Data to be reported for ops round trips and processing.\n\tprivate opPerfData: Partial<IOpPerfTelemetryProperties> = {};\n\n\tprivate firstConnection = true;\n\tprivate connectionOpSeqNumber: number | undefined;\n\tprivate readonly bootTime = performance.now();\n\tprivate connectionStartTime = 0;\n\tprivate gap = 0;\n\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tpublic constructor(\n\t\tprivate clientId: string | undefined,\n\t\tprivate readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,\n\t\tlogger: ITelemetryLoggerExt,\n\t) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpPerf\" });\n\n\t\tthis.deltaManager.on(\"pong\", (latency) => this.recordPingTime(latency));\n\t\tthis.deltaManager.on(\"submitOp\", (message) => this.beforeOpSubmit(message));\n\n\t\tthis.deltaManager.on(\"op\", (message) => this.afterProcessingOp(message));\n\n\t\tthis.deltaManager.on(\"connect\", (details, opsBehind) => {\n\t\t\tthis.clientId = details.clientId;\n\t\t\tif (opsBehind !== undefined) {\n\t\t\t\tthis.connectionOpSeqNumber = this.deltaManager.lastKnownSeqNumber;\n\t\t\t\tthis.gap = opsBehind;\n\t\t\t\tthis.connectionStartTime = performance.now();\n\n\t\t\t\t// We might be already up-today. If so, report it right away.\n\t\t\t\tif (this.gap <= 0) {\n\t\t\t\t\tthis.reportGettingUpToDate();\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\t\tthis.deltaManager.on(\"disconnect\", () => {\n\t\t\tthis.sequenceNumberForMsnTracking = undefined;\n\t\t\tthis.clientSequenceNumberForLatencyStatistics = undefined;\n\t\t\tthis.opProcessingTimes = {};\n\t\t\tthis.opPerfData = {};\n\t\t\tthis.connectionOpSeqNumber = undefined;\n\t\t\tthis.firstConnection = false;\n\t\t\tthis.pongCount = 0;\n\t\t});\n\n\t\tthis.deltaManager.outbound.on(\"push\", (messages) => {\n\t\t\tfor (const msg of messages) {\n\t\t\t\tif (\n\t\t\t\t\tmsg.type === MessageType.Operation &&\n\t\t\t\t\tthis.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber\n\t\t\t\t) {\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.opProcessingTimes.outboundPushEventTime === undefined,\n\t\t\t\t\t\t0x2c8 /* \"outboundPushEventTime should be undefined\" */,\n\t\t\t\t\t);\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.opPerfData.durationNetwork === undefined,\n\t\t\t\t\t\t0x2c9 /* \"durationNetwork should be undefined\" */,\n\t\t\t\t\t);\n\t\t\t\t\tthis.opProcessingTimes.outboundPushEventTime = Date.now();\n\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.opPerfData.durationOutboundBatching === undefined,\n\t\t\t\t\t\t0x2ca /* \"durationOutboundBatching should be undefined\" */,\n\t\t\t\t\t);\n\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.opProcessingTimes.submitOpEventTime !== undefined,\n\t\t\t\t\t\t0x2cb /* \"submitOpEventTime should be undefined\" */,\n\t\t\t\t\t);\n\n\t\t\t\t\tthis.opPerfData.durationOutboundBatching =\n\t\t\t\t\t\tthis.opProcessingTimes.outboundPushEventTime -\n\t\t\t\t\t\tthis.opProcessingTimes.submitOpEventTime;\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\n\t\tthis.deltaManager.inbound.on(\"push\", (message: ISequencedDocumentMessage) => {\n\t\t\tif (\n\t\t\t\tthis.clientId === message.clientId &&\n\t\t\t\tmessage.type === MessageType.Operation &&\n\t\t\t\tthis.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber &&\n\t\t\t\tthis.opProcessingTimes.outboundPushEventTime !== undefined\n\t\t\t) {\n\t\t\t\tthis.opProcessingTimes.inboundPushEventTime = Date.now();\n\t\t\t\tthis.opPerfData.durationNetwork =\n\t\t\t\t\tthis.opProcessingTimes.inboundPushEventTime -\n\t\t\t\t\tthis.opProcessingTimes.outboundPushEventTime;\n\t\t\t\tthis.opProcessingTimes.outboundPushEventTime = undefined;\n\t\t\t\tthis.opPerfData.lengthInboundQueue = this.deltaManager.inbound.length;\n\t\t\t}\n\t\t});\n\n\t\tthis.deltaManager.inbound.on(\"idle\", (count: number, duration: number) => {\n\t\t\t// Do not want to log zero for sure.\n\t\t\t// We are more interested in aggregates, so logging only if we are processing some number of ops\n\t\t\t// Cut-off is arbitrary - can be increased or decreased based on amount of data collected and questions we\n\t\t\t// want to get answered\n\t\t\t// back-compat: Once 0.36 loader version saturates (count & duration args were added there),\n\t\t\t// we can remove typeof check.\n\t\t\tif (typeof count === \"number\" && count >= 100) {\n\t\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\t\teventName: \"GetDeltas_OpProcessing\",\n\t\t\t\t\tcount,\n\t\t\t\t\tduration,\n\t\t\t\t});\n\t\t\t}\n\t\t});\n\t}\n\n\tprivate reportGettingUpToDate() {\n\t\tthis.connectionOpSeqNumber = undefined;\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\teventName: \"ConnectionSpeed\",\n\t\t\tduration: performance.now() - this.connectionStartTime,\n\t\t\tops: this.gap,\n\t\t\t// track time to connect only for first connection.\n\t\t\ttimeToConnect: this.firstConnection\n\t\t\t\t? formatTick(this.connectionStartTime - this.bootTime)\n\t\t\t\t: undefined,\n\t\t\tfirstConnection: this.firstConnection,\n\t\t});\n\t}\n\n\tprivate recordPingTime(latency: number) {\n\t\tthis.pingLatency = latency;\n\n\t\t// Log if latency is longer than 1 min\n\t\tif (latency > 1000 * 60) {\n\t\t\tthis.logger.sendErrorEvent({\n\t\t\t\teventName: \"LatencyTooLong\",\n\t\t\t\tduration: latency,\n\t\t\t});\n\t\t}\n\n\t\t// logging one in every 100 pongs, including the first time, if it is a \"write\" client.\n\t\tif (this.pongCount % 100 === 0 && this.deltaManager.active) {\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"DeltaLatency\",\n\t\t\t\tduration: latency,\n\t\t\t});\n\t\t}\n\t\tthis.pongCount++;\n\t}\n\n\tprivate beforeOpSubmit(message: IDocumentMessage) {\n\t\t// start with first client op and measure latency every 500 client ops\n\t\tif (\n\t\t\tthis.clientSequenceNumberForLatencyStatistics === undefined &&\n\t\t\tmessage.clientSequenceNumber % 500 === 1\n\t\t) {\n\t\t\tassert(\n\t\t\t\tthis.opProcessingTimes.outboundPushEventTime === undefined,\n\t\t\t\t0x2cc /* \"OpTimeSittingInboundQueue should be undefined\" */,\n\t\t\t);\n\t\t\tassert(\n\t\t\t\tthis.opPerfData.durationNetwork === undefined,\n\t\t\t\t0x2cd /* \"durationNetwork should be undefined\" */,\n\t\t\t);\n\t\t\tthis.opProcessingTimes.submitOpEventTime = Date.now();\n\t\t\tthis.clientSequenceNumberForLatencyStatistics = message.clientSequenceNumber;\n\t\t}\n\t}\n\n\tprivate afterProcessingOp(message: ISequencedDocumentMessage) {\n\t\tconst sequenceNumber = message.sequenceNumber;\n\n\t\tif (sequenceNumber === this.connectionOpSeqNumber) {\n\t\t\tthis.reportGettingUpToDate();\n\t\t}\n\n\t\t// Record collab window max size after every 1000th op.\n\t\tif (this.sequenceNumberForMsnTracking === undefined && sequenceNumber % 1000 === 0) {\n\t\t\tthis.sequenceNumberForMsnTracking = sequenceNumber;\n\t\t\tthis.msnTrackingTimestamp = message.timestamp;\n\t\t}\n\t\tif (\n\t\t\tthis.sequenceNumberForMsnTracking !== undefined &&\n\t\t\tmessage.minimumSequenceNumber >= this.sequenceNumberForMsnTracking\n\t\t) {\n\t\t\tassert(\n\t\t\t\tthis.msnTrackingTimestamp !== undefined,\n\t\t\t\t0x2ce /* \"msnTrackingTimestamp should not be undefined\" */,\n\t\t\t);\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"MsnStatistics\",\n\t\t\t\tsequenceNumber,\n\t\t\t\tmsnDistance: sequenceNumber - this.sequenceNumberForMsnTracking,\n\t\t\t\tduration: message.timestamp - this.msnTrackingTimestamp,\n\t\t\t});\n\t\t\tthis.sequenceNumberForMsnTracking = undefined;\n\t\t}\n\n\t\tif (\n\t\t\tthis.clientId === message.clientId &&\n\t\t\tthis.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber\n\t\t) {\n\t\t\tassert(\n\t\t\t\tthis.opProcessingTimes.submitOpEventTime !== undefined,\n\t\t\t\t0x120 /* \"Undefined latency statistics (op send time)\" */,\n\t\t\t);\n\t\t\tconst currentTime = Date.now();\n\n\t\t\tif (this.opProcessingTimes.inboundPushEventTime !== undefined) {\n\t\t\t\tthis.opPerfData.durationInboundToProcessing =\n\t\t\t\t\tcurrentTime - this.opProcessingTimes.inboundPushEventTime;\n\t\t\t}\n\n\t\t\tconst duration = currentTime - this.opProcessingTimes.submitOpEventTime;\n\n\t\t\t// One of the core expectations for Fluid service is to be fast.\n\t\t\t// When it's not the case, we want to learn about it and be able to investigate, so\n\t\t\t// raise awareness.\n\t\t\t// This also helps identify cases where it's due to client behavior (sending too many ops)\n\t\t\t// that results in overwhelming ordering service and thus starting to see long latencies.\n\t\t\t// The threshold could be adjusted, but ideally it stays  workload-agnostic, as service\n\t\t\t// performance impacts all workloads relying on service.\n\t\t\tconst category = duration > latencyThreshold ? \"error\" : \"performance\";\n\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"OpRoundtripTime\",\n\t\t\t\tsequenceNumber,\n\t\t\t\treferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\t\tduration,\n\t\t\t\tcategory,\n\t\t\t\tpingLatency: this.pingLatency,\n\t\t\t\tmsnDistance:\n\t\t\t\t\tthis.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,\n\t\t\t\t...this.opPerfData,\n\t\t\t});\n\t\t\tthis.clientSequenceNumberForLatencyStatistics = undefined;\n\t\t\tthis.opPerfData = {};\n\t\t\tthis.opProcessingTimes = {};\n\t\t}\n\t}\n}\nexport interface IPerfSignalReport {\n\t/**\n\t * Identifier for the signal being submitted in order to\n\t * allow collection of data around the roundtrip of signal messages.\n\t */\n\tsignalSequenceNumber: number;\n\t/**\n\t * Number of signals that were expected but not received.\n\t */\n\tsignalsLost: number;\n\n\t/**\n\t * Timestamp before submitting the signal we will trace.\n\t */\n\tsignalTimestamp: number;\n\n\t/**\n\t * Expected Signal Sequence to be received.\n\t */\n\ttrackingSignalSequenceNumber: number | undefined;\n}\n\nexport function ReportOpPerfTelemetry(\n\tclientId: string | undefined,\n\tdeltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,\n\tlogger: ITelemetryLoggerExt,\n) {\n\tnew OpPerfTelemetry(clientId, deltaManager, logger);\n}\n"]}
+{"version":3,"file":"connectionTelemetry.js","sourceRoot":"","sources":["../src/connectionTelemetry.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,qEAOyC;AAEzC,+EAI8C;AAC9C,2DAAoD;AACpD,+DAA2D;AAE3D;;GAEG;AACU,QAAA,gBAAgB,GAAG,IAAI,CAAC;AAgCrC,MAAM,eAAe;IA+BpB,YACS,QAA4B,EACnB,YAAwE,EACzF,MAA2B;QAFnB,aAAQ,GAAR,QAAQ,CAAoB;QACnB,iBAAY,GAAZ,YAAY,CAA4D;QA5BlF,yBAAoB,GAAW,CAAC,CAAC;QAGzC,sEAAsE;QACrD,sBAAiB,GAAG,IAAI,GAAG,EAMzC,CAAC;QAEI,oBAAe,GAAG,IAAI,CAAC;QAEd,aAAQ,GAAG,0BAAW,CAAC,GAAG,EAAE,CAAC;QACtC,wBAAmB,GAAG,CAAC,CAAC;QACxB,QAAG,GAAG,CAAC,CAAC;QAef,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEjE,MAAM,wBAAwB,GAAkB,CAAC,GAAG,EAAE;YACrD,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;YACpB,OAAO;gBACN,MAAM,EAAE,GAAG,EAAE;oBACZ,UAAU,EAAE,CAAC;oBACb,MAAM,YAAY,GACjB,UAAU,GAAG,eAAe,CAAC,yBAAyB,KAAK,CAAC,CAAC;oBAC9D,IAAI,YAAY,EAAE;wBACjB,UAAU,GAAG,CAAC,CAAC;qBACf;oBACD,OAAO,YAAY,CAAC;gBACrB,CAAC;aACD,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QAEL,IAAI,CAAC,kBAAkB,GAAG,IAAA,qCAAmB,EAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QAEhF,4FAA4F;QAC5F,4GAA4G;QAC5G,wFAAwF;QACxF,IAAI,CAAC,eAAe,GAAG,IAAA,qCAAmB,EAAC,MAAM,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5E,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAEzE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE;YACtD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;YACjC,IAAI,SAAS,KAAK,SAAS,EAAE;gBAC5B,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBAClE,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;gBACrB,IAAI,CAAC,mBAAmB,GAAG,0BAAW,CAAC,GAAG,EAAE,CAAC;gBAE7C,6DAA6D;gBAC7D,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE;oBAClB,IAAI,CAAC,qBAAqB,EAAE,CAAC;iBAC7B;aACD;QACF,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE;YACvC,IAAI,CAAC,4BAA4B,GAAG,SAAS,CAAC;YAC9C,IAAI,CAAC,wCAAwC,GAAG,SAAS,CAAC;YAC1D,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;YACvC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC7B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE;YAClD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE;gBAC3B,IACC,GAAG,CAAC,IAAI,KAAK,kCAAW,CAAC,SAAS;oBAClC,CAAC,IAAI,CAAC,eAAe,CAAC,kBAAkB;wBACvC,IAAI,CAAC,wCAAwC,KAAK,GAAG,CAAC,oBAAoB,CAAC,EAC3E;oBACD,oEAAoE;oBACpE,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAE,CAAC;oBAC3E,IAAA,mBAAM,EACL,YAAY,KAAK,SAAS,EAC1B,KAAK,CAAC,uCAAuC,CAC7C,CAAC;oBACF,IAAA,mBAAM,EACL,YAAY,CAAC,iBAAiB,CAAC,qBAAqB,KAAK,SAAS,EAClE,KAAK,CAAC,iDAAiD,CACvD,CAAC;oBACF,IAAA,mBAAM,EACL,YAAY,CAAC,UAAU,CAAC,eAAe,KAAK,SAAS,EACrD,KAAK,CAAC,2CAA2C,CACjD,CAAC;oBACF,YAAY,CAAC,iBAAiB,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAElE,IAAA,mBAAM,EACL,YAAY,CAAC,UAAU,CAAC,wBAAwB,KAAK,SAAS,EAC9D,KAAK,CAAC,oDAAoD,CAC1D,CAAC;oBAEF,IAAA,mBAAM,EACL,YAAY,CAAC,iBAAiB,CAAC,iBAAiB,KAAK,SAAS,EAC9D,KAAK,CAAC,6CAA6C,CACnD,CAAC;oBAEF,YAAY,CAAC,UAAU,CAAC,wBAAwB;wBAC/C,YAAY,CAAC,iBAAiB,CAAC,qBAAqB;4BACpD,YAAY,CAAC,iBAAiB,CAAC,iBAAiB,CAAC;iBAClD;aACD;QACF,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAkC,EAAE,EAAE;YAC3E,IACC,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ;gBAClC,OAAO,CAAC,IAAI,KAAK,kCAAW,CAAC,SAAS;gBACtC,CAAC,IAAI,CAAC,eAAe,CAAC,kBAAkB;oBACvC,IAAI,CAAC,wCAAwC,KAAK,OAAO,CAAC,oBAAoB,CAAC,EAC/E;gBACD,yDAAyD;gBACzD,oEAAoE;gBACpE,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAE,CAAC;gBAC/E,IAAA,mBAAM,EAAC,YAAY,KAAK,SAAS,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBAClF,IAAI,YAAY,CAAC,iBAAiB,CAAC,qBAAqB,KAAK,SAAS,EAAE;oBACvE,YAAY,CAAC,iBAAiB,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACjE,YAAY,CAAC,UAAU,CAAC,eAAe;wBACtC,YAAY,CAAC,iBAAiB,CAAC,oBAAoB;4BACnD,YAAY,CAAC,iBAAiB,CAAC,qBAAqB,CAAC;oBACtD,YAAY,CAAC,UAAU,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC;iBAC9E;aACD;QACF,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,QAAgB,EAAE,EAAE;YACxE,oCAAoC;YACpC,gGAAgG;YAChG,0GAA0G;YAC1G,uBAAuB;YACvB,4FAA4F;YAC5F,8BAA8B;YAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,IAAI,GAAG,EAAE;gBAC9C,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAChC,SAAS,EAAE,wBAAwB;oBACnC,KAAK;oBACL,QAAQ;iBACR,CAAC,CAAC;aACH;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,qBAAqB;QAC5B,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,SAAS,EAAE,iBAAiB;YAC5B,QAAQ,EAAE,0BAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,mBAAmB;YACtD,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,mDAAmD;YACnD,aAAa,EAAE,IAAI,CAAC,eAAe;gBAClC,CAAC,CAAC,IAAA,4BAAU,EAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,QAAQ,CAAC;gBACtD,CAAC,CAAC,SAAS;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;SACrC,CAAC,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,OAAe;QACrC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;QAE3B,sCAAsC;QACtC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAE,EAAE;YACxB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBAC1B,SAAS,EAAE,gBAAgB;gBAC3B,QAAQ,EAAE,OAAO;aACjB,CAAC,CAAC;SACH;QAED,6GAA6G;QAC7G,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;YAC7B,IAAI,CAAC,kBAAkB,CAAC,oBAAoB,CAAC;gBAC5C,SAAS,EAAE,cAAc;gBACzB,QAAQ,EAAE,OAAO;aACjB,CAAC,CAAC;SACH;IACF,CAAC;IAEO,cAAc,CAAC,OAAyB;QAC/C,sEAAsE;QACtE,IACC,IAAI,CAAC,eAAe,CAAC,kBAAkB;YACvC,CAAC,IAAI,CAAC,wCAAwC,KAAK,SAAS;gBAC3D,OAAO,CAAC,oBAAoB,GAAG,eAAe,CAAC,sBAAsB,KAAK,CAAC,CAAC,EAC5E;YACD,IAAA,mBAAM,EACL,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAC,KAAK,SAAS,EACtE,KAAK,CAAC,sDAAsD,CAC5D,CAAC;YACF,IAAI,CAAC,wCAAwC,GAAG,OAAO,CAAC,oBAAoB,CAAC;YAC7E,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,oBAAoB,EAAE;gBACxD,iBAAiB,EAAE;oBAClB,iBAAiB,EAAE,IAAI,CAAC,GAAG,EAAE;iBAC7B;gBACD,UAAU,EAAE,EAAE;aACd,CAAC,CAAC;SACH;IACF,CAAC;IAEO,iBAAiB,CAAC,OAAkC;QAC3D,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAE9C,IAAI,cAAc,KAAK,IAAI,CAAC,qBAAqB,EAAE;YAClD,IAAI,CAAC,qBAAqB,EAAE,CAAC;SAC7B;QAED,uDAAuD;QACvD,IAAI,IAAI,CAAC,4BAA4B,KAAK,SAAS,IAAI,cAAc,GAAG,IAAI,KAAK,CAAC,EAAE;YACnF,IAAI,CAAC,4BAA4B,GAAG,cAAc,CAAC;YACnD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,SAAS,CAAC;SAC9C;QACD,IACC,IAAI,CAAC,4BAA4B,KAAK,SAAS;YAC/C,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,4BAA4B,EACjE;YACD,IAAA,mBAAM,EACL,IAAI,CAAC,oBAAoB,KAAK,SAAS,EACvC,KAAK,CAAC,oDAAoD,CAC1D,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,eAAe;gBAC1B,cAAc;gBACd,WAAW,EAAE,cAAc,GAAG,IAAI,CAAC,4BAA4B;gBAC/D,QAAQ,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,oBAAoB;aACvD,CAAC,CAAC;YACH,IAAI,CAAC,4BAA4B,GAAG,SAAS,CAAC;SAC9C;QAED,IACC,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ;YAClC,CAAC,IAAI,CAAC,eAAe,CAAC,kBAAkB;gBACvC,IAAI,CAAC,wCAAwC,KAAK,OAAO,CAAC,oBAAoB,CAAC,EAC/E;YACD,yDAAyD;YACzD,oEAAoE;YACpE,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAE,CAAC;YAC9E,IAAA,mBAAM,EAAC,WAAW,KAAK,SAAS,EAAE,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACnF,IAAA,mBAAM,EACL,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,KAAK,SAAS,EAC7D,KAAK,CAAC,0DAA0D,CAChE,CAAC;YACF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,WAAW,CAAC,iBAAiB,CAAC,oBAAoB,KAAK,SAAS,EAAE;gBACrE,WAAW,CAAC,UAAU,CAAC,2BAA2B;oBACjD,WAAW,GAAG,WAAW,CAAC,iBAAiB,CAAC,oBAAoB,CAAC;aAClE;YACD,MAAM,QAAQ,GAAG,WAAW,GAAG,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,CAAC;YAE/E,gEAAgE;YAChE,mFAAmF;YACnF,mBAAmB;YACnB,0FAA0F;YAC1F,yFAAyF;YACzF,uFAAuF;YACvF,wDAAwD;YACxD,MAAM,QAAQ,GAAG,QAAQ,GAAG,wBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC;YACvE,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC;gBACzC,SAAS,EAAE,iBAAiB;gBAC5B,cAAc;gBACd,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,QAAQ;gBACR,QAAQ;gBACR,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EACV,IAAI,CAAC,YAAY,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,qBAAqB;gBAC/E,GAAG,WAAW,CAAC,UAAU;aACzB,CAAC,CAAC;YACH,IAAI,CAAC,wCAAwC,GAAG,SAAS,CAAC;YAC1D,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;SAC5D;IACF,CAAC;;AAzQuB,sCAAsB,GAAG,GAAG,AAAN,CAAO;AAG7B,yCAAyB,GAAG,GAAG,AAAN,CAAO;AA8RzD,SAAgB,qBAAqB,CACpC,QAA4B,EAC5B,YAAwE,EACxE,MAA2B;IAE3B,IAAI,eAAe,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AAND,sDAMC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport {\n\tIEventSampler,\n\tISampledTelemetryLogger,\n\tITelemetryLoggerExt,\n\tcreateChildLogger,\n\tcreateSampledLogger,\n\tformatTick,\n} from \"@fluidframework/telemetry-utils\";\nimport { IDeltaManager } from \"@fluidframework/container-definitions\";\nimport {\n\tIDocumentMessage,\n\tISequencedDocumentMessage,\n\tMessageType,\n} from \"@fluidframework/protocol-definitions\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { performance } from \"@fluid-internal/client-utils\";\n\n/**\n * We report various latency-related errors when waiting for op roundtrip takes longer than that amout of time.\n */\nexport const latencyThreshold = 5000;\n\n// Phases in OpPerfTelemetry:\n// 1.\tOp is added to DeltaManager (DM) buffer.\n// 2.\tOp is sent to service (op leaves outbound queue).\n// \t - Note: We do not know for sure when op is sent, we only track when it is added to outbound queue.\n//     If outbound queue is paused, time queue is paused is counted as network time.\n// 3.\tOp received from service back (pushed to inbound queue).\n// 4.\tOp is processed.\ninterface IOpPerfTelemetryProperties {\n\t/** Measure time between (1) and (2) - Measure time outbound op is sitting in queue due to active batch */\n\tdurationOutboundBatching: number; // was durationOutboundQueue in previous versions\n\t/** Measure time between (2) and (3) - Track how long it took for op to be acked by service */\n\tdurationNetwork: number; // was durationInboundQueue\n\t/** Measure time between (3) and (4) - Time between DM's inbound \"push\" event until DM's \"op\" event */\n\tdurationInboundToProcessing: number;\n\t/** Length of the DeltaManager's inbound queue at the time of the DM's inbound \"push\" event (3) */\n\tlengthInboundQueue: number;\n}\n\n/**\n * Timings collected at various moments during the op processing.\n */\ninterface IOpPerfTimings {\n\t/** Starting time for (1) */\n\tsubmitOpEventTime: number;\n\t/** Starting time for (2) */\n\toutboundPushEventTime: number;\n\t/** Starting time for (3) */\n\tinboundPushEventTime: number;\n}\n\nclass OpPerfTelemetry {\n\tprivate pingLatency: number | undefined;\n\n\t// Collab window tracking. This is timestamp of %1000 message.\n\tprivate sequenceNumberForMsnTracking: number | undefined;\n\tprivate msnTrackingTimestamp: number = 0;\n\t// To track round trip time for every %500 client message.\n\tprivate clientSequenceNumberForLatencyStatistics: number | undefined;\n\t// Performance Data to be reported for ops round trips and processing.\n\tprivate readonly latencyStatistics = new Map<\n\t\tnumber,\n\t\t{\n\t\t\topProcessingTimes: Partial<IOpPerfTimings>;\n\t\t\topPerfData: Partial<IOpPerfTelemetryProperties>;\n\t\t}\n\t>();\n\n\tprivate firstConnection = true;\n\tprivate connectionOpSeqNumber: number | undefined;\n\tprivate readonly bootTime = performance.now();\n\tprivate connectionStartTime = 0;\n\tprivate gap = 0;\n\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tprivate static readonly OP_LATENCY_SAMPLE_RATE = 500;\n\tprivate readonly opLatencyLogger: ISampledTelemetryLogger;\n\n\tprivate static readonly DELTA_LATENCY_SAMPLE_RATE = 100;\n\tprivate readonly deltaLatencyLogger: ISampledTelemetryLogger;\n\n\tpublic constructor(\n\t\tprivate clientId: string | undefined,\n\t\tprivate readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,\n\t\tlogger: ITelemetryLoggerExt,\n\t) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpPerf\" });\n\n\t\tconst deltaLatencyEventSampler: IEventSampler = (() => {\n\t\t\tlet eventCount = -1;\n\t\t\treturn {\n\t\t\t\tsample: () => {\n\t\t\t\t\teventCount++;\n\t\t\t\t\tconst shouldSample =\n\t\t\t\t\t\teventCount % OpPerfTelemetry.DELTA_LATENCY_SAMPLE_RATE === 0;\n\t\t\t\t\tif (shouldSample) {\n\t\t\t\t\t\teventCount = 0;\n\t\t\t\t\t}\n\t\t\t\t\treturn shouldSample;\n\t\t\t\t},\n\t\t\t};\n\t\t})();\n\n\t\tthis.deltaLatencyLogger = createSampledLogger(logger, deltaLatencyEventSampler);\n\n\t\t// The SampledLogger here is used get access to the isSamplingDisabled property dervied from\n\t\t// telemetry config properties. The actual sampling logic for op messages happens outside this SampledLogger\n\t\t// due to complexity of the different asynchronus scenarios of the op message lifecycle.\n\t\tthis.opLatencyLogger = createSampledLogger(logger);\n\n\t\tthis.deltaManager.on(\"pong\", (latency) => this.recordPingTime(latency));\n\t\tthis.deltaManager.on(\"submitOp\", (message) => this.beforeOpSubmit(message));\n\n\t\tthis.deltaManager.on(\"op\", (message) => this.afterProcessingOp(message));\n\n\t\tthis.deltaManager.on(\"connect\", (details, opsBehind) => {\n\t\t\tthis.clientId = details.clientId;\n\t\t\tif (opsBehind !== undefined) {\n\t\t\t\tthis.connectionOpSeqNumber = this.deltaManager.lastKnownSeqNumber;\n\t\t\t\tthis.gap = opsBehind;\n\t\t\t\tthis.connectionStartTime = performance.now();\n\n\t\t\t\t// We might be already up-today. If so, report it right away.\n\t\t\t\tif (this.gap <= 0) {\n\t\t\t\t\tthis.reportGettingUpToDate();\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\t\tthis.deltaManager.on(\"disconnect\", () => {\n\t\t\tthis.sequenceNumberForMsnTracking = undefined;\n\t\t\tthis.clientSequenceNumberForLatencyStatistics = undefined;\n\t\t\tthis.connectionOpSeqNumber = undefined;\n\t\t\tthis.firstConnection = false;\n\t\t\tthis.latencyStatistics.clear();\n\t\t});\n\n\t\tthis.deltaManager.outbound.on(\"push\", (messages) => {\n\t\t\tfor (const msg of messages) {\n\t\t\t\tif (\n\t\t\t\t\tmsg.type === MessageType.Operation &&\n\t\t\t\t\t(this.opLatencyLogger.isSamplingDisabled ||\n\t\t\t\t\t\tthis.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber)\n\t\t\t\t) {\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\t\t\t\tconst latencyStats = this.latencyStatistics.get(msg.clientSequenceNumber)!;\n\t\t\t\t\tassert(\n\t\t\t\t\t\tlatencyStats !== undefined,\n\t\t\t\t\t\t0x7c2 /* Latency stats for op should exist */,\n\t\t\t\t\t);\n\t\t\t\t\tassert(\n\t\t\t\t\t\tlatencyStats.opProcessingTimes.outboundPushEventTime === undefined,\n\t\t\t\t\t\t0x2c8 /* \"outboundPushEventTime should be undefined\" */,\n\t\t\t\t\t);\n\t\t\t\t\tassert(\n\t\t\t\t\t\tlatencyStats.opPerfData.durationNetwork === undefined,\n\t\t\t\t\t\t0x2c9 /* \"durationNetwork should be undefined\" */,\n\t\t\t\t\t);\n\t\t\t\t\tlatencyStats.opProcessingTimes.outboundPushEventTime = Date.now();\n\n\t\t\t\t\tassert(\n\t\t\t\t\t\tlatencyStats.opPerfData.durationOutboundBatching === undefined,\n\t\t\t\t\t\t0x2ca /* \"durationOutboundBatching should be undefined\" */,\n\t\t\t\t\t);\n\n\t\t\t\t\tassert(\n\t\t\t\t\t\tlatencyStats.opProcessingTimes.submitOpEventTime !== undefined,\n\t\t\t\t\t\t0x2cb /* \"submitOpEventTime should be undefined\" */,\n\t\t\t\t\t);\n\n\t\t\t\t\tlatencyStats.opPerfData.durationOutboundBatching =\n\t\t\t\t\t\tlatencyStats.opProcessingTimes.outboundPushEventTime -\n\t\t\t\t\t\tlatencyStats.opProcessingTimes.submitOpEventTime;\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\n\t\tthis.deltaManager.inbound.on(\"push\", (message: ISequencedDocumentMessage) => {\n\t\t\tif (\n\t\t\t\tthis.clientId === message.clientId &&\n\t\t\t\tmessage.type === MessageType.Operation &&\n\t\t\t\t(this.opLatencyLogger.isSamplingDisabled ||\n\t\t\t\t\tthis.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber)\n\t\t\t) {\n\t\t\t\t// We do an explicit check for undefined right after this\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\t\t\tconst latencyStats = this.latencyStatistics.get(message.clientSequenceNumber)!;\n\t\t\t\tassert(latencyStats !== undefined, 0x7c3 /* Latency stats for op should exist */);\n\t\t\t\tif (latencyStats.opProcessingTimes.outboundPushEventTime !== undefined) {\n\t\t\t\t\tlatencyStats.opProcessingTimes.inboundPushEventTime = Date.now();\n\t\t\t\t\tlatencyStats.opPerfData.durationNetwork =\n\t\t\t\t\t\tlatencyStats.opProcessingTimes.inboundPushEventTime -\n\t\t\t\t\t\tlatencyStats.opProcessingTimes.outboundPushEventTime;\n\t\t\t\t\tlatencyStats.opPerfData.lengthInboundQueue = this.deltaManager.inbound.length;\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\n\t\tthis.deltaManager.inbound.on(\"idle\", (count: number, duration: number) => {\n\t\t\t// Do not want to log zero for sure.\n\t\t\t// We are more interested in aggregates, so logging only if we are processing some number of ops\n\t\t\t// Cut-off is arbitrary - can be increased or decreased based on amount of data collected and questions we\n\t\t\t// want to get answered\n\t\t\t// back-compat: Once 0.36 loader version saturates (count & duration args were added there),\n\t\t\t// we can remove typeof check.\n\t\t\tif (typeof count === \"number\" && count >= 100) {\n\t\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\t\teventName: \"GetDeltas_OpProcessing\",\n\t\t\t\t\tcount,\n\t\t\t\t\tduration,\n\t\t\t\t});\n\t\t\t}\n\t\t});\n\t}\n\n\tprivate reportGettingUpToDate() {\n\t\tthis.connectionOpSeqNumber = undefined;\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\teventName: \"ConnectionSpeed\",\n\t\t\tduration: performance.now() - this.connectionStartTime,\n\t\t\tops: this.gap,\n\t\t\t// track time to connect only for first connection.\n\t\t\ttimeToConnect: this.firstConnection\n\t\t\t\t? formatTick(this.connectionStartTime - this.bootTime)\n\t\t\t\t: undefined,\n\t\t\tfirstConnection: this.firstConnection,\n\t\t});\n\t}\n\n\tprivate recordPingTime(latency: number) {\n\t\tthis.pingLatency = latency;\n\n\t\t// Log if latency is longer than 1 min\n\t\tif (latency > 1000 * 60) {\n\t\t\tthis.logger.sendErrorEvent({\n\t\t\t\teventName: \"LatencyTooLong\",\n\t\t\t\tduration: latency,\n\t\t\t});\n\t\t}\n\n\t\t// logging one in every DELTA_LATENCY_SAMPLE_RATE pongs, including the first time, if it is a \"write\" client.\n\t\tif (this.deltaManager.active) {\n\t\t\tthis.deltaLatencyLogger.sendPerformanceEvent({\n\t\t\t\teventName: \"DeltaLatency\",\n\t\t\t\tduration: latency,\n\t\t\t});\n\t\t}\n\t}\n\n\tprivate beforeOpSubmit(message: IDocumentMessage) {\n\t\t// start with first client op and measure latency every 500 client ops\n\t\tif (\n\t\t\tthis.opLatencyLogger.isSamplingDisabled ||\n\t\t\t(this.clientSequenceNumberForLatencyStatistics === undefined &&\n\t\t\t\tmessage.clientSequenceNumber % OpPerfTelemetry.OP_LATENCY_SAMPLE_RATE === 1)\n\t\t) {\n\t\t\tassert(\n\t\t\t\tthis.latencyStatistics.get(message.clientSequenceNumber) === undefined,\n\t\t\t\t0x7c4 /* Existing op perf data for client sequence number */,\n\t\t\t);\n\t\t\tthis.clientSequenceNumberForLatencyStatistics = message.clientSequenceNumber;\n\t\t\tthis.latencyStatistics.set(message.clientSequenceNumber, {\n\t\t\t\topProcessingTimes: {\n\t\t\t\t\tsubmitOpEventTime: Date.now(),\n\t\t\t\t},\n\t\t\t\topPerfData: {},\n\t\t\t});\n\t\t}\n\t}\n\n\tprivate afterProcessingOp(message: ISequencedDocumentMessage) {\n\t\tconst sequenceNumber = message.sequenceNumber;\n\n\t\tif (sequenceNumber === this.connectionOpSeqNumber) {\n\t\t\tthis.reportGettingUpToDate();\n\t\t}\n\n\t\t// Record collab window max size after every 1000th op.\n\t\tif (this.sequenceNumberForMsnTracking === undefined && sequenceNumber % 1000 === 0) {\n\t\t\tthis.sequenceNumberForMsnTracking = sequenceNumber;\n\t\t\tthis.msnTrackingTimestamp = message.timestamp;\n\t\t}\n\t\tif (\n\t\t\tthis.sequenceNumberForMsnTracking !== undefined &&\n\t\t\tmessage.minimumSequenceNumber >= this.sequenceNumberForMsnTracking\n\t\t) {\n\t\t\tassert(\n\t\t\t\tthis.msnTrackingTimestamp !== undefined,\n\t\t\t\t0x2ce /* \"msnTrackingTimestamp should not be undefined\" */,\n\t\t\t);\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"MsnStatistics\",\n\t\t\t\tsequenceNumber,\n\t\t\t\tmsnDistance: sequenceNumber - this.sequenceNumberForMsnTracking,\n\t\t\t\tduration: message.timestamp - this.msnTrackingTimestamp,\n\t\t\t});\n\t\t\tthis.sequenceNumberForMsnTracking = undefined;\n\t\t}\n\n\t\tif (\n\t\t\tthis.clientId === message.clientId &&\n\t\t\t(this.opLatencyLogger.isSamplingDisabled ||\n\t\t\t\tthis.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber)\n\t\t) {\n\t\t\t// We do an explicit check for undefined right after this\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\t\tconst latencyData = this.latencyStatistics.get(message.clientSequenceNumber)!;\n\t\t\tassert(latencyData !== undefined, 0x7c5 /* Undefined latency statistics for op */);\n\t\t\tassert(\n\t\t\t\tlatencyData.opProcessingTimes.submitOpEventTime !== undefined,\n\t\t\t\t0x120 /* \"Undefined latency statistics for op (op send time)\" */,\n\t\t\t);\n\t\t\tconst currentTime = Date.now();\n\t\t\tif (latencyData.opProcessingTimes.inboundPushEventTime !== undefined) {\n\t\t\t\tlatencyData.opPerfData.durationInboundToProcessing =\n\t\t\t\t\tcurrentTime - latencyData.opProcessingTimes.inboundPushEventTime;\n\t\t\t}\n\t\t\tconst duration = currentTime - latencyData.opProcessingTimes.submitOpEventTime;\n\n\t\t\t// One of the core expectations for Fluid service is to be fast.\n\t\t\t// When it's not the case, we want to learn about it and be able to investigate, so\n\t\t\t// raise awareness.\n\t\t\t// This also helps identify cases where it's due to client behavior (sending too many ops)\n\t\t\t// that results in overwhelming ordering service and thus starting to see long latencies.\n\t\t\t// The threshold could be adjusted, but ideally it stays  workload-agnostic, as service\n\t\t\t// performance impacts all workloads relying on service.\n\t\t\tconst category = duration > latencyThreshold ? \"error\" : \"performance\";\n\t\t\tthis.opLatencyLogger.sendPerformanceEvent({\n\t\t\t\teventName: \"OpRoundtripTime\",\n\t\t\t\tsequenceNumber,\n\t\t\t\treferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\t\tduration,\n\t\t\t\tcategory,\n\t\t\t\tpingLatency: this.pingLatency,\n\t\t\t\tmsnDistance:\n\t\t\t\t\tthis.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,\n\t\t\t\t...latencyData.opPerfData,\n\t\t\t});\n\t\t\tthis.clientSequenceNumberForLatencyStatistics = undefined;\n\t\t\tthis.latencyStatistics.delete(message.clientSequenceNumber);\n\t\t}\n\t}\n}\nexport interface IPerfSignalReport {\n\t/**\n\t * Identifier for the signal being submitted in order to\n\t * allow collection of data around the roundtrip of signal messages.\n\t */\n\tsignalSequenceNumber: number;\n\t/**\n\t * Number of signals that were expected but not received.\n\t */\n\tsignalsLost: number;\n\n\t/**\n\t * Timestamp before submitting the signal we will trace.\n\t */\n\tsignalTimestamp: number;\n\n\t/**\n\t * Expected Signal Sequence to be received.\n\t */\n\ttrackingSignalSequenceNumber: number | undefined;\n}\n\nexport function ReportOpPerfTelemetry(\n\tclientId: string | undefined,\n\tdeltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,\n\tlogger: ITelemetryLoggerExt,\n) {\n\tnew OpPerfTelemetry(clientId, deltaManager, logger);\n}\n"]}

package/dist/containerRuntime.d.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 import { FluidObject, IFluidHandle, IFluidHandleContext, IFluidRouter, IRequest, IResponse, IProvideFluidHandleContext } from "@fluidframework/core-interfaces";
-import { IAudience, IContainerContext, IDeltaManager, IRuntime, ICriticalContainerError, AttachState, ILoaderOptions } from "@fluidframework/container-definitions";
+import { IAudience, IContainerContext, IDeltaManager, IRuntime, ICriticalContainerError, AttachState, ILoaderOptions, ILoader, IGetPendingLocalStateProps } from "@fluidframework/container-definitions";
 import { IContainerRuntime, IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions";
 import { TypedEventEmitter } from "@fluid-internal/client-utils";
 import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
@@ -12,9 +12,12 @@ import { IClientDetails, IDocumentMessage, IQuorumClients, ISequencedDocumentMes
 import { FlushMode, IFluidDataStoreContextDetached, IFluidDataStoreRegistry, IGarbageCollectionData, NamedFluidDataStoreRegistryEntries, ISummaryTreeWithStats, IDataStore, ITelemetryContext, IIdCompressor, IIdCompressorCore } from "@fluidframework/runtime-definitions";
 import { IPendingLocalState } from "./pendingStateManager";
 import { IBlobManagerLoadInfo, IPendingBlobs } from "./blobManager";
-import { IContainerRuntimeMetadata, ISerializedElection, SubmitSummaryResult, ISubmitSummaryOptions, ISummarizerInternalsProvider, ISummarizerRuntime, IRefreshSummaryAckOptions, IOnDemandSummarizeOptions, ISummarizeResults, IEnqueueSummarizeOptions, EnqueueSummarizeResult, ISummarizerEvents } from "./summary";
+import { IContainerRuntimeMetadata, ISerializedElection, SubmitSummaryResult, ISubmitSummaryOptions, ISummarizerInternalsProvider, ISummarizerRuntime, IRefreshSummaryAckOptions, IOnDemandSummarizeOptions, ISummarizeResults, IEnqueueSummarizeOptions, EnqueueSummarizeResult, ISummarizerEvents, ISummarizer } from "./summary";
 import { GCNodeType, IGCRuntimeOptions, IGCStats } from "./gc";
 import { IBatch } from "./opLifecycle";
+/**
+ * @public
+ */
 export interface ISummaryBaseConfiguration {
     /**
      * Delay before first attempt to spawn summarizing container.
@@ -32,6 +35,9 @@ export interface ISummaryBaseConfiguration {
      */
     maxOpsSinceLastSummary: number;
 }
+/**
+ * @public
+ */
 export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfiguration {
     state: "enabled";
     /**
@@ -90,14 +96,29 @@ export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfigurati
      */
     nonRuntimeHeuristicThreshold?: number;
 }
+/**
+ * @public
+ */
 export interface ISummaryConfigurationDisableSummarizer {
     state: "disabled";
 }
+/**
+ * @public
+ */
 export interface ISummaryConfigurationDisableHeuristics extends ISummaryBaseConfiguration {
     state: "disableHeuristics";
 }
+/**
+ * @public
+ */
 export type ISummaryConfiguration = ISummaryConfigurationDisableSummarizer | ISummaryConfigurationDisableHeuristics | ISummaryConfigurationHeuristics;
+/**
+ * @public
+ */
 export declare const DefaultSummaryConfiguration: ISummaryConfiguration;
+/**
+ * @public
+ */
 export interface ISummaryRuntimeOptions {
     /** Override summary configurations set by the server. */
     summaryConfigOverrides?: ISummaryConfiguration;
@@ -111,20 +132,23 @@ export interface ISummaryRuntimeOptions {
 }
 /**
  * Options for op compression.
- * @experimental - Not ready for use
+ * @public
  */
 export interface ICompressionRuntimeOptions {
     /**
-     * The minimum size the batch's payload must exceed before the batch's contents will be compressed.
+     * The value the batch's content size must exceed for the batch to be compressed.
+     * By default the value is 600 * 1024 = 614400 bytes. If the value is set to `Infinity`, compression will be disabled.
      */
     readonly minimumBatchSizeInBytes: number;
     /**
      * The compression algorithm that will be used to compress the op.
+     * By default the value is `lz4` which is the only compression algorithm currently supported.
      */
     readonly compressionAlgorithm: CompressionAlgorithms;
 }
 /**
  * Options for container runtime.
+ * @public
  */
 export interface IContainerRuntimeOptions {
     readonly summaryOptions?: ISummaryRuntimeOptions;
@@ -146,8 +170,7 @@ export interface IContainerRuntimeOptions {
      */
     readonly flushMode?: FlushMode;
     /**
-     * Enables the runtime to compress ops. Compression is disabled when undefined.
-     * @experimental Not ready for use.
+     * Enables the runtime to compress ops. See {@link ICompressionRuntimeOptions}.
      */
     readonly compressionOptions?: ICompressionRuntimeOptions;
     /**
@@ -164,12 +187,15 @@ export interface IContainerRuntimeOptions {
     /**
      * If the op payload needs to be chunked in order to work around the maximum size of the batch, this value represents
      * how large the individual chunks will be. This is only supported when compression is enabled. If after compression, the
-     * batch size exceeds this value, it will be chunked into smaller ops of this size.
+     * batch content size exceeds this value, it will be chunked into smaller ops of this exact size.
      *
-     * If unspecified, if a batch exceeds `maxBatchSizeInBytes` after compression, the container will close with an instance
-     * of `GenericError` with the `BatchTooLarge` message.
+     * This value is a trade-off between having many small chunks vs fewer larger chunks and by default, the runtime is configured to use
+     * 200 * 1024 = 204800 bytes. This default value ensures that no compressed payload's content is able to exceed {@link IContainerRuntimeOptions.maxBatchSizeInBytes}
+     * regardless of the overhead of an individual op.
      *
-     * @experimental Not ready for use.
+     * Any value of `chunkSizeInBytes` exceeding {@link IContainerRuntimeOptions.maxBatchSizeInBytes} will disable this feature, therefore if a compressed batch's content
+     * size exceeds {@link IContainerRuntimeOptions.maxBatchSizeInBytes} after compression, the container will close with an instance of `GenericError` with
+     * the `BatchTooLarge` message.
      */
     readonly chunkSizeInBytes?: number;
     /**
@@ -201,6 +227,7 @@ export interface IContainerRuntimeOptions {
 }
 /**
  * Accepted header keys for requests coming to the runtime.
+ * @public
  */
 export declare enum RuntimeHeaders {
     /** True to wait for a data store to be created and loaded before returning it. */
@@ -208,13 +235,24 @@ export declare enum RuntimeHeaders {
     /** True if the request is coming from an IFluidHandle. */
     viaHandle = "viaHandle"
 }
-/** True if a tombstoned object should be returned without erroring */
+/** True if a tombstoned object should be returned without erroring
+ * @public
+ */
 export declare const AllowTombstoneRequestHeaderKey = "allowTombstone";
-/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
+/**
+ * [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring
+ * @public
+ */
 export declare const AllowInactiveRequestHeaderKey = "allowInactive";
-/** Tombstone error responses will have this header set to true */
+/**
+ * Tombstone error responses will have this header set to true
+ * @public
+ */
 export declare const TombstoneResponseHeaderKey = "isTombstoned";
-/** Inactive error responses will have this header set to true */
+/**
+ * Inactive error responses will have this header set to true
+ * @public
+ */
 export declare const InactiveResponseHeaderKey = "isInactive";
 /**
  * The full set of parsed header data that may be found on Runtime requests
@@ -228,6 +266,7 @@ export interface RuntimeHeaderData {
 export declare const defaultRuntimeHeaderData: Required<RuntimeHeaderData>;
 /**
  * Available compression algorithms for op compression.
+ * @public
  */
 export declare enum CompressionAlgorithms {
     lz4 = "lz4"
@@ -252,7 +291,8 @@ export declare const defaultPendingOpsWaitTimeoutMs = 1000;
 /** The default time to delay a summarization retry attempt when there are pending ops */
 export declare const defaultPendingOpsRetryDelayMs = 1000;
 /**
- * @deprecated - use ContainerRuntimeMessageType instead
+ * @deprecated use ContainerRuntimeMessageType instead
+ * @public
  */
 export declare enum RuntimeMessage {
     FluidDataStoreOp = "component",
@@ -264,13 +304,15 @@ export declare enum RuntimeMessage {
     Operation = "op"
 }
 /**
- * @deprecated - please use version in driver-utils
+ * @deprecated please use version in driver-utils
+ * @public
  */
 export declare function isRuntimeMessage(message: ISequencedDocumentMessage): boolean;
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
  * special-case for document dirty state.  Ultimately we should have no special-cases from the
  * ContainerRuntime's perspective.
+ * @public
  */
 export declare const agentSchedulerId = "_scheduler";
 export declare function getDeviceSpec(): {
@@ -286,9 +328,15 @@ export declare function getDeviceSpec(): {
  * we can provide a partially-applied function to keep those items private to the ContainerRuntime.
  */
 export declare const makeLegacySendBatchFn: (submitFn: (type: MessageType, contents: any, batch: boolean, appData?: any) => number, deltaManager: Pick<IDeltaManager<unknown, unknown>, "flush">) => (batch: IBatch) => void;
+/**
+ * This function is not supported publicly and exists for e2e testing
+ * @internal
+ */
+export declare function TEST_requestSummarizer(loader: ILoader, url: string): Promise<ISummarizer>;
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
+ * @public
  */
 export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents & ISummarizerEvents> implements IContainerRuntime, IRuntime, ISummarizerRuntime, ISummarizerInternalsProvider, IProvideFluidHandleContext {
     private readonly registry;
@@ -299,11 +347,11 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
     private readonly requestHandler?;
     private readonly summaryConfiguration;
     /**
-     * @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
+     * @deprecated Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
      */
     get IFluidRouter(): this;
     /**
-     * @deprecated - use loadRuntime instead.
+     * @deprecated use loadRuntime instead.
      * Load the stores from a snapshot and returns the runtime.
      * @param context - Context of the container.
      * @param registryEntries - Mapping to the stores.
@@ -461,10 +509,12 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * a summary is generated.
      */
     private nextSummaryNumber;
-    /**
-     * If false, loading or using a Tombstoned object should merely log, not fail
-     */
-    readonly gcTombstoneEnforcementAllowed: boolean;
+    /** If false, loading or using a Tombstoned object should merely log, not fail */
+    get gcTombstoneEnforcementAllowed(): boolean;
+    /** If true, throw an error when a tombstone data store is retrieved */
+    get gcThrowOnTombstoneLoad(): boolean;
+    /** If true, throw an error when a tombstone data store is used. */
+    get gcThrowOnTombstoneUsage(): boolean;
     /**
      * GUID to identify a document in telemetry
      * ! Note: should not be used for anything other than telemetry and is not considered a stable GUID
@@ -494,7 +544,7 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
     /**
      * Notifies this object about the request made to the container.
      * @param request - Request made to the handler.
-     * @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
+     * @deprecated Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
      */
     request(request: IRequest): Promise<IResponse>;
     /**
@@ -556,7 +606,7 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * Returns the runtime of the data store.
      * @param id - Id supplied during creating the data store.
      * @param wait - True if you want to wait for it.
-     * @deprecated - Use getAliasedDataStoreEntryPoint instead to get an aliased data store's entry point.
+     * @deprecated Use getAliasedDataStoreEntryPoint instead to get an aliased data store's entry point.
      */
     getRootDataStore(id: string, wait?: boolean): Promise<IFluidRouter>;
     private getRootDataStoreChannel;
@@ -601,9 +651,16 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * Submits the signal to be sent to other clients.
      * @param type - Type of the signal.
      * @param content - Content of the signal.
+     * @param targetClientId - When specified, the signal is only sent to the provided client id.
+     */
+    submitSignal(type: string, content: any, targetClientId?: string): void;
+    /**
+     * Submits the signal to be sent to other clients.
+     * @param type - Type of the signal.
+     * @param content - Content of the signal.
+     * @param targetClientId - When specified, the signal is only sent to the provided client id.
      */
-    submitSignal(type: string, content: any): void;
-    submitDataStoreSignal(address: string, type: string, content: any): void;
+    submitDataStoreSignal(address: string, type: string, content: any, targetClientId?: string): void;
     setAttachState(attachState: AttachState.Attaching | AttachState.Attached): void;
     /**
      * Create a summary. Used when attaching or serializing a detached container.
@@ -659,7 +716,7 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      */
     updateUnusedRoutes(unusedRoutes: string[]): void;
     /**
-     * @deprecated - Replaced by deleteSweepReadyNodes.
+     * @deprecated Replaced by deleteSweepReadyNodes.
      */
     deleteUnusedNodes(unusedRoutes: string[]): string[];
     /**
@@ -774,22 +831,19 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
     private prefetchLatestSummaryThenClose;
     private closeStaleSummarizer;
     /**
-     * Downloads snapshot from storage with the given versionId or latest if versionId is null.
+     * Downloads the latest snapshot from storage.
      * By default, it also closes the container after downloading the snapshot. However, this may be
      * overridden via options.
      */
-    private fetchSnapshotFromStorage;
+    private fetchLatestSnapshotFromStorage;
     notifyAttaching(): void;
-    getPendingLocalState(props?: {
-        notifyImminentClosure: boolean;
-    }): Promise<unknown>;
+    getPendingLocalState(props?: IGetPendingLocalStateProps): Promise<unknown>;
     summarizeOnDemand(options: IOnDemandSummarizeOptions): ISummarizeResults;
     enqueueSummarize(options: IEnqueueSummarizeOptions): EnqueueSummarizeResult;
     /**
-     * * Forms a function that will request a Summarizer.
-     * @param loaderRouter - the loader acting as an IFluidRouter
-     * */
-    private formRequestSummarizerFn;
+     * Forms a function that will create and retrieve a Summarizer.
+     */
+    private formCreateSummarizerFn;
     private validateSummaryHeuristicConfiguration;
     private get groupedBatchingEnabled();
 }