@fluidframework/container-runtime 2.3.0-288113 → 2.4.0-294316

package/lib/containerRuntime.js

@@ -11,7 +11,7 @@ import { DriverHeader, FetchSource, MessageType, } from "@fluidframework/driver-
 import { readAndParse } from "@fluidframework/driver-utils/internal";
 import { FlushMode, FlushModeExperimental, channelsTreeName, gcTreeKey, } from "@fluidframework/runtime-definitions/internal";
 import { GCDataBuilder, RequestParser, TelemetryContext, addBlobToSummary, addSummarizeResultToSummary, calculateStats, create404Response, exceptionToResponse, responseToException, seqFromTree, } from "@fluidframework/runtime-utils/internal";
-import { DataCorruptionError, DataProcessingError, GenericError, LoggingError, PerformanceEvent, 
+import { DataCorruptionError, DataProcessingError, extractSafePropertiesFromMessage, GenericError, LoggingError, PerformanceEvent, 
 // eslint-disable-next-line import/no-deprecated
 TaggedLoggerAdapter, UsageError, createChildLogger, createChildMonitoringContext, createSampledLogger, loggerToMonitoringContext, raiseConnectedEvent, wrapError, tagCodeArtifacts, } from "@fluidframework/telemetry-utils/internal";
 import { v4 as uuid } from "uuid";
@@ -25,7 +25,7 @@ import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
 import { DeltaManagerPendingOpsProxy, DeltaManagerSummarizerProxy, } from "./deltaManagerProxies.js";
 import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
 import { ContainerMessageType, } from "./messageTypes.js";
-import { ensureContentsDeserialized, OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
+import { DuplicateBatchDetector, ensureContentsDeserialized, OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
 import { pkgVersion } from "./packageVersion.js";
 import { PendingStateManager, } from "./pendingStateManager.js";
 import { ScheduleManager } from "./scheduleManager.js";
@@ -228,15 +228,25 @@ function lastMessageFromMetadata(metadata) {
  * We only want to log this once, to avoid spamming telemetry if we are wrong and these cases are hit commonly.
  */
 let getSingleUseLegacyLogCallback = (logger, type) => {
-    // We only want to log this once per ContainerRuntime instance, to avoid spamming telemetry.
-    getSingleUseLegacyLogCallback = () => () => { };
     return (codePath) => {
         logger.sendTelemetryEvent({
             eventName: "LegacyMessageFormat",
             details: { codePath, type },
         });
+        // Now that we've logged, prevent future logging (globally).
+        getSingleUseLegacyLogCallback = () => () => { };
     };
 };
+/**
+ * This is meant to be used by a {@link @fluidframework/container-definitions#IRuntimeFactory} to instantiate a container runtime.
+ * @param params - An object which specifies all required and optional params necessary to instantiate a runtime.
+ * @returns A runtime which provides all the functionality necessary to bind with the loader layer via the {@link @fluidframework/container-definitions#IRuntime} interface and provide a runtime environment via the {@link @fluidframework/container-runtime-definitions#IContainerRuntime} interface.
+ * @legacy
+ * @alpha
+ */
+export async function loadContainerRuntime(params) {
+    return ContainerRuntime.loadRuntime(params);
+}
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
@@ -588,11 +598,16 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
         this.defaultTelemetrySignalSampleCount = 100;
-        this._perfSignalData = {
+        this._signalTracking = {
+            totalSignalsSentInLatencyWindow: 0,
             signalsLost: 0,
-            signalSequenceNumber: 0,
+            signalsOutOfOrder: 0,
+            signalsSentSinceLastLatencyMeasurement: 0,
+            broadcastSignalSequenceNumber: 0,
             signalTimestamp: 0,
+            roundTripSignalSequenceNumber: undefined,
             trackingSignalSequenceNumber: undefined,
+            minimumTrackingSignalSequenceNumber: undefined,
         };
         /**
          * It a cache for holding mapping for loading groupIds with its snapshot from the service. Add expiry policy of 1 minute.
@@ -700,13 +715,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             isAttached: () => this.attachState !== AttachState.Detached,
         }, pendingRuntimeState?.pending, this.logger);
         let outerDeltaManager;
-        const useDeltaManagerOpsProxy = this.mc.config.getBoolean("Fluid.ContainerRuntime.DeltaManagerOpsProxy") !== false;
+        this.useDeltaManagerOpsProxy =
+            this.mc.config.getBoolean("Fluid.ContainerRuntime.DeltaManagerOpsProxy") === true;
         // The summarizerDeltaManager Proxy is used to lie to the summarizer to convince it is in the right state as a summarizer client.
         const summarizerDeltaManagerProxy = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
         outerDeltaManager = summarizerDeltaManagerProxy;
         // The DeltaManagerPendingOpsProxy is used to control the minimum sequence number
         // It allows us to lie to the layers below so that they can maintain enough local state for rebasing ops.
-        if (useDeltaManagerOpsProxy) {
+        if (this.useDeltaManagerOpsProxy) {
             const pendingOpsDeltaManagerProxy = new DeltaManagerPendingOpsProxy(summarizerDeltaManagerProxy, this.pendingStateManager);
             outerDeltaManager = pendingOpsDeltaManagerProxy;
         }
@@ -737,6 +753,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.closeFn(error);
             throw error;
         }
+        // DuplicateBatchDetection is only enabled if Offline Load is enabled
+        // It maintains a cache of all batchIds/sequenceNumbers within the collab window.
+        // Don't waste resources doing so if not needed.
+        if (this.offlineEnabled) {
+            this.duplicateBatchDetector = new DuplicateBatchDetector();
+        }
         if (context.attachState === AttachState.Attached) {
             const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
             if (maxSnapshotCacheDurationMs !== undefined &&
@@ -1134,13 +1156,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         let childTree = snapshotTree;
         for (const part of pathParts) {
             if (hasIsolatedChannels) {
-                // TODO Why are we non null asserting here
-                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                childTree = childTree.trees[channelsTreeName];
+                childTree = childTree?.trees[channelsTreeName];
             }
-            // TODO Why are we non null asserting here
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            childTree = childTree.trees[part];
+            childTree = childTree?.trees[part];
         }
         return childTree;
     }
@@ -1187,8 +1205,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return this.resolveHandle(requestParser.createSubRequest(1));
             }
             if (id === blobManagerBasePath && requestParser.isLeaf(2)) {
-                // TODO why are we non null asserting here?
-                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                 const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
                 return blob
                     ? {
@@ -1461,9 +1477,14 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         this._connected = connected;
         if (!connected) {
-            this._perfSignalData.signalsLost = 0;
-            this._perfSignalData.signalTimestamp = 0;
-            this._perfSignalData.trackingSignalSequenceNumber = undefined;
+            this._signalTracking.signalsLost = 0;
+            this._signalTracking.signalsOutOfOrder = 0;
+            this._signalTracking.signalTimestamp = 0;
+            this._signalTracking.signalsSentSinceLastLatencyMeasurement = 0;
+            this._signalTracking.totalSignalsSentInLatencyWindow = 0;
+            this._signalTracking.roundTripSignalSequenceNumber = undefined;
+            this._signalTracking.trackingSignalSequenceNumber = undefined;
+            this._signalTracking.minimumTrackingSignalSequenceNumber = undefined;
         }
         else {
             assert(this.attachState === AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
@@ -1510,158 +1531,159 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (hasModernRuntimeMessageEnvelope) {
             // If the message has the modern message envelope, then process it here.
             // Here we unpack the message (decompress, unchunk, and/or ungroup) into a batch of messages with ContainerMessageType
-            const inboundBatch = this.remoteMessageProcessor.process(messageCopy, logLegacyCase);
-            if (inboundBatch === undefined) {
+            const inboundResult = this.remoteMessageProcessor.process(messageCopy, logLegacyCase);
+            if (inboundResult === undefined) {
                 // This means the incoming message is an incomplete part of a message or batch
                 // and we need to process more messages before the rest of the system can understand it.
                 return;
             }
-            // Reach out to PendingStateManager to zip localOpMetadata into the message list if it's a local batch
-            const messagesWithPendingState = this.pendingStateManager.processInboundBatch(inboundBatch, local);
-            if (messagesWithPendingState.length > 0) {
-                messagesWithPendingState.forEach(({ message, localOpMetadata }) => {
-                    const msg = {
-                        message,
-                        local,
-                        isRuntimeMessage: true,
-                        savedOp,
-                        localOpMetadata,
-                    };
-                    this.ensureNoDataModelChanges(() => this.processRuntimeMessage(msg));
-                });
+            if ("batchStart" in inboundResult) {
+                const batchStart = inboundResult.batchStart;
+                const result = this.duplicateBatchDetector?.processInboundBatch(batchStart);
+                if (result?.duplicate) {
+                    const error = new DataCorruptionError("Duplicate batch - The same batch was sequenced twice", { batchId: batchStart.batchId });
+                    this.mc.logger.sendTelemetryEvent({
+                        eventName: "DuplicateBatch",
+                        details: {
+                            batchId: batchStart.batchId,
+                            clientId: batchStart.clientId,
+                            batchStartCsn: batchStart.batchStartCsn,
+                            size: inboundResult.length,
+                            duplicateBatchSequenceNumber: result.otherSequenceNumber,
+                            ...extractSafePropertiesFromMessage(batchStart.keyMessage),
+                        },
+                    }, error);
+                    throw error;
+                }
             }
-            else {
-                this.ensureNoDataModelChanges(() => this.processEmptyBatch(inboundBatch, local));
+            let runtimeBatch = true;
+            // Reach out to PendingStateManager, either to zip localOpMetadata into the *local* message list,
+            // or to check to ensure the *remote* messages don't match the batchId of a pending local batch.
+            // This latter case would indicate that the container has forked - two copies are trying to persist the same local changes.
+            let messagesWithPendingState = this.pendingStateManager.processInboundMessages(inboundResult, local);
+            if (inboundResult.type !== "fullBatch") {
+                assert(messagesWithPendingState.length === 1, 0xa3d /* Partial batch should have exactly one message */);
+            }
+            if (messagesWithPendingState.length === 0) {
+                assert(inboundResult.type === "fullBatch", 0xa3e /* Empty batch is always considered a full batch */);
+                /**
+                 * We need to process an empty batch, which will execute expected actions while processing even if there
+                 * are no inner runtime messages.
+                 *
+                 * Empty batches are produced by the outbox on resubmit when the resubmit flow resulted in no runtime
+                 * messages.
+                 * This can happen if changes from a remote client "cancel out" the pending changes being resubmitted by
+                 * this client.  We submit an empty batch if "offline load" (aka rehydrating from stashed state) is
+                 * enabled, to ensure we account for this batch when comparing batchIds, checking for a forked container.
+                 * Otherwise, we would not realize this container has forked in the case where it did fork, and a batch
+                 * became empty but wasn't submitted as such.
+                 */
+                messagesWithPendingState = [
+                    {
+                        message: inboundResult.batchStart.keyMessage,
+                        localOpMetadata: undefined,
+                    },
+                ];
+                // Empty batch message is a non-runtime message as it was generated by the op grouping manager.
+                runtimeBatch = false;
             }
+            const locationInBatch = inboundResult.type === "fullBatch"
+                ? { batchStart: true, batchEnd: true }
+                : inboundResult.type === "batchStartingMessage"
+                    ? { batchStart: true, batchEnd: false }
+                    : { batchStart: false, batchEnd: inboundResult.batchEnd === true };
+            this.processInboundMessages(messagesWithPendingState, locationInBatch, local, savedOp, runtimeBatch);
         }
         else {
-            // Check if message.type is one of values in ContainerMessageType
-            // eslint-disable-next-line import/no-deprecated
-            if (isRuntimeMessage(messageCopy)) {
-                // Legacy op received
-                this.ensureNoDataModelChanges(() => this.processRuntimeMessage({
-                    message: messageCopy,
-                    local,
-                    isRuntimeMessage: true,
-                    savedOp,
-                }));
-            }
-            else {
-                // A non container runtime message (like other system ops - join, ack, leave, nack etc.)
-                this.ensureNoDataModelChanges(() => this.observeNonRuntimeMessage({
-                    message: messageCopy,
-                    local,
-                    isRuntimeMessage: false,
-                    savedOp,
-                }));
-            }
+            this.processInboundMessages([{ message: messageCopy, localOpMetadata: undefined }], { batchStart: true, batchEnd: true }, // Single message
+            local, savedOp, isRuntimeMessage(messageCopy) /* runtimeBatch */);
+        }
+        if (local) {
+            // If we have processed a local op, this means that the container is
+            // making progress and we can reset the counter for how many times
+            // we have consecutively replayed the pending states
+            this.resetReconnectCount();
         }
     }
     /**
-     * Processes messages that are intended for the runtime layer to process.
-     * It redirects the message to the correct subsystem for processing, and implement other side effects
-     * @param messageWithContext - message to process with additional context and isRuntimeMessage prop as true
+     * Processes inbound message(s). It calls schedule manager according to the messages' location in the batch.
+     * @param messages - messages to process.
+     * @param locationInBatch - Are we processing the start and/or end of a batch?
+     * @param local - true if the messages were originally generated by the client receiving it.
+     * @param savedOp - true if the message is a replayed saved op.
+     * @param runtimeBatch - true if these are runtime messages.
      */
-    processRuntimeMessage(messageWithContext) {
-        const { message, local, localOpMetadata } = messageWithContext;
-        // Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
-        // Sequence numbers are not guaranteed to follow any sort of order. Re-entrancy is one of those situations
-        if (this.deltaManager.minimumSequenceNumber <
-            messageWithContext.message.minimumSequenceNumber) {
-            messageWithContext.message.minimumSequenceNumber =
-                this.deltaManager.minimumSequenceNumber;
-        }
-        // Surround the actual processing of the operation with messages to the schedule manager indicating
-        // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
-        // messages once a batch has been fully processed.
-        this.scheduleManager.beforeOpProcessing(message);
-        this._processedClientSequenceNumber = message.clientSequenceNumber;
+    processInboundMessages(messages, locationInBatch, local, savedOp, runtimeBatch) {
+        if (locationInBatch.batchStart) {
+            const firstMessage = messages[0]?.message;
+            assert(firstMessage !== undefined, 0xa31 /* Batch must have at least one message */);
+            this.scheduleManager.batchBegin(firstMessage);
+        }
+        let error;
         try {
-            // RemoteMessageProcessor would have already reconstituted Chunked Ops into the original op type
-            assert(message.type !== ContainerMessageType.ChunkedOp, 0x93b /* we should never get here with chunked ops */);
-            // If there are no more pending messages after processing a local message,
-            // the document is no longer dirty.
-            if (!this.hasPendingMessages()) {
-                this.updateDocumentDirtyState(false);
-            }
-            this.validateAndProcessRuntimeMessage(messageWithContext, localOpMetadata);
-            this.emit("op", message, messageWithContext.isRuntimeMessage);
-            this.scheduleManager.afterOpProcessing(undefined, message);
-            if (local) {
-                // If we have processed a local op, this means that the container is
-                // making progress and we can reset the counter for how many times
-                // we have consecutively replayed the pending states
-                this.resetReconnectCount();
-            }
+            messages.forEach(({ message, localOpMetadata }) => {
+                this.ensureNoDataModelChanges(() => {
+                    if (runtimeBatch) {
+                        this.validateAndProcessRuntimeMessage({
+                            message: message,
+                            local,
+                            savedOp,
+                            localOpMetadata,
+                        });
+                    }
+                    else {
+                        this.observeNonRuntimeMessage(message);
+                    }
+                });
+            });
         }
         catch (e) {
-            this.scheduleManager.afterOpProcessing(e, message);
-            throw e;
-        }
-    }
-    /**
-     * Process an empty batch, which will execute expected actions while processing even if there are no messages.
-     * This is a separate function because the processCore function expects at least one message to process.
-     * It is expected to happen only when the outbox produces an empty batch due to a resubmit flow.
-     */
-    processEmptyBatch(emptyBatch, local) {
-        const { emptyBatchSequenceNumber: sequenceNumber, batchStartCsn } = emptyBatch;
-        assert(sequenceNumber !== undefined, 0x9fa /* emptyBatchSequenceNumber must be defined */);
-        this.emit("batchBegin", { sequenceNumber });
-        this._processedClientSequenceNumber = batchStartCsn;
-        if (!this.hasPendingMessages()) {
-            this.updateDocumentDirtyState(false);
+            error = e;
+            throw error;
         }
-        this.emit("batchEnd", undefined, { sequenceNumber });
-        if (local) {
-            this.resetReconnectCount();
+        finally {
+            if (locationInBatch.batchEnd) {
+                const lastMessage = messages[messages.length - 1]?.message;
+                assert(lastMessage !== undefined, 0xa32 /* Batch must have at least one message */);
+                this.scheduleManager.batchEnd(error, lastMessage);
+            }
         }
     }
     /**
      * Observes messages that are not intended for the runtime layer, updating/notifying Runtime systems as needed.
-     * @param messageWithContext - non-runtime messages to process with additional context and isRuntimeMessage prop as false
+     * @param message - non-runtime message to process.
      */
-    observeNonRuntimeMessage(messageWithContext) {
-        const { message, local } = messageWithContext;
-        // Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
-        // Sequence numbers are not guaranteed to follow any sort of order. Re-entrancy is one of those situations
-        if (this.deltaManager.minimumSequenceNumber <
-            messageWithContext.message.minimumSequenceNumber) {
-            messageWithContext.message.minimumSequenceNumber =
-                this.deltaManager.minimumSequenceNumber;
-        }
-        // Surround the actual processing of the operation with messages to the schedule manager indicating
-        // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
-        // messages once a batch has been fully processed.
-        this.scheduleManager.beforeOpProcessing(message);
-        this._processedClientSequenceNumber = message.clientSequenceNumber;
-        try {
-            // If there are no more pending messages after processing a local message,
-            // the document is no longer dirty.
-            if (!this.hasPendingMessages()) {
-                this.updateDocumentDirtyState(false);
-            }
-            this.emit("op", message, messageWithContext.isRuntimeMessage);
-            this.scheduleManager.afterOpProcessing(undefined, message);
-            if (local) {
-                // If we have processed a local op, this means that the container is
-                // making progress and we can reset the counter for how many times
-                // we have consecutively replayed the pending states
-                this.resetReconnectCount();
-            }
+    observeNonRuntimeMessage(message) {
+        // Set the minimum sequence number to the containerRuntime's understanding of minimum sequence number.
+        if (this.deltaManager.minimumSequenceNumber < message.minimumSequenceNumber) {
+            message.minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
         }
-        catch (e) {
-            this.scheduleManager.afterOpProcessing(e, message);
-            throw e;
+        this._processedClientSequenceNumber = message.clientSequenceNumber;
+        // If there are no more pending messages after processing a local message,
+        // the document is no longer dirty.
+        if (!this.hasPendingMessages()) {
+            this.updateDocumentDirtyState(false);
         }
+        this.emit("op", message, false /* runtimeMessage */);
     }
     /**
      * Assuming the given message is also a TypedContainerRuntimeMessage,
      * checks its type and dispatches the message to the appropriate handler in the runtime.
      * Throws a DataProcessingError if the message looks like but doesn't conform to a known TypedContainerRuntimeMessage type.
      */
-    validateAndProcessRuntimeMessage(messageWithContext, localOpMetadata) {
-        const { local, message, savedOp } = messageWithContext;
+    validateAndProcessRuntimeMessage(messageWithContext) {
+        const { local, message, savedOp, localOpMetadata } = messageWithContext;
+        // Set the minimum sequence number to the containerRuntime's understanding of minimum sequence number.
+        if (this.useDeltaManagerOpsProxy &&
+            this.deltaManager.minimumSequenceNumber < message.minimumSequenceNumber) {
+            message.minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
+        }
+        this._processedClientSequenceNumber = message.clientSequenceNumber;
+        // If there are no more pending messages after processing a local message,
+        // the document is no longer dirty.
+        if (!this.hasPendingMessages()) {
+            this.updateDocumentDirtyState(false);
+        }
         switch (message.type) {
             case ContainerMessageType.Attach:
             case ContainerMessageType.Alias:
@@ -1722,20 +1744,25 @@ export class ContainerRuntime extends TypedEventEmitter {
                 }
             }
         }
+        this.emit("op", message, true /* runtimeMessage */);
     }
     /**
      * Emits the Signal event and update the perf signal data.
-     * @param clientSignalSequenceNumber - is the client signal sequence number to be uploaded.
      */
-    sendSignalTelemetryEvent(clientSignalSequenceNumber) {
-        const duration = Date.now() - this._perfSignalData.signalTimestamp;
+    sendSignalTelemetryEvent() {
+        const duration = Date.now() - this._signalTracking.signalTimestamp;
         this.mc.logger.sendPerformanceEvent({
             eventName: "SignalLatency",
-            duration,
-            signalsLost: this._perfSignalData.signalsLost,
+            duration, // Roundtrip duration of the tracked signal in milliseconds.
+            signalsSent: this._signalTracking.totalSignalsSentInLatencyWindow, // Signals sent since the last logged SignalLatency event.
+            signalsLost: this._signalTracking.signalsLost, // Signals lost since the last logged SignalLatency event.
+            outOfOrderSignals: this._signalTracking.signalsOutOfOrder, // Out of order signals since the last logged SignalLatency event.
+            reconnectCount: this.consecutiveReconnects, // Container reconnect count.
         });
-        this._perfSignalData.signalsLost = 0;
-        this._perfSignalData.signalTimestamp = 0;
+        this._signalTracking.signalsLost = 0;
+        this._signalTracking.signalsOutOfOrder = 0;
+        this._signalTracking.signalTimestamp = 0;
+        this._signalTracking.totalSignalsSentInLatencyWindow = 0;
     }
     processSignal(message, local) {
         const envelope = message.content;
@@ -1743,29 +1770,55 @@ export class ContainerRuntime extends TypedEventEmitter {
             clientId: message.clientId,
             content: envelope.contents.content,
             type: envelope.contents.type,
+            targetClientId: message.targetClientId,
         };
-        // Only collect signal telemetry for messages sent by the current client.
-        if (message.clientId === this.clientId && this.connected) {
-            // Check to see if the signal was lost.
-            if (this._perfSignalData.trackingSignalSequenceNumber !== undefined &&
-                envelope.clientSignalSequenceNumber > this._perfSignalData.trackingSignalSequenceNumber) {
-                this._perfSignalData.signalsLost++;
-                this._perfSignalData.trackingSignalSequenceNumber = undefined;
-                this.mc.logger.sendErrorEvent({
-                    eventName: "SignalLost",
-                    type: envelope.contents.type,
-                    signalsLost: this._perfSignalData.signalsLost,
-                    trackingSequenceNumber: this._perfSignalData.trackingSignalSequenceNumber,
-                    clientSignalSequenceNumber: envelope.clientSignalSequenceNumber,
-                });
-            }
-            else if (envelope.clientSignalSequenceNumber ===
-                this._perfSignalData.trackingSignalSequenceNumber) {
-                // only logging for the first connection and the trackingSignalSequenceNUmber.
-                if (this.consecutiveReconnects === 0) {
-                    this.sendSignalTelemetryEvent(envelope.clientSignalSequenceNumber);
+        // Only collect signal telemetry for broadcast messages sent by the current client.
+        if (message.clientId === this.clientId &&
+            this.connected &&
+            envelope.clientBroadcastSignalSequenceNumber !== undefined) {
+            if (this._signalTracking.trackingSignalSequenceNumber !== undefined &&
+                this._signalTracking.minimumTrackingSignalSequenceNumber !== undefined) {
+                if (envelope.clientBroadcastSignalSequenceNumber >=
+                    this._signalTracking.trackingSignalSequenceNumber) {
+                    // Calculate the number of signals lost and log the event.
+                    const signalsLost = envelope.clientBroadcastSignalSequenceNumber -
+                        this._signalTracking.trackingSignalSequenceNumber;
+                    if (signalsLost > 0) {
+                        this._signalTracking.signalsLost += signalsLost;
+                        this.mc.logger.sendErrorEvent({
+                            eventName: "SignalLost",
+                            signalsLost, // Number of lost signals detected.
+                            trackingSequenceNumber: this._signalTracking.trackingSignalSequenceNumber, // The next expected signal sequence number.
+                            clientBroadcastSignalSequenceNumber: envelope.clientBroadcastSignalSequenceNumber, // Actual signal sequence number received.
+                        });
+                    }
+                    // Update the tracking signal sequence number to the next expected signal in the sequence.
+                    this._signalTracking.trackingSignalSequenceNumber =
+                        envelope.clientBroadcastSignalSequenceNumber + 1;
+                }
+                else if (envelope.clientBroadcastSignalSequenceNumber >=
+                    this._signalTracking.minimumTrackingSignalSequenceNumber) {
+                    this._signalTracking.signalsOutOfOrder++;
+                    this.mc.logger.sendTelemetryEvent({
+                        eventName: "SignalOutOfOrder",
+                        type: envelope.contents.type, // Type of signal that was received out of order.
+                        trackingSequenceNumber: this._signalTracking.trackingSignalSequenceNumber, // The next expected signal sequence number.
+                        clientBroadcastSignalSequenceNumber: envelope.clientBroadcastSignalSequenceNumber, // Sequence number of the out of order signal.
+                    });
+                }
+                if (this._signalTracking.roundTripSignalSequenceNumber !== undefined &&
+                    envelope.clientBroadcastSignalSequenceNumber >=
+                        this._signalTracking.roundTripSignalSequenceNumber) {
+                    if (envelope.clientBroadcastSignalSequenceNumber ===
+                        this._signalTracking.roundTripSignalSequenceNumber) {
+                        // Latency tracked signal has been received.
+                        // We now log the roundtrip duration of the tracked signal.
+                        // This telemetry event also logs metrics for signals sent, signals lost, and out of order signals received.
+                        // These metrics are reset after logging the telemetry event.
+                        this.sendSignalTelemetryEvent();
+                    }
+                    this._signalTracking.roundTripSignalSequenceNumber = undefined;
                 }
-                this._perfSignalData.trackingSignalSequenceNumber = undefined;
             }
         }
         if (envelope.address === undefined) {
@@ -1941,18 +1994,36 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         return true;
     }
-    createNewSignalEnvelope(address, type, content) {
-        const newSequenceNumber = ++this._perfSignalData.signalSequenceNumber;
+    createNewSignalEnvelope(address, type, content, targetClientId) {
         const newEnvelope = {
             address,
-            clientSignalSequenceNumber: newSequenceNumber,
             contents: { type, content },
         };
-        // We should not track any signals in case we already have a tracking number.
-        if (newSequenceNumber % this.defaultTelemetrySignalSampleCount === 1 &&
-            this._perfSignalData.trackingSignalSequenceNumber === undefined) {
-            this._perfSignalData.signalTimestamp = Date.now();
-            this._perfSignalData.trackingSignalSequenceNumber = newSequenceNumber;
+        const isBroadcastSignal = targetClientId === undefined;
+        if (isBroadcastSignal) {
+            const clientBroadcastSignalSequenceNumber = ++this._signalTracking
+                .broadcastSignalSequenceNumber;
+            newEnvelope.clientBroadcastSignalSequenceNumber = clientBroadcastSignalSequenceNumber;
+            this._signalTracking.signalsSentSinceLastLatencyMeasurement++;
+            if (this._signalTracking.minimumTrackingSignalSequenceNumber === undefined ||
+                this._signalTracking.trackingSignalSequenceNumber === undefined) {
+                // Signal monitoring window is undefined
+                // Initialize tracking to expect the next signal sent by the connected client.
+                this._signalTracking.minimumTrackingSignalSequenceNumber =
+                    clientBroadcastSignalSequenceNumber;
+                this._signalTracking.trackingSignalSequenceNumber =
+                    clientBroadcastSignalSequenceNumber;
+            }
+            // We should not track the round trip of a new signal in the case we are already tracking one.
+            if (clientBroadcastSignalSequenceNumber % this.defaultTelemetrySignalSampleCount === 1 &&
+                this._signalTracking.roundTripSignalSequenceNumber === undefined) {
+                this._signalTracking.signalTimestamp = Date.now();
+                this._signalTracking.roundTripSignalSequenceNumber =
+                    clientBroadcastSignalSequenceNumber;
+                this._signalTracking.totalSignalsSentInLatencyWindow +=
+                    this._signalTracking.signalsSentSinceLastLatencyMeasurement;
+                this._signalTracking.signalsSentSinceLastLatencyMeasurement = 0;
+            }
         }
         return newEnvelope;
     }
@@ -1961,10 +2032,16 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param type - Type of the signal.
      * @param content - Content of the signal. Should be a JSON serializable object or primitive.
      * @param targetClientId - When specified, the signal is only sent to the provided client id.
+     *
+     * @remarks
+     *
+     * The `targetClientId` parameter here is currently intended for internal testing purposes only.
+     * Support for this option at container runtime is planned to be deprecated in the future.
+     *
      */
     submitSignal(type, content, targetClientId) {
         this.verifyNotClosed();
-        const envelope = this.createNewSignalEnvelope(undefined /* address */, type, content);
+        const envelope = this.createNewSignalEnvelope(undefined /* address */, type, content, targetClientId);
         return this.submitSignalFn(envelope, targetClientId);
     }
     setAttachState(attachState) {
@@ -2367,8 +2444,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Counting dataStores and handles
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
-            // TODO why are we non null asserting here
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const dataStoreTree = summaryTree.tree[channelsTreeName];
             assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === SummaryType.Handle).length;
@@ -2691,6 +2766,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw new Error("Runtime is closed");
         }
     }
+    /**
+     * Resubmits each message in the batch, and then flushes the outbox.
+     *
+     * @remarks - If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
+     * for correlation to detect container forking.
+     */
     reSubmitBatch(batch, batchId) {
         this.orderSequentially(() => {
             for (const message of batch) {