@fluidframework/container-loader 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.225277

package/lib/{deltaManager.js → deltaManager.mjs}

@@ -2,97 +2,64 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { default as AbortController } from "abort-controller";
 import { v4 as uuid } from "uuid";
-import { assert, TypedEventEmitter } from "@fluidframework/common-utils";
-import { normalizeError, logIfFalse, safeRaiseEvent, } from "@fluidframework/telemetry-utils";
-import { DriverErrorType, } from "@fluidframework/driver-definitions";
+import { TypedEventEmitter } from "@fluid-internal/client-utils";
+import { assert } from "@fluidframework/core-utils";
+import { DataProcessingError, extractSafePropertiesFromMessage, normalizeError, safeRaiseEvent, isFluidError, DataCorruptionError, UsageError, } from "@fluidframework/telemetry-utils";
+import { DriverErrorTypes, } from "@fluidframework/driver-definitions";
 import { MessageType, } from "@fluidframework/protocol-definitions";
-import { NonRetryableError, isClientMessage, } from "@fluidframework/driver-utils";
-import { ThrottlingWarning, DataCorruptionError, extractSafePropertiesFromMessage, DataProcessingError, } from "@fluidframework/container-utils";
-import { DeltaQueue } from "./deltaQueue";
+import { NonRetryableError, isRuntimeMessage, MessageType2 } from "@fluidframework/driver-utils";
+import { DeltaQueue } from "./deltaQueue.mjs";
+import { ThrottlingWarning } from "./error.mjs";
+/**
+ * Determines if message was sent by client, not service
+ */
+function isClientMessage(message) {
+    if (isRuntimeMessage(message)) {
+        return true;
+    }
+    switch (message.type) {
+        case MessageType.Propose:
+        case MessageType.Reject:
+        case MessageType.NoOp:
+        case MessageType2.Accept:
+        case MessageType.Summarize:
+            return true;
+        default:
+            return false;
+    }
+}
+/**
+ * Like assert, but logs only if the condition is false, rather than throwing
+ * @param condition - The condition to attest too
+ * @param logger - The logger to log with
+ * @param event - The string or event to log
+ * @returns The outcome of the condition
+ */
+function logIfFalse(condition, logger, event) {
+    if (condition) {
+        return true;
+    }
+    const newEvent = typeof event === "string"
+        ? { eventName: event, category: "error" }
+        : { category: "error", ...event };
+    logger.send(newEvent);
+    return false;
+}
 /**
  * Manages the flow of both inbound and outbound messages. This class ensures that shared objects receive delta
  * messages in order regardless of possible network conditions or timings causing out of order delivery.
  */
 export class DeltaManager extends TypedEventEmitter {
-    constructor(serviceProvider, logger, _active, createConnectionManager) {
-        super();
-        this.serviceProvider = serviceProvider;
-        this.logger = logger;
-        this._active = _active;
-        this.pending = [];
-        // The minimum sequence number and last sequence number received from the server
-        this.minSequenceNumber = 0;
-        // There are three numbers we track
-        // * lastQueuedSequenceNumber is the last queued sequence number. If there are gaps in seq numbers, then this number
-        //   is not updated until we cover that gap, so it increases each time by 1.
-        // * lastObservedSeqNumber is  an estimation of last known sequence number for container in storage. It's initially
-        //   populated at web socket connection time (if storage provides that info) and is  updated once ops shows up.
-        //   It's never less than lastQueuedSequenceNumber
-        // * lastProcessedSequenceNumber - last processed sequence number
-        this.lastQueuedSequenceNumber = 0;
-        this.lastObservedSeqNumber = 0;
-        this.lastProcessedSequenceNumber = 0;
-        this.baseTerm = 0;
-        /**
-         * Track down the ops size.
-        */
-        this.opsSize = 0;
-        // The sequence number we initially loaded from
-        this.initSequenceNumber = 0;
-        this.closed = false;
-        this.throttlingIdSet = new Set();
-        this.timeTillThrottling = 0;
-        this.closeAbortController = new AbortController();
-        this.deltaStorageDelayId = uuid();
-        this.deltaStreamDelayId = uuid();
-        this.messageBuffer = [];
-        const props = {
-            incomingOpHandler: (messages, reason) => {
-                try {
-                    this.enqueueMessages(messages, reason);
-                }
-                catch (error) {
-                    this.logger.sendErrorEvent({ eventName: "EnqueueMessages_Exception" }, error);
-                    this.close(normalizeError(error));
-                }
-            },
-            signalHandler: (message) => this._inboundSignal.push(message),
-            reconnectionDelayHandler: (delayMs, error) => this.emitDelayInfo(this.deltaStreamDelayId, delayMs, error),
-            closeHandler: (error) => this.close(error),
-            disconnectHandler: (reason) => this.disconnectHandler(reason),
-            connectHandler: (connection) => this.connectHandler(connection),
-            pongHandler: (latency) => this.emit("pong", latency),
-            readonlyChangeHandler: (readonly) => safeRaiseEvent(this, this.logger, "readonly", readonly),
-        };
-        this.connectionManager = createConnectionManager(props);
-        this._inbound = new DeltaQueue((op) => {
-            this.processInboundMessage(op);
-        });
-        this._inbound.on("error", (error) => {
-            this.close(DataProcessingError.wrapIfUnrecognized(error, "deltaManagerInboundErrorHandler", this.lastMessage));
-        });
-        // Inbound signal queue
-        this._inboundSignal = new DeltaQueue((message) => {
-            if (this.handler === undefined) {
-                throw new Error("Attempted to process an inbound signal without a handler attached");
-            }
-            this.handler.processSignal({
-                clientId: message.clientId,
-                content: JSON.parse(message.content),
-            });
-        });
-        this._inboundSignal.on("error", (error) => {
-            this.close(normalizeError(error));
-        });
-        // Initially, all queues are created paused.
-        // - outbound is flipped back and forth in setupNewSuccessfulConnection / disconnectFromDeltaStream
-        // - inbound & inboundSignal are resumed in attachOpHandler() when we have handler setup
+    get active() {
+        return this._active();
+    }
+    get disposed() {
+        return this._closed;
+    }
+    get IDeltaSender() {
+        return this;
     }
-    get active() { return this._active(); }
-    get disposed() { return this.closed; }
-    get IDeltaSender() { return this; }
     get inbound() {
         return this._inbound;
     }
@@ -111,9 +78,6 @@ export class DeltaManager extends TypedEventEmitter {
     get lastKnownSeqNumber() {
         return this.lastObservedSeqNumber;
     }
-    get referenceTerm() {
-        return this.baseTerm;
-    }
     get minimumSequenceNumber() {
         return this.minSequenceNumber;
     }
@@ -127,18 +91,33 @@ export class DeltaManager extends TypedEventEmitter {
         return this._checkpointSequenceNumber !== undefined;
     }
     // Forwarding connection manager properties / IDeltaManager implementation
-    get maxMessageSize() { return this.connectionManager.maxMessageSize; }
-    get version() { return this.connectionManager.version; }
-    get serviceConfiguration() { return this.connectionManager.serviceConfiguration; }
-    get outbound() { return this.connectionManager.outbound; }
-    get readOnlyInfo() { return this.connectionManager.readOnlyInfo; }
-    get clientDetails() { return this.connectionManager.clientDetails; }
-    submit(type, contents, batch = false, metadata) {
+    get maxMessageSize() {
+        return this.connectionManager.maxMessageSize;
+    }
+    get version() {
+        return this.connectionManager.version;
+    }
+    get serviceConfiguration() {
+        return this.connectionManager.serviceConfiguration;
+    }
+    get outbound() {
+        return this.connectionManager.outbound;
+    }
+    get readOnlyInfo() {
+        return this.connectionManager.readOnlyInfo;
+    }
+    get clientDetails() {
+        return this.connectionManager.clientDetails;
+    }
+    submit(type, contents, batch = false, metadata, compression, referenceSequenceNumber) {
+        // Back-compat ADO:3455
+        const backCompatRefSeqNum = referenceSequenceNumber ?? this.lastProcessedSequenceNumber;
         const messagePartial = {
-            contents: JSON.stringify(contents),
+            contents,
             metadata,
-            referenceSequenceNumber: this.lastProcessedSequenceNumber,
+            referenceSequenceNumber: backCompatRefSeqNum,
             type,
+            compression,
         };
         if (!batch) {
             this.flush();
@@ -147,26 +126,48 @@ export class DeltaManager extends TypedEventEmitter {
         if (message === undefined) {
             return -1;
         }
-        this.opsSize += message.contents.length;
+        assert(isClientMessage(message), 0x419 /* client sends non-client message */);
+        if (contents !== undefined) {
+            this.opsSize += contents.length;
+        }
         this.messageBuffer.push(message);
+        if (message.type === MessageType.NoOp) {
+            this.noOpCount++;
+        }
         this.emit("submitOp", message);
         if (!batch) {
             this.flush();
         }
         return message.clientSequenceNumber;
     }
-    submitSignal(content) { return this.connectionManager.submitSignal(content); }
+    submitSignal(content, targetClientId) {
+        return this.connectionManager.submitSignal(content, targetClientId);
+    }
     flush() {
-        if (this.messageBuffer.length === 0) {
+        const batch = this.messageBuffer;
+        if (batch.length === 0) {
             return;
         }
-        // The prepareFlush event allows listeners to append metadata to the batch prior to submission.
-        this.emit("prepareSend", this.messageBuffer);
-        this.connectionManager.sendMessages(this.messageBuffer);
         this.messageBuffer = [];
+        // The prepareFlush event allows listeners to append metadata to the batch prior to submission.
+        this.emit("prepareSend", batch);
+        if (batch.length === 1) {
+            assert(batch[0].metadata?.batch === undefined, 0x3c9 /* no batch markup on single message */);
+        }
+        else {
+            assert(batch[0].metadata?.batch === true, 0x3ca /* no start batch markup */);
+            assert(batch[batch.length - 1].metadata?.batch === false, 0x3cb /* no end batch markup */);
+        }
+        this.connectionManager.sendMessages(batch);
+        assert(this.messageBuffer.length === 0, 0x3cc /* reentrancy */);
     }
     get connectionProps() {
-        return Object.assign({ sequenceNumber: this.lastSequenceNumber, opsSize: this.opsSize > 0 ? this.opsSize : undefined }, this.connectionManager.connectionProps);
+        return {
+            sequenceNumber: this.lastSequenceNumber,
+            opsSize: this.opsSize > 0 ? this.opsSize : undefined,
+            deltaManagerState: this._disposed ? "disposed" : this._closed ? "closed" : "open",
+            ...this.connectionManager.connectionProps,
+        };
     }
     /**
      * Log error event with a bunch of internal to DeltaManager information about state of op processing
@@ -175,15 +176,121 @@ export class DeltaManager extends TypedEventEmitter {
      * @param event - Event to log.
      */
     logConnectionIssue(event) {
-        var _a;
         assert(this.connectionManager.connected, 0x238 /* "called only in connected state" */);
         const pendingSorted = this.pending.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
-        this.logger.sendErrorEvent(Object.assign(Object.assign(Object.assign(Object.assign({}, event), { 
+        this.logger.sendTelemetryEvent({
+            ...event,
             // This directly tells us if fetching ops is in flight, and thus likely the reason of
             // stalled op processing
-            fetchReason: this.fetchReason, 
+            fetchReason: this.fetchReason,
             // A bunch of useful sequence numbers to understand if we are holding some ops from processing
-            lastQueuedSequenceNumber: this.lastQueuedSequenceNumber, lastProcessedSequenceNumber: this.lastProcessedSequenceNumber, lastObserved: this.lastObservedSeqNumber }), this.connectionManager.connectionVerboseProps), { pendingOps: this.pending.length, pendingFirst: (_a = pendingSorted[0]) === null || _a === void 0 ? void 0 : _a.sequenceNumber, haveHandler: this.handler !== undefined, inboundLength: this.inbound.length, inboundPaused: this.inbound.paused }));
+            lastQueuedSequenceNumber: this.lastQueuedSequenceNumber,
+            lastProcessedSequenceNumber: this.lastProcessedSequenceNumber,
+            lastObserved: this.lastObservedSeqNumber,
+            // connection info
+            ...this.connectionManager.connectionVerboseProps,
+            pendingOps: this.pending.length,
+            pendingFirst: pendingSorted[0]?.sequenceNumber,
+            haveHandler: this.handler !== undefined,
+            inboundLength: this.inbound.length,
+            inboundPaused: this.inbound.paused,
+        });
+    }
+    constructor(serviceProvider, logger, _active, createConnectionManager) {
+        super();
+        this.serviceProvider = serviceProvider;
+        this.logger = logger;
+        this._active = _active;
+        this.pending = [];
+        // A boolean used to assert that ops are not being sent while processing another op.
+        this.currentlyProcessingOps = false;
+        // The minimum sequence number and last sequence number received from the server
+        this.minSequenceNumber = 0;
+        // There are three numbers we track
+        // * lastQueuedSequenceNumber is the last queued sequence number. If there are gaps in seq numbers, then this number
+        //   is not updated until we cover that gap, so it increases each time by 1.
+        // * lastObservedSeqNumber is an estimation of last known sequence number for container in storage. It's initially
+        //   populated at web socket connection time (if storage provides that info) and is updated once ops shows up.
+        //   It's never less than lastQueuedSequenceNumber
+        // * lastProcessedSequenceNumber - last processed sequence number
+        this.lastQueuedSequenceNumber = 0;
+        this.lastObservedSeqNumber = 0;
+        this.lastProcessedSequenceNumber = 0;
+        /** count number of noops sent by the client which may not be acked */
+        this.noOpCount = 0;
+        /** Track clientSequenceNumber of the last op */
+        this.lastClientSequenceNumber = 0;
+        /**
+         * Track down the ops size.
+         */
+        this.opsSize = 0;
+        // The sequence number we initially loaded from
+        // In case of reading from a snapshot or pending state, its value will be equal to
+        // the last message that got serialized.
+        this.initSequenceNumber = 0;
+        this._closed = false;
+        this._disposed = false;
+        this.throttlingIdSet = new Set();
+        this.timeTillThrottling = 0;
+        this.closeAbortController = new AbortController();
+        this.deltaStorageDelayId = uuid();
+        this.deltaStreamDelayId = uuid();
+        this.messageBuffer = [];
+        const props = {
+            incomingOpHandler: (messages, reason) => {
+                try {
+                    this.enqueueMessages(messages, reason);
+                }
+                catch (error) {
+                    this.logger.sendErrorEvent({ eventName: "EnqueueMessages_Exception" }, error);
+                    this.close(normalizeError(error));
+                }
+            },
+            signalHandler: (signals) => {
+                for (const signal of signals) {
+                    this._inboundSignal.push(signal);
+                }
+            },
+            reconnectionDelayHandler: (delayMs, error) => this.emitDelayInfo(this.deltaStreamDelayId, delayMs, error),
+            closeHandler: (error) => this.close(error),
+            disconnectHandler: (reason) => this.disconnectHandler(reason),
+            connectHandler: (connection) => this.connectHandler(connection),
+            pongHandler: (latency) => this.emit("pong", latency),
+            readonlyChangeHandler: (readonly, readonlyConnectionReason) => {
+                safeRaiseEvent(this, this.logger, "readonly", readonly, readonlyConnectionReason);
+            },
+            establishConnectionHandler: (reason) => this.establishingConnection(reason),
+            cancelConnectionHandler: (reason) => this.cancelEstablishingConnection(reason),
+        };
+        this.connectionManager = createConnectionManager(props);
+        this._inbound = new DeltaQueue((op) => {
+            this.processInboundMessage(op);
+        });
+        this._inbound.on("error", (error) => {
+            this.close(DataProcessingError.wrapIfUnrecognized(error, "deltaManagerInboundErrorHandler", this.lastMessage));
+        });
+        // Inbound signal queue
+        this._inboundSignal = new DeltaQueue((message) => {
+            if (this.handler === undefined) {
+                throw new Error("Attempted to process an inbound signal without a handler attached");
+            }
+            this.handler.processSignal({
+                clientId: message.clientId,
+                content: JSON.parse(message.content),
+            });
+        });
+        this._inboundSignal.on("error", (error) => {
+            this.close(normalizeError(error));
+        });
+        // Initially, all queues are created paused.
+        // - outbound is flipped back and forth in setupNewSuccessfulConnection / disconnectFromDeltaStream
+        // - inbound & inboundSignal are resumed in attachOpHandler() when we have handler setup
+    }
+    cancelEstablishingConnection(reason) {
+        this.emit("cancelEstablishingConnection", reason);
+    }
+    establishingConnection(reason) {
+        this.emit("establishingConnection", reason);
     }
     connectHandler(connection) {
         this.refreshDelayInfo(this.deltaStreamDelayId);
@@ -202,8 +309,10 @@ export class DeltaManager extends TypedEventEmitter {
         // state. As requirements change, so should these checks.
         assert(this.messageBuffer.length === 0, 0x0e9 /* "messageBuffer is not empty on new connection" */);
         this.opsSize = 0;
-        this.emit("connect", connection, checkpointSequenceNumber !== undefined ?
-            this.lastObservedSeqNumber - this.lastSequenceNumber : undefined);
+        this.noOpCount = 0;
+        this.emit("connect", connection, checkpointSequenceNumber !== undefined
+            ? this.lastObservedSeqNumber - this.lastSequenceNumber
+            : undefined);
         // If we got some initial ops, then we know the gap and call above fetched ops to fill it.
         // Same is true for "write" mode even if we have no ops - we will get "join" own op very very soon.
         // However if we are connecting as view-only, then there is no good signal to realize if client is behind.
@@ -219,16 +328,12 @@ export class DeltaManager extends TypedEventEmitter {
             this.fetchMissingDeltas("AfterReadConnection");
         }
     }
-    dispose() {
-        throw new Error("Not implemented.");
-    }
     /**
      * Sets the sequence number from which inbound messages should be returned
      */
-    async attachOpHandler(minSequenceNumber, sequenceNumber, term, handler, prefetchType = "none") {
+    async attachOpHandler(minSequenceNumber, sequenceNumber, handler, prefetchType = "none") {
         this.initSequenceNumber = sequenceNumber;
         this.lastProcessedSequenceNumber = sequenceNumber;
-        this.baseTerm = term;
         this.minSequenceNumber = minSequenceNumber;
         this.lastQueuedSequenceNumber = sequenceNumber;
         this.lastObservedSeqNumber = sequenceNumber;
@@ -243,7 +348,7 @@ export class DeltaManager extends TypedEventEmitter {
         // setupNewSuccessfulConnection. But it should do nothing, because there is no way to fetch ops before
         // we know snapshot sequence number that is set in attachOpHandler. So all such calls should be noop.
         assert(this.fetchReason === undefined, 0x268 /* "There can't be pending fetch that early in boot sequence!" */);
-        if (this.closed) {
+        if (this._closed) {
             return;
         }
         this._inbound.resume();
@@ -264,8 +369,7 @@ export class DeltaManager extends TypedEventEmitter {
         assert(this.fetchReason !== undefined || this.pending.length === 0, 0x269 /* "pending ops are not dropped" */);
     }
     connect(args) {
-        var _a;
-        const fetchOpsFromStorage = (_a = args.fetchOpsFromStorage) !== null && _a !== void 0 ? _a : true;
+        const fetchOpsFromStorage = args.fetchOpsFromStorage ?? true;
         logIfFalse(this.handler !== undefined || !fetchOpsFromStorage, this.logger, "CantFetchWithoutBaseline"); // can't fetch if no baseline
         // Note: There is race condition here.
         // We want to issue request to storage as soon as possible, to
@@ -278,9 +382,9 @@ export class DeltaManager extends TypedEventEmitter {
         // on the wire, we might be always behind.
         // See comment at the end of "connect" handler
         if (fetchOpsFromStorage) {
-            this.fetchMissingDeltas(args.reason);
+            this.fetchMissingDeltas(args.reason.text);
         }
-        this.connectionManager.connect(args.mode);
+        this.connectionManager.connect(args.reason, args.mode);
     }
     async getDeltas(from, // inclusive
     to, // exclusive
@@ -298,8 +402,14 @@ export class DeltaManager extends TypedEventEmitter {
             // It is possible that due to asynchrony (including await above), required ops were already
             // received through delta stream. Validate that before moving forward.
             if (this.lastQueuedSequenceNumber >= lastExpectedOp) {
-                this.logger.sendPerformanceEvent(Object.assign({ reason: fetchReason, eventName: "ExtraStorageCall", early: true, from,
-                    to }, this.connectionManager.connectionVerboseProps));
+                this.logger.sendPerformanceEvent({
+                    reason: fetchReason,
+                    eventName: "ExtraStorageCall",
+                    early: true,
+                    from,
+                    to,
+                    ...this.connectionManager.connectionVerboseProps,
+                });
                 return;
             }
             // Be prepared for the case where webSocket would receive the ops that we are trying to fill through
@@ -324,14 +434,14 @@ export class DeltaManager extends TypedEventEmitter {
             // This is useless for known ranges (to is defined) as it means request is over either way.
             // And it will cancel unbound request too early, not allowing us to learn where the end of the file is.
             if (!opsFromFetch && cancelFetch(op)) {
-                controller.abort();
+                controller.abort("DeltaManager getDeltas fetch cancelled");
                 this._inbound.off("push", opListener);
             }
         };
         try {
             this._inbound.on("push", opListener);
             assert(this.closeAbortController.signal.onabort === null, 0x1e8 /* "reentrancy" */);
-            this.closeAbortController.signal.onabort = () => controller.abort();
+            this.closeAbortController.signal.onabort = () => controller.abort(this.closeAbortController.signal.reason);
             const stream = this.deltaStorage.fetchMessages(from, // inclusive
             to, // exclusive
             controller.signal, cacheOnly, fetchReason);
@@ -351,6 +461,13 @@ export class DeltaManager extends TypedEventEmitter {
             }
         }
         finally {
+            if (controller.signal.aborted) {
+                this.logger.sendTelemetryEvent({
+                    eventName: "DeltaManager_GetDeltasAborted",
+                    fetchReason,
+                    reason: controller.signal.reason,
+                });
+            }
             this.closeAbortController.signal.onabort = null;
             this._inbound.off("push", opListener);
             assert(!opsFromFetch, 0x289 /* "logic error" */);
@@ -358,14 +475,47 @@ export class DeltaManager extends TypedEventEmitter {
     }
     /**
      * Closes the connection and clears inbound & outbound queues.
+     *
+     * Differences from dispose:
+     * - close will trigger readonly notification
+     * - close emits "closed"
+     * - close cannot be called after dispose
      */
     close(error) {
-        if (this.closed) {
+        if (this._closed) {
+            return;
+        }
+        this._closed = true;
+        this.connectionManager.dispose(error, true /* switchToReadonly */);
+        this.clearQueues();
+        this.emit("closed", error);
+    }
+    /**
+     * Disposes the connection and clears the inbound & outbound queues.
+     *
+     * Differences from close:
+     * - dispose will emit "disposed"
+     * - dispose will remove all listeners
+     * - dispose can be called after closure
+     */
+    dispose(error) {
+        if (this._disposed) {
             return;
         }
-        this.closed = true;
-        this.connectionManager.dispose(error);
-        this.closeAbortController.abort();
+        if (error !== undefined && !isFluidError(error)) {
+            throw new UsageError("Error must be a Fluid error");
+        }
+        this._disposed = true;
+        this._closed = true; // We consider "disposed" as a further state than "closed"
+        this.connectionManager.dispose(error, false /* switchToReadonly */);
+        this.clearQueues();
+        // This needs to be the last thing we do (before removing listeners), as it causes
+        // Container to dispose context and break ability of data stores / runtime to "hear" from delta manager.
+        this.emit("disposed", error);
+        this.removeAllListeners();
+    }
+    clearQueues() {
+        this.closeAbortController.abort("DeltaManager is closed");
         this._inbound.clear();
         this._inboundSignal.clear();
         // eslint-disable-next-line @typescript-eslint/no-floating-promises
@@ -374,11 +524,6 @@ export class DeltaManager extends TypedEventEmitter {
         this._inboundSignal.pause();
         // Drop pending messages - this will ensure catchUp() does not go into infinite loop
         this.pending = [];
-        // This needs to be the last thing we do (before removing listeners), as it causes
-        // Container to dispose context and break ability of data stores / runtime to "hear"
-        // from delta manager, including notification (above) about readonly state.
-        this.emit("closed", error);
-        this.removeAllListeners();
     }
     refreshDelayInfo(id) {
         this.throttlingIdSet.delete(id);
@@ -399,7 +544,7 @@ export class DeltaManager extends TypedEventEmitter {
     emitDelayInfo(id, delayMs, error) {
         const timeNow = Date.now();
         this.throttlingIdSet.add(id);
-        if (delayMs > 0 && (timeNow + delayMs > this.timeTillThrottling)) {
+        if (delayMs > 0 && timeNow + delayMs > this.timeTillThrottling) {
             this.timeTillThrottling = timeNow + delayMs;
             const throttlingWarning = ThrottlingWarning.wrap(error, delayMs / 1000 /* retryAfterSeconds */, this.logger);
             this.emit("throttled", throttlingWarning);
@@ -410,13 +555,12 @@ export class DeltaManager extends TypedEventEmitter {
     // for example, it's not clear if serverMetadata or timestamp property is a property of message or server state.
     // We only extract the most obvious fields that are sufficient (with high probability) to detect sequence number
     // reuse.
-    // Also payload goes to telemetry, so no PII, including content!!
+    // Also payload goes to telemetry, so no content or anything else that shouldn't be logged for privacy reasons
     // Note: It's possible for a duplicate op to be broadcasted and have everything the same except the timestamp.
     comparableMessagePayload(m) {
         return `${m.clientId}-${m.type}-${m.minimumSequenceNumber}-${m.referenceSequenceNumber}-${m.timestamp}`;
     }
     enqueueMessages(messages, reason, allowGaps = false) {
-        var _a, _b;
         if (this.handler === undefined) {
             // We did not setup handler yet.
             // This happens when we connect to web socket faster than we get attributes for container
@@ -463,31 +607,48 @@ export class DeltaManager extends TypedEventEmitter {
             }
             let eventName;
             // Report if we found some issues
-            if (duplicate !== 0 || gap !== 0 && !allowGaps || initialGap > 0 && this.fetchReason === undefined) {
+            if (duplicate !== 0 ||
+                (gap !== 0 && !allowGaps) ||
+                (initialGap > 0 && this.fetchReason === undefined)) {
                 eventName = "enqueueMessages";
                 // Also report if we are fetching ops, and same range comes in, thus making this fetch obsolete.
             }
-            else if (this.fetchReason !== undefined && this.fetchReason !== reason &&
-                (from <= this.lastQueuedSequenceNumber + 1 && last > this.lastQueuedSequenceNumber)) {
+            else if (this.fetchReason !== undefined &&
+                this.fetchReason !== reason &&
+                from <= this.lastQueuedSequenceNumber + 1 &&
+                last > this.lastQueuedSequenceNumber) {
                 eventName = "enqueueMessagesExtraFetch";
             }
             // Report if there is something to report
             // Do not report when pending fetch is in progress, as such reporting will not
             // correctly take into account pending ops.
             if (eventName !== undefined) {
-                this.logger.sendPerformanceEvent(Object.assign({ eventName,
-                    reason, previousReason: this.prevEnqueueMessagesReason, from, to: last + 1, length: messages.length, fetchReason: this.fetchReason, duplicate: duplicate > 0 ? duplicate : undefined, initialGap: initialGap !== 0 ? initialGap : undefined, gap: gap > 0 ? gap : undefined, firstMissing, dmInitialSeqNumber: this.initialSequenceNumber }, this.connectionManager.connectionVerboseProps));
+                this.logger.sendPerformanceEvent({
+                    eventName,
+                    reason,
+                    previousReason: this.prevEnqueueMessagesReason,
+                    from,
+                    to: last + 1,
+                    length: messages.length,
+                    fetchReason: this.fetchReason,
+                    duplicate: duplicate > 0 ? duplicate : undefined,
+                    initialGap: initialGap !== 0 ? initialGap : undefined,
+                    gap: gap > 0 ? gap : undefined,
+                    firstMissing,
+                    dmInitialSeqNumber: this.initialSequenceNumber,
+                    ...this.connectionManager.connectionVerboseProps,
+                });
             }
         }
         this.updateLatestKnownOpSeqNumber(messages[messages.length - 1].sequenceNumber);
-        const n = (_a = this.previouslyProcessedMessage) === null || _a === void 0 ? void 0 : _a.sequenceNumber;
+        const n = this.previouslyProcessedMessage?.sequenceNumber;
         assert(n === undefined || n === this.lastQueuedSequenceNumber, 0x0ec /* "Unexpected value for previously processed message's sequence number" */);
         for (const message of messages) {
             // Check that the messages are arriving in the expected order
             if (message.sequenceNumber <= this.lastQueuedSequenceNumber) {
                 // Validate that we do not have data loss, i.e. sequencing is reset and started again
                 // with numbers that this client already observed before.
-                if (((_b = this.previouslyProcessedMessage) === null || _b === void 0 ? void 0 : _b.sequenceNumber) === message.sequenceNumber) {
+                if (this.previouslyProcessedMessage?.sequenceNumber === message.sequenceNumber) {
                     const message1 = this.comparableMessagePayload(this.previouslyProcessedMessage);
                     const message2 = this.comparableMessagePayload(message);
                     if (message1 !== message2) {
@@ -499,8 +660,8 @@ export class DeltaManager extends TypedEventEmitter {
                         // hit. One example is that some clients could be submitting ops to two different service
                         // instances such that the same sequence number is reused for two different ops.
                         // pre-0.58 error message: twoMessagesWithSameSeqNumAndDifferentPayload
-                        "Found two messages with the same sequenceNumber but different payloads. Likely to be a "
-                            + "service issue", DriverErrorType.fileOverwrittenInStorage, {
+                        "Found two messages with the same sequenceNumber but different payloads. Likely to be a " +
+                            "service issue", DriverErrorTypes.fileOverwrittenInStorage, {
                             clientId: this.connectionManager.clientId,
                             sequenceNumber: message.sequenceNumber,
                             message1,
@@ -528,40 +689,73 @@ export class DeltaManager extends TypedEventEmitter {
     }
     processInboundMessage(message) {
         const startTime = Date.now();
+        assert(!this.currentlyProcessingOps, 0x3af /* Already processing ops. */);
+        this.currentlyProcessingOps = true;
         this.lastProcessedMessage = message;
-        // All non-system messages are coming from some client, and should have clientId
-        // System messages may have no clientId (but some do, like propose, noop, summarize)
-        assert(message.clientId !== undefined
-            || !(isClientMessage(message)), 0x0ed /* "non-system message have to have clientId" */);
+        const isString = typeof message.clientId === "string";
+        assert(message.clientId === null || isString, 0x41a /* undefined or string */);
+        // All client messages are coming from some client, and should have clientId,
+        // and non-client message should not have clientId. But, there are two exceptions:
+        // 1. (Legacy) We can see message.type === "attach" or "chunkedOp" for legacy files before RTM
+        // 2. Non-immediate noops (contents: null) can be sent by service without clientId
+        if (!isString && isClientMessage(message) && message.type !== MessageType.NoOp) {
+            throw new DataCorruptionError("Mismatch in clientId", {
+                ...extractSafePropertiesFromMessage(message),
+                messageType: message.type,
+            });
+        }
         // TODO Remove after SPO picks up the latest build.
-        if (typeof message.contents === "string"
-            && message.contents !== ""
-            && message.type !== MessageType.ClientLeave) {
+        if (typeof message.contents === "string" &&
+            message.contents !== "" &&
+            message.type !== MessageType.ClientLeave) {
             message.contents = JSON.parse(message.contents);
         }
+        // Validate client sequence number has no gap. Decrement the noOpCount by gap
+        // If the count ends up negative, that means we have a real gap and throw error
+        if (this.connectionManager.clientId !== undefined &&
+            this.connectionManager.clientId === message.clientId) {
+            if (message.type === MessageType.NoOp) {
+                this.noOpCount--;
+            }
+            const clientSeqNumGap = message.clientSequenceNumber - this.lastClientSequenceNumber - 1;
+            this.noOpCount -= clientSeqNumGap;
+            if (this.noOpCount < 0) {
+                throw new Error(`gap in client sequence number: ${clientSeqNumGap}`);
+            }
+            this.lastClientSequenceNumber = message.clientSequenceNumber;
+        }
         this.connectionManager.beforeProcessingIncomingOp(message);
         // Watch the minimum sequence number and be ready to update as needed
         if (this.minSequenceNumber > message.minimumSequenceNumber) {
             // pre-0.58 error message: msnMovesBackwards
-            throw new DataCorruptionError("Found a lower minimumSequenceNumber (msn) than previously recorded", Object.assign(Object.assign({}, extractSafePropertiesFromMessage(message)), { clientId: this.connectionManager.clientId }));
+            throw new DataCorruptionError("Found a lower minimumSequenceNumber (msn) than previously recorded", {
+                ...extractSafePropertiesFromMessage(message),
+                clientId: this.connectionManager.clientId,
+            });
+        }
+        // Client ops: MSN has to be lower than sequence #, as client can continue to send ops with same
+        // reference sequence number as this op.
+        // System ops (when no clients are connected) are the only ops where equation is possible.
+        const diff = message.sequenceNumber - message.minimumSequenceNumber;
+        if (diff < 0 || (diff === 0 && message.clientId !== null)) {
+            throw new DataCorruptionError("MSN has to be lower than sequence #", extractSafePropertiesFromMessage(message));
         }
         this.minSequenceNumber = message.minimumSequenceNumber;
         if (message.sequenceNumber !== this.lastProcessedSequenceNumber + 1) {
             // pre-0.58 error message: nonSequentialSequenceNumber
-            throw new DataCorruptionError("Found a non-Sequential sequenceNumber", Object.assign(Object.assign({}, extractSafePropertiesFromMessage(message)), { clientId: this.connectionManager.clientId }));
+            throw new DataCorruptionError("Found a non-Sequential sequenceNumber", {
+                ...extractSafePropertiesFromMessage(message),
+                clientId: this.connectionManager.clientId,
+            });
         }
         this.lastProcessedSequenceNumber = message.sequenceNumber;
         // a bunch of code assumes that this is true
         assert(this.lastProcessedSequenceNumber <= this.lastObservedSeqNumber, 0x267 /* "lastObservedSeqNumber should be updated first" */);
-        // Back-compat for older server with no term
-        if (message.term === undefined) {
-            message.term = 1;
-        }
-        this.baseTerm = message.term;
         if (this.handler === undefined) {
             throw new Error("Attempted to process an inbound message without a handler attached");
         }
         this.handler.process(message);
+        this.currentlyProcessingOps = false;
         const endTime = Date.now();
         // Should be last, after changing this.lastProcessedSequenceNumber above, as many callers
         // test this.lastProcessedSequenceNumber instead of using op.sequenceNumber itself.
@@ -576,16 +770,18 @@ export class DeltaManager extends TypedEventEmitter {
         });
     }
     /**
-    * Retrieves the missing deltas between the given sequence numbers
-    */
+     * Retrieves the missing deltas between the given sequence numbers
+     */
     async fetchMissingDeltasCore(reason, cacheOnly, to) {
-        var _a;
         // Exit out early if we're already fetching deltas
         if (this.fetchReason !== undefined) {
             return;
         }
-        if (this.closed) {
-            this.logger.sendTelemetryEvent({ eventName: "fetchMissingDeltasClosedConnection", reason });
+        if (this._closed) {
+            this.logger.sendTelemetryEvent({
+                eventName: "fetchMissingDeltasClosedConnection",
+                reason,
+            });
             return;
         }
         if (this.handler === undefined) {
@@ -595,7 +791,7 @@ export class DeltaManager extends TypedEventEmitter {
         }
         try {
             let from = this.lastQueuedSequenceNumber + 1;
-            const n = (_a = this.previouslyProcessedMessage) === null || _a === void 0 ? void 0 : _a.sequenceNumber;
+            const n = this.previouslyProcessedMessage?.sequenceNumber;
             if (n !== undefined) {
                 // If we already processed at least one op, then we have this.previouslyProcessedMessage populated
                 // and can use it to validate that we are operating on same file, i.e. it was not overwritten.
@@ -627,7 +823,7 @@ export class DeltaManager extends TypedEventEmitter {
      * Sorts pending ops and attempts to apply them
      */
     processPendingOps(reason) {
-        if (this.closed) {
+        if (this._closed) {
             return;
         }
         assert(this.handler !== undefined, 0x26c /* "handler should be installed" */);
@@ -660,4 +856,4 @@ export class DeltaManager extends TypedEventEmitter {
         }
     }
 }
-//# sourceMappingURL=deltaManager.js.map
+//# sourceMappingURL=deltaManager.mjs.map