@fluidframework/container-runtime 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

package/lib/packageVersion.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"packageVersion.mjs","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,2BAA2B,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.0.0-dev-rc.1.0.0.224419\";\n"]}

package/lib/{pendingStateManager.d.ts → pendingStateManager.d.mts}

@@ -2,109 +2,85 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { IDisposable } from "@fluidframework/common-definitions";
+import { IDisposable } from "@fluidframework/core-interfaces";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { FlushMode } from "@fluidframework/runtime-definitions";
-import { ContainerMessageType } from "./containerRuntime";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+import { InboundSequencedContainerRuntimeMessage } from "./messageTypes.mjs";
 /**
  * This represents a message that has been submitted and is added to the pending queue when `submit` is called on the
  * ContainerRuntime. This message has either not been ack'd by the server or has not been submitted to the server yet.
  */
 export interface IPendingMessage {
     type: "message";
-    messageType: ContainerMessageType;
-    clientSequenceNumber: number;
     referenceSequenceNumber: number;
-    content: any;
+    content: string;
     localOpMetadata: unknown;
     opMetadata: Record<string, unknown> | undefined;
 }
-/**
- * This represents a FlushMode update and is added to the pending queue when `setFlushMode` is called on the
- * ContainerRuntime and the FlushMode changes.
- */
-export interface IPendingFlushMode {
-    type: "flushMode";
-    flushMode: FlushMode;
-}
-/**
- * This represents an explicit flush call and is added to the pending queue when flush is called on the ContainerRuntime
- * to flush pending messages.
- */
-export interface IPendingFlush {
-    type: "flush";
-}
-export declare type IPendingState = IPendingMessage | IPendingFlushMode | IPendingFlush;
 export interface IPendingLocalState {
     /**
      * list of pending states, including ops and batch information
      */
-    pendingStates: IPendingState[];
+    pendingStates: IPendingMessage[];
+}
+export interface IPendingBatchMessage {
+    content: string;
+    localOpMetadata: unknown;
+    opMetadata: Record<string, unknown> | undefined;
 }
 export interface IRuntimeStateHandler {
     connected(): boolean;
     clientId(): string | undefined;
-    flushMode(): FlushMode;
-    setFlushMode(mode: FlushMode): void;
     close(error?: ICriticalContainerError): void;
-    applyStashedOp: (type: ContainerMessageType, content: ISequencedDocumentMessage) => Promise<unknown>;
-    flush(): void;
-    reSubmit(type: ContainerMessageType, content: any, localOpMetadata: unknown, opMetadata: Record<string, unknown> | undefined): void;
-    rollback(type: ContainerMessageType, content: any, localOpMetadata: unknown): void;
+    applyStashedOp(content: string): Promise<unknown>;
+    reSubmit(message: IPendingBatchMessage): void;
+    reSubmitBatch(batch: IPendingBatchMessage[]): void;
+    isActiveConnection: () => boolean;
 }
 /**
  * PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
  * acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
  * batches along with the messages.
- * When the Container reconnects, it replays the pending states, which includes setting the FlushMode, manual flushing
+ * When the Container reconnects, it replays the pending states, which includes manual flushing
  * of messages and triggering resubmission of unacked ops.
  *
  * It verifies that all the ops are acked, are received in the right order and batch information is correct.
  */
 export declare class PendingStateManager implements IDisposable {
     private readonly stateHandler;
-    private readonly pendingStates;
-    private readonly initialStates;
+    private readonly logger;
+    private readonly pendingMessages;
+    private readonly initialMessages;
+    /**
+     * Sequenced local ops that are saved when stashing since pending ops may depend on them
+     */
+    private savedOps;
     private readonly disposeOnce;
-    private _pendingMessagesCount;
-    get pendingMessagesCount(): number;
     private isProcessingBatch;
     private pendingBatchBeginMessage;
+    private clientId;
     /**
-     * This tracks the flush mode for the next message in the pending state queue. When replaying messages, we need to
-     * first set the flush mode to this value and then send ops. It is important to do this info because the flush
-     * mode could have been updated.
+     * The pending messages count. Includes `pendingMessages` and `initialMessages` to keep in sync with
+     * 'hasPendingMessages'.
      */
-    private flushModeForNextMessage;
-    private clientId;
+    get pendingMessagesCount(): number;
     /**
-     * Called to check if there are any pending messages in the pending state queue.
+     * Called to check if there are any pending messages in the pending message queue.
      * @returns A boolean indicating whether there are messages or not.
      */
     hasPendingMessages(): boolean;
     getLocalState(): IPendingLocalState | undefined;
-    constructor(stateHandler: IRuntimeStateHandler, initialFlushMode: FlushMode, initialLocalState: IPendingLocalState | undefined);
+    constructor(stateHandler: IRuntimeStateHandler, initialLocalState: IPendingLocalState | undefined, logger: ITelemetryLoggerExt | undefined);
     get disposed(): boolean;
     readonly dispose: () => void;
     /**
      * Called when a message is submitted locally. Adds the message and the associated details to the pending state
      * queue.
      * @param type - The container message type.
-     * @param clientSequenceNumber - The clientSequenceNumber associated with the message.
      * @param content - The message content.
      * @param localOpMetadata - The local metadata associated with the message.
      */
-    onSubmitMessage(type: ContainerMessageType, clientSequenceNumber: number, referenceSequenceNumber: number, content: any, localOpMetadata: unknown, opMetadata: Record<string, unknown> | undefined): void;
-    /**
-     * Called when the FlushMode is updated. Adds the FlushMode to the pending state queue.
-     * @param flushMode - The flushMode that was updated.
-     */
-    onFlushModeUpdated(flushMode: FlushMode): void;
-    /**
-     * Called when flush() is called on the ContainerRuntime to manually flush messages.
-     */
-    onFlush(): void;
+    onSubmitMessage(content: string, referenceSequenceNumber: number, localOpMetadata: unknown, opMetadata: Record<string, unknown> | undefined): void;
     /**
      * Applies stashed ops at their reference sequence number so they are ready to be ACKed or resubmitted
      * @param seqNum - Sequence number at which to apply ops. Will apply all ops if seqNum is undefined.
@@ -115,7 +91,7 @@ export declare class PendingStateManager implements IDisposable {
      * the batch information was preserved for batch messages.
      * @param message - The message that got ack'd and needs to be processed.
      */
-    processPendingLocalMessage(message: ISequencedDocumentMessage): unknown;
+    processPendingLocalMessage(message: InboundSequencedContainerRuntimeMessage): unknown;
     /**
      * This message could be the first message in batch. If so, set batch state marking the beginning of a batch.
      * @param message - The message that is being processed.
@@ -126,23 +102,10 @@ export declare class PendingStateManager implements IDisposable {
      * @param message - The message that is being processed.
      */
     private maybeProcessBatchEnd;
-    /**
-     * Capture the pending state at this point
-     */
-    checkpoint(): {
-        rollback: () => void;
-    };
-    /**
-     * Returns the next pending state from the pending state queue.
-     */
-    private peekNextPendingState;
-    /**
-     * Undo the last pending state
-     */
-    private rollbackNextPendingState;
     /**
      * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
-     * states in its queue. This includes setting the FlushMode and triggering resubmission of unacked ops.
+     * states in its queue. This includes triggering resubmission of unacked ops.
+     * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
      */
     replayPendingStates(): void;
 }

package/lib/pendingStateManager.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pendingStateManager.d.ts","sourceRoot":"","sources":["../src/pendingStateManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAII,EAAE,WAAW,EAAE,MAAM,iCAAiC;OAEtD,EAAE,uBAAuB,EAAE,MAAM,uCAAuC;OAExE,EAAuB,mBAAmB,EAAE,MAAM,iCAAiC;OAEnF,EAAE,uCAAuC,EAAE;AAIlD;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,SAAS,CAAC;IAChB,uBAAuB,EAAE,MAAM,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAChD;AAED,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,aAAa,EAAE,eAAe,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAChD;AAED,MAAM,WAAW,oBAAoB;IACpC,SAAS,IAAI,OAAO,CAAC;IACrB,QAAQ,IAAI,MAAM,GAAG,SAAS,CAAC;IAC/B,KAAK,CAAC,KAAK,CAAC,EAAE,uBAAuB,GAAG,IAAI,CAAC;IAC7C,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClD,QAAQ,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAC9C,aAAa,CAAC,KAAK,EAAE,oBAAoB,EAAE,GAAG,IAAI,CAAC;IACnD,kBAAkB,EAAE,MAAM,OAAO,CAAC;CAClC;AAsBD;;;;;;;;GAQG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IA0DrD,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,MAAM;IA3DxB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAChE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAEhE;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAyB;IAEzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAGzB;IAGH,OAAO,CAAC,iBAAiB,CAAkB;IAI3C,OAAO,CAAC,wBAAwB,CAAwC;IAExE,OAAO,CAAC,QAAQ,CAAqB;IAErC;;;OAGG;IACH,IAAW,oBAAoB,IAAI,MAAM,CAExC;IAED;;;OAGG;IACI,kBAAkB,IAAI,OAAO;IAI7B,aAAa,IAAI,kBAAkB,GAAG,SAAS;gBAmBpC,YAAY,EAAE,oBAAoB,EACnD,iBAAiB,EAAE,kBAAkB,GAAG,SAAS,EAChC,MAAM,EAAE,mBAAmB,GAAG,SAAS;IAOzD,IAAW,QAAQ,YAElB;IACD,SAAgB,OAAO,aAAgC;IAEvD;;;;;;OAMG;IACI,eAAe,CACrB,OAAO,EAAE,MAAM,EACf,uBAAuB,EAAE,MAAM,EAC/B,eAAe,EAAE,OAAO,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAahD;;;OAGG;IACU,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM;IA4B9C;;;;OAIG;IACI,0BAA0B,CAAC,OAAO,EAAE,uCAAuC,GAAG,OAAO;IAqC5F;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAe9B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAwD5B;;;;OAIG;IACI,mBAAmB;CA4F1B"}

package/lib/pendingStateManager.mjs

@@ -0,0 +1,275 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import Deque from "double-ended-queue";
+import { assert, Lazy } from "@fluidframework/core-utils";
+import { DataProcessingError } from "@fluidframework/telemetry-utils";
+import { pkgVersion } from "./packageVersion.mjs";
+function buildPendingMessageContent(
+// AnyComboFromUnion is needed need to gain access to compatDetails that
+// is only defined for some cases.
+message) {
+    // IMPORTANT: Order matters here, this must match the order of the properties used
+    // when submitting the message.
+    const { type, contents, compatDetails } = message;
+    // Any properties that are not defined, won't be emitted by stringify.
+    return JSON.stringify({ type, contents, compatDetails });
+}
+/**
+ * PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
+ * acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
+ * batches along with the messages.
+ * When the Container reconnects, it replays the pending states, which includes manual flushing
+ * of messages and triggering resubmission of unacked ops.
+ *
+ * It verifies that all the ops are acked, are received in the right order and batch information is correct.
+ */
+export class PendingStateManager {
+    /**
+     * The pending messages count. Includes `pendingMessages` and `initialMessages` to keep in sync with
+     * 'hasPendingMessages'.
+     */
+    get pendingMessagesCount() {
+        return this.pendingMessages.length + this.initialMessages.length;
+    }
+    /**
+     * Called to check if there are any pending messages in the pending message queue.
+     * @returns A boolean indicating whether there are messages or not.
+     */
+    hasPendingMessages() {
+        return this.pendingMessagesCount !== 0;
+    }
+    getLocalState() {
+        assert(this.initialMessages.isEmpty(), 0x2e9 /* "Must call getLocalState() after applying initial states" */);
+        if (!this.pendingMessages.isEmpty()) {
+            return {
+                pendingStates: [...this.savedOps, ...this.pendingMessages.toArray()].map((message) => {
+                    // delete localOpMetadata since it may not be serializable
+                    // and will be regenerated by applyStashedOp()
+                    return { ...message, localOpMetadata: undefined };
+                }),
+            };
+        }
+    }
+    constructor(stateHandler, initialLocalState, logger) {
+        this.stateHandler = stateHandler;
+        this.logger = logger;
+        this.pendingMessages = new Deque();
+        this.initialMessages = new Deque();
+        /**
+         * Sequenced local ops that are saved when stashing since pending ops may depend on them
+         */
+        this.savedOps = [];
+        this.disposeOnce = new Lazy(() => {
+            this.initialMessages.clear();
+            this.pendingMessages.clear();
+        });
+        // Indicates whether we are processing a batch.
+        this.isProcessingBatch = false;
+        this.dispose = () => this.disposeOnce.value;
+        if (initialLocalState?.pendingStates) {
+            this.initialMessages.push(...initialLocalState.pendingStates);
+        }
+    }
+    get disposed() {
+        return this.disposeOnce.evaluated;
+    }
+    /**
+     * Called when a message is submitted locally. Adds the message and the associated details to the pending state
+     * queue.
+     * @param type - The container message type.
+     * @param content - The message content.
+     * @param localOpMetadata - The local metadata associated with the message.
+     */
+    onSubmitMessage(content, referenceSequenceNumber, localOpMetadata, opMetadata) {
+        const pendingMessage = {
+            type: "message",
+            referenceSequenceNumber,
+            content,
+            localOpMetadata,
+            opMetadata,
+        };
+        this.pendingMessages.push(pendingMessage);
+    }
+    /**
+     * Applies stashed ops at their reference sequence number so they are ready to be ACKed or resubmitted
+     * @param seqNum - Sequence number at which to apply ops. Will apply all ops if seqNum is undefined.
+     */
+    async applyStashedOpsAt(seqNum) {
+        // apply stashed ops at sequence number
+        while (!this.initialMessages.isEmpty()) {
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            const nextMessage = this.initialMessages.peekFront();
+            if (seqNum !== undefined) {
+                if (nextMessage.referenceSequenceNumber > seqNum) {
+                    break; // nothing left to do at this sequence number
+                }
+                if (nextMessage.referenceSequenceNumber < seqNum) {
+                    throw new Error("loaded from snapshot too recent to apply stashed ops");
+                }
+            }
+            try {
+                // applyStashedOp will cause the DDS to behave as if it has sent the op but not actually send it
+                const localOpMetadata = await this.stateHandler.applyStashedOp(nextMessage.content);
+                nextMessage.localOpMetadata = localOpMetadata;
+            }
+            catch (error) {
+                throw DataProcessingError.wrapIfUnrecognized(error, "applyStashedOp", nextMessage);
+            }
+            // then we push onto pendingMessages which will cause PendingStateManager to resubmit when we connect
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            this.pendingMessages.push(this.initialMessages.shift());
+        }
+    }
+    /**
+     * Processes a local message once its ack'd by the server. It verifies that there was no data corruption and that
+     * the batch information was preserved for batch messages.
+     * @param message - The message that got ack'd and needs to be processed.
+     */
+    processPendingLocalMessage(message) {
+        // Pre-processing part - This may be the start of a batch.
+        this.maybeProcessBatchBegin(message);
+        // Get the next message from the pending queue. Verify a message exists.
+        const pendingMessage = this.pendingMessages.peekFront();
+        assert(pendingMessage !== undefined, 0x169 /* "No pending message found for this remote message" */);
+        this.savedOps.push(pendingMessage);
+        this.pendingMessages.shift();
+        const messageContent = buildPendingMessageContent(message);
+        // Stringified content should match
+        if (pendingMessage.content !== messageContent) {
+            this.stateHandler.close(DataProcessingError.create("pending local message content mismatch", "unexpectedAckReceived", message, {
+                expectedMessageType: JSON.parse(pendingMessage.content).type,
+            }));
+            return;
+        }
+        // Post-processing part - If we are processing a batch then this could be the last message in the batch.
+        this.maybeProcessBatchEnd(message);
+        return pendingMessage.localOpMetadata;
+    }
+    /**
+     * This message could be the first message in batch. If so, set batch state marking the beginning of a batch.
+     * @param message - The message that is being processed.
+     */
+    maybeProcessBatchBegin(message) {
+        // This message is the first in a batch if the "batch" property on the metadata is set to true
+        if (message.metadata?.batch) {
+            // We should not already be processing a batch and there should be no pending batch begin message.
+            assert(!this.isProcessingBatch && this.pendingBatchBeginMessage === undefined, 0x16b /* "The pending batch state indicates we are already processing a batch" */);
+            // Set the pending batch state indicating we have started processing a batch.
+            this.pendingBatchBeginMessage = message;
+            this.isProcessingBatch = true;
+        }
+    }
+    /**
+     * This message could be the last message in batch. If so, clear batch state since the batch is complete.
+     * @param message - The message that is being processed.
+     */
+    maybeProcessBatchEnd(message) {
+        if (!this.isProcessingBatch) {
+            return;
+        }
+        // There should be a pending batch begin message.
+        assert(this.pendingBatchBeginMessage !== undefined, 0x16d /* "There is no pending batch begin message" */);
+        const batchEndMetadata = message.metadata?.batch;
+        if (this.pendingMessages.isEmpty() || batchEndMetadata === false) {
+            // Get the batch begin metadata from the first message in the batch.
+            const batchBeginMetadata = this.pendingBatchBeginMessage.metadata?.batch;
+            // There could be just a single message in the batch. If so, it should not have any batch metadata. If there
+            // are multiple messages in the batch, verify that we got the correct batch begin and end metadata.
+            if (this.pendingBatchBeginMessage === message) {
+                assert(batchBeginMetadata === undefined, 0x16e /* "Batch with single message should not have batch metadata" */);
+            }
+            else {
+                if (batchBeginMetadata !== true || batchEndMetadata !== false) {
+                    this.stateHandler.close(DataProcessingError.create("Pending batch inconsistency", // Formerly known as asserts 0x16f and 0x170
+                    "processPendingLocalMessage", message, {
+                        runtimeVersion: pkgVersion,
+                        batchClientId: 
+                        // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+                        this.pendingBatchBeginMessage.clientId === null
+                            ? "null"
+                            : this.pendingBatchBeginMessage.clientId,
+                        clientId: this.stateHandler.clientId(),
+                        hasBatchStart: batchBeginMetadata === true,
+                        hasBatchEnd: batchEndMetadata === false,
+                        messageType: message.type,
+                        pendingMessagesCount: this.pendingMessagesCount,
+                    }));
+                }
+            }
+            // Clear the pending batch state now that we have processed the entire batch.
+            this.pendingBatchBeginMessage = undefined;
+            this.isProcessingBatch = false;
+        }
+    }
+    /**
+     * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
+     * states in its queue. This includes triggering resubmission of unacked ops.
+     * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
+     */
+    replayPendingStates() {
+        assert(this.stateHandler.connected(), 0x172 /* "The connection state is not consistent with the runtime" */);
+        // This assert suggests we are about to send same ops twice, which will result in data loss.
+        assert(this.clientId !== this.stateHandler.clientId(), 0x173 /* "replayPendingStates called twice for same clientId!" */);
+        this.clientId = this.stateHandler.clientId();
+        assert(this.initialMessages.isEmpty(), 0x174 /* "initial states should be empty before replaying pending" */);
+        const initialPendingMessagesCount = this.pendingMessages.length;
+        let remainingPendingMessagesCount = this.pendingMessages.length;
+        // Process exactly `pendingMessagesCount` items in the queue as it represents the number of messages that were
+        // pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
+        // which must not be replayed.
+        while (remainingPendingMessagesCount > 0) {
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            let pendingMessage = this.pendingMessages.shift();
+            remainingPendingMessagesCount--;
+            assert(pendingMessage.opMetadata?.batch !== false, 0x41b /* We cannot process batches in chunks */);
+            /**
+             * We want to ensure grouped messages get processed in a batch.
+             * Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will
+             * either receive the whole batch ack or nothing at all.
+             */
+            if (pendingMessage.opMetadata?.batch) {
+                assert(remainingPendingMessagesCount > 0, 0x554 /* Last pending message cannot be a batch begin */);
+                const batch = [];
+                // check is >= because batch end may be last pending message
+                while (remainingPendingMessagesCount >= 0) {
+                    batch.push({
+                        content: pendingMessage.content,
+                        localOpMetadata: pendingMessage.localOpMetadata,
+                        opMetadata: pendingMessage.opMetadata,
+                    });
+                    if (pendingMessage.opMetadata?.batch === false) {
+                        break;
+                    }
+                    assert(remainingPendingMessagesCount > 0, 0x555 /* No batch end found */);
+                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                    pendingMessage = this.pendingMessages.shift();
+                    remainingPendingMessagesCount--;
+                    assert(pendingMessage.opMetadata?.batch !== true, 0x556 /* Batch start needs a corresponding batch end */);
+                }
+                this.stateHandler.reSubmitBatch(batch);
+            }
+            else {
+                this.stateHandler.reSubmit({
+                    content: pendingMessage.content,
+                    localOpMetadata: pendingMessage.localOpMetadata,
+                    opMetadata: pendingMessage.opMetadata,
+                });
+            }
+        }
+        // pending ops should no longer depend on previous sequenced local ops after resubmit
+        this.savedOps = [];
+        // We replayPendingStates on read connections too - we expect these to get nack'd though, and to then reconnect
+        // on a write connection and replay again. This filters out the replay that happens on the read connection so
+        // we only see the replays on write connections (that have a chance to go through).
+        if (this.stateHandler.isActiveConnection()) {
+            this.logger?.sendTelemetryEvent({
+                eventName: "PendingStatesReplayed",
+                count: initialPendingMessagesCount,
+                clientId: this.stateHandler.clientId(),
+            });
+        }
+    }
+}
+//# sourceMappingURL=pendingStateManager.mjs.map

package/lib/pendingStateManager.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"pendingStateManager.mjs","sourceRoot":"","sources":["../src/pendingStateManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAEI,KAAK,MAAM,oBAAoB;OAG/B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,4BAA4B;OAGlD,EAAE,mBAAmB,EAAuB,MAAM,iCAAiC;OAGnF,EAAE,UAAU,EAAE;AA8CrB,SAAS,0BAA0B;AAClC,wEAAwE;AACxE,kCAAkC;AAClC,OAAmE;IAEnE,kFAAkF;IAClF,+BAA+B;IAC/B,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAClD,sEAAsE;IACtE,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;AAC1D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,OAAO,mBAAmB;IAuB/B;;;OAGG;IACH,IAAW,oBAAoB;QAC9B,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IAClE,CAAC;IAED;;;OAGG;IACI,kBAAkB;QACxB,OAAO,IAAI,CAAC,oBAAoB,KAAK,CAAC,CAAC;IACxC,CAAC;IAEM,aAAa;QACnB,MAAM,CACL,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAC9B,KAAK,CAAC,+DAA+D,CACrE,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE;YACpC,OAAO;gBACN,aAAa,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CACvE,CAAC,OAAO,EAAE,EAAE;oBACX,0DAA0D;oBAC1D,8CAA8C;oBAC9C,OAAO,EAAE,GAAG,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC;gBACnD,CAAC,CACD;aACD,CAAC;SACF;IACF,CAAC;IAED,YACkB,YAAkC,EACnD,iBAAiD,EAChC,MAAuC;QAFvC,iBAAY,GAAZ,YAAY,CAAsB;QAElC,WAAM,GAAN,MAAM,CAAiC;QA3DxC,oBAAe,GAAG,IAAI,KAAK,EAAmB,CAAC;QAC/C,oBAAe,GAAG,IAAI,KAAK,EAAmB,CAAC;QAEhE;;WAEG;QACK,aAAQ,GAAsB,EAAE,CAAC;QAExB,gBAAW,GAAG,IAAI,IAAI,CAAO,GAAG,EAAE;YAClD,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,+CAA+C;QACvC,sBAAiB,GAAY,KAAK,CAAC;QAuD3B,YAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;QARtD,IAAI,iBAAiB,EAAE,aAAa,EAAE;YACrC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,aAAa,CAAC,CAAC;SAC9D;IACF,CAAC;IAED,IAAW,QAAQ;QAClB,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;IACnC,CAAC;IAGD;;;;;;OAMG;IACI,eAAe,CACrB,OAAe,EACf,uBAA+B,EAC/B,eAAwB,EACxB,UAA+C;QAE/C,MAAM,cAAc,GAAoB;YACvC,IAAI,EAAE,SAAS;YACf,uBAAuB;YACvB,OAAO;YACP,eAAe;YACf,UAAU;SACV,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,iBAAiB,CAAC,MAAe;QAC7C,uCAAuC;QACvC,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE;YACvC,oEAAoE;YACpE,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAG,CAAC;YACtD,IAAI,MAAM,KAAK,SAAS,EAAE;gBACzB,IAAI,WAAW,CAAC,uBAAuB,GAAG,MAAM,EAAE;oBACjD,MAAM,CAAC,6CAA6C;iBACpD;gBACD,IAAI,WAAW,CAAC,uBAAuB,GAAG,MAAM,EAAE;oBACjD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;iBACxE;aACD;YAED,IAAI;gBACH,gGAAgG;gBAChG,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBACpF,WAAW,CAAC,eAAe,GAAG,eAAe,CAAC;aAC9C;YAAC,OAAO,KAAK,EAAE;gBACf,MAAM,mBAAmB,CAAC,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC;aACnF;YAED,qGAAqG;YACrG,oEAAoE;YACpE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAG,CAAC,CAAC;SACzD;IACF,CAAC;IAED;;;;OAIG;IACI,0BAA0B,CAAC,OAAgD;QACjF,0DAA0D;QAC1D,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAErC,wEAAwE;QACxE,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,CAAC;QACxD,MAAM,CACL,cAAc,KAAK,SAAS,EAC5B,KAAK,CAAC,wDAAwD,CAC9D,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAEnC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,MAAM,cAAc,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;QAE3D,mCAAmC;QACnC,IAAI,cAAc,CAAC,OAAO,KAAK,cAAc,EAAE;YAC9C,IAAI,CAAC,YAAY,CAAC,KAAK,CACtB,mBAAmB,CAAC,MAAM,CACzB,wCAAwC,EACxC,uBAAuB,EACvB,OAAO,EACP;gBACC,mBAAmB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,IAAI;aAC5D,CACD,CACD,CAAC;YACF,OAAO;SACP;QAED,wGAAwG;QACxG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO,cAAc,CAAC,eAAe,CAAC;IACvC,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,OAAkC;QAChE,8FAA8F;QAC9F,IAAK,OAAO,CAAC,QAAuC,EAAE,KAAK,EAAE;YAC5D,kGAAkG;YAClG,MAAM,CACL,CAAC,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,wBAAwB,KAAK,SAAS,EACtE,KAAK,CAAC,2EAA2E,CACjF,CAAC;YAEF,6EAA6E;YAC7E,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC;YACxC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;SAC9B;IACF,CAAC;IAED;;;OAGG;IACK,oBAAoB,CAAC,OAAkC;QAC9D,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC5B,OAAO;SACP;QAED,iDAAiD;QACjD,MAAM,CACL,IAAI,CAAC,wBAAwB,KAAK,SAAS,EAC3C,KAAK,CAAC,+CAA+C,CACrD,CAAC;QAEF,MAAM,gBAAgB,GAAI,OAAO,CAAC,QAAuC,EAAE,KAAK,CAAC;QACjF,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,IAAI,gBAAgB,KAAK,KAAK,EAAE;YACjE,oEAAoE;YACpE,MAAM,kBAAkB,GACvB,IAAI,CAAC,wBAAwB,CAAC,QAC9B,EAAE,KAAK,CAAC;YAET,4GAA4G;YAC5G,mGAAmG;YACnG,IAAI,IAAI,CAAC,wBAAwB,KAAK,OAAO,EAAE;gBAC9C,MAAM,CACL,kBAAkB,KAAK,SAAS,EAChC,KAAK,CAAC,gEAAgE,CACtE,CAAC;aACF;iBAAM;gBACN,IAAI,kBAAkB,KAAK,IAAI,IAAI,gBAAgB,KAAK,KAAK,EAAE;oBAC9D,IAAI,CAAC,YAAY,CAAC,KAAK,CACtB,mBAAmB,CAAC,MAAM,CACzB,6BAA6B,EAAE,4CAA4C;oBAC3E,4BAA4B,EAC5B,OAAO,EACP;wBACC,cAAc,EAAE,UAAU;wBAC1B,aAAa;wBACZ,wEAAwE;wBACxE,IAAI,CAAC,wBAAwB,CAAC,QAAQ,KAAK,IAAI;4BAC9C,CAAC,CAAC,MAAM;4BACR,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,QAAQ;wBAC1C,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;wBACtC,aAAa,EAAE,kBAAkB,KAAK,IAAI;wBAC1C,WAAW,EAAE,gBAAgB,KAAK,KAAK;wBACvC,WAAW,EAAE,OAAO,CAAC,IAAI;wBACzB,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;qBAC/C,CACD,CACD,CAAC;iBACF;aACD;YAED,6EAA6E;YAC7E,IAAI,CAAC,wBAAwB,GAAG,SAAS,CAAC;YAC1C,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;SAC/B;IACF,CAAC;IAED;;;;OAIG;IACI,mBAAmB;QACzB,MAAM,CACL,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,EAC7B,KAAK,CAAC,+DAA+D,CACrE,CAAC;QAEF,4FAA4F;QAC5F,MAAM,CACL,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,EAC9C,KAAK,CAAC,2DAA2D,CACjE,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC;QAE7C,MAAM,CACL,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAC9B,KAAK,CAAC,+DAA+D,CACrE,CAAC;QAEF,MAAM,2BAA2B,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;QAChE,IAAI,6BAA6B,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;QAEhE,8GAA8G;QAC9G,0GAA0G;QAC1G,8BAA8B;QAC9B,OAAO,6BAA6B,GAAG,CAAC,EAAE;YACzC,oEAAoE;YACpE,IAAI,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAG,CAAC;YACnD,6BAA6B,EAAE,CAAC;YAChC,MAAM,CACL,cAAc,CAAC,UAAU,EAAE,KAAK,KAAK,KAAK,EAC1C,KAAK,CAAC,yCAAyC,CAC/C,CAAC;YAEF;;;;eAIG;YACH,IAAI,cAAc,CAAC,UAAU,EAAE,KAAK,EAAE;gBACrC,MAAM,CACL,6BAA6B,GAAG,CAAC,EACjC,KAAK,CAAC,kDAAkD,CACxD,CAAC;gBAEF,MAAM,KAAK,GAA2B,EAAE,CAAC;gBAEzC,4DAA4D;gBAC5D,OAAO,6BAA6B,IAAI,CAAC,EAAE;oBAC1C,KAAK,CAAC,IAAI,CAAC;wBACV,OAAO,EAAE,cAAc,CAAC,OAAO;wBAC/B,eAAe,EAAE,cAAc,CAAC,eAAe;wBAC/C,UAAU,EAAE,cAAc,CAAC,UAAU;qBACrC,CAAC,CAAC;oBAEH,IAAI,cAAc,CAAC,UAAU,EAAE,KAAK,KAAK,KAAK,EAAE;wBAC/C,MAAM;qBACN;oBACD,MAAM,CAAC,6BAA6B,GAAG,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,CAAC;oBAE1E,oEAAoE;oBACpE,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAG,CAAC;oBAC/C,6BAA6B,EAAE,CAAC;oBAChC,MAAM,CACL,cAAc,CAAC,UAAU,EAAE,KAAK,KAAK,IAAI,EACzC,KAAK,CAAC,iDAAiD,CACvD,CAAC;iBACF;gBAED,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;aACvC;iBAAM;gBACN,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC;oBAC1B,OAAO,EAAE,cAAc,CAAC,OAAO;oBAC/B,eAAe,EAAE,cAAc,CAAC,eAAe;oBAC/C,UAAU,EAAE,cAAc,CAAC,UAAU;iBACrC,CAAC,CAAC;aACH;SACD;QAED,qFAAqF;QACrF,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QAEnB,+GAA+G;QAC/G,6GAA6G;QAC7G,mFAAmF;QACnF,IAAI,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,EAAE;YAC3C,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC;gBAC/B,SAAS,EAAE,uBAAuB;gBAClC,KAAK,EAAE,2BAA2B;gBAClC,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;aACtC,CAAC,CAAC;SACH;IACF,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport Deque from \"double-ended-queue\";\n\nimport { IDisposable } from \"@fluidframework/core-interfaces\";\nimport { assert, Lazy } from \"@fluidframework/core-utils\";\nimport { ICriticalContainerError } from \"@fluidframework/container-definitions\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { DataProcessingError, ITelemetryLoggerExt } from \"@fluidframework/telemetry-utils\";\n\nimport { InboundSequencedContainerRuntimeMessage } from \"./messageTypes\";\nimport { pkgVersion } from \"./packageVersion\";\nimport { IBatchMetadata } from \"./metadata\";\n\n/**\n * This represents a message that has been submitted and is added to the pending queue when `submit` is called on the\n * ContainerRuntime. This message has either not been ack'd by the server or has not been submitted to the server yet.\n */\nexport interface IPendingMessage {\n\ttype: \"message\";\n\treferenceSequenceNumber: number;\n\tcontent: string;\n\tlocalOpMetadata: unknown;\n\topMetadata: Record<string, unknown> | undefined;\n}\n\nexport interface IPendingLocalState {\n\t/**\n\t * list of pending states, including ops and batch information\n\t */\n\tpendingStates: IPendingMessage[];\n}\n\nexport interface IPendingBatchMessage {\n\tcontent: string;\n\tlocalOpMetadata: unknown;\n\topMetadata: Record<string, unknown> | undefined;\n}\n\nexport interface IRuntimeStateHandler {\n\tconnected(): boolean;\n\tclientId(): string | undefined;\n\tclose(error?: ICriticalContainerError): void;\n\tapplyStashedOp(content: string): Promise<unknown>;\n\treSubmit(message: IPendingBatchMessage): void;\n\treSubmitBatch(batch: IPendingBatchMessage[]): void;\n\tisActiveConnection: () => boolean;\n}\n\n/** Union of keys of T */\ntype KeysOfUnion<T extends object> = T extends T ? keyof T : never;\n/** *Partial* type all possible combinations of properties and values of union T.\n * This loosens typing allowing access to all possible properties without\n * narrowing.\n */\ntype AnyComboFromUnion<T extends object> = { [P in KeysOfUnion<T>]?: T[P] };\n\nfunction buildPendingMessageContent(\n\t// AnyComboFromUnion is needed need to gain access to compatDetails that\n\t// is only defined for some cases.\n\tmessage: AnyComboFromUnion<InboundSequencedContainerRuntimeMessage>,\n): string {\n\t// IMPORTANT: Order matters here, this must match the order of the properties used\n\t// when submitting the message.\n\tconst { type, contents, compatDetails } = message;\n\t// Any properties that are not defined, won't be emitted by stringify.\n\treturn JSON.stringify({ type, contents, compatDetails });\n}\n\n/**\n * PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been\n * acknowledged by the server. It also maintains the batch information for both automatically and manually flushed\n * batches along with the messages.\n * When the Container reconnects, it replays the pending states, which includes manual flushing\n * of messages and triggering resubmission of unacked ops.\n *\n * It verifies that all the ops are acked, are received in the right order and batch information is correct.\n */\nexport class PendingStateManager implements IDisposable {\n\tprivate readonly pendingMessages = new Deque<IPendingMessage>();\n\tprivate readonly initialMessages = new Deque<IPendingMessage>();\n\n\t/**\n\t * Sequenced local ops that are saved when stashing since pending ops may depend on them\n\t */\n\tprivate savedOps: IPendingMessage[] = [];\n\n\tprivate readonly disposeOnce = new Lazy<void>(() => {\n\t\tthis.initialMessages.clear();\n\t\tthis.pendingMessages.clear();\n\t});\n\n\t// Indicates whether we are processing a batch.\n\tprivate isProcessingBatch: boolean = false;\n\n\t// This stores the first message in the batch that we are processing. This is used to verify that we get\n\t// the correct batch metadata.\n\tprivate pendingBatchBeginMessage: ISequencedDocumentMessage | undefined;\n\n\tprivate clientId: string | undefined;\n\n\t/**\n\t * The pending messages count. Includes `pendingMessages` and `initialMessages` to keep in sync with\n\t * 'hasPendingMessages'.\n\t */\n\tpublic get pendingMessagesCount(): number {\n\t\treturn this.pendingMessages.length + this.initialMessages.length;\n\t}\n\n\t/**\n\t * Called to check if there are any pending messages in the pending message queue.\n\t * @returns A boolean indicating whether there are messages or not.\n\t */\n\tpublic hasPendingMessages(): boolean {\n\t\treturn this.pendingMessagesCount !== 0;\n\t}\n\n\tpublic getLocalState(): IPendingLocalState | undefined {\n\t\tassert(\n\t\t\tthis.initialMessages.isEmpty(),\n\t\t\t0x2e9 /* \"Must call getLocalState() after applying initial states\" */,\n\t\t);\n\t\tif (!this.pendingMessages.isEmpty()) {\n\t\t\treturn {\n\t\t\t\tpendingStates: [...this.savedOps, ...this.pendingMessages.toArray()].map(\n\t\t\t\t\t(message) => {\n\t\t\t\t\t\t// delete localOpMetadata since it may not be serializable\n\t\t\t\t\t\t// and will be regenerated by applyStashedOp()\n\t\t\t\t\t\treturn { ...message, localOpMetadata: undefined };\n\t\t\t\t\t},\n\t\t\t\t),\n\t\t\t};\n\t\t}\n\t}\n\n\tconstructor(\n\t\tprivate readonly stateHandler: IRuntimeStateHandler,\n\t\tinitialLocalState: IPendingLocalState | undefined,\n\t\tprivate readonly logger: ITelemetryLoggerExt | undefined,\n\t) {\n\t\tif (initialLocalState?.pendingStates) {\n\t\t\tthis.initialMessages.push(...initialLocalState.pendingStates);\n\t\t}\n\t}\n\n\tpublic get disposed() {\n\t\treturn this.disposeOnce.evaluated;\n\t}\n\tpublic readonly dispose = () => this.disposeOnce.value;\n\n\t/**\n\t * Called when a message is submitted locally. Adds the message and the associated details to the pending state\n\t * queue.\n\t * @param type - The container message type.\n\t * @param content - The message content.\n\t * @param localOpMetadata - The local metadata associated with the message.\n\t */\n\tpublic onSubmitMessage(\n\t\tcontent: string,\n\t\treferenceSequenceNumber: number,\n\t\tlocalOpMetadata: unknown,\n\t\topMetadata: Record<string, unknown> | undefined,\n\t) {\n\t\tconst pendingMessage: IPendingMessage = {\n\t\t\ttype: \"message\",\n\t\t\treferenceSequenceNumber,\n\t\t\tcontent,\n\t\t\tlocalOpMetadata,\n\t\t\topMetadata,\n\t\t};\n\n\t\tthis.pendingMessages.push(pendingMessage);\n\t}\n\n\t/**\n\t * Applies stashed ops at their reference sequence number so they are ready to be ACKed or resubmitted\n\t * @param seqNum - Sequence number at which to apply ops. Will apply all ops if seqNum is undefined.\n\t */\n\tpublic async applyStashedOpsAt(seqNum?: number) {\n\t\t// apply stashed ops at sequence number\n\t\twhile (!this.initialMessages.isEmpty()) {\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\t\tconst nextMessage = this.initialMessages.peekFront()!;\n\t\t\tif (seqNum !== undefined) {\n\t\t\t\tif (nextMessage.referenceSequenceNumber > seqNum) {\n\t\t\t\t\tbreak; // nothing left to do at this sequence number\n\t\t\t\t}\n\t\t\t\tif (nextMessage.referenceSequenceNumber < seqNum) {\n\t\t\t\t\tthrow new Error(\"loaded from snapshot too recent to apply stashed ops\");\n\t\t\t\t}\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\t// applyStashedOp will cause the DDS to behave as if it has sent the op but not actually send it\n\t\t\t\tconst localOpMetadata = await this.stateHandler.applyStashedOp(nextMessage.content);\n\t\t\t\tnextMessage.localOpMetadata = localOpMetadata;\n\t\t\t} catch (error) {\n\t\t\t\tthrow DataProcessingError.wrapIfUnrecognized(error, \"applyStashedOp\", nextMessage);\n\t\t\t}\n\n\t\t\t// then we push onto pendingMessages which will cause PendingStateManager to resubmit when we connect\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\t\tthis.pendingMessages.push(this.initialMessages.shift()!);\n\t\t}\n\t}\n\n\t/**\n\t * Processes a local message once its ack'd by the server. It verifies that there was no data corruption and that\n\t * the batch information was preserved for batch messages.\n\t * @param message - The message that got ack'd and needs to be processed.\n\t */\n\tpublic processPendingLocalMessage(message: InboundSequencedContainerRuntimeMessage): unknown {\n\t\t// Pre-processing part - This may be the start of a batch.\n\t\tthis.maybeProcessBatchBegin(message);\n\n\t\t// Get the next message from the pending queue. Verify a message exists.\n\t\tconst pendingMessage = this.pendingMessages.peekFront();\n\t\tassert(\n\t\t\tpendingMessage !== undefined,\n\t\t\t0x169 /* \"No pending message found for this remote message\" */,\n\t\t);\n\t\tthis.savedOps.push(pendingMessage);\n\n\t\tthis.pendingMessages.shift();\n\n\t\tconst messageContent = buildPendingMessageContent(message);\n\n\t\t// Stringified content should match\n\t\tif (pendingMessage.content !== messageContent) {\n\t\t\tthis.stateHandler.close(\n\t\t\t\tDataProcessingError.create(\n\t\t\t\t\t\"pending local message content mismatch\",\n\t\t\t\t\t\"unexpectedAckReceived\",\n\t\t\t\t\tmessage,\n\t\t\t\t\t{\n\t\t\t\t\t\texpectedMessageType: JSON.parse(pendingMessage.content).type,\n\t\t\t\t\t},\n\t\t\t\t),\n\t\t\t);\n\t\t\treturn;\n\t\t}\n\n\t\t// Post-processing part - If we are processing a batch then this could be the last message in the batch.\n\t\tthis.maybeProcessBatchEnd(message);\n\n\t\treturn pendingMessage.localOpMetadata;\n\t}\n\n\t/**\n\t * This message could be the first message in batch. If so, set batch state marking the beginning of a batch.\n\t * @param message - The message that is being processed.\n\t */\n\tprivate maybeProcessBatchBegin(message: ISequencedDocumentMessage) {\n\t\t// This message is the first in a batch if the \"batch\" property on the metadata is set to true\n\t\tif ((message.metadata as IBatchMetadata | undefined)?.batch) {\n\t\t\t// We should not already be processing a batch and there should be no pending batch begin message.\n\t\t\tassert(\n\t\t\t\t!this.isProcessingBatch && this.pendingBatchBeginMessage === undefined,\n\t\t\t\t0x16b /* \"The pending batch state indicates we are already processing a batch\" */,\n\t\t\t);\n\n\t\t\t// Set the pending batch state indicating we have started processing a batch.\n\t\t\tthis.pendingBatchBeginMessage = message;\n\t\t\tthis.isProcessingBatch = true;\n\t\t}\n\t}\n\n\t/**\n\t * This message could be the last message in batch. If so, clear batch state since the batch is complete.\n\t * @param message - The message that is being processed.\n\t */\n\tprivate maybeProcessBatchEnd(message: ISequencedDocumentMessage) {\n\t\tif (!this.isProcessingBatch) {\n\t\t\treturn;\n\t\t}\n\n\t\t// There should be a pending batch begin message.\n\t\tassert(\n\t\t\tthis.pendingBatchBeginMessage !== undefined,\n\t\t\t0x16d /* \"There is no pending batch begin message\" */,\n\t\t);\n\n\t\tconst batchEndMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;\n\t\tif (this.pendingMessages.isEmpty() || batchEndMetadata === false) {\n\t\t\t// Get the batch begin metadata from the first message in the batch.\n\t\t\tconst batchBeginMetadata = (\n\t\t\t\tthis.pendingBatchBeginMessage.metadata as IBatchMetadata | undefined\n\t\t\t)?.batch;\n\n\t\t\t// There could be just a single message in the batch. If so, it should not have any batch metadata. If there\n\t\t\t// are multiple messages in the batch, verify that we got the correct batch begin and end metadata.\n\t\t\tif (this.pendingBatchBeginMessage === message) {\n\t\t\t\tassert(\n\t\t\t\t\tbatchBeginMetadata === undefined,\n\t\t\t\t\t0x16e /* \"Batch with single message should not have batch metadata\" */,\n\t\t\t\t);\n\t\t\t} else {\n\t\t\t\tif (batchBeginMetadata !== true || batchEndMetadata !== false) {\n\t\t\t\t\tthis.stateHandler.close(\n\t\t\t\t\t\tDataProcessingError.create(\n\t\t\t\t\t\t\t\"Pending batch inconsistency\", // Formerly known as asserts 0x16f and 0x170\n\t\t\t\t\t\t\t\"processPendingLocalMessage\",\n\t\t\t\t\t\t\tmessage,\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\truntimeVersion: pkgVersion,\n\t\t\t\t\t\t\t\tbatchClientId:\n\t\t\t\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing\n\t\t\t\t\t\t\t\t\tthis.pendingBatchBeginMessage.clientId === null\n\t\t\t\t\t\t\t\t\t\t? \"null\"\n\t\t\t\t\t\t\t\t\t\t: this.pendingBatchBeginMessage.clientId,\n\t\t\t\t\t\t\t\tclientId: this.stateHandler.clientId(),\n\t\t\t\t\t\t\t\thasBatchStart: batchBeginMetadata === true,\n\t\t\t\t\t\t\t\thasBatchEnd: batchEndMetadata === false,\n\t\t\t\t\t\t\t\tmessageType: message.type,\n\t\t\t\t\t\t\t\tpendingMessagesCount: this.pendingMessagesCount,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Clear the pending batch state now that we have processed the entire batch.\n\t\t\tthis.pendingBatchBeginMessage = undefined;\n\t\t\tthis.isProcessingBatch = false;\n\t\t}\n\t}\n\n\t/**\n\t * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending\n\t * states in its queue. This includes triggering resubmission of unacked ops.\n\t * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)\n\t */\n\tpublic replayPendingStates() {\n\t\tassert(\n\t\t\tthis.stateHandler.connected(),\n\t\t\t0x172 /* \"The connection state is not consistent with the runtime\" */,\n\t\t);\n\n\t\t// This assert suggests we are about to send same ops twice, which will result in data loss.\n\t\tassert(\n\t\t\tthis.clientId !== this.stateHandler.clientId(),\n\t\t\t0x173 /* \"replayPendingStates called twice for same clientId!\" */,\n\t\t);\n\t\tthis.clientId = this.stateHandler.clientId();\n\n\t\tassert(\n\t\t\tthis.initialMessages.isEmpty(),\n\t\t\t0x174 /* \"initial states should be empty before replaying pending\" */,\n\t\t);\n\n\t\tconst initialPendingMessagesCount = this.pendingMessages.length;\n\t\tlet remainingPendingMessagesCount = this.pendingMessages.length;\n\n\t\t// Process exactly `pendingMessagesCount` items in the queue as it represents the number of messages that were\n\t\t// pending when we connected. This is important because the `reSubmitFn` might add more items in the queue\n\t\t// which must not be replayed.\n\t\twhile (remainingPendingMessagesCount > 0) {\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\t\tlet pendingMessage = this.pendingMessages.shift()!;\n\t\t\tremainingPendingMessagesCount--;\n\t\t\tassert(\n\t\t\t\tpendingMessage.opMetadata?.batch !== false,\n\t\t\t\t0x41b /* We cannot process batches in chunks */,\n\t\t\t);\n\n\t\t\t/**\n\t\t\t * We want to ensure grouped messages get processed in a batch.\n\t\t\t * Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will\n\t\t\t * either receive the whole batch ack or nothing at all.\n\t\t\t */\n\t\t\tif (pendingMessage.opMetadata?.batch) {\n\t\t\t\tassert(\n\t\t\t\t\tremainingPendingMessagesCount > 0,\n\t\t\t\t\t0x554 /* Last pending message cannot be a batch begin */,\n\t\t\t\t);\n\n\t\t\t\tconst batch: IPendingBatchMessage[] = [];\n\n\t\t\t\t// check is >= because batch end may be last pending message\n\t\t\t\twhile (remainingPendingMessagesCount >= 0) {\n\t\t\t\t\tbatch.push({\n\t\t\t\t\t\tcontent: pendingMessage.content,\n\t\t\t\t\t\tlocalOpMetadata: pendingMessage.localOpMetadata,\n\t\t\t\t\t\topMetadata: pendingMessage.opMetadata,\n\t\t\t\t\t});\n\n\t\t\t\t\tif (pendingMessage.opMetadata?.batch === false) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\tassert(remainingPendingMessagesCount > 0, 0x555 /* No batch end found */);\n\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\t\t\t\tpendingMessage = this.pendingMessages.shift()!;\n\t\t\t\t\tremainingPendingMessagesCount--;\n\t\t\t\t\tassert(\n\t\t\t\t\t\tpendingMessage.opMetadata?.batch !== true,\n\t\t\t\t\t\t0x556 /* Batch start needs a corresponding batch end */,\n\t\t\t\t\t);\n\t\t\t\t}\n\n\t\t\t\tthis.stateHandler.reSubmitBatch(batch);\n\t\t\t} else {\n\t\t\t\tthis.stateHandler.reSubmit({\n\t\t\t\t\tcontent: pendingMessage.content,\n\t\t\t\t\tlocalOpMetadata: pendingMessage.localOpMetadata,\n\t\t\t\t\topMetadata: pendingMessage.opMetadata,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\n\t\t// pending ops should no longer depend on previous sequenced local ops after resubmit\n\t\tthis.savedOps = [];\n\n\t\t// We replayPendingStates on read connections too - we expect these to get nack'd though, and to then reconnect\n\t\t// on a write connection and replay again. This filters out the replay that happens on the read connection so\n\t\t// we only see the replays on write connections (that have a chance to go through).\n\t\tif (this.stateHandler.isActiveConnection()) {\n\t\t\tthis.logger?.sendTelemetryEvent({\n\t\t\t\teventName: \"PendingStatesReplayed\",\n\t\t\t\tcount: initialPendingMessagesCount,\n\t\t\t\tclientId: this.stateHandler.clientId(),\n\t\t\t});\n\t\t}\n\t}\n}\n"]}

package/lib/scheduleManager.d.mts

@@ -0,0 +1,27 @@
+/// <reference types="node" />
+import { EventEmitter } from "events";
+import { IDeltaManager } from "@fluidframework/container-definitions";
+import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+/**
+ * This class has the following responsibilities:
+ *
+ * 1. It tracks batches as we process ops and raises "batchBegin" and "batchEnd" events.
+ * As part of it, it validates batch correctness (i.e. no system ops in the middle of batch)
+ *
+ * 2. It creates instance of ScheduleManagerCore that ensures we never start processing ops from batch
+ * unless all ops of the batch are in.
+ */
+export declare class ScheduleManager {
+    private readonly deltaManager;
+    private readonly emitter;
+    readonly getClientId: () => string | undefined;
+    private readonly logger;
+    private readonly deltaScheduler;
+    private batchClientId;
+    private hitError;
+    constructor(deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>, emitter: EventEmitter, getClientId: () => string | undefined, logger: ITelemetryLoggerExt);
+    beforeOpProcessing(message: ISequencedDocumentMessage): void;
+    afterOpProcessing(error: any | undefined, message: ISequencedDocumentMessage): void;
+}
+//# sourceMappingURL=scheduleManager.d.ts.map

package/lib/scheduleManager.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduleManager.d.ts","sourceRoot":"","sources":["../src/scheduleManager.ts"],"names":[],"mappings":";OAIO,EAAE,YAAY,EAAE,MAAM,QAAQ;OAC9B,EAAE,aAAa,EAAE,MAAM,uCAAuC;OAC9D,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,sCAAsC;OAC3F,EAKN,mBAAmB,EACnB,MAAM,iCAAiC;AAcxC;;;;;;;;GAQG;AACH,qBAAa,eAAe;IAM1B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,MAAM,GAAG,SAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM;IARxB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,aAAa,CAAqB;IAC1C,OAAO,CAAC,QAAQ,CAAS;gBAGP,YAAY,EAAE,aAAa,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,EACxE,OAAO,EAAE,YAAY,EAC7B,WAAW,EAAE,MAAM,MAAM,GAAG,SAAS,EAC7B,MAAM,EAAE,mBAAmB;IAStC,kBAAkB,CAAC,OAAO,EAAE,yBAAyB;IAkBrD,iBAAiB,CAAC,KAAK,EAAE,GAAG,GAAG,SAAS,EAAE,OAAO,EAAE,yBAAyB;CAwBnF"}