@fluidframework/test-utils 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

package/dist/loaderContainerTracker.js

@@ -1,15 +1,22 @@
 "use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LoaderContainerTracker = void 0;
 /*!
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LoaderContainerTracker = void 0;
-const common_utils_1 = require("@fluidframework/common-utils");
+const core_utils_1 = require("@fluidframework/core-utils");
+const container_loader_1 = require("@fluidframework/container-loader");
+const driver_utils_1 = require("@fluidframework/driver-utils");
 const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
+const containerUtils_1 = require("./containerUtils");
 const debug_1 = require("./debug");
+const timeoutUtils_1 = require("./timeoutUtils");
 const debugOp = debug_1.debug.extend("ops");
 const debugWait = debug_1.debug.extend("wait");
+/**
+ * @alpha
+ */
 class LoaderContainerTracker {
     constructor(syncSummarizerClients = false) {
         this.syncSummarizerClients = syncSummarizerClients;
@@ -42,14 +49,28 @@ class LoaderContainerTracker {
      * @param container - container to add
      */
     addContainer(container) {
-        // ignore summarizer
-        if (!container.deltaManager.clientDetails.capabilities.interactive && !this.syncSummarizerClients) {
-            return;
-        }
         // don't add container that is already tracked
         if (this.containers.has(container)) {
             return;
         }
+        const containerWithClone = container;
+        // back-compat: Check for undefined because this function was added recently and older containers won't have it.
+        if (containerWithClone.clone !== undefined) {
+            const patch = (fn) => {
+                const boundFn = fn.bind(containerWithClone);
+                return async (...args) => {
+                    const newContainer = await boundFn(...args);
+                    this.addContainer(newContainer);
+                    return newContainer;
+                };
+            };
+            containerWithClone.clone = patch(containerWithClone.clone);
+        }
+        // ignore summarizer
+        if (!container.deltaManager.clientDetails.capabilities.interactive &&
+            !this.syncSummarizerClients) {
+            return;
+        }
         const record = {
             index: this.containers.size,
             paused: false,
@@ -72,7 +93,7 @@ class LoaderContainerTracker {
     trackTrailingNoOps(container, record) {
         container.deltaManager.outbound.on("op", (messages) => {
             for (const msg of messages) {
-                if (msg.type === protocol_definitions_1.MessageType.NoOp) {
+                if ((0, driver_utils_1.canBeCoalescedByService)(msg)) {
                     // Track the NoOp that was sent.
                     if (record.trailingNoOps === 0) {
                         // record the starting sequence number of the trailing no ops if we haven't been tracking yet.
@@ -88,15 +109,15 @@ class LoaderContainerTracker {
         });
         container.deltaManager.inbound.on("push", (message) => {
             // Received the no op back, update the record if we are tracking
-            if (message.type === protocol_definitions_1.MessageType.NoOp
-                && message.clientId === container.clientId
-                && record.trailingNoOps !== 0
-                && record.startTrailingNoOps <= message.clientSequenceNumber) {
+            if ((0, driver_utils_1.canBeCoalescedByService)(message) &&
+                message.clientId === container.clientId &&
+                record.trailingNoOps !== 0 &&
+                record.startTrailingNoOps <= message.clientSequenceNumber) {
                 // NoOp might have coalesced and skipped ahead some sequence number
                 // update the record and skip ahead as well
                 const oldStartTrailingNoOps = record.startTrailingNoOps;
                 record.startTrailingNoOps = message.clientSequenceNumber + 1;
-                record.trailingNoOps -= (record.startTrailingNoOps - oldStartTrailingNoOps);
+                record.trailingNoOps -= record.startTrailingNoOps - oldStartTrailingNoOps;
             }
         });
         container.on("disconnected", () => {
@@ -124,25 +145,34 @@ class LoaderContainerTracker {
     }
     /**
      * Make sure all the tracked containers are synchronized.
-     * - No isDirty (non-readonly) containers
-     * - No extra clientId in quorum of any container that is not tracked and still opened.
-     *      - i.e. no pending Join/Leave message.
-     * - No unresolved proposal (minSeqNum \>= lastProposalSeqNum)
-     * - lastSequenceNumber of all container is the same
-     * - clientSequenceNumberObserved is the same as clientSequenceNumber sent
-     *      - this overlaps with !isDirty, but include task scheduler ops.
-     *      - Trailing NoOp is tracked and don't count as pending ops.
+     *
+     * No isDirty (non-readonly) containers
+     * No extra clientId in quorum of any container that is not tracked and still opened.
+     * - i.e. no pending Join/Leave message.
+     * No unresolved proposal (minSeqNum \>= lastProposalSeqNum)
+     * lastSequenceNumber of all container is the same
+     * clientSequenceNumberObserved is the same as clientSequenceNumber sent
+     * - this overlaps with !isDirty, but include task scheduler ops.
+     * - Trailing NoOp is tracked and don't count as pending ops.
+     *
+     * Containers that are already pause will resume process and paused again once
+     * everything is synchronized.  Containers that aren't paused will remain unpaused when this
+     * function returns.
      */
     async ensureSynchronized(...containers) {
         const resumed = this.resumeProcessing(...containers);
-        let waitingSequenceNumberSynchronized = false;
+        let waitingSequenceNumberSynchronized;
         // eslint-disable-next-line no-constant-condition
         while (true) {
+            // yield a turn to allow side effect of resuming or the ops we just processed execute before we check
+            await new Promise((resolve) => {
+                setTimeout(resolve, 0);
+            });
             const containersToApply = this.getContainers(containers);
             if (containersToApply.length === 0) {
                 break;
             }
-            // Ignore readonly dirty containers, because it can't sent up and nothing can be done about it being dirty
+            // Ignore readonly dirty containers, because it can't sent ops and nothing can be done about it being dirty
             const dirtyContainers = containersToApply.filter((c) => {
                 const { deltaManager, isDirty } = c;
                 return deltaManager.readOnlyInfo.readonly !== true && isDirty;
@@ -151,37 +181,46 @@ class LoaderContainerTracker {
                 // Wait for all the leave messages
                 const pendingClients = this.getPendingClients(containersToApply);
                 if (pendingClients.length === 0) {
-                    if (this.isSequenceNumberSynchronized(containersToApply)) {
+                    const needSync = this.needSequenceNumberSynchronize(containersToApply);
+                    if (needSync === undefined) {
                         // done, we are in sync
                         break;
                     }
-                    if (!waitingSequenceNumberSynchronized) {
-                        // Only write it out once
-                        waitingSequenceNumberSynchronized = true;
-                        debugWait("Waiting for sequence number synchronized");
-                        await this.waitForAnyInboundOps(containersToApply);
+                    if (waitingSequenceNumberSynchronized !== needSync.reason) {
+                        // Don't repeat writing to console if it is the same reason
+                        waitingSequenceNumberSynchronized = needSync.reason;
+                        debugWait(needSync.message);
                     }
+                    // Wait for one inbounds ops which might change the state of things
+                    await (0, timeoutUtils_1.timeoutAwait)(this.waitForAnyInboundOps(containersToApply), {
+                        errorMsg: `Timeout on ${needSync.message}`,
+                    });
                 }
                 else {
-                    waitingSequenceNumberSynchronized = false;
-                    await this.waitForPendingClients(pendingClients);
+                    waitingSequenceNumberSynchronized = undefined;
+                    await (0, timeoutUtils_1.timeoutAwait)(this.waitForPendingClients(pendingClients), {
+                        errorMsg: "Timeout on waiting for pending join or leave op",
+                    });
                 }
             }
             else {
                 // Wait for all the containers to be saved
-                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                debugWait(`Waiting container to be saved ${dirtyContainers.map((c) => this.containers.get(c).index)}`);
-                waitingSequenceNumberSynchronized = false;
-                await Promise.all(dirtyContainers.map(async (c) => Promise.race([new Promise((resolve) => c.once("saved", resolve)),
-                    new Promise((resolve) => c.once("closed", resolve))])));
+                debugWait(`Waiting container to be saved ${this.containerIndexStrings(dirtyContainers)}`);
+                waitingSequenceNumberSynchronized = undefined;
+                await Promise.all(dirtyContainers.map(async (c) => Promise.race([
+                    (0, timeoutUtils_1.timeoutPromise)((resolve) => c.once("saved", () => resolve()), {
+                        errorMsg: "Timeout on waiting a container to be saved",
+                    }),
+                    new Promise((resolve) => c.once("closed", resolve)),
+                ])));
             }
-            // yield a turn to allow side effect of the ops we just processed execute before we check again
-            await new Promise((resolve) => { setTimeout(resolve, 0); });
         }
         // Pause all container that was resumed
         // don't call pause if resumed is empty and pause everything, which is not what we want
         if (resumed.length !== 0) {
-            await this.pauseProcessing(...resumed);
+            await (0, timeoutUtils_1.timeoutAwait)(this.pauseProcessing(...resumed), {
+                errorMsg: "Timeout on waiting for pausing all resumed containers",
+            });
         }
         debugWait("Synchronized");
     }
@@ -202,7 +241,8 @@ class LoaderContainerTracker {
             const quorum = container.getQuorum();
             quorum.getMembers().forEach((client, clientId) => {
                 // ignore summarizer
-                if (!client.client.details.capabilities.interactive && !this.syncSummarizerClients) {
+                if (!client.client.details.capabilities.interactive &&
+                    !this.syncSummarizerClients) {
                     return;
                 }
                 if (!openedClientId.includes(clientId)) {
@@ -221,42 +261,62 @@ class LoaderContainerTracker {
      *
      * @param containersToApply - the set of containers to check
      */
-    isSequenceNumberSynchronized(containersToApply) {
+    needSequenceNumberSynchronize(containersToApply) {
+        // If there is a pending proposal, wait for it to be accepted
+        const minSeqNum = containersToApply[0].deltaManager.minimumSequenceNumber;
+        if (minSeqNum < this.lastProposalSeqNum) {
+            return {
+                reason: "Proposal",
+                message: `waiting for MSN to advance to proposal at sequence number ${this.lastProposalSeqNum}`,
+            };
+        }
         // clientSequenceNumber check detects ops in flight, both on the wire and in the outbound queue
         // We need both client sequence number and isDirty check because:
         // - Currently isDirty flag ignores ops for task scheduler, so we need the client sequence number check
         // - But isDirty flags include ops during forceReadonly and disconnected, because we don't submit
         //   the ops in the first place, clientSequenceNumber is not assigned
-        const isClientSequenceNumberSynchronized = containersToApply.every((container) => {
+        const containerWithInflightOps = containersToApply.filter((container) => {
             if (container.deltaManager.readOnlyInfo.readonly === true) {
                 // Ignore readonly container. the clientSeqNum and clientSeqNumObserved might be out of sync
                 // because we transition to readonly when outbound is not empty or the in transit op got lost
-                return true;
+                return false;
             }
             // Note that in read only mode, the op won't be submitted
             let deltaManager = container.deltaManager;
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const { trailingNoOps } = this.containers.get(container);
-            // Back-compat: clientSequenceNumber & clientSequenceNumberObserved moved to ConnectionManager in 0.53
+            // Back-compat: lastSubmittedClientId/clientSequenceNumber/clientSequenceNumberObserved moved to ConnectionManager in 0.53
             if (!("clientSequenceNumber" in deltaManager)) {
                 deltaManager = deltaManager.connectionManager;
             }
-            (0, common_utils_1.assert)("clientSequenceNumber" in deltaManager, "no clientSequenceNumber");
-            (0, common_utils_1.assert)("clientSequenceNumberObserved" in deltaManager, "no clientSequenceNumber");
-            return deltaManager.clientSequenceNumber ===
-                deltaManager.clientSequenceNumberObserved + trailingNoOps;
+            (0, core_utils_1.assert)("clientSequenceNumber" in deltaManager, "no clientSequenceNumber");
+            (0, core_utils_1.assert)("clientSequenceNumberObserved" in deltaManager, "no clientSequenceNumber");
+            // If last submittedClientId isn't the current clientId, then we haven't send any ops
+            return (deltaManager.lastSubmittedClientId === container.clientId &&
+                deltaManager.clientSequenceNumber !==
+                    deltaManager.clientSequenceNumberObserved + trailingNoOps);
         });
-        if (!isClientSequenceNumberSynchronized) {
-            return false;
-        }
-        const minSeqNum = containersToApply[0].deltaManager.minimumSequenceNumber;
-        if (minSeqNum < this.lastProposalSeqNum) {
-            // There is an unresolved proposal
-            return false;
+        if (containerWithInflightOps.length !== 0) {
+            return {
+                reason: "InflightOps",
+                message: `waiting for containers with inflight ops: ${this.containerIndexStrings(containerWithInflightOps)}`,
+            };
         }
         // Check to see if all the container has process the same number of ops.
-        const seqNum = containersToApply[0].deltaManager.lastSequenceNumber;
-        return containersToApply.every((c) => c.deltaManager.lastSequenceNumber === seqNum);
+        const maxSeqNum = Math.max(...containersToApply.map((c) => c.deltaManager.lastSequenceNumber));
+        const containerWithPendingIncoming = containersToApply.filter((c) => c.deltaManager.lastSequenceNumber !== maxSeqNum);
+        if (containerWithPendingIncoming.length !== 0) {
+            return {
+                reason: "Pending",
+                message: `waiting for containers with pending incoming ops up to sequence number ${maxSeqNum}: ${this.containerIndexStrings(containerWithPendingIncoming)}`,
+            };
+        }
+        return undefined;
+    }
+    containerIndexStrings(containers) {
+        return containers.map(
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        (c) => this.containers.get(c).index);
     }
     /**
      * Utility to wait for any clientId in quorum that is NOT associated with any container we
@@ -267,7 +327,7 @@ class LoaderContainerTracker {
      * @param containersToApply - the set of containers to wait for any inbound ops for
      */
     async waitForPendingClients(pendingClients) {
-        const unconnectedClients = Array.from(this.containers.keys()).filter((c) => !c.closed && !c.connected);
+        const unconnectedClients = Array.from(this.containers.keys()).filter((c) => !c.closed && c.connectionState !== container_loader_1.ConnectionState.Connected);
         return Promise.all(pendingClients.map(async ([container, pendingClientId]) => {
             return new Promise((resolve) => {
                 const cleanup = () => {
@@ -318,7 +378,8 @@ class LoaderContainerTracker {
         const containersToApply = this.getContainers(containers);
         for (const container of containersToApply) {
             const record = this.containers.get(container);
-            if ((record === null || record === void 0 ? void 0 : record.paused) === true) {
+            (0, core_utils_1.assert)(record?.pauseP === undefined, "Cannot resume container while pausing is in progress");
+            if (record?.paused === true) {
                 debugWait(`${record.index}: container resumed`);
                 container.deltaManager.inbound.resume();
                 container.deltaManager.outbound.resume();
@@ -331,25 +392,86 @@ class LoaderContainerTracker {
     /**
      * Pause all queue activities on the containers given, or all tracked containers
      * Any containers given that is not tracked will be ignored.
+     *
+     * When a container is paused, it is assumed that we want fine grain control over op
+     * sequencing.  This function will prepare the container and force it into write mode to
+     * avoid missing join messages or change the sequence of event when switching from read to
+     * write mode.
      */
     async pauseProcessing(...containers) {
-        const pauseP = [];
+        const waitP = [];
         const containersToApply = this.getContainers(containers);
         for (const container of containersToApply) {
             const record = this.containers.get(container);
             if (record !== undefined && !record.paused) {
-                debugWait(`${record.index}: container paused`);
-                pauseP.push(container.deltaManager.inbound.pause());
-                pauseP.push(container.deltaManager.outbound.pause());
-                record.paused = true;
+                if (record.pauseP === undefined) {
+                    record.pauseP = this.pauseContainer(container, record);
+                }
+                waitP.push(record.pauseP);
             }
         }
-        await Promise.all(pauseP);
+        await Promise.all(waitP);
+    }
+    /**
+     * When a container is paused, it is assumed that we want fine grain control over op
+     * sequencing.  This function will prepare the container and force it into write mode to
+     * avoid missing join messages or change the sequence of event when switching from read to
+     * write mode.
+     *
+     * @param container - the container to pause
+     * @param record - the record for the container
+     */
+    async pauseContainer(container, record) {
+        debugWait(`${record.index}: pausing container`);
+        (0, core_utils_1.assert)(!container.deltaManager.outbound.paused, "Container should not be paused yet");
+        (0, core_utils_1.assert)(!container.deltaManager.inbound.paused, "Container should not be paused yet");
+        // Pause outbound
+        debugWait(`${record.index}: pausing container outbound queues`);
+        await container.deltaManager.outbound.pause();
+        // Ensure the container is connected first.
+        if (container.connectionState !== container_loader_1.ConnectionState.Connected) {
+            debugWait(`${record.index}: Wait for container connection`);
+            await (0, containerUtils_1.waitForContainerConnection)(container);
+        }
+        // Check if the container is in write mode
+        if (!container.deltaManager.active) {
+            let proposalP;
+            if (container.deltaManager.outbound.idle) {
+                // Need to generate an op to force write mode
+                debugWait(`${record.index}: container force write connection`);
+                const maybeContainer = container;
+                const codeProposal = maybeContainer.getLoadedCodeDetails
+                    ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                        container.getLoadedCodeDetails()
+                    : container.chaincodePackage;
+                proposalP = container.proposeCodeDetails(codeProposal);
+            }
+            // Wait for nack
+            debugWait(`${record.index}: Wait for container disconnect`);
+            container.deltaManager.outbound.resume();
+            await new Promise((resolve) => container.once("disconnected", resolve));
+            const accepted = proposalP ? await proposalP : false;
+            (0, core_utils_1.assert)(!accepted, "A proposal in read mode should be rejected");
+            await container.deltaManager.outbound.pause();
+            // Ensure the container is reconnect.
+            if (container.connectionState !== container_loader_1.ConnectionState.Connected) {
+                debugWait(`${record.index}: Wait for container reconnection`);
+                await (0, containerUtils_1.waitForContainerConnection)(container);
+            }
+        }
+        debugWait(`${record.index}: pausing container inbound queues`);
+        // Pause inbound
+        await container.deltaManager.inbound.pause();
+        debugWait(`${record.index}: container paused`);
+        record.pauseP = undefined;
+        record.paused = true;
     }
     /**
      * Pause all queue activities on all tracked containers, and resume only
      * inbound to process ops until it is idle. All queues are left in the paused state
-     * after the function
+     * after the function.
+     *
+     * Pausing will switch the container to write mode. See `pauseProcessing`
      */
     async processIncoming(...containers) {
         return this.processQueue(containers, (container) => container.deltaManager.inbound);
@@ -357,7 +479,9 @@ class LoaderContainerTracker {
     /**
      * Pause all queue activities on all tracked containers, and resume only
      * outbound to process ops until it is idle. All queues are left in the paused state
-     * after the function
+     * after the function.
+     *
+     * Pausing will switch the container to write mode. See `pauseProcessing`
      */
     async processOutgoing(...containers) {
         return this.processQueue(containers, (container) => container.deltaManager.outbound);
@@ -372,6 +496,7 @@ class LoaderContainerTracker {
         const inflightTracker = new Map();
         const cleanup = [];
         for (const container of containersToApply) {
+            (0, core_utils_1.assert)(container.deltaManager.active, "Container should be connected in write mode already");
             const queue = getQueue(container);
             // track the outgoing ops (if any) to make sure they make the round trip to at least to the same client
             // to make sure they are sequenced.
@@ -381,7 +506,9 @@ class LoaderContainerTracker {
         }
         while (resumed.some((queue) => !queue.idle)) {
             debugWait("Wait until queue is idle");
-            await new Promise((resolve) => { setTimeout(resolve, 0); });
+            await new Promise((resolve) => {
+                setTimeout(resolve, 0);
+            });
         }
         // Make sure all the op that we sent out are acked first
         // This is no op if we are processing incoming
@@ -405,15 +532,15 @@ class LoaderContainerTracker {
     setupInOutTracker(container, inflightTracker) {
         const outHandler = (messages) => {
             for (const message of messages) {
-                if (message.type !== protocol_definitions_1.MessageType.NoOp) {
+                if (!(0, driver_utils_1.canBeCoalescedByService)(message)) {
                     inflightTracker.set(container, message.clientSequenceNumber);
                 }
             }
         };
         const inHandler = (message) => {
-            if (message.type !== protocol_definitions_1.MessageType.NoOp
-                && message.clientId === container.clientId
-                && inflightTracker.get(container) === message.clientSequenceNumber) {
+            if (!(0, driver_utils_1.canBeCoalescedByService)(message) &&
+                message.clientId === container.clientId &&
+                inflightTracker.get(container) === message.clientSequenceNumber) {
                 inflightTracker.delete(container);
             }
         };
@@ -430,7 +557,6 @@ class LoaderContainerTracker {
     setupTrace(container, index) {
         if (debugOp.enabled) {
             const getContentsString = (type, msgContents) => {
-                var _a, _b;
                 try {
                     if (type !== protocol_definitions_1.MessageType.Operation) {
                         if (typeof msgContents === "string") {
@@ -441,14 +567,13 @@ class LoaderContainerTracker {
                     let address = "";
                     // contents comes in the wire as JSON string ("push" event)
                     // But already parsed when apply ("op" event)
-                    let contents = typeof msgContents === "string" ?
-                        JSON.parse(msgContents) : msgContents;
+                    let contents = typeof msgContents === "string" ? JSON.parse(msgContents) : msgContents;
                     while (contents !== undefined && contents !== null) {
-                        if (((_a = contents.contents) === null || _a === void 0 ? void 0 : _a.address) !== undefined) {
+                        if (contents.contents?.address !== undefined) {
                             address += `/${contents.contents.address}`;
                             contents = contents.contents.contents;
                         }
-                        else if (((_b = contents.content) === null || _b === void 0 ? void 0 : _b.address) !== undefined) {
+                        else if (contents.content?.address !== undefined) {
                             address += `/${contents.content.address}`;
                             contents = contents.content.contents;
                         }
@@ -468,19 +593,22 @@ class LoaderContainerTracker {
             debugOp(`${index}: ADD: clientId: ${container.clientId}`);
             container.deltaManager.outbound.on("op", (messages) => {
                 for (const msg of messages) {
-                    debugOp(`${index}: OUT:          `
-                        + `cli: ${msg.clientSequenceNumber.toString().padStart(3)} `
-                        + `rsq: ${msg.referenceSequenceNumber.toString().padStart(3)} `
-                        + `${msg.type} ${getContentsString(msg.type, msg.contents)}`);
+                    debugOp(`${index}: OUT:          ` +
+                        `cli: ${msg.clientSequenceNumber.toString().padStart(3)} ` +
+                        `rsq: ${msg.referenceSequenceNumber.toString().padStart(3)} ` +
+                        `${msg.type} ${getContentsString(msg.type, msg.contents)}`);
                 }
             });
             const getInboundHandler = (type) => {
                 return (msg) => {
-                    const clientSeq = msg.clientId === container.clientId ?
-                        `cli: ${msg.clientSequenceNumber.toString().padStart(3)}` : "        ";
-                    debugOp(`${index}: ${type}: seq: ${msg.sequenceNumber.toString().padStart(3)} `
-                        + `${clientSeq} min: ${msg.minimumSequenceNumber.toString().padStart(3)} `
-                        + `${msg.type} ${getContentsString(msg.type, msg.contents)}`);
+                    const clientSeq = msg.clientId === container.clientId
+                        ? `cli: ${msg.clientSequenceNumber.toString().padStart(3)}`
+                        : "        ";
+                    debugOp(`${index}: ${type}: seq: ${msg.sequenceNumber.toString().padStart(3)} ` +
+                        `${clientSeq} min: ${msg.minimumSequenceNumber
+                            .toString()
+                            .padStart(3)} ` +
+                        `${msg.type} ${getContentsString(msg.type, msg.contents)}`);
                 };
             };
             container.deltaManager.inbound.on("push", getInboundHandler("IN "));