@fluidframework/test-utils 2.0.0-internal.4.0.6 → 2.0.0-internal.4.1.0

package/CHANGELOG.md

@@ -0,0 +1,5 @@
+# @fluidframework/test-utils
+
+## 2.0.0-internal.4.1.0
+
+Dependency updates only.

package/dist/loaderContainerTracker.d.ts

@@ -29,34 +29,29 @@ export declare class LoaderContainerTracker implements IOpProcessingController {
      * Reset the tracker, closing all containers and stop tracking them.
      */
     reset(): void;
-    /**
-     * Ensure all tracked containers are synchronized
-     */
-    ensureSynchronized(...containers: IContainer[]): Promise<void>;
     /**
      * Ensure all tracked containers are synchronized with a time limit
+     *
+     * @deprecated - this method is equivalent to @see {@link LoaderContainerTracker.ensureSynchronized}, please configure the test timeout instead
      */
     ensureSynchronizedWithTimeout?(timeoutDuration: number | undefined, ...containers: IContainer[]): Promise<void>;
     /**
      * Make sure all the tracked containers are synchronized.
      *
      * No isDirty (non-readonly) containers
-     *
      * No extra clientId in quorum of any container that is not tracked and still opened.
-     *
      * - i.e. no pending Join/Leave message.
-     *
      * No unresolved proposal (minSeqNum \>= lastProposalSeqNum)
-     *
      * lastSequenceNumber of all container is the same
-     *
      * clientSequenceNumberObserved is the same as clientSequenceNumber sent
-     *
      * - this overlaps with !isDirty, but include task scheduler ops.
-     *
      * - Trailing NoOp is tracked and don't count as pending ops.
+     *
+     * Containers that are already pause will resume process and paused again once
+     * everything is synchronized.  Containers that aren't paused will remain unpaused when this
+     * function returns.
      */
-    private processSynchronized;
+    ensureSynchronized(...containers: IContainer[]): Promise<void>;
     /**
      * Utility to calculate the set of clientId per container in quorum that is NOT associated with
      * any container we tracked, indicating there is a pending join or leave op that we need to wait.
@@ -70,7 +65,8 @@ export declare class LoaderContainerTracker implements IOpProcessingController {
      *
      * @param containersToApply - the set of containers to check
      */
-    private isSequenceNumberSynchronized;
+    private needSequenceNumberSynchronize;
+    private containerIndexStrings;
     /**
      * Utility to wait for any clientId in quorum that is NOT associated with any container we
      * tracked, indicating there is a pending join or leave op that we need to wait.
@@ -92,18 +88,37 @@ export declare class LoaderContainerTracker implements IOpProcessingController {
     /**
      * Pause all queue activities on the containers given, or all tracked containers
      * Any containers given that is not tracked will be ignored.
+     *
+     * When a container is paused, it is assumed that we want fine grain control over op
+     * sequencing.  This function will prepare the container and force it into write mode to
+     * avoid missing join messages or change the sequence of event when switching from read to
+     * write mode.
      */
     pauseProcessing(...containers: IContainer[]): Promise<void>;
+    /**
+     * When a container is paused, it is assumed that we want fine grain control over op
+     * sequencing.  This function will prepare the container and force it into write mode to
+     * avoid missing join messages or change the sequence of event when switching from read to
+     * write mode.
+     *
+     * @param container - the container to pause
+     * @param record - the record for the container
+     */
+    private pauseContainer;
     /**
      * Pause all queue activities on all tracked containers, and resume only
      * inbound to process ops until it is idle. All queues are left in the paused state
-     * after the function
+     * after the function.
+     *
+     * Pausing will switch the container to write mode. See `pauseProcessing`
      */
     processIncoming(...containers: IContainer[]): Promise<void>;
     /**
      * Pause all queue activities on all tracked containers, and resume only
      * outbound to process ops until it is idle. All queues are left in the paused state
-     * after the function
+     * after the function.
+     *
+     * Pausing will switch the container to write mode. See `pauseProcessing`
      */
     processOutgoing(...containers: IContainer[]): Promise<void>;
     /**

package/dist/loaderContainerTracker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loaderContainerTracker.d.ts","sourceRoot":"","sources":["../src/loaderContainerTracker.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAe,WAAW,EAAE,MAAM,uCAAuC,CAAC;AAS7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAsB/D,qBAAa,sBAAuB,YAAW,uBAAuB;IAIzD,OAAO,CAAC,QAAQ,CAAC,qBAAqB;IAHlD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA0C;IACrE,OAAO,CAAC,kBAAkB,CAAa;gBAEV,qBAAqB,GAAE,OAAe;IAEnE;;;OAGG;IACI,GAAG,CAAC,UAAU,SAAS,WAAW,EAAE,MAAM,EAAE,UAAU;IAmB7D;;;;OAIG;IACH,OAAO,CAAC,YAAY;IA2BpB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IAuC1B,OAAO,CAAC,iBAAiB;IAQzB;;OAEG;IACI,KAAK;IAUZ;;OAEG;IACU,kBAAkB,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACU,6BAA6B,CAAC,CAC1C,eAAe,EAAE,MAAM,GAAG,SAAS,EACnC,GAAG,UAAU,EAAE,UAAU,EAAE;IAK5B;;;;;;;;;;;;;;;;;;OAkBG;YACW,mBAAmB;IA+EjC;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IA8BzB;;;;;OAKG;IACH,OAAO,CAAC,4BAA4B;IA4CpC;;;;;;;OAOG;YACW,qBAAqB;IAoCnC;;;OAGG;YACW,oBAAoB;IAclC;;OAEG;IACI,gBAAgB,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAgBnD;;;OAGG;IACU,eAAe,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAexD;;;;OAIG;IACU,eAAe,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAIxD;;;;OAIG;IACU,eAAe,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAIxD;;OAEG;YACW,YAAY;IA0C1B;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;IA2BzB;;OAEG;IACH,OAAO,CAAC,UAAU;IAwElB;;;;OAIG;IACH,OAAO,CAAC,aAAa;CAKrB"}
+{"version":3,"file":"loaderContainerTracker.d.ts","sourceRoot":"","sources":["../src/loaderContainerTracker.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAe,WAAW,EAAE,MAAM,uCAAuC,CAAC;AAU7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAuB/D,qBAAa,sBAAuB,YAAW,uBAAuB;IAIzD,OAAO,CAAC,QAAQ,CAAC,qBAAqB;IAHlD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA0C;IACrE,OAAO,CAAC,kBAAkB,CAAa;gBAEV,qBAAqB,GAAE,OAAe;IAEnE;;;OAGG;IACI,GAAG,CAAC,UAAU,SAAS,WAAW,EAAE,MAAM,EAAE,UAAU;IAmB7D;;;;OAIG;IACH,OAAO,CAAC,YAAY;IA2BpB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IAuC1B,OAAO,CAAC,iBAAiB;IAQzB;;OAEG;IACI,KAAK;IAUZ;;;;OAIG;IACU,6BAA6B,CAAC,CAC1C,eAAe,EAAE,MAAM,GAAG,SAAS,EACnC,GAAG,UAAU,EAAE,UAAU,EAAE;IAK5B;;;;;;;;;;;;;;;OAeG;IACU,kBAAkB,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA2E3E;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IA8BzB;;;;;OAKG;IACH,OAAO,CAAC,6BAA6B;IAmErC,OAAO,CAAC,qBAAqB;IAO7B;;;;;;;OAOG;YACW,qBAAqB;IAoCnC;;;OAGG;YACW,oBAAoB;IAclC;;OAEG;IACI,gBAAgB,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAoBnD;;;;;;;;OAQG;IACU,eAAe,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAexD;;;;;;;;OAQG;YACW,cAAc;IAwD5B;;;;;;OAMG;IACU,eAAe,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAIxD;;;;;;OAMG;IACU,eAAe,CAAC,GAAG,UAAU,EAAE,UAAU,EAAE;IAIxD;;OAEG;YACW,YAAY;IAgD1B;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;IA2BzB;;OAEG;IACH,OAAO,CAAC,UAAU;IAwElB;;;;OAIG;IACH,OAAO,CAAC,aAAa;CAKrB"}

package/dist/loaderContainerTracker.js

@@ -9,6 +9,7 @@ const common_utils_1 = require("@fluidframework/common-utils");
 const container_loader_1 = require("@fluidframework/container-loader");
 const driver_utils_1 = require("@fluidframework/driver-utils");
 const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
+const containerUtils_1 = require("./containerUtils");
 const debug_1 = require("./debug");
 const timeoutUtils_1 = require("./timeoutUtils");
 const debugOp = debug_1.debug.extend("ops");
@@ -126,47 +127,44 @@ class LoaderContainerTracker {
         this.containers.clear();
         // REVIEW: do we need to unpatch the loaders?
     }
-    /**
-     * Ensure all tracked containers are synchronized
-     */
-    async ensureSynchronized(...containers) {
-        await this.processSynchronized(undefined, ...containers);
-    }
     /**
      * Ensure all tracked containers are synchronized with a time limit
+     *
+     * @deprecated - this method is equivalent to @see {@link LoaderContainerTracker.ensureSynchronized}, please configure the test timeout instead
      */
     async ensureSynchronizedWithTimeout(timeoutDuration, ...containers) {
-        await this.processSynchronized(timeoutDuration, ...containers);
+        await this.ensureSynchronized(...containers);
     }
     /**
      * Make sure all the tracked containers are synchronized.
      *
      * No isDirty (non-readonly) containers
-     *
      * No extra clientId in quorum of any container that is not tracked and still opened.
-     *
      * - i.e. no pending Join/Leave message.
-     *
      * No unresolved proposal (minSeqNum \>= lastProposalSeqNum)
-     *
      * lastSequenceNumber of all container is the same
-     *
      * clientSequenceNumberObserved is the same as clientSequenceNumber sent
-     *
      * - this overlaps with !isDirty, but include task scheduler ops.
-     *
      * - Trailing NoOp is tracked and don't count as pending ops.
+     *
+     * Containers that are already pause will resume process and paused again once
+     * everything is synchronized.  Containers that aren't paused will remain unpaused when this
+     * function returns.
      */
-    async processSynchronized(timeoutDuration, ...containers) {
+    async ensureSynchronized(...containers) {
         const resumed = this.resumeProcessing(...containers);
-        let waitingSequenceNumberSynchronized = false;
+        let waitingSequenceNumberSynchronized;
         // eslint-disable-next-line no-constant-condition
         while (true) {
+            // yield a turn to allow side effect of resuming or the ops we just processed execute before we check
+            await new Promise((resolve) => {
+                setTimeout(resolve, 0);
+            });
             const containersToApply = this.getContainers(containers);
             if (containersToApply.length === 0) {
                 break;
             }
-            // Ignore readonly dirty containers, because it can't sent up and nothing can be done about it being dirty
+            // Ignore readonly dirty containers, because it can't sent ops and nothing can be done about it being dirty
             const dirtyContainers = containersToApply.filter((c) => {
                 const { deltaManager, isDirty } = c;
                 return deltaManager.readOnlyInfo.readonly !== true && isDirty;
@@ -175,21 +173,23 @@ class LoaderContainerTracker {
                 // Wait for all the leave messages
                 const pendingClients = this.getPendingClients(containersToApply);
                 if (pendingClients.length === 0) {
-                    if (this.isSequenceNumberSynchronized(containersToApply)) {
+                    const needSync = this.needSequenceNumberSynchronize(containersToApply);
+                    if (needSync === undefined) {
                         // done, we are in sync
                         break;
                     }
-                    if (!waitingSequenceNumberSynchronized) {
-                        // Only write it out once
-                        waitingSequenceNumberSynchronized = true;
-                        debugWait("Waiting for sequence number synchronized");
-                        await (0, timeoutUtils_1.timeoutAwait)(this.waitForAnyInboundOps(containersToApply), {
-                            errorMsg: "Timeout on waiting for sequence number synchronized",
-                        });
+                    if (waitingSequenceNumberSynchronized !== needSync.reason) {
+                        // Don't repeat writing to console if it is the same reason
+                        waitingSequenceNumberSynchronized = needSync.reason;
+                        debugWait(needSync.message);
                     }
+                    // Wait for one inbounds ops which might change the state of things
+                    await (0, timeoutUtils_1.timeoutAwait)(this.waitForAnyInboundOps(containersToApply), {
+                        errorMsg: `Timeout on ${needSync.message}`,
+                    });
                 }
                 else {
-                    waitingSequenceNumberSynchronized = false;
+                    waitingSequenceNumberSynchronized = undefined;
                     await (0, timeoutUtils_1.timeoutAwait)(this.waitForPendingClients(pendingClients), {
                         errorMsg: "Timeout on waiting for pending join or leave op",
                     });
@@ -197,10 +197,8 @@ class LoaderContainerTracker {
             }
             else {
                 // Wait for all the containers to be saved
-                debugWait(`Waiting container to be saved ${dirtyContainers.map(
-                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                (c) => this.containers.get(c).index)}`);
-                waitingSequenceNumberSynchronized = false;
+                debugWait(`Waiting container to be saved ${this.containerIndexStrings(dirtyContainers)}`);
+                waitingSequenceNumberSynchronized = undefined;
                 await Promise.all(dirtyContainers.map(async (c) => Promise.race([
                     (0, timeoutUtils_1.timeoutPromise)((resolve) => c.once("saved", () => resolve()), {
                         errorMsg: "Timeout on waiting a container to be saved",
@@ -208,10 +206,6 @@ class LoaderContainerTracker {
                     new Promise((resolve) => c.once("closed", resolve)),
                 ])));
             }
-            // yield a turn to allow side effect of the ops we just processed execute before we check again
-            await new Promise((resolve) => {
-                setTimeout(resolve, 0);
-            });
         }
         // Pause all container that was resumed
         // don't call pause if resumed is empty and pause everything, which is not what we want
@@ -259,42 +253,62 @@ class LoaderContainerTracker {
      *
      * @param containersToApply - the set of containers to check
      */
-    isSequenceNumberSynchronized(containersToApply) {
+    needSequenceNumberSynchronize(containersToApply) {
+        // If there is a pending proposal, wait for it to be accepted
+        const minSeqNum = containersToApply[0].deltaManager.minimumSequenceNumber;
+        if (minSeqNum < this.lastProposalSeqNum) {
+            return {
+                reason: "Proposal",
+                message: `waiting for MSN to advance to proposal at sequence number ${this.lastProposalSeqNum}`,
+            };
+        }
         // clientSequenceNumber check detects ops in flight, both on the wire and in the outbound queue
         // We need both client sequence number and isDirty check because:
         // - Currently isDirty flag ignores ops for task scheduler, so we need the client sequence number check
         // - But isDirty flags include ops during forceReadonly and disconnected, because we don't submit
         //   the ops in the first place, clientSequenceNumber is not assigned
-        const isClientSequenceNumberSynchronized = containersToApply.every((container) => {
+        const containerWithInflightOps = containersToApply.filter((container) => {
             if (container.deltaManager.readOnlyInfo.readonly === true) {
                 // Ignore readonly container. the clientSeqNum and clientSeqNumObserved might be out of sync
                 // because we transition to readonly when outbound is not empty or the in transit op got lost
-                return true;
+                return false;
             }
             // Note that in read only mode, the op won't be submitted
             let deltaManager = container.deltaManager;
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const { trailingNoOps } = this.containers.get(container);
-            // Back-compat: clientSequenceNumber & clientSequenceNumberObserved moved to ConnectionManager in 0.53
+            // Back-compat: lastSubmittedClientId/clientSequenceNumber/clientSequenceNumberObserved moved to ConnectionManager in 0.53
             if (!("clientSequenceNumber" in deltaManager)) {
                 deltaManager = deltaManager.connectionManager;
             }
             (0, common_utils_1.assert)("clientSequenceNumber" in deltaManager, "no clientSequenceNumber");
             (0, common_utils_1.assert)("clientSequenceNumberObserved" in deltaManager, "no clientSequenceNumber");
-            return (deltaManager.clientSequenceNumber ===
-                deltaManager.clientSequenceNumberObserved + trailingNoOps);
+            // If last submittedClientId isn't the current clientId, then we haven't send any ops
+            return (deltaManager.lastSubmittedClientId === container.clientId &&
+                deltaManager.clientSequenceNumber !==
+                    deltaManager.clientSequenceNumberObserved + trailingNoOps);
         });
-        if (!isClientSequenceNumberSynchronized) {
-            return false;
-        }
-        const minSeqNum = containersToApply[0].deltaManager.minimumSequenceNumber;
-        if (minSeqNum < this.lastProposalSeqNum) {
-            // There is an unresolved proposal
-            return false;
+        if (containerWithInflightOps.length !== 0) {
+            return {
+                reason: "InflightOps",
+                message: `waiting for containers with inflight ops: ${this.containerIndexStrings(containerWithInflightOps)}`,
+            };
         }
         // Check to see if all the container has process the same number of ops.
-        const seqNum = containersToApply[0].deltaManager.lastSequenceNumber;
-        return containersToApply.every((c) => c.deltaManager.lastSequenceNumber === seqNum);
+        const maxSeqNum = Math.max(...containersToApply.map((c) => c.deltaManager.lastSequenceNumber));
+        const containerWithPendingIncoming = containersToApply.filter((c) => c.deltaManager.lastSequenceNumber !== maxSeqNum);
+        if (containerWithPendingIncoming.length !== 0) {
+            return {
+                reason: "Pending",
+                message: `waiting for containers with pending incoming ops up to sequence number ${maxSeqNum}: ${this.containerIndexStrings(containerWithPendingIncoming)}`,
+            };
+        }
+        return undefined;
+    }
+    containerIndexStrings(containers) {
+        return containers.map(
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        (c) => this.containers.get(c).index);
     }
     /**
      * Utility to wait for any clientId in quorum that is NOT associated with any container we
@@ -356,6 +370,7 @@ class LoaderContainerTracker {
         const containersToApply = this.getContainers(containers);
         for (const container of containersToApply) {
             const record = this.containers.get(container);
+            (0, common_utils_1.assert)((record === null || record === void 0 ? void 0 : record.pauseP) === undefined, "Cannot resume container while pausing is in progress");
             if ((record === null || record === void 0 ? void 0 : record.paused) === true) {
                 debugWait(`${record.index}: container resumed`);
                 container.deltaManager.inbound.resume();
@@ -369,25 +384,86 @@ class LoaderContainerTracker {
     /**
      * Pause all queue activities on the containers given, or all tracked containers
      * Any containers given that is not tracked will be ignored.
+     *
+     * When a container is paused, it is assumed that we want fine grain control over op
+     * sequencing.  This function will prepare the container and force it into write mode to
+     * avoid missing join messages or change the sequence of event when switching from read to
+     * write mode.
      */
     async pauseProcessing(...containers) {
-        const pauseP = [];
+        const waitP = [];
         const containersToApply = this.getContainers(containers);
         for (const container of containersToApply) {
             const record = this.containers.get(container);
             if (record !== undefined && !record.paused) {
-                debugWait(`${record.index}: container paused`);
-                pauseP.push(container.deltaManager.inbound.pause());
-                pauseP.push(container.deltaManager.outbound.pause());
-                record.paused = true;
+                if (record.pauseP === undefined) {
+                    record.pauseP = this.pauseContainer(container, record);
+                }
+                waitP.push(record.pauseP);
             }
         }
-        await Promise.all(pauseP);
+        await Promise.all(waitP);
+    }
+    /**
+     * When a container is paused, it is assumed that we want fine grain control over op
+     * sequencing.  This function will prepare the container and force it into write mode to
+     * avoid missing join messages or change the sequence of event when switching from read to
+     * write mode.
+     *
+     * @param container - the container to pause
+     * @param record - the record for the container
+     */
+    async pauseContainer(container, record) {
+        debugWait(`${record.index}: pausing container`);
+        (0, common_utils_1.assert)(!container.deltaManager.outbound.paused, "Container should not be paused yet");
+        (0, common_utils_1.assert)(!container.deltaManager.inbound.paused, "Container should not be paused yet");
+        // Pause outbound
+        debugWait(`${record.index}: pausing container outbound queues`);
+        await container.deltaManager.outbound.pause();
+        // Ensure the container is connected first.
+        if (container.connectionState !== container_loader_1.ConnectionState.Connected) {
+            debugWait(`${record.index}: Wait for container connection`);
+            await (0, containerUtils_1.waitForContainerConnection)(container);
+        }
+        // Check if the container is in write mode
+        if (!container.deltaManager.active) {
+            let proposalP;
+            if (container.deltaManager.outbound.idle) {
+                // Need to generate an op to force write mode
+                debugWait(`${record.index}: container force write connection`);
+                const maybeContainer = container;
+                const codeProposal = maybeContainer.getLoadedCodeDetails
+                    ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                        container.getLoadedCodeDetails()
+                    : container.chaincodePackage;
+                proposalP = container.proposeCodeDetails(codeProposal);
+            }
+            // Wait for nack
+            debugWait(`${record.index}: Wait for container disconnect`);
+            container.deltaManager.outbound.resume();
+            await new Promise((resolve) => container.once("disconnected", resolve));
+            const accepted = proposalP ? await proposalP : false;
+            (0, common_utils_1.assert)(!accepted, "A proposal in read mode should be rejected");
+            await container.deltaManager.outbound.pause();
+            // Ensure the container is reconnect.
+            if (container.connectionState !== container_loader_1.ConnectionState.Connected) {
+                debugWait(`${record.index}: Wait for container reconnection`);
+                await (0, containerUtils_1.waitForContainerConnection)(container);
+            }
+        }
+        debugWait(`${record.index}: pausing container inbound queues`);
+        // Pause inbound
+        await container.deltaManager.inbound.pause();
+        debugWait(`${record.index}: container paused`);
+        record.pauseP = undefined;
+        record.paused = true;
     }
     /**
      * Pause all queue activities on all tracked containers, and resume only
      * inbound to process ops until it is idle. All queues are left in the paused state
-     * after the function
+     * after the function.
+     *
+     * Pausing will switch the container to write mode. See `pauseProcessing`
      */
     async processIncoming(...containers) {
         return this.processQueue(containers, (container) => container.deltaManager.inbound);
@@ -395,7 +471,9 @@ class LoaderContainerTracker {
     /**
      * Pause all queue activities on all tracked containers, and resume only
      * outbound to process ops until it is idle. All queues are left in the paused state
-     * after the function
+     * after the function.
+     *
+     * Pausing will switch the container to write mode. See `pauseProcessing`
      */
     async processOutgoing(...containers) {
         return this.processQueue(containers, (container) => container.deltaManager.outbound);
@@ -410,6 +488,7 @@ class LoaderContainerTracker {
         const inflightTracker = new Map();
         const cleanup = [];
         for (const container of containersToApply) {
+            (0, common_utils_1.assert)(container.deltaManager.active, "Container should be connected in write mode already");
             const queue = getQueue(container);
             // track the outgoing ops (if any) to make sure they make the round trip to at least to the same client
             // to make sure they are sequenced.