@fluidframework/container-runtime 0.51.2 → 0.52.1

package/lib/containerRuntime.js

@@ -3,12 +3,12 @@
  * Licensed under the MIT License.
  */
 import { AttachState, } from "@fluidframework/container-definitions";
-import { assert, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
+import { assert, Trace, TypedEventEmitter, unreachableCase, performance, } from "@fluidframework/common-utils";
 import { ChildLogger, raiseConnectedEvent, PerformanceEvent, normalizeError, TaggedLoggerAdapter, } from "@fluidframework/telemetry-utils";
 import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
-import { DataCorruptionError, GenericError } from "@fluidframework/container-utils";
+import { DataCorruptionError, GenericError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
 import { BlobTreeEntry, TreeTreeEntry, } from "@fluidframework/protocol-base";
-import { MessageType, } from "@fluidframework/protocol-definitions";
+import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
 import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
 import { addBlobToSummary, addTreeToSummary, convertToSummaryTree, createRootSummarizerNodeWithGC, FluidSerializer, RequestParser, create404Response, exceptionToResponse, responseToException, seqFromTree, } from "@fluidframework/runtime-utils";
 import { v4 as uuid } from "uuid";
@@ -17,7 +17,7 @@ import { FluidDataStoreRegistry } from "./dataStoreRegistry";
 import { Summarizer } from "./summarizer";
 import { formRequestSummarizerFn, SummaryManager } from "./summaryManager";
 import { DeltaScheduler } from "./deltaScheduler";
-import { ReportOpPerfTelemetry } from "./connectionTelemetry";
+import { ReportOpPerfTelemetry, latencyThreshold } from "./connectionTelemetry";
 import { PendingStateManager } from "./pendingStateManager";
 import { pkgVersion } from "./packageVersion";
 import { BlobManager } from "./blobManager";
@@ -92,13 +92,16 @@ export function unpackRuntimeMessage(message) {
     }
     return message;
 }
-export class ScheduleManager {
-    constructor(deltaManager, emitter, logger) {
+/**
+ * This class controls pausing and resuming of inbound queue to ensure that we never
+ * start processing ops in a batch IF we do not have all ops in the batch.
+ */
+class ScheduleManagerCore {
+    constructor(deltaManager, logger) {
         this.deltaManager = deltaManager;
-        this.emitter = emitter;
         this.logger = logger;
         this.localPaused = false;
-        this.deltaScheduler = new DeltaScheduler(this.deltaManager, ChildLogger.create(this.logger, "DeltaScheduler"));
+        this.timePaused = 0;
         // Listen for delta manager sends and add batch metadata to messages
         this.deltaManager.on("prepareSend", (messages) => {
             if (messages.length === 0) {
@@ -106,7 +109,7 @@ export class ScheduleManager {
             }
             // First message will have the batch flag set to true if doing a batched send
             const firstMessageMetadata = messages[0].metadata;
-            if (!firstMessageMetadata || !firstMessageMetadata.batch) {
+            if (!(firstMessageMetadata === null || firstMessageMetadata === void 0 ? void 0 : firstMessageMetadata.batch)) {
                 return;
             }
             // If the batch contains only a single op, clear the batch flag.
@@ -121,29 +124,143 @@ export class ScheduleManager {
         // Listen for updates and peek at the inbound
         this.deltaManager.inbound.on("push", (message) => {
             this.trackPending(message);
-            this.updatePauseState(message.sequenceNumber);
         });
+        // Start with baseline - empty inbound queue.
+        assert(!this.localPaused, 0x293 /* "initial state" */);
         const allPending = this.deltaManager.inbound.toArray();
         for (const pending of allPending) {
             this.trackPending(pending);
         }
-        // Based on track pending update the pause state
-        this.updatePauseState(this.deltaManager.lastSequenceNumber);
     }
-    beginOperation(message) {
+    /**
+     * The only public function in this class - called when we processed an op,
+     * to make decision if op processing should be paused or not afer that.
+     */
+    afterOpProcessing(sequenceNumber) {
+        assert(!this.localPaused, 0x294 /* "can't have op processing paused if we are processing an op" */);
+        // If the inbound queue is ever empty, nothing to do!
+        if (this.deltaManager.inbound.length === 0) {
+            assert(this.pauseSequenceNumber === undefined, 0x295 /* "there should be no pending batch if we have no ops" */);
+            return;
+        }
+        // The queue is
+        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
+        //    - here (processing ops until reaching start of incomplete batch)
+        //    - in trackPending(), when queue was empty and start of batch showed up.
+        // 2. resumed when batch end comes in (in trackPending())
+        // do we have incomplete batch to worry about?
+        if (this.pauseSequenceNumber !== undefined) {
+            assert(sequenceNumber < this.pauseSequenceNumber, 0x296 /* "we should never start processing incomplete batch!" */);
+            // If the next op is the start of incomplete batch, then we can't process it until it's fully in - pause!
+            if (sequenceNumber + 1 === this.pauseSequenceNumber) {
+                this.pauseQueue();
+            }
+        }
+    }
+    pauseQueue() {
+        assert(!this.localPaused, 0x297 /* "always called from resumed state" */);
+        this.localPaused = true;
+        this.timePaused = performance.now();
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        this.deltaManager.inbound.pause();
+    }
+    resumeQueue(startBatch, endBatch) {
+        // Return early if no change in value
+        if (!this.localPaused) {
+            return;
+        }
+        this.localPaused = false;
+        const duration = performance.now() - this.timePaused;
+        // Random round number - we want to know when batch waiting paused op processing.
+        if (duration > latencyThreshold) {
+            this.logger.sendErrorEvent({
+                eventName: "MaxBatchWaitTimeExceeded",
+                duration,
+                sequenceNumber: endBatch,
+                length: endBatch - startBatch,
+            });
+        }
+        this.deltaManager.inbound.resume();
+    }
+    /**
+     * Called for each incoming op (i.e. inbound "push" notification)
+     */
+    trackPending(message) {
+        assert(this.deltaManager.inbound.length !== 0, 0x298 /* "we have something in the queue that generates this event" */);
+        assert((this.currentBatchClientId === undefined) === (this.pauseSequenceNumber === undefined), 0x299 /* "non-synchronized state" */);
+        const metadata = message.metadata;
+        const batchMetadata = metadata === null || metadata === void 0 ? void 0 : metadata.batch;
+        // Protocol messages are never part of a runtime batch of messages
+        if (!isRuntimeMessage(message)) {
+            // Protocol messages should never show up in the middle of the batch!
+            assert(this.currentBatchClientId === undefined, 0x29a /* "System message in the middle of batch!" */);
+            assert(batchMetadata === undefined, 0x29b /* "system op in a batch?" */);
+            assert(!this.localPaused, 0x29c /* "we should be processing ops when there is no active batch" */);
+            return;
+        }
+        if (this.currentBatchClientId === undefined && batchMetadata === undefined) {
+            assert(!this.localPaused, 0x29d /* "we should be processing ops when there is no active batch" */);
+            return;
+        }
+        // If the client ID changes then we can move the pause point. If it stayed the same then we need to check.
+        // If batchMetadata is not undefined then if it's true we've begun a new batch - if false we've ended
+        // the previous one
+        if (this.currentBatchClientId !== undefined || batchMetadata === false) {
+            if (this.currentBatchClientId !== message.clientId) {
+                // "Batch not closed, yet message from another client!"
+                throw new DataCorruptionError("OpBatchIncomplete", Object.assign({ batchClientId: this.currentBatchClientId }, extractSafePropertiesFromMessage(message)));
+            }
+        }
+        // The queue is
+        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
+        //    - in afterOpProcessing() - processing ops until reaching start of incomplete batch
+        //    - here (batchMetadata == false below), when queue was empty and start of batch showed up.
+        // 2. resumed when batch end comes in (batchMetadata === true case below)
+        if (batchMetadata) {
+            assert(this.currentBatchClientId === undefined, 0x29e /* "there can't be active batch" */);
+            assert(!this.localPaused, 0x29f /* "we should be processing ops when there is no active batch" */);
+            this.pauseSequenceNumber = message.sequenceNumber;
+            this.currentBatchClientId = message.clientId;
+            // Start of the batch
+            // Only pause processing if queue has no other ops!
+            // If there are any other ops in the queue, processing will be stopped when they are processed!
+            if (this.deltaManager.inbound.length === 1) {
+                this.pauseQueue();
+            }
+        }
+        else if (batchMetadata === false) {
+            assert(this.pauseSequenceNumber !== undefined, 0x2a0 /* "batch presence was validated above" */);
+            // Batch is complete, we can process it!
+            this.resumeQueue(this.pauseSequenceNumber, message.sequenceNumber);
+            this.pauseSequenceNumber = undefined;
+            this.currentBatchClientId = undefined;
+        }
+        else {
+            // Continuation of current batch. Do nothing
+            assert(this.currentBatchClientId !== undefined, 0x2a1 /* "logic error" */);
+        }
+    }
+}
+/**
+ * This class has the following responsibilities:
+ * 1. It tracks batches as we process ops and raises "batchBegin" and "batchEnd" events.
+ *    As part of it, it validates batch correctness (i.e. no system ops in the middle of batch)
+ * 2. It creates instance of ScheduleManagerCore that ensures we never start processing ops from batch
+ *    unless all ops of the batch are in.
+ */
+export class ScheduleManager {
+    constructor(deltaManager, emitter, logger) {
+        this.deltaManager = deltaManager;
+        this.emitter = emitter;
+        this.logger = logger;
+        this.hitError = false;
+        this.deltaScheduler = new DeltaScheduler(this.deltaManager, ChildLogger.create(this.logger, "DeltaScheduler"));
+        this.scheduler = new ScheduleManagerCore(deltaManager, logger);
+    }
+    beforeOpProcessing(message) {
         var _a;
         if (this.batchClientId !== message.clientId) {
-            // As a back stop for any bugs marking the end of a batch - if the client ID flipped, we
-            // consider the previous batch over.
-            if (this.batchClientId) {
-                this.emitter.emit("batchEnd", "Did not receive real batchEnd message", undefined);
-                this.deltaScheduler.batchEnd();
-                this.logger.sendTelemetryEvent({
-                    eventName: "BatchEndNotReceived",
-                    clientId: this.batchClientId,
-                    sequenceNumber: message.sequenceNumber,
-                });
-            }
+            assert(this.batchClientId === undefined, 0x2a2 /* "Batch is interrupted by other client op. Should be caught by trackPending()" */);
             // This could be the beginning of a new batch or an individual message.
             this.emitter.emit("batchBegin", message);
             this.deltaScheduler.batchBegin();
@@ -156,79 +273,32 @@ export class ScheduleManager {
             }
         }
     }
-    endOperation(error, message) {
+    afterOpProcessing(error, message) {
         var _a;
+        // If this is no longer true, we need to revisit what we do where we set this.hitError.
+        assert(!this.hitError, 0x2a3 /* "container should be closed on any error" */);
+        // Let the scheduler know how far we progressed, to decide if op processing
+        // should be paused or not.
+        this.scheduler.afterOpProcessing(message.sequenceNumber);
         if (error) {
+            // We assume here that loader will close container and stop processing all future ops.
+            // This is implicit dependency. If this flow changes, this code might no longer be correct.
+            this.hitError = true;
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", error, message);
             this.deltaScheduler.batchEnd();
             return;
         }
-        this.updatePauseState(message.sequenceNumber);
         const batch = (_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.batch;
         // If no batchClientId has been set then we're in an individual batch. Else, if we get
         // batch end metadata, this is end of the current batch.
-        if (!this.batchClientId || batch === false) {
+        if (this.batchClientId === undefined || batch === false) {
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", undefined, message);
             this.deltaScheduler.batchEnd();
             return;
         }
     }
-    setPaused(localPaused) {
-        // Return early if no change in value
-        if (this.localPaused === localPaused) {
-            return;
-        }
-        this.localPaused = localPaused;
-        if (localPaused) {
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-            this.deltaManager.inbound.pause();
-        }
-        else {
-            this.deltaManager.inbound.resume();
-        }
-    }
-    updatePauseState(sequenceNumber) {
-        // If the inbound queue is ever empty we pause it and wait for new events
-        if (this.deltaManager.inbound.length === 0) {
-            this.setPaused(true);
-            return;
-        }
-        // If no message has caused the pause flag to be set, or the next message up is not the one we need to pause at
-        // then we simply continue processing
-        if (!this.pauseSequenceNumber || sequenceNumber + 1 < this.pauseSequenceNumber) {
-            this.setPaused(false);
-        }
-        else {
-            // Otherwise the next message requires us to pause
-            this.setPaused(true);
-        }
-    }
-    trackPending(message) {
-        const metadata = message.metadata;
-        // Protocol messages are never part of a runtime batch of messages
-        if (!isRuntimeMessage(message)) {
-            this.pauseSequenceNumber = undefined;
-            this.pauseClientId = undefined;
-            return;
-        }
-        const batchMetadata = metadata ? metadata.batch : undefined;
-        // If the client ID changes then we can move the pause point. If it stayed the same then we need to check.
-        if (this.pauseClientId === message.clientId) {
-            if (batchMetadata !== undefined) {
-                // If batchMetadata is not undefined then if it's true we've begun a new batch - if false we've ended
-                // the previous one
-                this.pauseSequenceNumber = batchMetadata ? message.sequenceNumber : undefined;
-                this.pauseClientId = batchMetadata ? this.pauseClientId : undefined;
-            }
-        }
-        else {
-            // We check the batch flag for the new clientID - if true we pause otherwise we reset the tracking data
-            this.pauseSequenceNumber = batchMetadata ? message.sequenceNumber : undefined;
-            this.pauseClientId = batchMetadata ? message.clientId : undefined;
-        }
-    }
 }
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
@@ -299,9 +369,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             else {
                 // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // generateSummaries is turned off. We are throwing instead of returning a failure here,
+                // disableSummaries is turned on. We are throwing instead of returning a failure here,
                 // because it is a misuse of the API rather than an expected failure.
-                throw new Error(`Can't summarize, generateSummaries: ${this.runtimeOptions.summaryOptions.generateSummaries}`);
+                throw new Error(`Can't summarize, disableSummaries: ${this.summariesDisabled()}`);
             }
         };
         this.enqueueSummarize = (...args) => {
@@ -315,7 +385,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // If we're not the summarizer, and we don't have a summaryManager, we expect that
                 // generateSummaries is turned off. We are throwing instead of returning a failure here,
                 // because it is a misuse of the API rather than an expected failure.
-                throw new Error(`Can't summarize, generateSummaries: ${this.runtimeOptions.summaryOptions.generateSummaries}`);
+                throw new Error(`Can't summarize, disableSummaries: ${this.summariesDisabled()}`);
             }
         };
         this.baseSummaryMessage = metadata === null || metadata === void 0 ? void 0 : metadata.message;
@@ -364,7 +434,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
         // Only create a SummaryManager if summaries are enabled and we are not the summarizer client
+        // Map the deprecated generateSummaries flag to disableSummaries.
         if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
+            this.runtimeOptions.summaryOptions.disableSummaries = true;
+        }
+        if (this.summariesDisabled()) {
             this._logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
         }
         else {
@@ -454,7 +528,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 runtimeVersion: pkgVersion,
             },
         });
-        const { summaryOptions = { generateSummaries: true }, gcOptions = {}, loadSequenceNumberVerification = "close", } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", } = runtimeOptions;
         // We pack at data store level only. If isolated channels are disabled,
         // then there are no .channel layers, we pack at level 1, otherwise we pack at level 2
         const packingLevel = summaryOptions.disableIsolatedChannels ? 1 : 2;
@@ -505,7 +579,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" && runtimeSequenceNumber !== protocolSequenceNumber) {
-                const error = new DataCorruptionError("Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber", { runtimeSequenceNumber, protocolSequenceNumber });
+                // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
+                const error = new DataCorruptionError("SummaryMetadataMismatch", { runtimeSequenceNumber, protocolSequenceNumber });
                 if (loadSequenceNumberVerification === "log") {
                     logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
                 }
@@ -842,11 +917,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         // We do not need to make deep copy, as each layer will just replace message.content itself,
         // but would not modify contents details
         let message = Object.assign({}, messageArg);
-        let error;
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
-        this.scheduleManager.beginOperation(message);
+        this.scheduleManager.beforeOpProcessing(message);
         try {
             message = unpackRuntimeMessage(message);
             // Chunk processing must come first given that we will transform the message to the unchunked version
@@ -874,14 +948,12 @@ export class ContainerRuntime extends TypedEventEmitter {
                 default:
             }
             this.emit("op", message);
+            this.scheduleManager.afterOpProcessing(undefined, message);
         }
         catch (e) {
-            error = e;
+            this.scheduleManager.afterOpProcessing(e, message);
             throw e;
         }
-        finally {
-            this.scheduleManager.endOperation(error, message);
-        }
     }
     processSignal(message, local) {
         const envelope = message.content;
@@ -1093,7 +1165,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
         }
         const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
-        assert(summarizeResult.summary.type === 1 /* Tree */, 0x12f /* "Container Runtime's summarize should always return a tree" */);
+        assert(summarizeResult.summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
         return summarizeResult;
     }
     /**
@@ -1209,8 +1281,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
             const dataStoreTree = this.disableIsolatedChannels ? summaryTree : summaryTree.tree[channelsTreeName];
-            assert(dataStoreTree.type === 1 /* Tree */, 0x1fc /* "summary is not a tree" */);
-            const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === 3 /* Handle */).length;
+            assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
+            const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === SummaryType.Handle).length;
             const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount }, partialStats);
             const generateSummaryData = {
                 referenceSequenceNumber: summaryRefSeqNum,
@@ -1487,6 +1559,14 @@ export class ContainerRuntime extends TypedEventEmitter {
     getPendingLocalState() {
         return this.pendingStateManager.getLocalState();
     }
+    /**
+     * @returns true if summaries are explicitly disabled for this ContainerRuntime, false otherwise
+     */
+    summariesDisabled() {
+        var _a;
+        return this.runtimeOptions.summaryOptions.disableSummaries === true ||
+            ((_a = this.runtimeOptions.summaryOptions.summaryConfigOverrides) === null || _a === void 0 ? void 0 : _a.disableSummaries) === true;
+    }
 }
 /**
  * Wait for a specific sequence number. Promise should resolve when we reach that number,