@fluidframework/container-runtime 0.57.2 → 0.58.1000

package/lib/containerRuntime.js

@@ -26,6 +26,7 @@ import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
 import { GarbageCollector, gcTreeKey, } from "./garbageCollection";
 import { channelToDataStore, isDataStoreAliasMessage, } from "./dataStore";
+import { BindBatchTracker } from "./batchTracker";
 export var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -68,10 +69,16 @@ export var RuntimeHeaders;
     /** True if the request is coming from an IFluidHandle. */
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders || (RuntimeHeaders = {}));
-// Local storage key to set the default flush mode to TurnBased
-const turnBasedFlushModeKey = "Fluid.ContainerRuntime.FlushModeTurnBased";
 const useDataStoreAliasingKey = "Fluid.ContainerRuntime.UseDataStoreAliasing";
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
+// Feature gate for the max op size. If the value is negative, chunking is enabled
+// and all ops over 16k would be chunked. If the value is positive, all ops with
+// a size strictly larger will be rejected and the container closed with an error.
+const maxOpSizeInBytesKey = "Fluid.ContainerRuntime.MaxOpSizeInBytes";
+// By default, we should reject any op larger than 768KB,
+// in order to account for some extra overhead from serialization
+// to not reach the 1MB limits in socket.io and Kafka.
+const defaultMaxOpSizeInBytes = 768000;
 export var RuntimeMessage;
 (function (RuntimeMessage) {
     RuntimeMessage["FluidDataStoreOp"] = "component";
@@ -356,13 +363,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._storage = _storage;
         this.defaultMaxConsecutiveReconnects = 15;
         this._orderSequentiallyCalls = 0;
+        this._flushMode = FlushMode.TurnBased;
         this.needsFlush = false;
         this.flushTrigger = false;
         this.paused = false;
         this.consecutiveReconnects = 0;
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
-        this.summarizerWarning = (warning) => this.mc.logger.sendTelemetryEvent({ eventName: "summarizerWarning" }, warning);
         /**
          * Used to apply stashed ops at their reference sequence number.
          * Normal op processing is synchronous, but applying stashed ops is async since the
@@ -434,11 +441,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.chunkMap = new Map(chunks);
         this.handleContext = new ContainerFluidHandleContext("", this);
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
-        this._flushMode =
-            ((_b = this.mc.config.getBoolean(turnBasedFlushModeKey)) !== null && _b !== void 0 ? _b : false) ? FlushMode.TurnBased : FlushMode.Immediate;
         this._aliasingEnabled =
-            ((_c = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _c !== void 0 ? _c : false) ||
-                ((_d = runtimeOptions.useDataStoreAliasing) !== null && _d !== void 0 ? _d : false);
+            ((_b = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _b !== void 0 ? _b : false) ||
+                ((_c = runtimeOptions.useDataStoreAliasing) !== null && _c !== void 0 ? _c : false);
+        this._maxOpSizeInBytes = ((_d = this.mc.config.getNumber(maxOpSizeInBytesKey)) !== null && _d !== void 0 ? _d : defaultMaxOpSizeInBytes);
         this.maxConsecutiveReconnects = (_e = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _e !== void 0 ? _e : this.defaultMaxConsecutiveReconnects;
         this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), (nodePath) => this.dataStores.getNodePackagePath(nodePath), 
         /**
@@ -523,7 +529,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 formExponentialFn({ coefficient: 20, initialDelay: 0 })), {
                     initialDelayMs: this.runtimeOptions.summaryOptions.initialSummarizerDelayMs,
                 }, this.runtimeOptions.summaryOptions.summarizerOptions);
-                this.summaryManager.on("summarizerWarning", this.summarizerWarning);
                 this.summaryManager.start();
             }
         }
@@ -554,6 +559,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // logging container load stats
         this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryCount: this.summaryCount, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature }));
         ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
+        BindBatchTracker(this, this.logger);
     }
     get IContainerRuntime() { return this; }
     get IFluidRouter() { return this; }
@@ -569,7 +575,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         var _a, _b, _c;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
-        const passLogger = (_a = context.taggedLogger) !== null && _a !== void 0 ? _a : new TaggedLoggerAdapter(context.logger);
+        const backCompatContext = context;
+        const passLogger = (_a = backCompatContext.taggedLogger) !== null && _a !== void 0 ? _a : new TaggedLoggerAdapter(backCompatContext.logger);
         const logger = ChildLogger.create(passLogger, undefined, {
             all: {
                 runtimeVersion: pkgVersion,
@@ -630,7 +637,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" && runtimeSequenceNumber !== protocolSequenceNumber) {
                 // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
-                const error = new DataCorruptionError("SummaryMetadataMismatch", { runtimeSequenceNumber, protocolSequenceNumber });
+                const error = new DataCorruptionError("Summary metadata mismatch", { runtimeSequenceNumber, protocolSequenceNumber });
                 if (loadSequenceNumberVerification === "log") {
                     logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
                 }
@@ -732,7 +739,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             attachState: this.attachState,
         }, error);
         if (this.summaryManager !== undefined) {
-            this.summaryManager.off("summarizerWarning", this.summarizerWarning);
             this.summaryManager.dispose();
         }
         this.garbageCollector.dispose();
@@ -896,7 +902,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (this.consecutiveReconnects === Math.floor(this.maxConsecutiveReconnects / 2)) {
             // If we're halfway through the max reconnects, send an event in order
             // to better identify false positives, if any. If the rate of this event
-            // matches `MaxReconnectsWithNoProgress`, we can safely cut down
+            // matches Container Close count below, we can safely cut down
             // maxConsecutiveReconnects to half.
             this.mc.logger.sendTelemetryEvent({
                 eventName: "ReconnectsWithNoProgress",
@@ -961,7 +967,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.deltaManager.off("op", this.onOp);
             this.context.pendingLocalState = undefined;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(new GenericError("MaxReconnectsWithNoProgress", undefined, // error
+                this.closeFn(new GenericError("Runtime detected too many reconnects with no progress syncing local ops", undefined, // error
                 { attempts: this.consecutiveReconnects }));
                 return;
             }
@@ -1107,7 +1113,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             callback();
         }
         catch (error) {
-            this.closeFn(new GenericError("orderSequentiallyCallbackException", error));
+            this.closeFn(new GenericError("orderSequentially callback exception", error));
             throw error; // throw the original error for the consumer of the runtime
         }
         finally {
@@ -1364,7 +1370,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param options - options controlling how the summary is generated or submitted
      */
     async submitSummary(options) {
-        var _a, _b;
+        var _a, _b, _c;
         const { fullTree, refreshLatestAck, summaryLogger } = options;
         if (refreshLatestAck) {
             const latestSummaryRefSeq = await this.refreshLatestSummaryAckFromServer(ChildLogger.create(summaryLogger, undefined, { all: { safeSummary: true } }));
@@ -1382,6 +1388,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             await this.deltaManager.inbound.pause();
             const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
+            // We should be here is we haven't processed be here. If we are of if the last message's sequence number
+            // doesn't match the last processed sequence number, log an error.
+            if (summaryRefSeqNum !== ((_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.sequenceNumber)) {
+                summaryLogger.sendErrorEvent({
+                    eventName: "LastSequenceMismatch",
+                    message,
+                });
+            }
             this.summarizerNode.startSummary(summaryRefSeqNum, summaryLogger);
             // Helper function to check whether we should still continue between each async step.
             const checkContinue = () => {
@@ -1446,7 +1460,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const dataStoreTree = this.disableIsolatedChannels ? summaryTree : summaryTree.tree[channelsTreeName];
             assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === SummaryType.Handle).length;
-            const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount, gcStateUpdatedDataStoreCount: (_a = summarizeResult.gcStats) === null || _a === void 0 ? void 0 : _a.updatedDataStoreCount }, partialStats);
+            const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount, gcStateUpdatedDataStoreCount: (_b = summarizeResult.gcStats) === null || _b === void 0 ? void 0 : _b.updatedDataStoreCount }, partialStats);
             const generateSummaryData = {
                 referenceSequenceNumber: summaryRefSeqNum,
                 summaryTree,
@@ -1462,7 +1476,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const summaryContext = lastAck === undefined
                 ? {
                     proposalHandle: undefined,
-                    ackHandle: (_b = this.context.getLoadedFromVersion()) === null || _b === void 0 ? void 0 : _b.id,
+                    ackHandle: (_c = this.context.getLoadedFromVersion()) === null || _c === void 0 ? void 0 : _c.id,
                     referenceSequenceNumber: summaryRefSeqNum,
                 }
                 : {
@@ -1593,16 +1607,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                     });
                 }
             }
-            // Note: Chunking will increase content beyond maxOpSize because we JSON'ing JSON payload -
-            // there will be a lot of escape characters that can make it up to 2x bigger!
-            // This is Ok, because DeltaManager.shouldSplit() will have 2 * maxMessageSize limit
-            if (!serializedContent || serializedContent.length <= maxOpSize) {
-                clientSequenceNumber = this.submitRuntimeMessage(type, content, 
-                /* batch: */ this._flushMode === FlushMode.TurnBased, opMetadataInternal);
-            }
-            else {
-                clientSequenceNumber = this.submitChunkedMessage(type, serializedContent, maxOpSize);
-            }
+            clientSequenceNumber = this.submitMaybeChunkedMessages(type, content, serializedContent, maxOpSize, this._flushMode === FlushMode.TurnBased, opMetadataInternal);
         }
         // Let the PendingStateManager know that a message was submitted.
         this.pendingStateManager.onSubmitMessage(type, clientSequenceNumber, this.deltaManager.lastSequenceNumber, content, localOpMetadata, opMetadataInternal);
@@ -1610,6 +1615,35 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.updateDocumentDirtyState(true);
         }
     }
+    submitMaybeChunkedMessages(type, content, serializedContent, serverMaxOpSize, batch, opMetadataInternal = undefined) {
+        if (this._maxOpSizeInBytes >= 0) {
+            // Chunking disabled
+            if (!serializedContent || serializedContent.length <= this._maxOpSizeInBytes) {
+                return this.submitRuntimeMessage(type, content, batch, opMetadataInternal);
+            }
+            // When chunking is disabled, we ignore the server max message size
+            // and if the content length is larger than the client configured message size
+            // instead of splitting the content, we will fail by explicitly close the container
+            this.closeFn(new GenericError("OpTooLarge", 
+            /* error */ undefined, {
+                length: {
+                    value: serializedContent.length,
+                    tag: TelemetryDataTag.PackageData,
+                },
+                limit: {
+                    value: this._maxOpSizeInBytes,
+                    tag: TelemetryDataTag.PackageData,
+                },
+            }));
+            return -1;
+        }
+        // Chunking enabled, fallback on the server's max message size
+        // and split the content accordingly
+        if (!serializedContent || serializedContent.length <= serverMaxOpSize) {
+            return this.submitRuntimeMessage(type, content, batch, opMetadataInternal);
+        }
+        return this.submitChunkedMessage(type, serializedContent, serverMaxOpSize);
+    }
     submitChunkedMessage(type, content, maxOpSize) {
         const contentLength = content.length;
         const chunkN = Math.floor((contentLength - 1) / maxOpSize) + 1;
@@ -1688,6 +1722,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
         const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, async () => this.fetchSnapshotFromStorage(ackHandle, summaryLogger, {
             eventName: "RefreshLatestSummaryGetSnapshot",
+            ackHandle,
+            summaryRefSeq,
             fetchLatest: false,
         }), readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.