@fluidframework/container-runtime 2.0.0-internal.2.2.1 → 2.0.0-internal.2.3.1

package/src/opLifecycle/outbox.ts

@@ -3,6 +3,7 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert } from "@fluidframework/common-utils";
 import { IContainerContext } from "@fluidframework/container-definitions";
 import { GenericError } from "@fluidframework/container-utils";
@@ -12,11 +13,13 @@ import { PendingStateManager } from "../pendingStateManager";
 import { BatchManager } from "./batchManager";
 import { BatchMessage, IBatch } from "./definitions";
 import { OpCompressor } from "./opCompressor";
+import { OpSplitter } from "./opSplitter";
 
 export interface IOutboxConfig {
     readonly compressionOptions: ICompressionRuntimeOptions;
     // The maximum size of a batch that we can send over the wire.
     readonly maxBatchSizeInBytes: number;
+    readonly enableOpReentryCheck?: boolean;
 };
 
 export interface IOutboxParameters {
@@ -25,6 +28,8 @@ export interface IOutboxParameters {
     readonly containerContext: IContainerContext,
     readonly config: IOutboxConfig,
     readonly compressor: OpCompressor;
+    readonly splitter: OpSplitter;
+    readonly logger: ITelemetryLogger;
 }
 
 export class Outbox {
@@ -41,10 +46,12 @@ export class Outbox {
         this.attachFlowBatch = new BatchManager({
             hardLimit,
             softLimit,
-        });
+            enableOpReentryCheck: params.config.enableOpReentryCheck,
+        }, params.logger);
         this.mainBatch = new BatchManager({
-            hardLimit
-        });
+            hardLimit,
+            enableOpReentryCheck: params.config.enableOpReentryCheck,
+        }, params.logger);
     }
 
     public get isEmpty(): boolean {
@@ -113,20 +120,25 @@ export class Outbox {
         }
 
         const compressedBatch = this.params.compressor.compressBatch(batch);
-        if (compressedBatch.contentSizeInBytes > this.params.config.maxBatchSizeInBytes) {
-            throw new GenericError(
-                "BatchTooLarge",
-                    /* error */ undefined,
-                {
-                    opSize: batch.contentSizeInBytes,
-                    count: batch.content.length,
-                    limit: this.params.config.maxBatchSizeInBytes,
-                    compressed: true,
-                });
+        if (compressedBatch.contentSizeInBytes <= this.params.config.maxBatchSizeInBytes) {
+            // If we don't reach the maximum supported size of a batch, it can safely be sent as is
+            return compressedBatch;
+        }
+
+        if (this.params.splitter.isBatchChunkingEnabled) {
+            return this.params.splitter.splitCompressedBatch(compressedBatch);
         }
 
-        // If we don't reach the maximum supported size of a batch, it safe to be sent as is
-        return compressedBatch;
+        // If we've reached this point, the runtime would attempt to send a batch larger than the allowed size
+        throw new GenericError(
+            "BatchTooLarge",
+            /* error */ undefined,
+            {
+                opSize: batch.contentSizeInBytes,
+                count: batch.content.length,
+                limit: this.params.config.maxBatchSizeInBytes,
+                compressed: true,
+            });
     }
 
     /**
@@ -166,7 +178,11 @@ export class Outbox {
         } else {
             // returns clientSequenceNumber of last message in a batch
             clientSequenceNumber = this.params.containerContext.submitBatchFn(
-                batch.content.map((message) => ({ contents: message.contents, metadata: message.metadata })));
+                batch.content.map((message) => ({
+                    contents: message.contents,
+                    metadata: message.metadata,
+                    compression: message.compression,
+                })));
         }
 
         // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -24,11 +24,27 @@ export class RemoteMessageProcessor {
 
     public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage {
         let message = copy(remoteMessage);
-
-        message = this.opDecompressor.processMessage(message);
+        message = this.opDecompressor.processMessage(message).message;
         unpackRuntimeMessage(message);
-        message = this.opSplitter.processRemoteMessage(message);
 
+        const chunkProcessingResult = this.opSplitter.processRemoteMessage(message);
+        message = chunkProcessingResult.message;
+        if (chunkProcessingResult.state !== "Processed") {
+            // If the message is not chunked or if the splitter is still rebuilding the original message,
+            // there is no need to continue processing
+            return message;
+        }
+
+        const decompressionAfterChunking = this.opDecompressor.processMessage(message);
+        message = decompressionAfterChunking.message;
+        if (decompressionAfterChunking.state === "Skipped") {
+            // After chunking, if the original message was not compressed,
+            // there is no need to continue processing
+            return message;
+        }
+
+        // The message needs to be unpacked after chunking + decompression
+        unpack(message);
         return message;
     }
 }

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.2.2.1";
+export const pkgVersion = "2.0.0-internal.2.3.1";

package/src/runningSummarizer.ts

@@ -279,7 +279,6 @@ export class RunningSummarizer implements IDisposable {
     }
 
     private nonRuntimeOpCanTriggerSummary(): boolean {
-        // eslint-disable-next-line max-len
         const opsSinceLastAck = this.heuristicData.lastOpSequenceNumber - this.heuristicData.lastSuccessfulSummary.refSequenceNumber;
         return this.configuration.state === "enabled"
             && (this.configuration.nonRuntimeHeuristicThreshold === undefined

package/src/scheduleManager.ts

@@ -16,7 +16,6 @@ import {
 } from "@fluidframework/container-utils";
 import { DeltaScheduler } from "./deltaScheduler";
 import { pkgVersion } from "./packageVersion";
-import { latencyThreshold } from "./connectionTelemetry";
 
 type IRuntimeMessageMetadata = undefined | {
     batch?: boolean;
@@ -212,16 +211,6 @@ class ScheduleManagerCore {
 
         this.localPaused = false;
 
-        // Random round number - we want to know when batch waiting paused op processing.
-        if (duration !== undefined && duration > latencyThreshold) {
-            this.logger.sendErrorEvent({
-                eventName: "MaxBatchWaitTimeExceeded",
-                duration,
-                sequenceNumber: endBatch,
-                length: endBatch - startBatch,
-            });
-        }
-
         this.deltaManager.inbound.resume();
     }
 
@@ -236,6 +225,8 @@ class ScheduleManagerCore {
             0x299 /* "non-synchronized state" */);
 
         const metadata = message.metadata as IRuntimeMessageMetadata;
+        // batchMetadata will be true for the message that starts a batch, false for the one that ends it, and
+        // undefined for all other messages.
         const batchMetadata = metadata?.batch;
 
         // Protocol messages are never part of a runtime batch of messages
@@ -265,30 +256,28 @@ class ScheduleManagerCore {
             return;
         }
 
-        // If the client ID changes then we can move the pause point. If it stayed the same then we need to check.
-        // If batchMetadata is not undefined then if it's true we've begun a new batch - if false we've ended
-        // the previous one
-        if (this.currentBatchClientId !== undefined || batchMetadata === false) {
-            if (this.currentBatchClientId !== message.clientId) {
-                // "Batch not closed, yet message from another client!"
-                throw new DataCorruptionError(
-                    "OpBatchIncomplete",
-                    {
-                        runtimeVersion: pkgVersion,
-                        batchClientId: this.currentBatchClientId,
-                        pauseSequenceNumber: this.pauseSequenceNumber,
-                        localBatch: this.currentBatchClientId === this.getClientId(),
-                        localMessage: message.clientId === this.getClientId(),
-                        ...extractSafePropertiesFromMessage(message),
-                    });
-            }
+        // If we got here, the message is part of a batch. Either starting, in progress, or ending.
+
+        // If this is not the start of the batch, error out if the message was sent by a client other than the one that
+        // started the current batch (it should not be possible for ops from other clients to get interleaved with a batch).
+        if (this.currentBatchClientId !== undefined && this.currentBatchClientId !== message.clientId) {
+            throw new DataCorruptionError(
+                "OpBatchIncomplete",
+                {
+                    runtimeVersion: pkgVersion,
+                    batchClientId: this.currentBatchClientId,
+                    pauseSequenceNumber: this.pauseSequenceNumber,
+                    localBatch: this.currentBatchClientId === this.getClientId(),
+                    localMessage: message.clientId === this.getClientId(),
+                    ...extractSafePropertiesFromMessage(message),
+                });
         }
 
         // The queue is
         // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
         //    - in afterOpProcessing() - processing ops until reaching start of incomplete batch
-        //    - here (batchMetadata == false below), when queue was empty and start of batch showed up.
-        // 2. resumed when batch end comes in (batchMetadata === true case below)
+        //    - here, when queue was empty and start of batch showed up (batchMetadata === true below).
+        // 2. resumed when batch end comes in (batchMetadata === false below)
 
         if (batchMetadata) {
             assert(this.currentBatchClientId === undefined, 0x29e /* "there can't be active batch" */);

package/src/summarizer.ts

@@ -160,7 +160,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         // This will result in "summarizerClientDisconnected" stop reason recorded in telemetry,
         // unless stop() was called earlier
         this.dispose();
-        this.runtime.closeFn();
+        (this.runtime.disposeFn ?? this.runtime.closeFn)()
     }
 
     private async runCore(onBehalfOf: string): Promise<SummarizerStopReason> {

package/src/summarizerTypes.ts

@@ -94,6 +94,7 @@ export interface ISummarizerRuntime extends IConnectableRuntime {
     readonly logger: ITelemetryLogger;
     /** clientId of parent (non-summarizing) container that owns summarizer container */
     readonly summarizerClientId: string | undefined;
+    disposeFn?(): void;
     closeFn(): void;
     /** @deprecated 1.0, please remove all implementations and usage */
     on(event: "batchEnd", listener: (error: any, op: ISequencedDocumentMessage) => void): this;

package/src/summaryFormat.ts

@@ -7,8 +7,7 @@ import { assert } from "@fluidframework/common-utils";
 import { IDocumentStorageService } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { ISequencedDocumentMessage, ISnapshotTree, SummaryType } from "@fluidframework/protocol-definitions";
-import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
-import { gcTreeKey } from "./garbageCollectionConstants";
+import { channelsTreeName, gcTreeKey, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 
 type OmitAttributesVersions<T> = Omit<T, "snapshotFormatVersion" | "summaryFormatVersion">;
 interface IFluidDataStoreAttributes0 {