@fluidframework/container-runtime 2.0.0-internal.2.2.1 → 2.0.0-internal.2.3.1

package/src/garbageCollectionConstants.ts

@@ -3,13 +3,12 @@
  * Licensed under the MIT License.
  */
 
+import { GCVersion } from "./summaryFormat";
 
-// The key for the GC tree in summary.
-export const gcTreeKey = "gc";
-// They prefix for GC blobs in the GC tree in summary.
-export const gcBlobPrefix = "__gc";
-// The key for tombstone blob in the GC tree in summary.
-export const gcTombstoneBlobKey = "__tombstones";
+/** The stable version of garbage collection in production. */
+export const stableGCVersion: GCVersion = 1;
+/** The current version of garbage collection. */
+export const currentGCVersion: GCVersion = 2;
 
 // Feature gate key to turn GC on / off.
 export const runGCKey = "Fluid.GarbageCollection.RunGC";
@@ -25,8 +24,12 @@ export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
 export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
 // Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
 export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
-// Feature gate to enable throwing an error when tombstone object is used.
+// Feature gate to enable throwing an error when tombstone object is loaded (requested).
+export const throwOnTombstoneLoadKey = "Fluid.GarbageCollection.ThrowOnTombstoneLoad";
+// Feature gate to enable throwing an error when tombstone object is used (e.g. outgoing or incoming ops).
 export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
+// Feature gate to enable GC version upgrade.
+export const gcVersionUpgradeToV2Key = "Fluid.GarbageCollection.GCVersionUpgradeToV2";
 
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;

package/src/garbageCollectionTombstoneUtils.ts

@@ -0,0 +1,28 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
+import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
+import { MonitoringContext } from "@fluidframework/telemetry-utils";
+import { disableTombstoneKey, throwOnTombstoneLoadKey, throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+
+/**
+ * Consolidates info / logic for logging when we encounter a Tombstone
+ */
+export function sendGCTombstoneEvent(
+    mc: MonitoringContext,
+    event: ITelemetryGenericEvent & { category: "error" | "generic", isSummarizerClient: boolean },
+    packagePath: readonly string[] | undefined,
+    error?: unknown,
+) {
+    event.pkg = packagePathToTelemetryProperty(packagePath);
+    event.tombstoneFlags = JSON.stringify({
+        DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
+        ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
+        ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
+    });
+
+    mc.logger.sendTelemetryEvent(event, error);
+}

package/src/index.ts

@@ -19,6 +19,8 @@ export {
     agentSchedulerId,
     ContainerRuntime,
     RuntimeHeaders,
+    AllowTombstoneRequestHeaderKey,
+    TombstoneResponseHeaderKey,
     ISummaryConfiguration,
     DefaultSummaryConfiguration,
     ICompressionRuntimeOptions,
@@ -28,11 +30,6 @@ export { FluidDataStoreRegistry } from "./dataStoreRegistry";
 export {
     IGCStats,
 } from "./garbageCollection";
-export {
-    gcBlobPrefix,
-    gcTombstoneBlobKey,
-    gcTreeKey,
-} from "./garbageCollectionConstants";
 export {
     IPendingFlush,
     IPendingLocalState,

package/src/opLifecycle/batchManager.ts

@@ -3,10 +3,14 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { UsageError } from "@fluidframework/driver-utils";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { ICompressionRuntimeOptions } from "../containerRuntime";
 import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions";
 
 export interface IBatchManagerOptions {
+    readonly enableOpReentryCheck?: boolean;
     readonly hardLimit: number;
     readonly softLimit?: number;
     readonly compressionOptions?: ICompressionRuntimeOptions;
@@ -16,15 +20,32 @@ export interface IBatchManagerOptions {
  * Helper class that manages partial batch & rollback.
  */
 export class BatchManager {
+    private readonly logger;
     private pendingBatch: BatchMessage[] = [];
     private batchContentSize = 0;
+    /**
+    * Track the number of ops which were detected to have a mismatched
+    * reference sequence number, in order to self-throttle the telemetry events.
+    *
+    * This should be removed as part of ADO:2322
+    */
+    private readonly maxMismatchedOpsToReport = 5;
+    private mismatchedOpsReported = 0;
+
 
     public get length() { return this.pendingBatch.length; }
     public get contentSizeInBytes() { return this.batchContentSize; }
 
-    constructor(public readonly options: IBatchManagerOptions) { }
+    constructor(
+        public readonly options: IBatchManagerOptions,
+        logger: ITelemetryLogger,
+    ) {
+        this.logger = ChildLogger.create(logger, "BatchManager");
+    }
 
     public push(message: BatchMessage): boolean {
+        this.checkReferenceSequenceNumber(message);
+
         const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
         const opCount = this.pendingBatch.length;
 
@@ -87,6 +108,43 @@ export class BatchManager {
             },
         };
     }
+
+    private checkReferenceSequenceNumber(message: BatchMessage) {
+        if (this.pendingBatch.length === 0 || message.referenceSequenceNumber === this.pendingBatch[0].referenceSequenceNumber) {
+            // The reference sequence numbers are stable
+            return;
+        }
+
+        const telemetryProperties = {
+            referenceSequenceNumber: this.pendingBatch[0].referenceSequenceNumber,
+            messageReferenceSequenceNumber: message.referenceSequenceNumber,
+            type: message.deserializedContent.type,
+            length: this.pendingBatch.length,
+            enableOpReentryCheck: this.options.enableOpReentryCheck === true,
+        };
+        const error = new UsageError("Submission of an out of order message");
+        const eventName = "ReferenceSequenceNumberMismatch";
+
+        if (this.options.enableOpReentryCheck === true) {
+            this.logger.sendErrorEvent(
+                { eventName, ...telemetryProperties },
+                error,
+            );
+            throw error;
+        }
+
+        if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
+            this.logger.sendErrorEvent(
+                {
+                    eventName,
+                    ...telemetryProperties,
+                    ops: this.mismatchedOpsReported,
+                    maxOps: this.maxMismatchedOpsToReport,
+                },
+                error,
+            );
+        }
+    }
 }
 
 const addBatchMetadata = (batch: IBatch): IBatch => {

package/src/opLifecycle/definitions.ts

@@ -4,7 +4,7 @@
  */
 
 import { IBatchMessage } from "@fluidframework/container-definitions";
-import { MessageType } from "@fluidframework/protocol-definitions";
+import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
 import { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from "..";
 
 /**
@@ -41,4 +41,30 @@ export interface IChunkedOp {
     totalChunks: number;
     contents: string;
     originalType: MessageType | ContainerMessageType;
+    originalMetadata?: Record<string, unknown>;
+    originalCompression?: string;
+}
+
+/**
+ * The state of remote message processing:
+ * `Processed` - the message can be considered processed
+ * `Skipped` - the message was ignored by the processor
+ * `Accepted` - the message was processed partially. Eventually, a message
+ * will make the processor return `Processed`.
+ */
+export type ProcessingState = "Processed" | "Skipped" | "Accepted";
+
+/**
+ * Return type for functions which process remote messages
+ */
+export interface IMessageProcessingResult {
+    /**
+     * A shallow copy of the input message if processing happened, or
+     * the original message otherwise
+     */
+    readonly message: ISequencedDocumentMessage;
+    /**
+     * Processing result of the input message.
+     */
+    readonly state: ProcessingState;
 }

package/src/opLifecycle/index.ts

@@ -9,9 +9,10 @@ export {
     IBatch,
     IBatchCheckpoint,
     IChunkedOp,
+    IMessageProcessingResult,
 } from "./definitions";
 export { Outbox } from "./outbox";
 export { OpCompressor } from "./opCompressor";
 export { OpDecompressor } from "./opDecompressor";
-export { OpSplitter } from "./opSplitter";
+export { OpSplitter, splitOp } from "./opSplitter";
 export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor";

package/src/opLifecycle/opCompressor.ts

@@ -5,9 +5,10 @@
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { IsoBuffer } from "@fluidframework/common-utils";
+import { UsageError } from "@fluidframework/container-utils";
 import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { compress } from "lz4js";
-import { CompressionAlgorithms, ContainerRuntimeMessage } from "../containerRuntime";
+import { CompressionAlgorithms } from "../containerRuntime";
 import { IBatch, BatchMessage } from "./definitions";
 
 /**
@@ -17,27 +18,19 @@ import { IBatch, BatchMessage } from "./definitions";
  */
 export class OpCompressor {
     private readonly logger;
-    private compressedBatchCount = 0;
 
     constructor(logger: ITelemetryLogger) {
         this.logger = ChildLogger.create(logger, "OpCompressor");
     }
 
     public compressBatch(batch: IBatch): IBatch {
-        const messages: BatchMessage[] = [];
-        this.compressedBatchCount++;
-        const contentToCompress: ContainerRuntimeMessage[] = [];
-        for (const message of batch.content) {
-            contentToCompress.push(message.deserializedContent);
-        }
-
         const compressionStart = Date.now();
-        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(contentToCompress));
+        const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
         const compressedContents = compress(contentsAsBuffer);
         const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
         const duration = Date.now() - compressionStart;
 
-        if (batch.contentSizeInBytes > 200000 || this.compressedBatchCount % 25) {
+        if (batch.contentSizeInBytes > 200000) {
             this.logger.sendPerformanceEvent({
                 eventName: "CompressedBatch",
                 duration,
@@ -46,9 +39,10 @@ export class OpCompressor {
             });
         }
 
+        const messages: BatchMessage[] = [];
         messages.push({
             ...batch.content[0], contents: JSON.stringify({ packedContents: compressedContent }),
-            metadata: { ...batch.content[0].metadata, compressed: true },
+            metadata: batch.content[0].metadata,
             compression: CompressionAlgorithms.lz4,
         });
 
@@ -61,4 +55,27 @@ export class OpCompressor {
             content: messages,
         };
     }
+
+    private serializeBatch(batch: IBatch): string {
+        try {
+            return JSON.stringify(batch.content.map((message) => message.deserializedContent))
+        } catch (e: any) {
+            if (e.message === "Invalid string length") {
+                // This is how JSON.stringify signals that
+                // the content size exceeds its capacity
+                const error = new UsageError("Payload too large");
+                this.logger.sendErrorEvent(
+                    {
+                        eventName: "BatchTooLarge",
+                        size: batch.contentSizeInBytes,
+                        length: batch.content.length,
+                    },
+                    error,
+                );
+                throw error;
+            }
+
+            throw e;
+        }
+    }
 }

package/src/opLifecycle/opDecompressor.ts

@@ -7,6 +7,7 @@ import { decompress } from "lz4js";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
 import { CompressionAlgorithms } from "../containerRuntime";
+import { IMessageProcessingResult } from "./definitions";
 
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
@@ -21,13 +22,12 @@ export class OpDecompressor {
     private rootMessageContents: any | undefined;
     private processedCount = 0;
 
-    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
-        // We're checking for compression = true or top level compression property so
-        // that we can enable compression without waiting on all ordering services
-        // to pick up protocol change. Eventually only the top level property should
-        // be used.
-        if (message.metadata?.batch === true
-            && (message.metadata?.compressed || message.compression !== undefined)) {
+    public processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
+        assert(
+            message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
+            0x511 /* Only lz4 compression is supported */);
+
+        if (message.metadata?.batch === true && message.compression === CompressionAlgorithms.lz4) {
             // Beginning of a compressed batch
             assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
             if (message.compression) {
@@ -44,30 +44,37 @@ export class OpDecompressor {
             const asObj = JSON.parse(intoString);
             this.rootMessageContents = asObj;
 
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
         }
 
         if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {
+            assert(message.contents === undefined, 0x512 /* Expecting empty message */);
+
             // Continuation of compressed batch
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
         }
 
         if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
             // End of compressed batch
-            const returnMessage = {
-                ...message,
-                contents: this.rootMessageContents[this.processedCount++]
-            };
+            const returnMessage = newMessage(message, this.rootMessageContents[this.processedCount++]);
 
             this.activeBatch = false;
             this.rootMessageContents = undefined;
             this.processedCount = 0;
 
-            return returnMessage;
+            return {
+                message: returnMessage,
+                state: "Processed",
+            };
         }
 
-        if (message.metadata?.batch === undefined &&
-            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
+        if (message.metadata?.batch === undefined && message.compression === CompressionAlgorithms.lz4) {
             // Single compressed message
             assert(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
 
@@ -76,9 +83,23 @@ export class OpDecompressor {
             const intoString = new TextDecoder().decode(decompressedMessage);
             const asObj = JSON.parse(intoString);
 
-            return { ...message, contents: asObj[0] };
+            return {
+                message: newMessage(message, asObj[0]),
+                state: "Processed",
+            };
         }
 
-        return message;
+        return {
+            message,
+            state: "Skipped",
+        };
     }
 }
+
+// We should not be mutating the input message nor its metadata
+const newMessage = (originalMessage: ISequencedDocumentMessage, contents: any): ISequencedDocumentMessage => ({
+    ...originalMessage,
+    contents,
+    compression: undefined,
+    metadata: { ...originalMessage.metadata },
+});

package/src/opLifecycle/opSplitter.ts

@@ -3,10 +3,14 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { assert } from "@fluidframework/common-utils";
+import { IBatchMessage } from "@fluidframework/container-definitions";
 import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ContainerMessageType } from "../containerRuntime";
-import { IChunkedOp } from "./definitions";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
+import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
+import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
 
 /**
  * Responsible for creating and reconstructing chunked messages.
@@ -14,18 +18,33 @@ import { IChunkedOp } from "./definitions";
 export class OpSplitter {
     // Local copy of incomplete received chunks.
     private readonly chunkMap: Map<string, string[]>;
+    private readonly logger;
 
-    constructor(chunks: [string, string[]][]) {
+    constructor(
+        chunks: [string, string[]][],
+        private readonly submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined,
+        private readonly chunkSizeInBytes: number,
+        private readonly maxBatchSizeInBytes: number,
+        logger: ITelemetryLogger,
+    ) {
         this.chunkMap = new Map<string, string[]>(chunks);
+        this.logger = ChildLogger.create(logger, "OpSplitter");
+    }
+
+    public get isBatchChunkingEnabled(): boolean {
+        return this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;
     }
 
     public get chunks(): ReadonlyMap<string, string[]> {
         return this.chunkMap;
     }
 
-    public processRemoteMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
+    public processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
         if (message.type !== ContainerMessageType.ChunkedOp) {
-            return message;
+            return {
+                message,
+                state: "Skipped",
+            };
         }
 
         const clientId = message.clientId;
@@ -35,7 +54,10 @@ export class OpSplitter {
         if (chunkedContent.chunkId < chunkedContent.totalChunks) {
             // We are processing the op in chunks but haven't reached
             // the last chunk yet in order to reconstruct the original op
-            return message;
+            return {
+                message,
+                state: "Accepted",
+            };
         }
 
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -45,7 +67,12 @@ export class OpSplitter {
         const newMessage = { ...message };
         newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
         newMessage.type = chunkedContent.originalType;
-        return newMessage;
+        newMessage.metadata = chunkedContent.originalMetadata;
+        newMessage.compression = chunkedContent.originalCompression;
+        return {
+            message: newMessage,
+            state: "Processed",
+        };
     }
 
     public clearPartialChunks(clientId: string) {
@@ -75,4 +102,111 @@ export class OpSplitter {
 
         map.push(chunkedContent.contents);
     }
+
+    /**
+     * Splits the first op of a compressed batch in chunks, sends the chunks separately and
+     * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
+     *
+     * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
+     * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
+     * uncompressed ops at ingestion in the runtime.
+     *
+     * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+     * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
+     *
+     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
+     * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
+     * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
+     * (as it is the last chunk).
+     *
+     * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
+     *
+     * @param batch - the compressed batch which needs to be processed
+     * @returns A new adjusted batch which can be sent over the wire
+     */
+    public splitCompressedBatch(batch: IBatch): IBatch {
+        assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
+        assert(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x514 /* Batch needs to be non-empty */);
+        assert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
+        assert(this.chunkSizeInBytes < this.maxBatchSizeInBytes, 0x516 /* Chunk size needs to be smaller than the max batch size */);
+
+        const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
+        assert(firstMessage.metadata?.compressed === true || firstMessage.compression !== undefined, 0x517 /* Batch needs to be compressed */);
+        assert((firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes, 0x518 /* First message in the batch needs to be chunkable */);
+
+        const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
+        const chunks = splitOp(firstMessage, this.chunkSizeInBytes);
+
+        assert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
+        // Send the first N-1 chunks immediately
+        for (const chunk of chunks.slice(0, -1)) {
+            this.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);
+        }
+
+        // The last chunk will be part of the new batch and needs to
+        // preserve the batch metadata of the original batch
+        const lastChunk = chunkToBatchMessage(
+            chunks[chunks.length - 1],
+            firstMessage.referenceSequenceNumber,
+            { batch: firstMessage.metadata?.batch });
+
+        this.logger.sendPerformanceEvent({
+            eventName: "Chunked compressed batch",
+            length: batch.content.length,
+            sizeInBytes: batch.contentSizeInBytes,
+            chunks: chunks.length,
+            chunkSizeInBytes: this.chunkSizeInBytes,
+        });
+
+        return {
+            content: [lastChunk, ...restOfMessages],
+            contentSizeInBytes: lastChunk.contents?.length ?? 0,
+        };
+    }
 }
+
+const chunkToBatchMessage = (
+    chunk: IChunkedOp,
+    referenceSequenceNumber: number,
+    metadata: Record<string, unknown> | undefined = undefined,
+): BatchMessage => {
+    const payload: ContainerRuntimeMessage = { type: ContainerMessageType.ChunkedOp, contents: chunk };
+    return {
+        contents: JSON.stringify(payload),
+        deserializedContent: payload,
+        metadata,
+        localOpMetadata: undefined,
+        referenceSequenceNumber,
+    };
+}
+
+export const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[] => {
+    const chunks: IChunkedOp[] = [];
+    assert(op.contents !== undefined && op.contents !== null, 0x51a /* We should have something to chunk */);
+
+    const contentLength = op.contents.length;
+    const chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;
+    let offset = 0;
+    for (let i = 1; i <= chunkN; i++) {
+        const chunk: IChunkedOp = {
+            chunkId: i,
+            contents: op.contents.substr(offset, chunkSizeInBytes),
+            originalType: op.deserializedContent.type,
+            totalChunks: chunkN,
+        }
+
+        if (i === chunkN) {
+            // We don't need to port these to all the chunks,
+            // as we rebuild the original op when we process the
+            // last chunk, therefore it is the only one that needs it.
+            chunk.originalMetadata = op.metadata;
+            chunk.originalCompression = op.compression;
+        }
+
+        chunks.push(chunk);
+        offset += chunkSizeInBytes;
+    }
+
+    return chunks;
+};