@fluidframework/container-runtime 1.3.0 → 2.0.0-dev.1.4.5.105745

package/src/scheduleManager.ts

@@ -0,0 +1,314 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { EventEmitter } from "events";
+import { IDeltaManager } from "@fluidframework/container-definitions";
+import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
+import { assert, performance } from "@fluidframework/common-utils";
+import { isRuntimeMessage } from "@fluidframework/driver-utils";
+import {
+    DataCorruptionError,
+    DataProcessingError,
+    extractSafePropertiesFromMessage,
+} from "@fluidframework/container-utils";
+import { DeltaScheduler } from "./deltaScheduler";
+import { pkgVersion } from "./packageVersion";
+import { latencyThreshold } from "./connectionTelemetry";
+
+type IRuntimeMessageMetadata = undefined | {
+    batch?: boolean;
+};
+
+/**
+ * This class has the following responsibilities:
+ *
+ * 1. It tracks batches as we process ops and raises "batchBegin" and "batchEnd" events.
+ * As part of it, it validates batch correctness (i.e. no system ops in the middle of batch)
+ *
+ * 2. It creates instance of ScheduleManagerCore that ensures we never start processing ops from batch
+ * unless all ops of the batch are in.
+ */
+export class ScheduleManager {
+    private readonly deltaScheduler: DeltaScheduler;
+    private batchClientId: string | undefined;
+    private hitError = false;
+
+    constructor(
+        private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
+        private readonly emitter: EventEmitter,
+        readonly getClientId: () => string | undefined,
+        private readonly logger: ITelemetryLogger,
+    ) {
+        this.deltaScheduler = new DeltaScheduler(
+            this.deltaManager,
+            ChildLogger.create(this.logger, "DeltaScheduler"),
+        );
+        void new ScheduleManagerCore(deltaManager, getClientId, logger);
+    }
+
+    public beforeOpProcessing(message: ISequencedDocumentMessage) {
+        if (this.batchClientId !== message.clientId) {
+            assert(this.batchClientId === undefined,
+                0x2a2 /* "Batch is interrupted by other client op. Should be caught by trackPending()" */);
+
+            // This could be the beginning of a new batch or an individual message.
+            this.emitter.emit("batchBegin", message);
+            this.deltaScheduler.batchBegin(message);
+
+            const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
+            this.batchClientId = batch ? message.clientId : undefined;
+        }
+    }
+
+    public afterOpProcessing(error: any | undefined, message: ISequencedDocumentMessage) {
+        // If this is no longer true, we need to revisit what we do where we set this.hitError.
+        assert(!this.hitError, 0x2a3 /* "container should be closed on any error" */);
+
+        if (error) {
+            // We assume here that loader will close container and stop processing all future ops.
+            // This is implicit dependency. If this flow changes, this code might no longer be correct.
+            this.hitError = true;
+            this.batchClientId = undefined;
+            this.emitter.emit("batchEnd", error, message);
+            this.deltaScheduler.batchEnd(message);
+            return;
+        }
+
+        const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
+        // If no batchClientId has been set then we're in an individual batch. Else, if we get
+        // batch end metadata, this is end of the current batch.
+        if (this.batchClientId === undefined || batch === false) {
+            this.batchClientId = undefined;
+            this.emitter.emit("batchEnd", undefined, message);
+            this.deltaScheduler.batchEnd(message);
+            return;
+        }
+    }
+}
+
+/**
+ * This class controls pausing and resuming of inbound queue to ensure that we never
+ * start processing ops in a batch IF we do not have all ops in the batch.
+ */
+class ScheduleManagerCore {
+    private pauseSequenceNumber: number | undefined;
+    private currentBatchClientId: string | undefined;
+    private localPaused = false;
+    private timePaused = 0;
+    private batchCount = 0;
+
+    constructor(
+        private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
+        private readonly getClientId: () => string | undefined,
+        private readonly logger: ITelemetryLogger,
+    ) {
+        // Listen for delta manager sends and add batch metadata to messages
+        this.deltaManager.on("prepareSend", (messages: IDocumentMessage[]) => {
+            if (messages.length === 0) {
+                return;
+            }
+
+            // First message will have the batch flag set to true if doing a batched send
+            const firstMessageMetadata = messages[0].metadata as IRuntimeMessageMetadata;
+            if (!firstMessageMetadata?.batch) {
+                return;
+            }
+
+            // If the batch contains only a single op, clear the batch flag.
+            if (messages.length === 1) {
+                delete firstMessageMetadata.batch;
+                return;
+            }
+
+            // Set the batch flag to false on the last message to indicate the end of the send batch
+            const lastMessage = messages[messages.length - 1];
+            lastMessage.metadata = { ...lastMessage.metadata, batch: false };
+        });
+
+        // Listen for updates and peek at the inbound
+        this.deltaManager.inbound.on(
+            "push",
+            (message: ISequencedDocumentMessage) => {
+                this.trackPending(message);
+            });
+
+        // Start with baseline - empty inbound queue.
+        assert(!this.localPaused, 0x293 /* "initial state" */);
+
+        const allPending = this.deltaManager.inbound.toArray();
+        for (const pending of allPending) {
+            this.trackPending(pending);
+        }
+
+        // We are intentionally directly listening to the "op" to inspect system ops as well.
+        // If we do not observe system ops, we are likely to hit 0x296 assert when system ops
+        // precedes start of incomplete batch.
+        this.deltaManager.on("op", (message) => this.afterOpProcessing(message.sequenceNumber));
+    }
+
+    /**
+     * The only public function in this class - called when we processed an op,
+     * to make decision if op processing should be paused or not after that.
+     */
+    public afterOpProcessing(sequenceNumber: number) {
+        assert(!this.localPaused, 0x294 /* "can't have op processing paused if we are processing an op" */);
+
+        // If the inbound queue is ever empty, nothing to do!
+        if (this.deltaManager.inbound.length === 0) {
+            assert(this.pauseSequenceNumber === undefined,
+                0x295 /* "there should be no pending batch if we have no ops" */);
+            return;
+        }
+
+        // The queue is
+        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
+        //    - here (processing ops until reaching start of incomplete batch)
+        //    - in trackPending(), when queue was empty and start of batch showed up.
+        // 2. resumed when batch end comes in (in trackPending())
+
+        // do we have incomplete batch to worry about?
+        if (this.pauseSequenceNumber !== undefined) {
+            assert(sequenceNumber < this.pauseSequenceNumber,
+                0x296 /* "we should never start processing incomplete batch!" */);
+            // If the next op is the start of incomplete batch, then we can't process it until it's fully in - pause!
+            if (sequenceNumber + 1 === this.pauseSequenceNumber) {
+                this.pauseQueue();
+            }
+        }
+    }
+
+    private pauseQueue() {
+        assert(!this.localPaused, 0x297 /* "always called from resumed state" */);
+        this.localPaused = true;
+        this.timePaused = performance.now();
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        this.deltaManager.inbound.pause();
+    }
+
+    private resumeQueue(startBatch: number, messageEndBatch: ISequencedDocumentMessage) {
+        const endBatch = messageEndBatch.sequenceNumber;
+        const duration = this.localPaused ? (performance.now() - this.timePaused) : undefined;
+
+        this.batchCount++;
+        if (this.batchCount % 1000 === 1) {
+            this.logger.sendTelemetryEvent({
+                eventName: "BatchStats",
+                sequenceNumber: endBatch,
+                length: endBatch - startBatch + 1,
+                msnDistance: endBatch - messageEndBatch.minimumSequenceNumber,
+                duration,
+                batchCount: this.batchCount,
+                interrupted: this.localPaused,
+            });
+        }
+
+        // Return early if no change in value
+        if (!this.localPaused) {
+            return;
+        }
+
+        this.localPaused = false;
+
+        // Random round number - we want to know when batch waiting paused op processing.
+        if (duration !== undefined && duration > latencyThreshold) {
+            this.logger.sendErrorEvent({
+                eventName: "MaxBatchWaitTimeExceeded",
+                duration,
+                sequenceNumber: endBatch,
+                length: endBatch - startBatch,
+            });
+        }
+        this.deltaManager.inbound.resume();
+    }
+
+    /**
+     * Called for each incoming op (i.e. inbound "push" notification)
+     */
+    private trackPending(message: ISequencedDocumentMessage) {
+        assert(this.deltaManager.inbound.length !== 0,
+            0x298 /* "we have something in the queue that generates this event" */);
+
+        assert((this.currentBatchClientId === undefined) === (this.pauseSequenceNumber === undefined),
+            0x299 /* "non-synchronized state" */);
+
+        const metadata = message.metadata as IRuntimeMessageMetadata;
+        const batchMetadata = metadata?.batch;
+
+        // Protocol messages are never part of a runtime batch of messages
+        if (!isRuntimeMessage(message)) {
+            // Protocol messages should never show up in the middle of the batch!
+            if (this.currentBatchClientId !== undefined) {
+                throw DataProcessingError.create(
+                    "Received a system message during batch processing", // Formerly known as assert 0x29a
+                    "trackPending",
+                    message,
+                    {
+                        runtimeVersion: pkgVersion,
+                        batchClientId: this.currentBatchClientId,
+                        pauseSequenceNumber: this.pauseSequenceNumber,
+                        localBatch: this.currentBatchClientId === this.getClientId(),
+                        messageType: message.type,
+                    });
+            }
+
+            assert(batchMetadata === undefined, 0x29b /* "system op in a batch?" */);
+            assert(!this.localPaused, 0x29c /* "we should be processing ops when there is no active batch" */);
+            return;
+        }
+
+        if (this.currentBatchClientId === undefined && batchMetadata === undefined) {
+            assert(!this.localPaused, 0x29d /* "we should be processing ops when there is no active batch" */);
+            return;
+        }
+
+        // If the client ID changes then we can move the pause point. If it stayed the same then we need to check.
+        // If batchMetadata is not undefined then if it's true we've begun a new batch - if false we've ended
+        // the previous one
+        if (this.currentBatchClientId !== undefined || batchMetadata === false) {
+            if (this.currentBatchClientId !== message.clientId) {
+                // "Batch not closed, yet message from another client!"
+                throw new DataCorruptionError(
+                    "OpBatchIncomplete",
+                    {
+                        runtimeVersion: pkgVersion,
+                        batchClientId: this.currentBatchClientId,
+                        pauseSequenceNumber: this.pauseSequenceNumber,
+                        localBatch: this.currentBatchClientId === this.getClientId(),
+                        localMessage: message.clientId === this.getClientId(),
+                        ...extractSafePropertiesFromMessage(message),
+                    });
+            }
+        }
+
+        // The queue is
+        // 1. paused only when the next message to be processed is the beginning of a batch. Done in two places:
+        //    - in afterOpProcessing() - processing ops until reaching start of incomplete batch
+        //    - here (batchMetadata == false below), when queue was empty and start of batch showed up.
+        // 2. resumed when batch end comes in (batchMetadata === true case below)
+
+        if (batchMetadata) {
+            assert(this.currentBatchClientId === undefined, 0x29e /* "there can't be active batch" */);
+            assert(!this.localPaused, 0x29f /* "we should be processing ops when there is no active batch" */);
+            this.pauseSequenceNumber = message.sequenceNumber;
+            this.currentBatchClientId = message.clientId;
+            // Start of the batch
+            // Only pause processing if queue has no other ops!
+            // If there are any other ops in the queue, processing will be stopped when they are processed!
+            if (this.deltaManager.inbound.length === 1) {
+                this.pauseQueue();
+            }
+        } else if (batchMetadata === false) {
+            assert(this.pauseSequenceNumber !== undefined, 0x2a0 /* "batch presence was validated above" */);
+            // Batch is complete, we can process it!
+            this.resumeQueue(this.pauseSequenceNumber, message);
+            this.pauseSequenceNumber = undefined;
+            this.currentBatchClientId = undefined;
+        } else {
+            // Continuation of current batch. Do nothing
+            assert(this.currentBatchClientId !== undefined, 0x2a1 /* "logic error" */);
+        }
+    }
+}

package/src/summarizer.ts

@@ -23,9 +23,6 @@ import {
     IFluidHandle,
     IRequest,
 } from "@fluidframework/core-interfaces";
-import {
-    ISequencedDocumentMessage,
-} from "@fluidframework/protocol-definitions";
 import { ISummaryConfiguration } from "./containerRuntime";
 import { ICancellableSummarizerController } from "./runWhileConnectedCoordinator";
 import { summarizerClientType } from "./summarizerClientElection";
@@ -76,8 +73,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
 
     private readonly logger: ITelemetryLogger;
     private runningSummarizer?: RunningSummarizer;
-    private systemOpListener?: (op: ISequencedDocumentMessage) => void;
-    private opListener?: (error: any, op: ISequencedDocumentMessage) => void;
     private _disposed: boolean = false;
     private starting: boolean = false;
 
@@ -283,6 +278,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
             this.summaryCollection,
             runCoordinator /* cancellationToken */,
             (reason) => runCoordinator.stop(reason), /* stopSummarizerCallback */
+            this.runtime,
         );
         this.runningSummarizer = runningSummarizer;
         this.starting = false;
@@ -293,13 +289,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
             this.logger.sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
         });
 
-        // Listen for ops
-        this.systemOpListener = (op: ISequencedDocumentMessage) => runningSummarizer.handleSystemOp(op);
-        this.runtime.deltaManager.inbound.on("op", this.systemOpListener);
-
-        this.opListener = (error: any, op: ISequencedDocumentMessage) => runningSummarizer.handleOp(error, op);
-        this.runtime.on("batchEnd", this.opListener);
-
         return runningSummarizer;
     }
 
@@ -318,12 +307,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
             this.runningSummarizer.dispose();
             this.runningSummarizer = undefined;
         }
-        if (this.systemOpListener) {
-            this.runtime.deltaManager.inbound.off("op", this.systemOpListener);
-        }
-        if (this.opListener) {
-            this.runtime.removeListener("batchEnd", this.opListener);
-        }
     }
 
     public readonly summarizeOnDemand: ISummarizer["summarizeOnDemand"] = (...args) => {

package/src/summarizerHeuristics.ts

@@ -6,11 +6,11 @@
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { Timer } from "@fluidframework/common-utils";
 import { ISummaryConfigurationHeuristics } from "./containerRuntime";
-
 import {
     ISummarizeHeuristicData,
     ISummarizeHeuristicRunner,
     ISummarizeAttempt,
+    ISummaryHeuristicStrategy,
 } from "./summarizerTypes";
 import { SummarizeReason } from "./summaryGenerator";
 
@@ -26,6 +26,30 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
         return this._lastSuccessfulSummary;
     }
 
+    public numNonRuntimeOps: number = 0;
+    public totalOpsSize: number = 0;
+    public hasMissingOpData: boolean = false;
+
+    /**
+     * Cumulative size in bytes of all the ops at the beginning of the summarization attempt.
+     * Is used to adjust totalOpsSize appropriately after successful summarization.
+     */
+     /** */
+    private totalOpsSizeBefore: number = 0;
+
+    /**
+     * Number of system ops at beginning of attempting to summarize.
+     * Is used to adjust numSystemOps appropriately after successful summarization.
+     */
+    private numSystemOpsBefore: number = 0;
+
+    public numRuntimeOps: number = 0;
+    /**
+     * Number of non-system ops at beginning of attempting to summarize.
+     * Is used to adjust numNonSystemOps appropriately after successful summarization.
+     */
+    private numNonSystemOpsBefore: number = 0;
+
     constructor(
         public lastOpSequenceNumber: number,
         /** Baseline attempt data used for comparisons with subsequent attempts/calculations. */
@@ -45,10 +69,23 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
             refSequenceNumber: refSequenceNumber ?? this.lastOpSequenceNumber,
             summaryTime: Date.now(),
         };
+
+        this.numSystemOpsBefore = this.numNonRuntimeOps;
+        this.numNonSystemOpsBefore = this.numRuntimeOps;
+        this.totalOpsSizeBefore = this.totalOpsSize;
     }
 
     public markLastAttemptAsSuccessful() {
         this._lastSuccessfulSummary = { ...this.lastAttempt };
+
+        this.numNonRuntimeOps -= this.numSystemOpsBefore;
+        this.numSystemOpsBefore = 0;
+
+        this.numRuntimeOps -= this.numNonSystemOpsBefore;
+        this.numNonSystemOpsBefore = 0;
+
+        this.totalOpsSize -= this.totalOpsSizeBefore;
+        this.totalOpsSizeBefore = 0;
     }
 }
 
@@ -56,42 +93,74 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
  * This class contains the heuristics for when to summarize.
  */
 export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
-    private readonly idleTimer: Timer;
-    private readonly minOpsForLastSummaryAttempt: number;
+    private readonly idleTimer: Timer | undefined;
+    private readonly runSummarize: (reason: SummarizeReason) => void;
 
     public constructor(
         private readonly heuristicData: ISummarizeHeuristicData,
         private readonly configuration: ISummaryConfigurationHeuristics,
-        private readonly trySummarize: (reason: SummarizeReason) => void,
+        trySummarize: (reason: SummarizeReason) => void,
         private readonly logger: ITelemetryLogger,
+        private readonly summarizeStrategies: ISummaryHeuristicStrategy[] = getDefaultSummaryHeuristicStrategies(),
     ) {
         this.idleTimer = new Timer(
-            this.configuration.idleTime,
-            () => this.trySummarize("idle"));
-        this.minOpsForLastSummaryAttempt = this.configuration.minOpsForLastSummaryAttempt;
+            this.idleTime,
+            () => this.runSummarize("idle"));
+
+        this.runSummarize = (reason: SummarizeReason) => {
+            this.idleTimer?.clear();
+
+            // We shouldn't attempt a summary if there are no new processed ops
+            const opsSinceLastAck = this.opsSinceLastAck;
+            if (opsSinceLastAck > 0) {
+                trySummarize(reason);
+            }
+        };
+    }
+
+    public get idleTime(): number {
+        if (this.configuration.idleTime !== undefined) {
+            return this.configuration.idleTime;
+        }
+        const maxIdleTime = this.configuration.maxIdleTime;
+        const minIdleTime = this.configuration.minIdleTime;
+        const weightedNumOfOps = getWeightedNumberOfOps(
+            this.heuristicData.numRuntimeOps,
+            this.heuristicData.numNonRuntimeOps,
+            this.configuration.runtimeOpWeight,
+            this.configuration.nonRuntimeOpWeight,
+        );
+        const pToMaxOps = weightedNumOfOps * 1.0 / this.configuration.maxOps;
+
+        if (pToMaxOps >= 1) {
+            return minIdleTime;
+        }
+
+        // Return a ratioed idle time based on the percentage of ops
+        return maxIdleTime - ((maxIdleTime - minIdleTime) * pToMaxOps);
     }
 
     public get opsSinceLastAck(): number {
         return this.heuristicData.lastOpSequenceNumber - this.heuristicData.lastSuccessfulSummary.refSequenceNumber;
     }
 
+    public start() {
+        this.idleTimer?.start(this.idleTime);
+    }
+
     public run() {
-        const timeSinceLastSummary = Date.now() - this.heuristicData.lastSuccessfulSummary.summaryTime;
-        const opsSinceLastAck = this.opsSinceLastAck;
-        if (timeSinceLastSummary > this.configuration.maxTime) {
-            this.idleTimer.clear();
-            this.trySummarize("maxTime");
-        } else if (opsSinceLastAck > this.configuration.maxOps) {
-            this.idleTimer.clear();
-            this.trySummarize("maxOps");
-        } else {
-            this.idleTimer.restart();
+        for (const strategy of this.summarizeStrategies) {
+            if (strategy.shouldRunSummary(this.configuration, this.heuristicData)) {
+                return this.runSummarize(strategy.summarizeReason);
+            }
         }
+
+        this.idleTimer?.restart(this.idleTime);
     }
 
     public shouldRunLastSummary(): boolean {
         const opsSinceLastAck = this.opsSinceLastAck;
-        const minOpsForLastSummaryAttempt = this.minOpsForLastSummaryAttempt;
+        const minOpsForLastSummaryAttempt = this.configuration.minOpsForLastSummaryAttempt;
 
         this.logger.sendTelemetryEvent({
             eventName: "ShouldRunLastSummary",
@@ -103,6 +172,54 @@ export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
     }
 
     public dispose() {
-        this.idleTimer.clear();
+        this.idleTimer?.clear();
+    }
+}
+
+/** Strategy used to run a summary when it's been a while since our last successful summary */
+class MaxTimeSummaryHeuristicStrategy implements ISummaryHeuristicStrategy {
+    public readonly summarizeReason: Readonly<SummarizeReason> = "maxTime";
+
+    public shouldRunSummary(
+        configuration: ISummaryConfigurationHeuristics,
+        heuristicData: ISummarizeHeuristicData,
+    ): boolean {
+        const timeSinceLastSummary = Date.now() - heuristicData.lastSuccessfulSummary.summaryTime;
+        return timeSinceLastSummary > configuration.maxTime;
     }
 }
+
+function getWeightedNumberOfOps(
+    runtimeOpCount: number,
+    nonRuntimeOpCount: number,
+    runtimeOpWeight: number,
+    nonRuntimeOpWeight: number,
+): number {
+    return (runtimeOpWeight * runtimeOpCount)
+         + (nonRuntimeOpWeight * nonRuntimeOpCount);
+}
+
+/** Strategy used to do a weighted analysis on the ops we've processed since the last successful summary */
+class WeightedOpsSummaryHeuristicStrategy implements ISummaryHeuristicStrategy {
+    public readonly summarizeReason: Readonly<SummarizeReason> = "maxOps";
+
+    public shouldRunSummary(
+        configuration: ISummaryConfigurationHeuristics,
+        heuristicData: ISummarizeHeuristicData,
+    ): boolean {
+        const weightedNumOfOps = getWeightedNumberOfOps(
+            heuristicData.numRuntimeOps,
+            heuristicData.numNonRuntimeOps,
+            configuration.runtimeOpWeight,
+            configuration.nonRuntimeOpWeight,
+        );
+        return weightedNumOfOps > configuration.maxOps;
+    }
+}
+
+function getDefaultSummaryHeuristicStrategies() {
+    return [
+        new MaxTimeSummaryHeuristicStrategy(),
+        new WeightedOpsSummaryHeuristicStrategy(),
+    ];
+}