@fluidframework/container-runtime 1.2.7 → 2.0.0-dev.1.3.0.96595

package/src/gcSweepReadyUsageDetection.ts

@@ -0,0 +1,139 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryProperties } from "@fluidframework/common-definitions";
+import { ICriticalContainerError } from "@fluidframework/container-definitions";
+import {
+    IConfigProvider,
+    IFluidErrorBase,
+    LoggingError,
+    MonitoringContext,
+} from "@fluidframework/telemetry-utils";
+import { oneDayMs } from "./garbageCollection";
+
+/**
+ * Feature Gate Key -
+ * How many days between closing the container from this error (avoids locking user out of their file altogether)
+ */
+export const skipClosureForXDaysKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
+
+/**
+ * LocalStorage key (NOT via feature gate / monitoring context)
+ * A map from docId to info about the last time we closed due to this error
+ */
+export const closuresMapLocalStorageKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
+
+/**
+ * Feature gate key to enable closing the container if SweepReady objects are used.
+ * Value should contain keywords "interactiveClient" and/or "summarizer" to enable detection in each container type
+ */
+const sweepReadyUsageDetectionSetting = {
+    read(config: IConfigProvider) {
+        const sweepReadyUsageDetectionKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
+        const value = config.getString(sweepReadyUsageDetectionKey);
+        if (value === undefined) {
+            return { interactiveClient: false, summarizer: false };
+        }
+        return {
+            interactiveClient: value.includes("interactiveClient"),
+            summarizer: value.includes("summarizer"),
+        };
+    },
+};
+
+/**
+ * Error class raised when a SweepReady object is used, indicating a bug in how
+ * references are managed in the container by the application, or a bug in how
+ * GC tracks those references.
+ *
+ * There's a chance for false positives when this error is raised by an Interactive Container,
+ * since only the Summarizer has the latest truth about unreferenced node tracking
+ */
+export class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {
+    /** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
+    public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
+}
+
+/**
+ * This class encapsulates the logic around what to do when a SweepReady object is used.
+ * There are several tactics we plan to use in Dogfood environments to aid diagnosis of these cases:
+ * - Closing the interactive container when either the interactive or summarizer client detects this kind of violation
+ * (via sweepReadyUsageDetectionSetting above)
+ * - Throttling the frequency of these crashes via a "Skip Closure Period" per container per device
+ * (via skipClosureForXDaysKey above.  Uses localStorage and closuresMapLocalStorageKey to implement this behavior)
+ */
+export class SweepReadyUsageDetectionHandler {
+    private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
+
+    constructor(
+        private readonly uniqueContainerKey: string,
+        private readonly mc: MonitoringContext,
+        private readonly closeFn: (error?: ICriticalContainerError) => void,
+        localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
+    ) {
+        const noopStorage = { getItem: () => null, setItem: () => {} };
+        // localStorage is not defined in Node environment, so fall back to noopStorage if needed.
+        this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
+
+        if (this.localStorage === noopStorage) {
+            // This means the Skip Closure Period logic will not work.
+            this.mc.logger.sendTelemetryEvent({ eventName: "SweepReadyUsageDetectionHandlerNoopStorage" });
+        }
+    }
+
+    /**
+      * If SweepReady Usage Detection is enabled, close the interactive container.
+      * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
+      *
+      * Once Sweep is fully implemented, this will be removed since the objects will be gone
+      * and errors will arise elsewhere in the runtime
+     */
+    public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
+        if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
+            return;
+        }
+
+        // Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
+        // However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
+        // which will only allow the container to close once during that period, to avoid locking users out.
+        let shouldClose: boolean = true;
+        let pastClosuresMap: Record<string, { lastCloseTime: number; } | undefined> = {};
+        let lastCloseTime: number | undefined;
+        const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
+        if (skipClosureForXDays !== undefined) {
+            // Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
+            try {
+                const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
+                const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
+                if (typeof parsedValue === "object") {
+                    pastClosuresMap = parsedValue;
+                }
+            } catch (e) {
+            }
+            lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
+
+            // Don't close if we did already within the Skip Closure Period
+            if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs) {
+                shouldClose = false;
+            }
+        }
+
+        const error = new SweepReadyUsageError(
+            "SweepReady object used in Non-Summarizer Client",
+            { errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }) },
+        );
+        if (shouldClose) {
+            // Update closures map in localStorage before closing
+            // Note there is a race condition between different tabs updating localStorage and overwriting
+            // each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
+            pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
+            this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
+
+            this.closeFn(error);
+        } else {
+            this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
+        }
+    }
+}

package/src/index.ts

@@ -14,12 +14,10 @@ export {
     ISummaryConfigurationDisableSummarizer,
     ISummaryConfigurationDisableHeuristics,
     IContainerRuntimeOptions,
-    IPendingRuntimeState,
     IRootSummaryTreeWithStats,
     isRuntimeMessage,
     RuntimeMessage,
     unpackRuntimeMessage,
-    ScheduleManager,
     agentSchedulerId,
     ContainerRuntime,
     RuntimeHeaders,
@@ -41,6 +39,7 @@ export {
     IPendingMessage,
     IPendingState,
 } from "./pendingStateManager";
+export { ScheduleManager } from "./scheduleManager";
 export { Summarizer } from "./summarizer";
 export {
     EnqueueSummarizeResult,

package/src/opProperties.ts

@@ -0,0 +1,19 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ISequencedDocumentMessage, ISequencedDocumentSystemMessage } from "@fluidframework/protocol-definitions";
+
+export const opSize = (op: ISequencedDocumentMessage): number => {
+    // Some messages may already have string contents,
+    // so stringifying them again will add inaccurate overhead.
+    const content = typeof op.contents === "string" ?
+        op.contents :
+        JSON.stringify(op.contents) ?? "";
+    const data = opHasData(op) ? op.data : "";
+    return content.length + data.length;
+};
+
+const opHasData = (op: ISequencedDocumentMessage): op is ISequencedDocumentSystemMessage =>
+    (op as ISequencedDocumentSystemMessage).data !== undefined;

package/src/orderedClientElection.ts

@@ -208,11 +208,17 @@ export interface IOrderedClientElectionEvents extends IEvent {
 export interface ISerializedElection {
     /** Sequence number at the time of the latest election. */
     readonly electionSequenceNumber: number;
-    /** Most recently elected client id. This is either:
+
+    /**
+     * Most recently elected client id. This is either:
+     *
      * 1. the interactive elected parent client, in which case electedClientId === electedParentId,
-     *  and the SummaryManager on the elected client will spawn a summarizer client, or
-     * 2. the non-interactive summarizer client itself. */
+     * and the SummaryManager on the elected client will spawn a summarizer client, or
+     *
+     * 2. the non-interactive summarizer client itself.
+     */
     readonly electedClientId: string | undefined;
+
     /** Most recently elected parent client id. This is always an interactive client. */
     readonly electedParentId: string | undefined;
 }
@@ -221,10 +227,15 @@ export interface ISerializedElection {
 export interface IOrderedClientElection extends IEventProvider<IOrderedClientElectionEvents> {
     /** Count of eligible clients in the collection. */
     readonly eligibleCount: number;
-    /** Currently elected client. This is either:
+
+    /**
+     * Currently elected client. This is either:
+     *
      * 1. the interactive elected parent client, in which case electedClientId === electedParentId,
-     *  and the SummaryManager on the elected client will spawn a summarizer client, or
-     * 2. the non-interactive summarizer client itself. */
+     * and the SummaryManager on the elected client will spawn a summarizer client, or
+     *
+     * 2. the non-interactive summarizer client itself.
+     */
     readonly electedClient: ITrackedClient | undefined;
     /** Currently elected parent client. This is always an interactive client. */
     readonly electedParent: ITrackedClient | undefined;
@@ -283,13 +294,20 @@ export class OrderedClientElection
      * electedClient leaves the quorum.
      *
      * A typical sequence looks like this:
+     *
      * i. Begin by electing A. electedParent === A, electedClient === A.
+     *
      * ii. SummaryManager running on A spawns a summarizer client, A'. electedParent === A, electedClient === A'
+     *
      * iii. A' stops producing summaries. A new parent client, B, is elected. electedParent === B, electedClient === A'
+     *
      * iv. SummaryManager running on A detects the change to electedParent and tells the summarizer to stop, but A'
-     *      is in mid-summarization. No new summarizer is spawned, as electedParent !== electedClient.
+     * is in mid-summarization. No new summarizer is spawned, as electedParent !== electedClient.
+     *
      * v. A' completes its summary, and the summarizer and backing client are torn down.
+     *
      * vi. A' leaves the quorum, and B takes its place as electedClient. electedParent === B, electedClient === B
+     *
      * vii. SummaryManager running on B spawns a summarizer client, B'. electedParent === B, electedClient === B'
      */
     public get electedClient() {
@@ -357,7 +375,8 @@ export class OrderedClientElection
         }
     }
 
-    /** Tries changing the elected client, raising an event if it is different.
+    /**
+     * Tries changing the elected client, raising an event if it is different.
      * Note that this function does no eligibility or suitability checks. If we get here, then
      * we will set _electedClient, and we will set _electedParent if this is an interactive client.
      */
@@ -467,7 +486,8 @@ export class OrderedClientElection
         return this.orderedClientCollection.getAllClients().filter(this.isEligibleFn);
     }
 
-    /** Advance election to the next-oldest client. This is called if the current parent is leaving the quorum,
+    /**
+     * Advance election to the next-oldest client. This is called if the current parent is leaving the quorum,
      * or if the current summarizer is not responsive and we want to stop it and spawn a new one.
      */
     public incrementElectedClient(sequenceNumber: number): void {
@@ -482,7 +502,8 @@ export class OrderedClientElection
         }
     }
 
-    /** (Re-)start election with the oldest client in the quorum. This is called if we need to summarize
+    /**
+     * (Re-)start election with the oldest client in the quorum. This is called if we need to summarize
      * and no client has been elected.
      */
     public resetElectedClient(sequenceNumber: number): void {

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "1.2.7";
+export const pkgVersion = "2.0.0-dev.1.3.0.96595";

package/src/pendingStateManager.ts

@@ -11,9 +11,9 @@ import {
     ISequencedDocumentMessage,
 } from "@fluidframework/protocol-definitions";
 import { FlushMode } from "@fluidframework/runtime-definitions";
-import { wrapError } from "@fluidframework/telemetry-utils";
 import Deque from "double-ended-queue";
 import { ContainerMessageType } from "./containerRuntime";
+import { pkgVersion } from "./packageVersion";
 
 /**
  * This represents a message that has been submitted and is added to the pending queue when `submit` is called on the
@@ -68,10 +68,6 @@ export interface IRuntimeStateHandler{
         content: any,
         localOpMetadata: unknown,
         opMetadata: Record<string, unknown> | undefined): void;
-    rollback(
-        type: ContainerMessageType,
-        content: any,
-        localOpMetadata: unknown): void;
 }
 
 /**
@@ -232,7 +228,11 @@ export class PendingStateManager implements IDisposable {
 
             // then we push onto pendingStates which will cause PendingStateManager to resubmit when we connect
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            this.pendingStates.push(this.initialStates.shift()!);
+            const firstPendingState = this.initialStates.shift()!;
+            this.pendingStates.push(firstPendingState);
+            if (firstPendingState.type === "message") {
+                this._pendingMessagesCount++;
+            }
         }
     }
 
@@ -266,6 +266,7 @@ export class PendingStateManager implements IDisposable {
         }
 
         this._pendingMessagesCount--;
+        assert(this._pendingMessagesCount >= 0, 0x3d6 /* positive */);
 
         // Post-processing part - If we are processing a batch then this could be the last message in the batch.
         this.maybeProcessBatchEnd(message);
@@ -285,9 +286,11 @@ export class PendingStateManager implements IDisposable {
 
         /**
          * We are checking if the next message is the start of a batch. It can happen in the following scenarios:
+         *
          * 1. The FlushMode was set to TurnBased before this message was sent.
+         *
          * 2. The FlushMode was already TurnBased and a flush was called before this message was sent. This essentially
-         *    means that the flush marked the end of a previous batch and beginning of a new batch.
+         * means that the flush marked the end of a previous batch and beginning of a new batch.
          *
          * Keep reading pending states from the queue until we encounter a message. It's possible that the FlushMode was
          * updated a bunch of times without sending any messages.
@@ -368,8 +371,23 @@ export class PendingStateManager implements IDisposable {
         } else {
             // Get the batch metadata from the last message in the batch.
             const batchEndMetadata = message.metadata?.batch;
-            assert(batchBeginMetadata === true, 0x16f /* "Did not receive batch begin metadata" */);
-            assert(batchEndMetadata === false, 0x170 /* "Did not receive batch end metadata" */);
+            if (batchBeginMetadata !== true || batchEndMetadata !== false) {
+                this.stateHandler.close(DataProcessingError.create(
+                    "Pending batch inconsistency", // Formerly known as asserts 0x16f and 0x170
+                    "processPendingLocalMessage",
+                    message,
+                    {
+                        runtimeVersion: pkgVersion,
+                        batchClientId: this.pendingBatchBeginMessage.clientId,
+                        clientId: this.stateHandler.clientId(),
+                        hasBatchStart: batchBeginMetadata === true,
+                        hasBatchEnd: batchEndMetadata === false,
+                        messageType: message.type,
+                        batchStartSequenceNumber: this.pendingBatchBeginMessage.clientSequenceNumber,
+                        pendingMessagesCount: this.pendingMessagesCount,
+                        nextPendingState: nextPendingState.type,
+                    }));
+            }
         }
 
         // Clear the pending batch state now that we have processed the entire batch.
@@ -377,31 +395,6 @@ export class PendingStateManager implements IDisposable {
         this.isProcessingBatch = false;
     }
 
-    /**
-     * Capture the pending state at this point
-     */
-    public checkpoint() {
-        const checkpointHead = this.pendingStates.peekBack();
-        return {
-            rollback: () => {
-                try {
-                    while (this.pendingStates.peekBack() !== checkpointHead) {
-                        this.rollbackNextPendingState();
-                    }
-                } catch (err) {
-                    const error = wrapError(err, (message) => {
-                        return DataProcessingError.create(
-                            `RollbackError: ${message}`,
-                            "checkpointRollback",
-                            undefined) as DataProcessingError;
-                    });
-                    this.stateHandler.close(error);
-                    throw error;
-                }
-            },
-        };
-    }
-
     /**
      * Returns the next pending state from the pending state queue.
      */
@@ -411,31 +404,6 @@ export class PendingStateManager implements IDisposable {
         return nextPendingState;
     }
 
-    /**
-     * Undo the last pending state
-     */
-    private rollbackNextPendingState() {
-        const pendingStatesCount = this.pendingStates.length;
-        if (pendingStatesCount === 0) {
-            return;
-        }
-
-        this._pendingMessagesCount--;
-
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        const pendingState = this.pendingStates.pop()!;
-        switch (pendingState.type) {
-            case "message":
-                this.stateHandler.rollback(
-                    pendingState.messageType,
-                    pendingState.content,
-                    pendingState.localOpMetadata);
-                break;
-            default:
-                throw new Error(`Can't rollback state ${pendingState.type}`);
-        }
-    }
-
     /**
      * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
      * states in its queue. This includes setting the FlushMode and triggering resubmission of unacked ops.

package/src/runningSummarizer.ts

@@ -6,6 +6,7 @@
 import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
 import { UsageError } from "@fluidframework/container-utils";
+import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import {
     ISequencedDocumentMessage,
     MessageType,
@@ -14,6 +15,7 @@ import { ChildLogger } from "@fluidframework/telemetry-utils";
 import {
     ISummaryConfiguration,
 } from "./containerRuntime";
+import { opSize } from "./opProperties";
 import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
 import {
     IEnqueueSummarizeOptions,
@@ -28,6 +30,7 @@ import {
     ISummaryCancellationToken,
     ISummarizeResults,
     ISummarizeTelemetryProperties,
+    ISummarizerRuntime,
     ISummarizeRunnerTelemetry,
 } from "./summarizerTypes";
 import { IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
@@ -58,6 +61,7 @@ export class RunningSummarizer implements IDisposable {
         summaryCollection: SummaryCollection,
         cancellationToken: ISummaryCancellationToken,
         stopSummarizerCallback: (reason: SummarizerStopReason) => void,
+        runtime: ISummarizerRuntime,
     ): Promise<RunningSummarizer> {
         const summarizer = new RunningSummarizer(
             logger,
@@ -68,12 +72,36 @@ export class RunningSummarizer implements IDisposable {
             raiseSummarizingError,
             summaryCollection,
             cancellationToken,
-            stopSummarizerCallback);
+            stopSummarizerCallback,
+            runtime);
 
         await summarizer.waitStart();
 
-        // Run the heuristics after starting
+        // Update heuristic counts
+        // By the time we get here, there are potentially ops missing from the heuristic summary counts
+        // Examples of where this could happen:
+        // 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we
+        //    listen for the op events (will get missed by the handlers in the current workflow)
+        // 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)
+        const diff = runtime.deltaManager.lastSequenceNumber - (
+            heuristicData.lastSuccessfulSummary.refSequenceNumber
+            + heuristicData.numNonRuntimeOps
+            + heuristicData.numRuntimeOps);
+        heuristicData.hasMissingOpData = diff > 0;
+
+        if (heuristicData.hasMissingOpData) {
+            // Split the diff 50-50 and increment the counts appropriately
+            heuristicData.numNonRuntimeOps += Math.ceil(diff / 2);
+            heuristicData.numRuntimeOps += Math.floor(diff / 2);
+        }
+
+        // Update last seq number (in case the handlers haven't processed anything yet)
+        heuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;
+
+        // Start heuristics
+        summarizer.heuristicRunner?.start();
         summarizer.heuristicRunner?.run();
+
         return summarizer;
     }
 
@@ -95,6 +123,7 @@ export class RunningSummarizer implements IDisposable {
     } | undefined;
     private summarizeCount = 0;
     private totalSuccessfulAttempts = 0;
+    private initialized = false;
 
     private constructor(
         baseLogger: ITelemetryLogger,
@@ -106,6 +135,7 @@ export class RunningSummarizer implements IDisposable {
         private readonly summaryCollection: SummaryCollection,
         private readonly cancellationToken: ISummaryCancellationToken,
         private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
+        private readonly runtime: ISummarizerRuntime,
     ) {
         const telemetryProps: ISummarizeRunnerTelemetry = {
             summarizeCount: () => this.summarizeCount,
@@ -175,9 +205,13 @@ export class RunningSummarizer implements IDisposable {
             this.summaryWatcher,
             this.logger,
         );
+
+        // Listen for ops
+        this.runtime.deltaManager.on("op", (op) => { this.handleOp(op); });
     }
 
     public dispose(): void {
+        this.runtime.deltaManager.off("op", (op) => { this.handleOp(op); });
         this.summaryWatcher.dispose();
         this.heuristicRunner?.dispose();
         this.heuristicRunner = undefined;
@@ -199,30 +233,48 @@ export class RunningSummarizer implements IDisposable {
             ? this.logger
             : undefined;
 
-    public handleSystemOp(op: ISequencedDocumentMessage) {
-        switch (op.type) {
-            case MessageType.ClientLeave:
-            case MessageType.ClientJoin:
-            case MessageType.Propose: {
-                // Synchronously handle quorum ops like regular ops
-                this.handleOp(undefined, op);
-                return;
-            }
-            default: {
-                return;
-            }
-        }
-    }
+    /** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */
+    private heuristicRunnerMicroTaskExists = false;
 
-    public handleOp(error: any, { sequenceNumber, type, clientId, contents }: ISequencedDocumentMessage) {
-        if (error !== undefined) {
-            return;
+    public handleOp(op: ISequencedDocumentMessage) {
+        this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;
+
+        if (isRuntimeMessage(op)) {
+            this.heuristicData.numRuntimeOps++;
+        } else {
+            this.heuristicData.numNonRuntimeOps++;
         }
-        this.heuristicData.lastOpSequenceNumber = sequenceNumber;
+
+        this.heuristicData.totalOpsSize += opSize(op);
 
         // Check for enqueued on-demand summaries; Intentionally do nothing otherwise
-        if (!this.tryRunEnqueuedSummary()) {
-            this.heuristicRunner?.run();
+        if (this.initialized
+            && this.opCanTriggerSummary(op)
+            && !this.tryRunEnqueuedSummary()
+            && !this.heuristicRunnerMicroTaskExists) {
+            this.heuristicRunnerMicroTaskExists = true;
+            Promise.resolve().then(() => {
+                this.heuristicRunner?.run();
+            }).finally(() => {
+                this.heuristicRunnerMicroTaskExists = false;
+            });
+        }
+    }
+
+    /**
+     * Can the given op trigger a summary?
+     * # Currently only prevents summaries for Summarize and SummaryAck ops
+     * @param op - op to check
+     * @returns true if this type of op can trigger a summary
+     */
+    private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
+        switch (op.type) {
+            case MessageType.Summarize:
+            case MessageType.SummaryAck:
+            case MessageType.SummaryNack:
+                return false;
+            default:
+                return true;
         }
     }
 
@@ -274,6 +326,7 @@ export class RunningSummarizer implements IDisposable {
                 summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,
             });
         }
+        this.initialized = true;
     }
 
     /**