@fluidframework/container-runtime 0.59.4001 → 1.1.0-75972

package/src/garbageCollection.ts

@@ -22,8 +22,9 @@ import {
     IGarbageCollectionData,
     IGarbageCollectionState,
     IGarbageCollectionDetailsBase,
-    IGarbageCollectionNodeData,
     ISummarizeResult,
+    ITelemetryContext,
+    IGarbageCollectionNodeData,
 } from "@fluidframework/runtime-definitions";
 import {
     mergeStats,
@@ -41,7 +42,6 @@ import {
 
 import { IGCRuntimeOptions, RuntimeHeaders } from "./containerRuntime";
 import { getSummaryForDatastores } from "./dataStores";
-import { pkgVersion } from "./packageVersion";
 import {
     getGCVersion,
     GCVersion,
@@ -72,12 +72,8 @@ const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
 export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
 // Feature gate key to disable expiring session after a set period of time, even if expiry value is present
 export const disableSessionExpiryKey = "Fluid.GarbageCollection.DisableSessionExpiry";
-// Feature gate key to log error messages if GC reference validation fails.
-export const logUnknownOutboundReferencesKey = "Fluid.GarbageCollection.LogUnknownOutboundReferences";
 // Feature gate key to write the gc blob as a handle if the data is the same.
 export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
-// Feature gate key to limit which versions can write the gc blob as a handle if the data is the same.
-export const trackGCStateMinimumVersionKey = "Fluid.GarbageCollection.TrackGCState.MinVersion";
 
 const defaultInactiveTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
 export const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
@@ -162,7 +158,11 @@ export interface IGarbageCollector {
         options: { logger?: ITelemetryLogger; runGC?: boolean; runSweep?: boolean; fullGC?: boolean; },
     ): Promise<IGCStats>;
     /** Summarizes the GC data and returns it as a summary tree. */
-    summarize(fullTree: boolean, trackState: boolean): ISummarizeResult | undefined;
+    summarize(
+        fullTree: boolean,
+        trackState: boolean,
+        telemetryContext?: ITelemetryContext,
+    ): ISummarizeResult | undefined;
     /** Returns the garbage collector specific metadata to be written into the summary. */
     getMetadata(): IGCMetadata;
     /** Returns a map of each node id to its base GC details in the base summary. */
@@ -182,6 +182,20 @@ export interface IGarbageCollector {
     dispose(): void;
 }
 
+/** Parameters necessary for creating a GarbageCollector. */
+export interface IGarbageCollectorCreateParams {
+    readonly runtime: IGarbageCollectionRuntime;
+    readonly gcOptions: IGCRuntimeOptions;
+    readonly baseLogger: ITelemetryLogger;
+    readonly existing: boolean;
+    readonly metadata: IContainerRuntimeMetadata | undefined;
+    readonly baseSnapshot: ISnapshotTree | undefined;
+    readonly isSummarizerClient: boolean;
+    readonly getNodePackagePath: (nodePath: string) => readonly string[] | undefined;
+    readonly getLastSummaryTimestampMs: () => number | undefined;
+    readonly readAndParseBlob: ReadAndParseBlob;
+}
+
 /**
  * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced. It also sets
  * the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).
@@ -253,30 +267,8 @@ class UnreferencedStateTracker {
  *  NodeId = "dds1"     NodeId = "dds2"
  */
 export class GarbageCollector implements IGarbageCollector {
-    public static create(
-        provider: IGarbageCollectionRuntime,
-        gcOptions: IGCRuntimeOptions,
-        getNodePackagePath: (nodePath: string) => readonly string[] | undefined,
-        getLastSummaryTimestampMs: () => number | undefined,
-        baseSnapshot: ISnapshotTree | undefined,
-        readAndParseBlob: ReadAndParseBlob,
-        baseLogger: ITelemetryLogger,
-        existing: boolean,
-        metadata: IContainerRuntimeMetadata | undefined,
-        isSummarizerClient: boolean,
-    ): IGarbageCollector {
-        return new GarbageCollector(
-            provider,
-            gcOptions,
-            getNodePackagePath,
-            getLastSummaryTimestampMs,
-            baseSnapshot,
-            readAndParseBlob,
-            baseLogger,
-            existing,
-            metadata,
-            isSummarizerClient,
-        );
+    public static create(createParams: IGarbageCollectorCreateParams): IGarbageCollector {
+        return new GarbageCollector(createParams);
     }
 
     /**
@@ -327,10 +319,10 @@ export class GarbageCollector implements IGarbageCollector {
     /**
      * Tells whether the GC data should be written to the root of the summary tree.
      */
-    private _writeDataAtRoot: boolean = false;
+    private _writeDataAtRoot: boolean = true;
     public get writeDataAtRoot(): boolean {
         return this._writeDataAtRoot;
-     }
+    }
 
     /**
      * Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
@@ -383,23 +375,29 @@ export class GarbageCollector implements IGarbageCollector {
     // The number of times GC has successfully completed on this instance of GarbageCollector.
     private completedRuns = 0;
 
-    protected constructor(
-        private readonly runtime: IGarbageCollectionRuntime,
-        private readonly gcOptions: IGCRuntimeOptions,
-        /** For a given node path, returns the node's package path. */
-        private readonly getNodePackagePath: (nodePath: string) => readonly string[] | undefined,
-        /** Returns the timestamp of the last summary generated for this container. */
-        private readonly getLastSummaryTimestampMs: () => number | undefined,
-        baseSnapshot: ISnapshotTree | undefined,
-        readAndParseBlob: ReadAndParseBlob,
-        baseLogger: ITelemetryLogger,
-        existing: boolean,
-        metadata: IContainerRuntimeMetadata | undefined,
-        private readonly isSummarizerClient: boolean = true,
-    ) {
-        this.mc = loggerToMonitoringContext(
-            ChildLogger.create(baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } }),
-        );
+    private readonly runtime: IGarbageCollectionRuntime;
+    private readonly gcOptions: IGCRuntimeOptions;
+    private readonly isSummarizerClient: boolean;
+
+    /** For a given node path, returns the node's package path. */
+    private readonly getNodePackagePath: (nodePath: string) => readonly string[] | undefined;
+    /** Returns the timestamp of the last summary generated for this container. */
+    private readonly getLastSummaryTimestampMs: () => number | undefined;
+
+    protected constructor(createParams: IGarbageCollectorCreateParams) {
+        this.runtime = createParams.runtime;
+        this.isSummarizerClient = createParams.isSummarizerClient;
+        this.gcOptions = createParams.gcOptions;
+        this.getNodePackagePath = createParams.getNodePackagePath;
+        this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
+
+        const baseSnapshot = createParams.baseSnapshot;
+        const metadata = createParams.metadata;
+        const readAndParseBlob = createParams.readAndParseBlob;
+
+        this.mc = loggerToMonitoringContext(ChildLogger.create(
+            createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } },
+        ));
 
         let prevSummaryGCVersion: number | undefined;
 
@@ -410,7 +408,7 @@ export class GarbageCollector implements IGarbageCollector {
          * 3. Whether GC session expiry is enabled or not.
          * For existing containers, we get this information from the metadata blob of its summary.
          */
-        if (existing) {
+        if (createParams.existing) {
             prevSummaryGCVersion = getGCVersion(metadata);
             // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
             // other existing documents, GC is enabled.
@@ -420,13 +418,15 @@ export class GarbageCollector implements IGarbageCollector {
         } else {
             // Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
             // scenario but explicitly failing makes it clearer and promotes correct usage.
-            if (gcOptions.sweepAllowed && !gcOptions.gcAllowed) {
+            if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
                 throw new UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
             }
 
-            // For new documents, GC has to be explicitly enabled via the flags in GC options.
-            this.gcEnabled = gcOptions.gcAllowed === true;
-            this.sweepEnabled = gcOptions.sweepAllowed === true;
+            // For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
+            // flag in GC options to false.
+            this.gcEnabled = this.gcOptions.gcAllowed !== false;
+            // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
+            this.sweepEnabled = this.gcOptions.sweepAllowed === true;
 
             // Set the Session Expiry only if the flag is enabled or the test option is set.
             if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
@@ -468,13 +468,10 @@ export class GarbageCollector implements IGarbageCollector {
             // GC must be enabled for the document.
             this.gcEnabled
             // GC must not be disabled via GC options.
-            && !gcOptions.disableGC
+            && !this.gcOptions.disableGC
         );
 
-        const minimumVersion = this.mc.config.getString(trackGCStateMinimumVersionKey);
-        const shouldTrackStateForVersion = meetsMinimumVersionRequirement(pkgVersion, minimumVersion);
-
-        this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true && shouldTrackStateForVersion;
+        this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
 
         /**
          * Whether sweep should run or not. The following conditions have to be met to run sweep:
@@ -493,21 +490,17 @@ export class GarbageCollector implements IGarbageCollector {
             defaultInactiveTimeoutMs;
 
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
-        this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;
+        this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
 
-        /**
-         * Enable resetting initial state once the following issue is resolved:
-         * https://github.com/microsoft/FluidFramework/issues/8878.
-         * Currently, the GC tree is not written at root, so we don't know if the base snapshot contains GC tree or not.
-         */
-        // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't contain
-        // GC tree and GC is enabled.
-        // const gcTreePresent = baseSnapshot?.trees[gcTreeKey] !== undefined;
-        // this.initialStateNeedsReset = gcTreePresent ? !this.shouldRunGC : this.shouldRunGC;
+        // GC state is written into root of the summary tree by default. Can be overridden via feature flag for now.
+        this._writeDataAtRoot = this.mc.config.getBoolean(writeAtRootKey) ?? true;
 
-        // If `writeDataAtRoot` setting is true, write the GC data into the root of the summary tree. We do this so that
-        // the roll out can be staged. Once its rolled out everywhere, we will start writing at root by default.
-        this._writeDataAtRoot = this.mc.config.getBoolean(writeAtRootKey) ?? this.gcOptions.writeDataAtRoot === true;
+        if (this._writeDataAtRoot) {
+            // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
+            // contain GC tree and GC is enabled.
+            const gcTreePresent = baseSnapshot?.trees[gcTreeKey] !== undefined;
+            this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
+        }
 
         // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
         // this once since it involves fetching blobs from storage which is expensive.
@@ -588,7 +581,7 @@ export class GarbageCollector implements IGarbageCollector {
                 return;
             }
 
-            const gcNodes: { [ id: string ]: string[]; } = {};
+            const gcNodes: { [id: string]: string[]; } = {};
             for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
                 if (nodeData.unreferencedTimestampMs !== undefined) {
                     this.unreferencedNodesState.set(
@@ -613,18 +606,14 @@ export class GarbageCollector implements IGarbageCollector {
                 return new Map();
             }
 
-            const gcNodes: { [ id: string ]: string[]; } = {};
+            const gcNodes: { [id: string]: string[]; } = {};
             for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
             // Run GC on the nodes in the base summary to get the routes used in each node in the container.
             // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
             // each node in the summary.
-            const usedRoutes = runGarbageCollection(
-                gcNodes,
-                ["/"],
-                this.mc.logger,
-            ).referencedNodeIds;
+            const usedRoutes = runGarbageCollection(gcNodes, ["/"]).referencedNodeIds;
 
             const baseGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
             // Currently, the nodes may write the GC data. So, we need to update it's base GC details with the
@@ -651,7 +640,7 @@ export class GarbageCollector implements IGarbageCollector {
             testMode: this.testMode,
             sessionExpiry: this.sessionExpiryTimeoutMs,
             inactiveTimeout: this.inactiveTimeoutMs,
-            existing,
+            existing: createParams.existing,
             ...this.gcOptions,
         });
         if (this.isSummarizerClient) {
@@ -705,11 +694,7 @@ export class GarbageCollector implements IGarbageCollector {
 
             // Get the runtime's GC data and run GC on the reference graph in it.
             const gcData = await this.runtime.getGCData(fullGC);
-            const gcResult = runGarbageCollection(
-                gcData.gcNodes,
-                ["/"],
-                logger,
-            );
+            const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
             const gcStats = this.generateStatsAndLogEvents(gcResult, logger);
 
             // Update the state since the last GC run. There can be nodes that were referenced between the last and
@@ -737,8 +722,7 @@ export class GarbageCollector implements IGarbageCollector {
             this.completedRuns++;
 
             return gcStats;
-        },
-        { end: true, cancel: "error" });
+        }, { end: true, cancel: "error" });
     }
 
     /**
@@ -749,6 +733,7 @@ export class GarbageCollector implements IGarbageCollector {
     public summarize(
         fullTree: boolean,
         trackState: boolean,
+        telemetryContext?: ITelemetryContext,
     ): ISummarizeResult | undefined {
         if (!this.shouldRunGC || this.previousGCDataFromLastRun === undefined) {
             return;
@@ -822,10 +807,6 @@ export class GarbageCollector implements IGarbageCollector {
         result: RefreshSummaryResult,
         readAndParseBlob: ReadAndParseBlob,
     ): Promise<void> {
-        // After a summary is successfully submitted and ack'd by this client, the GC state should have been reset in
-        // the summary and doesn't need to be reset anymore.
-        this.initialStateNeedsReset = false;
-
         if (!this.shouldRunGC || !result.latestSummaryUpdated) {
             return;
         }
@@ -834,6 +815,7 @@ export class GarbageCollector implements IGarbageCollector {
         // Basically, it was written in the current GC version.
         if (result.wasSummaryTracked) {
             this.latestSummaryGCVersion = this.currentGCVersion;
+            this.initialStateNeedsReset = false;
             if (this.trackGCState) {
                 this.latestSerializedSummaryState = this.pendingSerializedSummaryState;
                 this.pendingSerializedSummaryState = undefined;
@@ -1000,10 +982,7 @@ export class GarbageCollector implements IGarbageCollector {
             this.newReferencesSinceLastRun,
         );
 
-        // The following log will be enabled once this issue is resolved:
-        // https://github.com/microsoft/FluidFramework/issues/8878.
-        if (this.mc.config.getBoolean(logUnknownOutboundReferencesKey) === true
-            && missingExplicitReferences.length > 0) {
+        if (this.writeDataAtRoot && missingExplicitReferences.length > 0) {
             missingExplicitReferences.forEach((missingExplicitReference) => {
                 const event: ITelemetryPerformanceEvent = {
                     eventName: "gcUnknownOutboundReferences",
@@ -1049,7 +1028,7 @@ export class GarbageCollector implements IGarbageCollector {
          * unreferenced, stop tracking them and remove from unreferenced list.
          * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
          */
-        const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/"], logger);
+        const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/"]);
         for (const nodeId of gcResult.referencedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker !== undefined) {
@@ -1147,11 +1126,10 @@ export class GarbageCollector implements IGarbageCollector {
 
         const updateNodeStats = (nodeId: string, referenced: boolean) => {
             gcStats.nodeCount++;
-            /**
-             * `this.unreferencedNodesState` has the previous unreferenced state of all nodes. `referenced` flag passed
-             * here is current state of the give node. Check if the reference state of the changed.
-             */
-            const stateUpdated = this.unreferencedNodesState.has(nodeId) ? referenced : !referenced;
+            // If there is no previous GC data, every node's state is generated and is considered as updated.
+            // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
+            const stateUpdated = this.previousGCDataFromLastRun === undefined ||
+                this.unreferencedNodesState.has(nodeId) === referenced;
             if (stateUpdated) {
                 gcStats.updatedNodeCount++;
             }
@@ -1244,7 +1222,7 @@ export class GarbageCollector implements IGarbageCollector {
             if (pkg !== undefined) {
                 this.mc.logger.sendErrorEvent({
                     ...event,
-                    pkg: { value: `/${pkg.join("/")}`, tag: TelemetryDataTag.PackageData },
+                    pkg: { value: pkg.join("/"), tag: TelemetryDataTag.PackageData },
                 });
             } else {
                 this.pendingEventsQueue.push(event);
@@ -1313,66 +1291,3 @@ function setLongTimeout(
     }
     setTimerFn(timer);
 }
-
-/**
- * meetsMinimumVersionRequirement is used determining if a feature version should be run. This is similar to feature
- * flags. The advantage of this is that if we ship a bug in version 0.1.1 and fix it in version 0.2.1. We can keep this
- * feature disabled for version 0.1.1 and enabled for 0.2.1. Older versions will run without the feature and new
- * versions will run with the feature.
- * @param currentVersion - the total time the timeout needs to last in ms
- * @param minimumVersion - the function to execute when the timer ends
- */
-function meetsMinimumVersionRequirement(currentVersion: string, minimumVersion: string | undefined) {
-    return minimumVersion === undefined || semverCompare(currentVersion, minimumVersion) >= 0;
-}
-
-/**
- * Compare semver versions.
- * @param currentVersion - assumed to be any valid semver version
- * @param minimumVersion - must be [major].[minor].[patch], where major, minor, and patch are all numbers
- *  as it complicates the algorithm if we allow comparisons against minimum pre-release versions.
- * @returns
- *  0 if the currentVersion equals the minimumVersion
- *  1 if the currentVersion is greater than the minimumVersion
- *  -1 if the minimumVersion is greater than the currentVersion
- */
-export function semverCompare(currentVersion: string, minimumVersion: string): number {
-    const minimumValues = minimumVersion.split(".").map((value): number => {
-        assert(isNaN(+value) === false, 0x2fa /* Expected real numbers in minimum version! */);
-        return Number.parseInt(value, 10);
-    });
-    assert(minimumValues.length === 3, 0x2fb /* Expected minimumVersion to be [major].[minor].[patch] */);
-    const [minMajor, minMinor, minPatch] = minimumValues;
-
-    const currentValuesString = currentVersion.split(/\W/);
-    assert(currentValuesString.length >= 3, 0x2fc /* Expected version to match semver rules! */);
-    const currentValues = currentValuesString.slice(0, 3).map((value) => {
-        assert(isNaN(+value) === false, 0x2fd /* Expected real numbers in minimum version! */);
-        return Number.parseInt(value, 10);
-    });
-    const [cMajor, cMinor, cPatch] = currentValues;
-
-    if (cMajor > minMajor) {
-        return 1;
-    } else if (minMajor > cMajor) {
-        return -1;
-    }
-
-    if (cMinor > minMinor) {
-        return 1;
-    } else if (minMinor > cMinor) {
-        return -1;
-    }
-
-    if (cPatch > minPatch) {
-        return 1;
-    } else if (minPatch > cPatch) {
-        return -1;
-    }
-
-    if (currentValuesString.length === 3) {
-        return 0;
-    }
-
-    return -1;
-}

package/src/index.ts

@@ -9,7 +9,12 @@ export {
     ContainerRuntimeMessage,
     IGCRuntimeOptions,
     ISummaryRuntimeOptions,
+    ISummaryBaseConfiguration,
+    ISummaryConfigurationHeuristics,
+    ISummaryConfigurationDisableSummarizer,
+    ISummaryConfigurationDisableHeuristics,
     IContainerRuntimeOptions,
+    IPendingRuntimeState,
     IRootSummaryTreeWithStats,
     isRuntimeMessage,
     RuntimeMessage,
@@ -18,6 +23,8 @@ export {
     agentSchedulerId,
     ContainerRuntime,
     RuntimeHeaders,
+    ISummaryConfiguration,
+    DefaultSummaryConfiguration,
 } from "./containerRuntime";
 export { DeltaScheduler } from "./deltaScheduler";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
@@ -55,7 +62,6 @@ export {
     ISummarizer,
     ISummarizerEvents,
     ISummarizerInternalsProvider,
-    ISummarizerOptions,
     ISummarizerRuntime,
     ISummarizingWarning,
     ISummaryCancellationToken,

package/src/orderedClientElection.ts

@@ -2,7 +2,7 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-
+/* eslint-disable @rushstack/no-new-null */
 import { IEvent, IEventProvider, ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, TypedEventEmitter } from "@fluidframework/common-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "0.59.4001";
+export const pkgVersion = "1.1.0-75972";