@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.3.1.0.125672

package/src/garbageCollectionConstants.ts

@@ -3,13 +3,12 @@
  * Licensed under the MIT License.
  */
 
+import { GCVersion } from "./summaryFormat";
 
-// The key for the GC tree in summary.
-export const gcTreeKey = "gc";
-// They prefix for GC blobs in the GC tree in summary.
-export const gcBlobPrefix = "__gc";
-// The key for tombstone blob in the GC tree in summary.
-export const gcTombstoneBlobKey = "__tombstones";
+/** The stable version of garbage collection in production. */
+export const stableGCVersion: GCVersion = 1;
+/** The current version of garbage collection. */
+export const currentGCVersion: GCVersion = 2;
 
 // Feature gate key to turn GC on / off.
 export const runGCKey = "Fluid.GarbageCollection.RunGC";
@@ -25,8 +24,12 @@ export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
 export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
 // Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
 export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
-// Feature gate to enable throwing an error when tombstone object is used.
+// Feature gate to enable throwing an error when tombstone object is loaded (requested).
+export const throwOnTombstoneLoadKey = "Fluid.GarbageCollection.ThrowOnTombstoneLoad";
+// Feature gate to enable throwing an error when tombstone object is used (e.g. outgoing or incoming ops).
 export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
+// Feature gate to enable GC version upgrade.
+export const gcVersionUpgradeToV2Key = "Fluid.GarbageCollection.GCVersionUpgradeToV2";
 
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;

package/src/garbageCollectionHelpers.ts

@@ -0,0 +1,37 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
+import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
+import { MonitoringContext } from "@fluidframework/telemetry-utils";
+import {
+	disableTombstoneKey,
+	runSweepKey,
+	throwOnTombstoneLoadKey,
+	throwOnTombstoneUsageKey,
+} from "./garbageCollectionConstants";
+
+/**
+ * Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
+ * tombstoned or deleted object is loaded.
+ */
+export function sendGCUnexpectedUsageEvent(
+	mc: MonitoringContext,
+	event: ITelemetryGenericEvent & { category: "error" | "generic"; isSummarizerClient: boolean },
+	packagePath: readonly string[] | undefined,
+	error?: unknown,
+) {
+	event.pkg = packagePathToTelemetryProperty(packagePath);
+	event.tombstoneFlags = JSON.stringify({
+		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
+		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
+		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
+	});
+	event.sweepFlags = JSON.stringify({
+		EnableSweepFlag: mc.config.getBoolean(runSweepKey),
+	});
+
+	mc.logger.sendTelemetryEvent(event, error);
+}

package/src/gcSweepReadyUsageDetection.ts

@@ -6,10 +6,10 @@
 import { ITelemetryProperties } from "@fluidframework/common-definitions";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import {
-    IConfigProvider,
-    IFluidErrorBase,
-    LoggingError,
-    MonitoringContext,
+	IConfigProvider,
+	IFluidErrorBase,
+	LoggingError,
+	MonitoringContext,
 } from "@fluidframework/telemetry-utils";
 import { oneDayMs } from "./garbageCollectionConstants";
 
@@ -17,30 +17,33 @@ import { oneDayMs } from "./garbageCollectionConstants";
  * Feature Gate Key -
  * How many days between closing the container from this error (avoids locking user out of their file altogether)
  */
-export const skipClosureForXDaysKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
+export const skipClosureForXDaysKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
 
 /**
  * LocalStorage key (NOT via feature gate / monitoring context)
  * A map from docId to info about the last time we closed due to this error
  */
-export const closuresMapLocalStorageKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
+export const closuresMapLocalStorageKey =
+	"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
 
 /**
  * Feature gate key to enable closing the container if SweepReady objects are used.
  * Value should contain keywords "interactiveClient" and/or "summarizer" to enable detection in each container type
  */
 const sweepReadyUsageDetectionSetting = {
-    read(config: IConfigProvider) {
-        const sweepReadyUsageDetectionKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
-        const value = config.getString(sweepReadyUsageDetectionKey);
-        if (value === undefined) {
-            return { interactiveClient: false, summarizer: false };
-        }
-        return {
-            interactiveClient: value.includes("interactiveClient"),
-            summarizer: value.includes("summarizer"),
-        };
-    },
+	read(config: IConfigProvider) {
+		const sweepReadyUsageDetectionKey =
+			"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
+		const value = config.getString(sweepReadyUsageDetectionKey);
+		if (value === undefined) {
+			return { interactiveClient: false, summarizer: false };
+		}
+		return {
+			interactiveClient: value.includes("interactiveClient"),
+			summarizer: value.includes("summarizer"),
+		};
+	},
 };
 
 /**
@@ -52,8 +55,8 @@ const sweepReadyUsageDetectionSetting = {
  * since only the Summarizer has the latest truth about unreferenced node tracking
  */
 export class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {
-    /** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
-    public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
+	/** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
+	public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
 }
 
 /**
@@ -65,75 +68,78 @@ export class SweepReadyUsageError extends LoggingError implements IFluidErrorBas
  * (via skipClosureForXDaysKey above.  Uses localStorage and closuresMapLocalStorageKey to implement this behavior)
  */
 export class SweepReadyUsageDetectionHandler {
-    private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
+	private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
 
-    constructor(
-        private readonly uniqueContainerKey: string,
-        private readonly mc: MonitoringContext,
-        private readonly closeFn: (error?: ICriticalContainerError) => void,
-        localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
-    ) {
-        const noopStorage = { getItem: () => null, setItem: () => {} };
-        // localStorage is not defined in Node environment, so fall back to noopStorage if needed.
-        this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
+	constructor(
+		private readonly uniqueContainerKey: string,
+		private readonly mc: MonitoringContext,
+		private readonly closeFn: (error?: ICriticalContainerError) => void,
+		localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
+	) {
+		const noopStorage = { getItem: () => null, setItem: () => {} };
+		// localStorage is not defined in Node environment, so fall back to noopStorage if needed.
+		this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
 
-        if (this.localStorage === noopStorage) {
-            // This means the Skip Closure Period logic will not work.
-            this.mc.logger.sendTelemetryEvent({ eventName: "SweepReadyUsageDetectionHandlerNoopStorage" });
-        }
-    }
+		if (this.localStorage === noopStorage) {
+			// This means the Skip Closure Period logic will not work.
+			this.mc.logger.sendTelemetryEvent({
+				eventName: "SweepReadyUsageDetectionHandlerNoopStorage",
+			});
+		}
+	}
 
-    /**
-      * If SweepReady Usage Detection is enabled, close the interactive container.
-      * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
-      *
-      * Once Sweep is fully implemented, this will be removed since the objects will be gone
-      * and errors will arise elsewhere in the runtime
-     */
-    public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
-        if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
-            return;
-        }
+	/**
+	 * If SweepReady Usage Detection is enabled, close the interactive container.
+	 * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
+	 *
+	 * Once Sweep is fully implemented, this will be removed since the objects will be gone
+	 * and errors will arise elsewhere in the runtime
+	 */
+	public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
+		if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
+			return;
+		}
 
-        // Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
-        // However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
-        // which will only allow the container to close once during that period, to avoid locking users out.
-        let shouldClose: boolean = true;
-        let pastClosuresMap: Record<string, { lastCloseTime: number; } | undefined> = {};
-        let lastCloseTime: number | undefined;
-        const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
-        if (skipClosureForXDays !== undefined) {
-            // Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
-            try {
-                const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
-                const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
-                if (typeof parsedValue === "object") {
-                    pastClosuresMap = parsedValue;
-                }
-            } catch (e) {
-            }
-            lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
+		// Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
+		// However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
+		// which will only allow the container to close once during that period, to avoid locking users out.
+		let shouldClose: boolean = true;
+		let pastClosuresMap: Record<string, { lastCloseTime: number } | undefined> = {};
+		let lastCloseTime: number | undefined;
+		const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
+		if (skipClosureForXDays !== undefined) {
+			// Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
+			try {
+				const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
+				const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
+				if (typeof parsedValue === "object") {
+					pastClosuresMap = parsedValue;
+				}
+			} catch (e) {}
+			lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
 
-            // Don't close if we did already within the Skip Closure Period
-            if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs) {
-                shouldClose = false;
-            }
-        }
+			// Don't close if we did already within the Skip Closure Period
+			if (
+				lastCloseTime !== undefined &&
+				Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs
+			) {
+				shouldClose = false;
+			}
+		}
 
-        const error = new SweepReadyUsageError(
-            "SweepReady object used in Non-Summarizer Client",
-            { errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }) },
-        );
-        if (shouldClose) {
-            // Update closures map in localStorage before closing
-            // Note there is a race condition between different tabs updating localStorage and overwriting
-            // each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
-            pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
-            this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
+		const error = new SweepReadyUsageError("SweepReady object used in Non-Summarizer Client", {
+			errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }),
+		});
+		if (shouldClose) {
+			// Update closures map in localStorage before closing
+			// Note there is a race condition between different tabs updating localStorage and overwriting
+			// each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
+			pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
+			this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
 
-            this.closeFn(error);
-        } else {
-            this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
-        }
-    }
+			this.closeFn(error);
+		} else {
+			this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
+		}
+	}
 }

package/src/index.ts

@@ -4,82 +4,80 @@
  */
 
 export {
-    ContainerMessageType,
-    ContainerRuntimeMessage,
-    IGCRuntimeOptions,
-    ISummaryRuntimeOptions,
-    ISummaryBaseConfiguration,
-    ISummaryConfigurationHeuristics,
-    ISummaryConfigurationDisableSummarizer,
-    ISummaryConfigurationDisableHeuristics,
-    IContainerRuntimeOptions,
-    IRootSummaryTreeWithStats,
-    isRuntimeMessage,
-    RuntimeMessage,
-    agentSchedulerId,
-    ContainerRuntime,
-    RuntimeHeaders,
-    ISummaryConfiguration,
-    DefaultSummaryConfiguration,
-    ICompressionRuntimeOptions,
-    CompressionAlgorithms,
+	ContainerMessageType,
+	ContainerRuntimeMessage,
+	IGCRuntimeOptions,
+	ISummaryRuntimeOptions,
+	ISummaryBaseConfiguration,
+	ISummaryConfigurationHeuristics,
+	ISummaryConfigurationDisableSummarizer,
+	ISummaryConfigurationDisableHeuristics,
+	IContainerRuntimeOptions,
+	IRootSummaryTreeWithStats,
+	isRuntimeMessage,
+	RuntimeMessage,
+	agentSchedulerId,
+	ContainerRuntime,
+	RuntimeHeaders,
+	AllowTombstoneRequestHeaderKey,
+	TombstoneResponseHeaderKey,
+	ISummaryConfiguration,
+	DefaultSummaryConfiguration,
+	ICompressionRuntimeOptions,
+	CompressionAlgorithms,
 } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
+export { IGCStats } from "./garbageCollection";
 export {
-    IGCStats,
-} from "./garbageCollection";
-export {
-    gcBlobPrefix,
-    gcTombstoneBlobKey,
-    gcTreeKey,
-} from "./garbageCollectionConstants";
-export {
-    IPendingFlush,
-    IPendingLocalState,
-    IPendingMessage,
-    IPendingState,
+	IPendingFlush,
+	IPendingLocalState,
+	IPendingMessage,
+	IPendingState,
 } from "./pendingStateManager";
 export { Summarizer } from "./summarizer";
 export {
-    EnqueueSummarizeResult,
-    IAckSummaryResult,
-    IBaseSummarizeResult,
-    IBroadcastSummaryResult,
-    ICancellationToken,
-    IConnectableRuntime,
-    IEnqueueSummarizeOptions,
-    IGenerateSummaryTreeResult,
-    IGeneratedSummaryStats,
-    INackSummaryResult,
-    IOnDemandSummarizeOptions,
-    IProvideSummarizer,
-    IRefreshSummaryAckOptions,
-    ISubmitSummaryOpResult,
-    ISubmitSummaryOptions,
-    ISummarizeOptions,
-    ISummarizeResults,
-    ISummarizer,
-    ISummarizerEvents,
-    ISummarizerInternalsProvider,
-    ISummarizerRuntime,
-    ISummarizingWarning,
-    ISummaryCancellationToken,
-    IUploadSummaryResult,
-    SubmitSummaryResult,
-    SummarizeResultPart,
-    SummarizerStopReason,
+	EnqueueSummarizeResult,
+	IAckSummaryResult,
+	IBaseSummarizeResult,
+	IBroadcastSummaryResult,
+	ICancellationToken,
+	IConnectableRuntime,
+	IEnqueueSummarizeOptions,
+	IGenerateSummaryTreeResult,
+	IGeneratedSummaryStats,
+	INackSummaryResult,
+	IOnDemandSummarizeOptions,
+	IProvideSummarizer,
+	IRefreshSummaryAckOptions,
+	ISubmitSummaryOpResult,
+	ISubmitSummaryOptions,
+	ISummarizeOptions,
+	ISummarizeResults,
+	ISummarizer,
+	ISummarizerEvents,
+	ISummarizerInternalsProvider,
+	ISummarizerRuntime,
+	ISummarizingWarning,
+	ISummaryCancellationToken,
+	IUploadSummaryResult,
+	SubmitSummaryResult,
+	SummarizeResultPart,
+	SummarizerStopReason,
 } from "./summarizerTypes";
 export {
-    IAckedSummary,
-    IClientSummaryWatcher,
-    ISummary,
-    ISummaryCollectionOpEvents,
-    ISummaryAckMessage,
-    ISummaryNackMessage,
-    ISummaryOpMessage,
-    OpActionEventListener,
-    OpActionEventName,
-    SummaryCollection,
+	IAckedSummary,
+	IClientSummaryWatcher,
+	ISummary,
+	ISummaryCollectionOpEvents,
+	ISummaryAckMessage,
+	ISummaryNackMessage,
+	ISummaryOpMessage,
+	OpActionEventListener,
+	OpActionEventName,
+	SummaryCollection,
 } from "./summaryCollection";
-export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
+export {
+	ICancellableSummarizerController,
+	neverCancelledSummaryToken,
+} from "./runWhileConnectedCoordinator";
 export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";