@fluidframework/container-runtime 2.0.0-internal.2.3.1 → 2.0.0-internal.2.4.1

package/src/garbageCollection.ts

@@ -30,6 +30,7 @@ import {
     IGarbageCollectionNodeData,
     IGarbageCollectionSummaryDetailsLegacy,
     ISummaryTreeWithStats,
+    gcDeletedBlobKey,
 } from "@fluidframework/runtime-definitions";
 import {
     mergeStats,
@@ -157,9 +158,8 @@ export interface IGarbageCollector {
     getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
     /** Called when the latest summary of the system has been refreshed. */
     refreshLatestSummary(
-        result: RefreshSummaryResult,
         proposalHandle: string | undefined,
-        summaryRefSeq: number,
+        result: RefreshSummaryResult,
         readAndParseBlob: ReadAndParseBlob,
     ): Promise<void>;
     /** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
@@ -172,6 +172,8 @@ export interface IGarbageCollector {
     ): void;
     /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
     addedOutboundReference(fromNodePath: string, toNodePath: string): void;
+    /** Returns true if this node has been deleted by GC during sweep phase. */
+    isNodeDeleted(nodePath: string): boolean;
     setConnectionState(connected: boolean, clientId?: string): void;
     dispose(): void;
 }
@@ -226,6 +228,7 @@ interface IUnreferencedEventProps {
 interface IGCSummaryTrackingData {
     serializedGCState: string | undefined;
     serializedTombstones: string | undefined;
+    serializedDeletedNodes: string | undefined;
 }
 
 /**
@@ -419,7 +422,10 @@ export class GarbageCollector implements IGarbageCollector {
     // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
     // outbound routes from that node.
     private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
+    // A list of nodes that have been tombstoned.
     private tombstones: string[] = [];
+    // A list of nodes that have been deleted during sweep phase.
+    private deletedNodes: Set<string> = new Set();
 
     /**
      * Keeps track of the GC data from the latest summary successfully submitted to and acked from the server.
@@ -565,8 +571,7 @@ export class GarbageCollector implements IGarbageCollector {
             // flag in GC options to false.
             this.gcEnabled = this.gcOptions.gcAllowed !== false;
             // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
-            // ...unless we're using the TestOverride
-            this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
+            this.sweepEnabled = this.gcOptions.sweepAllowed === true;
 
             // Set the Session Expiry only if the flag is enabled and GC is enabled.
             if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
@@ -640,8 +645,9 @@ export class GarbageCollector implements IGarbageCollector {
 
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
         this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
-        // Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
-        this.tombstoneMode = this.mc.config.getBoolean(disableTombstoneKey) !== true;
+        // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
+        // via feature flags.
+        this.tombstoneMode = !this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
 
         // If GC ran in the container that generated the base snapshot, it will have a GC tree.
         this.wasGCRunInLatestSummary = baseSnapshot?.trees[gcTreeKey] !== undefined;
@@ -704,7 +710,9 @@ export class GarbageCollector implements IGarbageCollector {
                 }
                 // If there is only one node (root node just added above), either GC is disabled or we are loading from
                 // the first summary generated by detached container. In both cases, GC was not run - return undefined.
-                return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
+                return Object.keys(gcState.gcNodes).length === 1
+                    ? undefined
+                    : { gcState, tombstones: undefined, deletedNodes: undefined };
             } catch (error) {
                 const dpe = DataProcessingError.wrapIfUnrecognized(
                     error,
@@ -782,24 +790,33 @@ export class GarbageCollector implements IGarbageCollector {
     }
 
     /**
-     * Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
-     * before they are loaded or used. This is important to get accurate information of whether tombstoned object are
-     * in use or not.
+     * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
+     * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
      */
     public async initializeBaseState(): Promise<void> {
         const baseSnapshotData = await this.baseSnapshotDataP;
         /**
-         * The base snapshot data or tombstone state will not be present if the container is loaded from:
+         * The base snapshot data will not be present if the container is loaded from:
          * 1. The first summary created by the detached container.
          * 2. A summary that was generated with GC disabled.
          * 3. A summary that was generated before GC even existed.
-         * 4. A summary that was generated with tombstone feature disabled.
          */
-        if (!this.tombstoneMode || baseSnapshotData?.tombstones === undefined) {
+        if (baseSnapshotData === undefined) {
             return;
         }
-        this.tombstones = Array.from(baseSnapshotData.tombstones);
-        this.runtime.updateTombstonedRoutes(this.tombstones);
+
+        // Initialize the deleted nodes from the snapshot. This is done irrespective of whether sweep is enabled or not
+        // to identify deleted nodes' usage.
+        if (baseSnapshotData.deletedNodes !== undefined) {
+            this.deletedNodes = new Set(baseSnapshotData.deletedNodes);
+        }
+
+        // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
+        // tombstone routes.
+        if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
+            this.tombstones = Array.from(baseSnapshotData.tombstones);
+            this.runtime.updateTombstonedRoutes(this.tombstones);
+        }
     }
 
     /**
@@ -831,9 +848,29 @@ export class GarbageCollector implements IGarbageCollector {
         };
         this.unreferencedNodesState.clear();
 
-        // If tombstone mode is enabled, update tombstone information and also update all tombstoned nodes in the
-        // container as per the state in the snapshot data.
-        if (this.tombstoneMode) {
+        // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
+        // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
+        // tombstones.
+        // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
+        // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
+        if (this.shouldRunSweep) {
+            const snapshotDeletedNodes = snapshotData?.tombstones ? new Set(snapshotData.tombstones) : undefined;
+            // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
+            // delete them.
+            if (snapshotDeletedNodes !== undefined) {
+                const newDeletedNodes: string[] = [];
+                for (const nodeId of snapshotDeletedNodes) {
+                    if (!this.deletedNodes.has(nodeId)) {
+                        newDeletedNodes.push(nodeId);
+                    }
+                }
+                if (newDeletedNodes.length > 0) {
+                    // Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
+                }
+            }
+        } else if (this.tombstoneMode) {
+            // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
+            // notify the runtime to update its state as well.
             this.tombstones = snapshotData?.tombstones ? Array.from(snapshotData.tombstones) : [];
             this.runtime.updateTombstonedRoutes(this.tombstones);
         }
@@ -869,6 +906,7 @@ export class GarbageCollector implements IGarbageCollector {
             this.latestSummaryData = {
                 serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
                 serializedTombstones: JSON.stringify(snapshotData.tombstones),
+                serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
             };
         }
     }
@@ -1020,18 +1058,26 @@ export class GarbageCollector implements IGarbageCollector {
         }
 
         const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
+        // Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
+        // to identify deleted nodes' usage.
+        const serializedDeletedNodes = this.deletedNodes.size > 0
+            ? JSON.stringify(Array.from(this.deletedNodes).sort())
+            : undefined;
+        // If running in tombstone mode, serialize and write tombstones, if any.
         const serializedTombstones = this.tombstoneMode
             ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
             : undefined;
 
         /**
-         * Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
-         * summary, send summary handles for them. Otherwise, send the data in summary blobs.
+         * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
+         * write summary handle instead of summary tree for GC.
+         * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
+         * for each of these that did not change, write a summary handle.
          */
         if (this.trackGCState) {
-            this.pendingSummaryData = { serializedGCState, serializedTombstones };
+            this.pendingSummaryData = { serializedGCState, serializedTombstones, serializedDeletedNodes };
             if (trackState && !fullTree && this.latestSummaryData !== undefined) {
-                // If neither GC state or tombstone state changed, send a summary handle for the entire GC data.
+                // If nothing changed since last summary, send a summary handle for the entire GC data.
                 if (this.latestSummaryData.serializedGCState === serializedGCState
                     && this.latestSummaryData.serializedTombstones === serializedTombstones) {
                     const stats = mergeStats();
@@ -1046,26 +1092,30 @@ export class GarbageCollector implements IGarbageCollector {
                     };
                 }
 
-                // If either or both of GC state or tombstone state changed, build a GC summary tree.
-                return this.buildGCSummaryTree(serializedGCState, serializedTombstones, true /* trackState */);
+                // If some state changed, build a GC summary tree.
+                return this.buildGCSummaryTree(
+                    serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
             }
         }
         // If not tracking GC state, build a GC summary tree without any summary handles.
-        return this.buildGCSummaryTree(serializedGCState, serializedTombstones, false /* trackState */);
+        return this.buildGCSummaryTree(
+            serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
     }
 
     /**
-     * Builds the GC summary tree which contains GC state and tombstone state.
-     * If trackState is false, both GC state and tombstone state are written as summary blobs.
-     * If trackState is true, summary blob is written for GC state or tombstone state if they changed.
+     * Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
+     * If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
+     * If trackState is true, only states that changed are written. Rest are written as handles.
      * @param serializedGCState - The GC state serialized as string.
-     * @param serializedTombstones - THe tombstone state serialized as string.
+     * @param serializedTombstones - The tombstone state serialized as string.
+     * @param serializedDeletedNodes - Deleted nodes serialized as string.
      * @param trackState - Whether we are tracking GC state across summaries.
      * @returns the GC summary tree.
      */
     private buildGCSummaryTree(
         serializedGCState: string,
         serializedTombstones: string | undefined,
+        serializedDeletedNodes: string | undefined,
         trackState: boolean,
     ): ISummaryTreeWithStats {
         const gcStateBlobKey = `${gcBlobPrefix}_root`;
@@ -1078,16 +1128,26 @@ export class GarbageCollector implements IGarbageCollector {
             builder.addBlob(gcStateBlobKey, serializedGCState);
         }
 
-        // If there is no tombstone data, return only the GC state.
-        if (serializedTombstones === undefined) {
+        // If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
+        // summary blob.
+        if (serializedTombstones !== undefined) {
+            if (this.latestSummaryData?.serializedTombstones === serializedTombstones && trackState) {
+                builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
+            } else {
+                builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
+            }
+        }
+
+        // If there are no deleted nodes, return the summary tree.
+        if (serializedDeletedNodes === undefined) {
             return builder.getSummaryTree();
         }
 
-        // If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
-        if (this.latestSummaryData?.serializedTombstones === serializedTombstones && trackState) {
-            builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
+        // If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
+        if (this.latestSummaryData?.serializedDeletedNodes === serializedDeletedNodes && trackState) {
+            builder.addHandle(gcDeletedBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcDeletedBlobKey}`);
         } else {
-            builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
+            builder.addBlob(gcDeletedBlobKey, serializedDeletedNodes);
         }
         return builder.getSummaryTree();
     }
@@ -1118,9 +1178,8 @@ export class GarbageCollector implements IGarbageCollector {
      * is downloaded and should be used to update the state.
      */
     public async refreshLatestSummary(
-        result: RefreshSummaryResult,
         proposalHandle: string | undefined,
-        summaryRefSeq: number,
+        result: RefreshSummaryResult,
         readAndParseBlob: ReadAndParseBlob,
     ): Promise<void> {
         // If the latest summary was updated and the summary was tracked, this client is the one that generated this
@@ -1147,8 +1206,8 @@ export class GarbageCollector implements IGarbageCollector {
         }
 
         // If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
-        const snapshot = result.snapshot;
-        const metadataBlobId = snapshot.blobs[metadataBlobName];
+        const snapshotTree = result.snapshotTree;
+        const metadataBlobId = snapshotTree.blobs[metadataBlobName];
         if (metadataBlobId) {
             const metadata = await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId);
             this.latestSummaryGCVersion = getGCVersion(metadata);
@@ -1162,10 +1221,10 @@ export class GarbageCollector implements IGarbageCollector {
                 "No reference timestamp when updating GC state from snapshot",
                 "refreshLatestSummary",
                 undefined,
-                { proposalHandle, summaryRefSeq, details: JSON.stringify(this.configs) },
+                { proposalHandle, summaryRefSeq: result.summaryRefSeq, details: JSON.stringify(this.configs) },
             );
         }
-        const gcSnapshotTree = snapshot.trees[gcTreeKey];
+        const gcSnapshotTree = snapshotTree.trees[gcTreeKey];
         // If GC ran in the container that generated this snapshot, it will have a GC tree.
         this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
         let latestGCData: IGarbageCollectionSnapshotData | undefined;
@@ -1257,6 +1316,14 @@ export class GarbageCollector implements IGarbageCollector {
         }
     }
 
+    /**
+     * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
+     * cases where objects are used after they are deleted and throw / log errors accordingly.
+     */
+    public isNodeDeleted(nodePath: string): boolean {
+        return this.deletedNodes.has(nodePath);
+    }
+
     public dispose(): void {
         this.sessionExpiryTimer?.clear();
         this.sessionExpiryTimer = undefined;
@@ -1640,7 +1707,7 @@ export class GarbageCollector implements IGarbageCollector {
                 }
             }
 
-            // If SweepReady Usage Detection is enabed, the handler may close the interactive container.
+            // If SweepReady Usage Detection is enabled, the handler may close the interactive container.
             // Once Sweep is fully implemented, this will be removed since the objects will be gone
             // and errors will arise elsewhere in the runtime
             if (state === UnreferencedState.SweepReady) {

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.2.3.1";
+export const pkgVersion = "2.0.0-internal.2.4.1";

package/src/summarizer.ts

@@ -160,7 +160,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         // This will result in "summarizerClientDisconnected" stop reason recorded in telemetry,
         // unless stop() was called earlier
         this.dispose();
-        (this.runtime.disposeFn ?? this.runtime.closeFn)()
+        (this.runtime.disposeFn ?? this.runtime.closeFn)();
     }
 
     private async runCore(onBehalfOf: string): Promise<SummarizerStopReason> {