@fluidframework/container-runtime 2.0.0-dev.7.4.0.217884 → 2.0.0-dev.7.4.0.221926

package/lib/gc/garbageCollection.d.ts

@@ -6,6 +6,7 @@ import { IRequest } from "@fluidframework/core-interfaces";
 import { IGarbageCollectionDetailsBase, ISummarizeResult, ITelemetryContext } from "@fluidframework/runtime-definitions";
 import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import { RuntimeHeaderData } from "../containerRuntime";
+import { ContainerRuntimeGCMessage } from "../messageTypes";
 import { IRefreshSummaryResult } from "../summary";
 import { IGarbageCollector, IGarbageCollectorCreateParams, IGCStats, IGCMetadata } from "./gcDefinitions";
 /**
@@ -61,6 +62,7 @@ export declare class GarbageCollector implements IGarbageCollector {
     private readonly getLastSummaryTimestampMs;
     /** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
     private readonly activeConnection;
+    private readonly submitMessage;
     get summaryStateNeedsReset(): boolean;
     /** Returns the count of data stores whose GC state updated since the last summary. */
     get updatedDSCountSinceLastSummary(): number;
@@ -81,7 +83,7 @@ export declare class GarbageCollector implements IGarbageCollector {
     private updateStateFromSnapshotData;
     /**
      * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
-     * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
+     * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
      * @param connected - Whether the runtime connected / disconnected.
      * @param clientId - The clientId of this runtime.
      */
@@ -105,10 +107,15 @@ export declare class GarbageCollector implements IGarbageCollector {
     }, telemetryContext?: ITelemetryContext): Promise<IGCStats | undefined>;
     /**
      * Runs garbage collection. It does the following:
+     *
      * 1. It generates / analyzes the runtime's reference graph.
+     *
      * 2. Generates mark phase stats.
+     *
      * 3. Runs Mark phase.
+     *
      * 4. Runs Sweep phase.
+     *
      * 5. Generates sweep phase stats.
      */
     private runGC;
@@ -124,19 +131,19 @@ export declare class GarbageCollector implements IGarbageCollector {
      * @param gcResult - The result of the GC run on the gcData.
      * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
      * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
-     * @returns A list of sweep ready nodes, i.e., nodes that ready to be deleted.
+     * @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
      */
     private runMarkPhase;
     /**
      * Runs the GC Sweep phase. It does the following:
-     * 1. Calls the runtime to delete nodes that are sweep ready.
-     * 2. Clears tracking for deleted nodes.
+     *
+     * 1. Marks tombstone-ready nodes as tombstones.
+     *
+     * 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
      *
      * @param gcResult - The result of the GC run on the gcData.
-     * @param sweepReadyNodes - List of nodes that are sweep ready.
-     * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
-     * @param logger - The logger to be used to log any telemetry.
-     * @returns A list of nodes that have been deleted.
+     * @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
+     * @param sweepReadyNodes - List of nodes that are sweep-ready.
      */
     private runSweepPhase;
     /**
@@ -166,6 +173,18 @@ export declare class GarbageCollector implements IGarbageCollector {
      * Called to refresh the latest summary state. This happens when either a pending summary is acked.
      */
     refreshLatestSummary(result: IRefreshSummaryResult): Promise<void>;
+    /**
+     * Process a GC message.
+     * @param message - The GC message from the container runtime.
+     * @param local - Whether it was send by this client.
+     */
+    processMessage(message: ContainerRuntimeGCMessage, local: boolean): void;
+    /**
+     * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
+     * tracking for nodes that are actually deleted by the runtime.
+     * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
+     */
+    private deleteSweepReadyNodes;
     /**
      * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
      * or throw an error if failing on incorrect usage is configured.
@@ -199,8 +218,8 @@ export declare class GarbageCollector implements IGarbageCollector {
     private getMarkPhaseStats;
     /**
      * Generates the stats of a garbage collection sweep phase run.
-     * @param allDeletedNodes - All the nodes that have been deleted across all GC runs.
-     * @param currentDeletedNodes - The nodes that have been deleted in this GC run.
+     * @param deletedNodes - The nodes that have been deleted until this run.
+     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
      * @param markPhaseStats - The stats of the mark phase run.
      * @returns the stats of the sweep phase run.
      */

package/lib/gc/garbageCollection.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEnD,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAIX,MAAM,iBAAiB,CAAC;AAQzB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IACrE,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgB;IAEjD,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IAqKjE;;;OAGG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjD;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA4EnC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAwEhC;;;;;;;OAOG;YACW,KAAK;IA6CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAgDpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IAiDrB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAyD/B;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAqBtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0DzB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;CAiD1B"}
+{"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAwB,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEnD,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAMX,MAAM,iBAAiB,CAAC;AAazB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IACrE,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgB;IAEjD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IAsKjE;;;OAGG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjD;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA6EnC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAwEhC;;;;;;;;;;;;OAYG;YACW,KAAK;IA2CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAqDpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IAgErB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;OAIG;IACI,cAAc,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAyBxE;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAqC7B;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAyD/B;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAqBtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0DzB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;CAkD1B"}

package/lib/gc/garbageCollection.js

@@ -8,9 +8,10 @@ import { createResponseError, responseToException } from "@fluidframework/runtim
 import { createChildLogger, createChildMonitoringContext, DataProcessingError, PerformanceEvent, } from "@fluidframework/telemetry-utils";
 import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey, } from "../containerRuntime";
 import { ClientSessionExpiredError } from "../error";
+import { ContainerMessageType } from "../messageTypes";
 import { generateGCConfigs } from "./gcConfigs";
-import { GCNodeType, UnreferencedState, } from "./gcDefinitions";
-import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot } from "./gcHelpers";
+import { GCNodeType, UnreferencedState, GarbageCollectionMessageType, } from "./gcDefinitions";
+import { cloneGCData, compatBehaviorAllowsGCMessageType, concatGarbageCollectionData, getGCDataFromSnapshot, } from "./gcHelpers";
 import { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
 import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
 import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
@@ -46,7 +47,7 @@ export class GarbageCollector {
     }
     /** If false, loading or using a Tombstoned object should merely log, not fail */
     get tombstoneEnforcementAllowed() {
-        return this.configs.tombstoneEnforcementAllowed;
+        return this.configs.sweepEnabled;
     }
     /** If true, throw an error when a tombstone data store is retrieved */
     get throwOnTombstoneLoad() {
@@ -80,6 +81,7 @@ export class GarbageCollector {
         this.getNodePackagePath = createParams.getNodePackagePath;
         this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
         this.activeConnection = createParams.activeConnection;
+        this.submitMessage = createParams.submitMessage;
         const baseSnapshot = createParams.baseSnapshot;
         const readAndParseBlob = createParams.readAndParseBlob;
         this.mc = createChildMonitoringContext({
@@ -141,7 +143,7 @@ export class GarbageCollector {
         /**
          * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
          * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
-         * GC state and updates their inactive or sweep ready state.
+         * GC state and updates their inactive or sweep-ready state.
          */
         this.initializeGCStateFromBaseSnapshotP = new LazyPromise(async () => {
             /**
@@ -291,7 +293,7 @@ export class GarbageCollector {
         const gcNodes = {};
         for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
             if (nodeData.unreferencedTimestampMs !== undefined) {
-                this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
+                this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
             }
             gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
         }
@@ -299,7 +301,7 @@ export class GarbageCollector {
     }
     /**
      * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
-     * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
+     * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
      * @param connected - Whether the runtime connected / disconnected.
      * @param clientId - The clientId of this runtime.
      */
@@ -315,7 +317,7 @@ export class GarbageCollector {
          * the receiving summarizer client.
          *
          * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
-         * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
+         * sweep in phases and we want to track when inactive and sweep-ready objects are used in any client.
          */
         if (this.activeConnection() && this.configs.shouldRunGC) {
             this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
@@ -392,10 +394,15 @@ export class GarbageCollector {
     }
     /**
      * Runs garbage collection. It does the following:
+     *
      * 1. It generates / analyzes the runtime's reference graph.
+     *
      * 2. Generates mark phase stats.
+     *
      * 3. Runs Mark phase.
+     *
      * 4. Runs Sweep phase.
+     *
      * 5. Generates sweep phase stats.
      */
     async runGC(fullGC, currentReferenceTimestampMs, logger) {
@@ -410,14 +417,15 @@ export class GarbageCollector {
         // This is done before running mark phase because we need the previous GC state before it is updated.
         const markPhaseStats = this.getMarkPhaseStats(gcResult);
         // 3. Run the Mark phase.
-        // It will mark nodes as referenced / unreferenced and return a list of node ids that are ready to be swept.
-        const sweepReadyNodeIds = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
+        // It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
+        const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
         // 4. Run the Sweep phase.
-        // It will delete sweep ready nodes and return a list of deleted node ids.
-        const deletedNodeIds = this.runSweepPhase(gcResult, sweepReadyNodeIds);
-        this.gcDataFromLastRun = cloneGCData(gcData, (id) => deletedNodeIds.includes(id) /* filter out deleted nodes */);
+        // It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
+        // sweep op. All clients, including this one, will delete these nodes once it processes the op.
+        this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
+        this.gcDataFromLastRun = cloneGCData(gcData);
         // 5. Get the sweep phase stats.
-        const sweepPhaseStats = this.getSweepPhaseStats(this.configs.shouldRunSweep ? this.deletedNodes : sweepReadyNodeIds, new Set(deletedNodeIds), markPhaseStats);
+        const sweepPhaseStats = this.getSweepPhaseStats(this.deletedNodes, sweepReadyNodeIds, markPhaseStats);
         return { ...markPhaseStats, ...sweepPhaseStats };
     }
     /**
@@ -432,7 +440,7 @@ export class GarbageCollector {
      * @param gcResult - The result of the GC run on the gcData.
      * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
      * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
-     * @returns A list of sweep ready nodes, i.e., nodes that ready to be deleted.
+     * @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
      */
     runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
         // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
@@ -446,17 +454,21 @@ export class GarbageCollector {
             }
         }
         // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
+        const tombstoneReadyNodeIds = new Set();
         const sweepReadyNodeIds = new Set();
         for (const nodeId of gcResult.deletedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker === undefined) {
-                this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
+                this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
             }
             else {
                 // If a node was already unreferenced, update its tracking information. Since the current reference time
                 // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
                 nodeStateTracker.updateTracking(currentReferenceTimestampMs);
-                // If a node is sweep ready, store it so it can be returned.
+                // If a node is tombstone or sweep-ready, store it so it can be returned.
+                if (nodeStateTracker.state === UnreferencedState.TombstoneReady) {
+                    tombstoneReadyNodeIds.add(nodeId);
+                }
                 if (nodeStateTracker.state === UnreferencedState.SweepReady) {
                     sweepReadyNodeIds.add(nodeId);
                 }
@@ -464,62 +476,72 @@ export class GarbageCollector {
         }
         // 3. Call the runtime to update referenced nodes in this run.
         this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
-        return sweepReadyNodeIds;
+        return { tombstoneReadyNodeIds, sweepReadyNodeIds };
     }
     /**
      * Runs the GC Sweep phase. It does the following:
-     * 1. Calls the runtime to delete nodes that are sweep ready.
-     * 2. Clears tracking for deleted nodes.
+     *
+     * 1. Marks tombstone-ready nodes as tombstones.
+     *
+     * 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
      *
      * @param gcResult - The result of the GC run on the gcData.
-     * @param sweepReadyNodes - List of nodes that are sweep ready.
-     * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
-     * @param logger - The logger to be used to log any telemetry.
-     * @returns A list of nodes that have been deleted.
+     * @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
+     * @param sweepReadyNodes - List of nodes that are sweep-ready.
      */
-    runSweepPhase(gcResult, sweepReadyNodes) {
+    runSweepPhase(gcResult, tombstoneReadyNodes, sweepReadyNodes) {
         /**
-         * Currently, there are 3 modes for sweep:
-         * Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
-         * Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted.
-         * Sweep mode - Sweep ready modes are deleted.
+         * Under "Test Mode", unreferenced nodes are immediately deleted without waiting for them to be sweep-ready.
          *
-         * These modes serve as staging for applications that want to enable sweep by providing an incremental
-         * way to test and validate sweep works as expected.
+         * Otherwise, depending on how long it's been since the node was unreferenced, it will either be
+         * marked as Tombstone, or deleted by Sweep.
          */
         if (this.configs.testMode) {
             // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
             this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
-            return [];
+            return;
         }
+        // If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
+        // This is important because a container may never load during a node's Sweep Grace Period,
+        // so that node would directly become sweep-ready skipping over tombstone-ready state,
+        // but should be Tombstoned since Sweep is disabled.
+        const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
+            ? {
+                nodesToTombstone: [...tombstoneReadyNodes],
+                nodesToDelete: [...sweepReadyNodes],
+            }
+            : {
+                nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
+                nodesToDelete: [],
+            };
         if (this.configs.tombstoneMode) {
-            this.tombstones = Array.from(sweepReadyNodes);
-            // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
-            // involving access to "deleted" data without actually deleting the data from summaries.
+            this.tombstones = nodesToTombstone;
+            // If we are running in GC tombstone mode, update tombstoned routes.
             this.runtime.updateTombstonedRoutes(this.tombstones);
-            return [];
         }
-        if (!this.configs.shouldRunSweep) {
-            return [];
-        }
-        // 1. Call the runtime to delete sweep ready nodes. The runtime returns a list of nodes it deleted.
-        // TODO: GC:Validation - validate that removed routes are not double delete and that the child routes of
-        // removed routes are deleted as well.
-        const deletedNodeIds = this.runtime.deleteSweepReadyNodes(Array.from(sweepReadyNodes));
-        // 2. Clear unreferenced state tracking for deleted nodes.
-        for (const nodeId of deletedNodeIds) {
-            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-            // TODO: GC:Validation - assert that the nodeStateTracker is defined
-            if (nodeStateTracker !== undefined) {
-                // Stop tracking so as to clear out any running timers.
-                nodeStateTracker.stopTracking();
-                // Delete the node as we don't need to track it any more.
-                this.unreferencedNodesState.delete(nodeId);
-            }
-            // TODO: GC:Validation - assert that the deleted node is not a duplicate
-            this.deletedNodes.add(nodeId);
+        if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
+            // Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
+            // to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
+            // local state when processing the op.
+            const sweepReadyDSAndBlobs = nodesToDelete.filter((nodeId) => {
+                const nodeType = this.runtime.getNodeType(nodeId);
+                return nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob;
+            });
+            const contents = {
+                type: GarbageCollectionMessageType.Sweep,
+                deletedNodeIds: sweepReadyDSAndBlobs,
+            };
+            // Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
+            // is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
+            // accidental usage. The clients will sync without the delete op too but it may take longer.
+            const containerGCMessage = {
+                type: ContainerMessageType.GC,
+                contents,
+                compatDetails: { behavior: "Ignore" },
+            };
+            this.submitMessage(containerGCMessage);
+            return;
         }
-        return deletedNodeIds;
     }
     /**
      * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
@@ -629,6 +651,65 @@ export class GarbageCollector {
     async refreshLatestSummary(result) {
         return this.summaryStateTracker.refreshLatestSummary(result);
     }
+    /**
+     * Process a GC message.
+     * @param message - The GC message from the container runtime.
+     * @param local - Whether it was send by this client.
+     */
+    processMessage(message, local) {
+        switch (message.contents.type) {
+            case "Sweep": {
+                // Delete the nodes whose ids are present in the contents.
+                this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
+                break;
+            }
+            default: {
+                if (!compatBehaviorAllowsGCMessageType(message.contents.type, message.compatDetails?.behavior)) {
+                    const error = DataProcessingError.create(`Garbage collection message of unknown type ${message.contents.type}`, "processMessage");
+                    throw error;
+                }
+                break;
+            }
+        }
+    }
+    /**
+     * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
+     * tracking for nodes that are actually deleted by the runtime.
+     * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyNodeIds) {
+        // Use a set for lookup because its much faster than array or map.
+        const sweepReadyNodesSet = new Set(sweepReadyNodeIds);
+        // The ids in the sweep-ready nodes do not contain DDS node ids. This is an optimization to reduce the size
+        // of the GC op. Since GC applies to data store only, all its DDSes are deleted along with it. So, get the
+        // DDS nodes ID from the unreferenced nodes state.
+        const allSweepReadyNodeIds = Array.from(sweepReadyNodeIds);
+        for (const [id] of this.unreferencedNodesState) {
+            // Ignore data store nodes since they would already be in the list.
+            const pathParts = id.split("/");
+            if (pathParts.length <= 2) {
+                continue;
+            }
+            // Get the data store id part. Note that this may include blobs but that's okay since the part would just
+            // be "_blobs" and it won't be found.
+            const dsId = `/${pathParts[1]}`;
+            if (sweepReadyNodesSet.has(dsId)) {
+                allSweepReadyNodeIds.push(id);
+            }
+        }
+        const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
+        // Clear unreferenced state tracking for deleted nodes.
+        for (const nodeId of deletedNodeIds) {
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker !== undefined) {
+                // Stop tracking so as to clear out any running timers.
+                nodeStateTracker.stopTracking();
+                // Delete the node as we don't need to track it any more.
+                this.unreferencedNodesState.delete(nodeId);
+            }
+            this.deletedNodes.add(nodeId);
+        }
+    }
     /**
      * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
      * or throw an error if failing on incorrect usage is configured.
@@ -774,12 +855,12 @@ export class GarbageCollector {
     }
     /**
      * Generates the stats of a garbage collection sweep phase run.
-     * @param allDeletedNodes - All the nodes that have been deleted across all GC runs.
-     * @param currentDeletedNodes - The nodes that have been deleted in this GC run.
+     * @param deletedNodes - The nodes that have been deleted until this run.
+     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
      * @param markPhaseStats - The stats of the mark phase run.
      * @returns the stats of the sweep phase run.
      */
-    getSweepPhaseStats(allDeletedNodes, currentDeletedNodes, markPhaseStats) {
+    getSweepPhaseStats(deletedNodes, sweepReadyNodes, markPhaseStats) {
         // Initialize the life time node counts to the mark phase node counts. If sweep is not enabled,
         // these will be the life time node count for this container.
         const sweepPhaseStats = {
@@ -790,7 +871,7 @@ export class GarbageCollector {
             deletedDataStoreCount: 0,
             deletedAttachmentBlobCount: 0,
         };
-        for (const nodeId of allDeletedNodes) {
+        for (const nodeId of deletedNodes) {
             sweepPhaseStats.deletedNodeCount++;
             const nodeType = this.runtime.getNodeType(nodeId);
             if (nodeType === GCNodeType.DataStore) {
@@ -800,24 +881,25 @@ export class GarbageCollector {
                 sweepPhaseStats.deletedAttachmentBlobCount++;
             }
         }
-        if (!this.configs.shouldRunSweep) {
-            return sweepPhaseStats;
-        }
         // If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
-        // deleted in previous runs. Add the deleted node counts to life time stats.
+        // deleted in previous runs. So, add the deleted node counts to life time stats.
         sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
         sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
         sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
-        // The node deleted in current run are counted twice - once in allDeletedNodes and again in
-        // markPhaseStats. So, remove them from the life time stats.
-        for (const nodeId of currentDeletedNodes) {
-            sweepPhaseStats.lifetimeNodeCount--;
+        if (this.configs.shouldRunSweep) {
+            return sweepPhaseStats;
+        }
+        // If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
+        // is the final state the node will be in.
+        // If sweep is enabled, this will happen in the run after the GC op round trips back.
+        for (const nodeId of sweepReadyNodes) {
+            sweepPhaseStats.deletedNodeCount++;
             const nodeType = this.runtime.getNodeType(nodeId);
             if (nodeType === GCNodeType.DataStore) {
-                sweepPhaseStats.lifetimeDataStoreCount--;
+                sweepPhaseStats.deletedDataStoreCount++;
             }
             else if (nodeType === GCNodeType.Blob) {
-                sweepPhaseStats.lifetimeAttachmentBlobCount--;
+                sweepPhaseStats.deletedAttachmentBlobCount++;
             }
         }
         return sweepPhaseStats;