@fluidframework/container-runtime 2.0.0-internal.7.3.0 → 2.0.0-internal.7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/api-extractor-lint.json +13 -0
- package/api-extractor.json +9 -1
- package/api-report/container-runtime.api.md +124 -107
- package/dist/blobManager.d.ts +4 -4
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/container-runtime-alpha.d.ts +1473 -0
- package/dist/container-runtime-beta.d.ts +300 -0
- package/dist/container-runtime-public.d.ts +300 -0
- package/dist/container-runtime-untrimmed.d.ts +1836 -0
- package/dist/containerRuntime.d.ts +30 -30
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +62 -40
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreRegistry.d.ts +1 -1
- package/dist/dataStoreRegistry.js +1 -1
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +10 -15
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +77 -40
- package/dist/dataStores.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +41 -13
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +215 -78
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts.map +1 -1
- package/dist/gc/gcConfigs.js +34 -37
- package/dist/gc/gcConfigs.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +121 -46
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +26 -18
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +18 -25
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +29 -45
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +0 -5
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +14 -42
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.d.ts +11 -5
- package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.js +43 -19
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +1 -1
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +4 -5
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +14 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +16 -5
- package/dist/index.js.map +1 -1
- package/dist/messageTypes.d.ts +15 -7
- package/dist/messageTypes.d.ts.map +1 -1
- package/dist/messageTypes.js +6 -1
- package/dist/messageTypes.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +1 -0
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +1 -0
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summary/orderedClientElection.d.ts +1 -1
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.d.ts +2 -2
- package/dist/summary/runWhileConnectedCoordinator.js +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/summarizer.d.ts +1 -1
- package/dist/summary/summarizer.js +1 -1
- package/dist/summary/summarizer.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +30 -30
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts +10 -10
- package/dist/summary/summaryCollection.js +1 -1
- package/dist/summary/summaryCollection.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +3 -3
- package/dist/summary/summaryFormat.js.map +1 -1
- package/lib/blobManager.d.ts +4 -4
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/container-runtime-alpha.d.ts +1473 -0
- package/lib/container-runtime-beta.d.ts +300 -0
- package/lib/container-runtime-public.d.ts +300 -0
- package/lib/container-runtime-untrimmed.d.ts +1836 -0
- package/lib/containerRuntime.d.ts +30 -30
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +64 -42
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreRegistry.d.ts +1 -1
- package/lib/dataStoreRegistry.js +1 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +10 -15
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +80 -43
- package/lib/dataStores.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +41 -13
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +217 -80
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts.map +1 -1
- package/lib/gc/gcConfigs.js +37 -40
- package/lib/gc/gcConfigs.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +121 -46
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +25 -17
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +18 -25
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +27 -43
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +0 -5
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +15 -43
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.d.ts +11 -5
- package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js +43 -19
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +1 -1
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +1 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +14 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +15 -1
- package/lib/index.js.map +1 -1
- package/lib/messageTypes.d.ts +15 -7
- package/lib/messageTypes.d.ts.map +1 -1
- package/lib/messageTypes.js +6 -1
- package/lib/messageTypes.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +1 -0
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +1 -0
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summary/orderedClientElection.d.ts +1 -1
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.d.ts +2 -2
- package/lib/summary/runWhileConnectedCoordinator.js +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/summarizer.d.ts +1 -1
- package/lib/summary/summarizer.js +1 -1
- package/lib/summary/summarizer.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +30 -30
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts +10 -10
- package/lib/summary/summaryCollection.js +1 -1
- package/lib/summary/summaryCollection.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +3 -3
- package/lib/summary/summaryFormat.js.map +1 -1
- package/package.json +42 -19
- package/src/blobManager.ts +5 -5
- package/src/containerRuntime.ts +86 -56
- package/src/dataStoreRegistry.ts +1 -1
- package/src/dataStores.ts +140 -69
- package/src/gc/garbageCollection.md +14 -15
- package/src/gc/garbageCollection.ts +256 -96
- package/src/gc/gcConfigs.ts +50 -52
- package/src/gc/gcDefinitions.ts +137 -52
- package/src/gc/gcHelpers.ts +31 -52
- package/src/gc/gcTelemetry.ts +16 -57
- package/src/gc/gcUnreferencedStateTracker.ts +61 -22
- package/src/gc/index.ts +6 -4
- package/src/index.ts +19 -1
- package/src/messageTypes.ts +19 -4
- package/src/opLifecycle/definitions.ts +1 -1
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +1 -0
- package/src/summary/orderedClientElection.ts +1 -1
- package/src/summary/runWhileConnectedCoordinator.ts +2 -2
- package/src/summary/summarizer.ts +1 -1
- package/src/summary/summarizerTypes.ts +30 -30
- package/src/summary/summaryCollection.ts +10 -10
- package/src/summary/summaryFormat.ts +3 -3
- package/dist/id-compressor/appendOnlySortedMap.d.ts +0 -124
- package/dist/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
- package/dist/id-compressor/appendOnlySortedMap.js +0 -318
- package/dist/id-compressor/appendOnlySortedMap.js.map +0 -1
- package/dist/id-compressor/finalSpace.d.ts +0 -29
- package/dist/id-compressor/finalSpace.d.ts.map +0 -1
- package/dist/id-compressor/finalSpace.js +0 -62
- package/dist/id-compressor/finalSpace.js.map +0 -1
- package/dist/id-compressor/idCompressor.d.ts +0 -54
- package/dist/id-compressor/idCompressor.d.ts.map +0 -1
- package/dist/id-compressor/idCompressor.js +0 -495
- package/dist/id-compressor/idCompressor.js.map +0 -1
- package/dist/id-compressor/identifiers.d.ts +0 -32
- package/dist/id-compressor/identifiers.d.ts.map +0 -1
- package/dist/id-compressor/identifiers.js +0 -15
- package/dist/id-compressor/identifiers.js.map +0 -1
- package/dist/id-compressor/index.d.ts +0 -13
- package/dist/id-compressor/index.d.ts.map +0 -1
- package/dist/id-compressor/index.js +0 -32
- package/dist/id-compressor/index.js.map +0 -1
- package/dist/id-compressor/persistanceUtilities.d.ts +0 -22
- package/dist/id-compressor/persistanceUtilities.d.ts.map +0 -1
- package/dist/id-compressor/persistanceUtilities.js +0 -43
- package/dist/id-compressor/persistanceUtilities.js.map +0 -1
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
- package/dist/id-compressor/sessionSpaceNormalizer.js +0 -80
- package/dist/id-compressor/sessionSpaceNormalizer.js.map +0 -1
- package/dist/id-compressor/sessions.d.ts +0 -115
- package/dist/id-compressor/sessions.d.ts.map +0 -1
- package/dist/id-compressor/sessions.js +0 -305
- package/dist/id-compressor/sessions.js.map +0 -1
- package/dist/id-compressor/utilities.d.ts +0 -52
- package/dist/id-compressor/utilities.d.ts.map +0 -1
- package/dist/id-compressor/utilities.js +0 -169
- package/dist/id-compressor/utilities.js.map +0 -1
- package/lib/id-compressor/appendOnlySortedMap.d.ts +0 -124
- package/lib/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
- package/lib/id-compressor/appendOnlySortedMap.js +0 -314
- package/lib/id-compressor/appendOnlySortedMap.js.map +0 -1
- package/lib/id-compressor/finalSpace.d.ts +0 -29
- package/lib/id-compressor/finalSpace.d.ts.map +0 -1
- package/lib/id-compressor/finalSpace.js +0 -58
- package/lib/id-compressor/finalSpace.js.map +0 -1
- package/lib/id-compressor/idCompressor.d.ts +0 -54
- package/lib/id-compressor/idCompressor.d.ts.map +0 -1
- package/lib/id-compressor/idCompressor.js +0 -491
- package/lib/id-compressor/idCompressor.js.map +0 -1
- package/lib/id-compressor/identifiers.d.ts +0 -32
- package/lib/id-compressor/identifiers.d.ts.map +0 -1
- package/lib/id-compressor/identifiers.js +0 -11
- package/lib/id-compressor/identifiers.js.map +0 -1
- package/lib/id-compressor/index.d.ts +0 -13
- package/lib/id-compressor/index.d.ts.map +0 -1
- package/lib/id-compressor/index.js +0 -13
- package/lib/id-compressor/index.js.map +0 -1
- package/lib/id-compressor/persistanceUtilities.d.ts +0 -22
- package/lib/id-compressor/persistanceUtilities.d.ts.map +0 -1
- package/lib/id-compressor/persistanceUtilities.js +0 -34
- package/lib/id-compressor/persistanceUtilities.js.map +0 -1
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
- package/lib/id-compressor/sessionSpaceNormalizer.js +0 -76
- package/lib/id-compressor/sessionSpaceNormalizer.js.map +0 -1
- package/lib/id-compressor/sessions.d.ts +0 -115
- package/lib/id-compressor/sessions.d.ts.map +0 -1
- package/lib/id-compressor/sessions.js +0 -290
- package/lib/id-compressor/sessions.js.map +0 -1
- package/lib/id-compressor/utilities.d.ts +0 -52
- package/lib/id-compressor/utilities.d.ts.map +0 -1
- package/lib/id-compressor/utilities.js +0 -151
- package/lib/id-compressor/utilities.js.map +0 -1
- package/src/id-compressor/README.md +0 -3
- package/src/id-compressor/appendOnlySortedMap.ts +0 -366
- package/src/id-compressor/finalSpace.ts +0 -67
- package/src/id-compressor/idCompressor.ts +0 -630
- package/src/id-compressor/identifiers.ts +0 -42
- package/src/id-compressor/index.ts +0 -26
- package/src/id-compressor/persistanceUtilities.ts +0 -58
- package/src/id-compressor/sessionSpaceNormalizer.ts +0 -83
- package/src/id-compressor/sessions.ts +0 -405
- package/src/id-compressor/utilities.ts +0 -190
|
@@ -6,6 +6,7 @@ import { IRequest } from "@fluidframework/core-interfaces";
|
|
|
6
6
|
import { IGarbageCollectionDetailsBase, ISummarizeResult, ITelemetryContext } from "@fluidframework/runtime-definitions";
|
|
7
7
|
import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
|
|
8
8
|
import { RuntimeHeaderData } from "../containerRuntime";
|
|
9
|
+
import { ContainerRuntimeGCMessage } from "../messageTypes";
|
|
9
10
|
import { IRefreshSummaryResult } from "../summary";
|
|
10
11
|
import { IGarbageCollector, IGarbageCollectorCreateParams, IGCStats, IGCMetadata } from "./gcDefinitions";
|
|
11
12
|
/**
|
|
@@ -61,6 +62,7 @@ export declare class GarbageCollector implements IGarbageCollector {
|
|
|
61
62
|
private readonly getLastSummaryTimestampMs;
|
|
62
63
|
/** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
|
|
63
64
|
private readonly activeConnection;
|
|
65
|
+
private readonly submitMessage;
|
|
64
66
|
get summaryStateNeedsReset(): boolean;
|
|
65
67
|
/** Returns the count of data stores whose GC state updated since the last summary. */
|
|
66
68
|
get updatedDSCountSinceLastSummary(): number;
|
|
@@ -81,7 +83,7 @@ export declare class GarbageCollector implements IGarbageCollector {
|
|
|
81
83
|
private updateStateFromSnapshotData;
|
|
82
84
|
/**
|
|
83
85
|
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
84
|
-
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep
|
|
86
|
+
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
|
|
85
87
|
* @param connected - Whether the runtime connected / disconnected.
|
|
86
88
|
* @param clientId - The clientId of this runtime.
|
|
87
89
|
*/
|
|
@@ -105,10 +107,16 @@ export declare class GarbageCollector implements IGarbageCollector {
|
|
|
105
107
|
}, telemetryContext?: ITelemetryContext): Promise<IGCStats | undefined>;
|
|
106
108
|
/**
|
|
107
109
|
* Runs garbage collection. It does the following:
|
|
110
|
+
*
|
|
108
111
|
* 1. It generates / analyzes the runtime's reference graph.
|
|
109
|
-
*
|
|
112
|
+
*
|
|
113
|
+
* 2. Generates mark phase stats.
|
|
114
|
+
*
|
|
110
115
|
* 3. Runs Mark phase.
|
|
116
|
+
*
|
|
111
117
|
* 4. Runs Sweep phase.
|
|
118
|
+
*
|
|
119
|
+
* 5. Generates sweep phase stats.
|
|
112
120
|
*/
|
|
113
121
|
private runGC;
|
|
114
122
|
/**
|
|
@@ -123,19 +131,19 @@ export declare class GarbageCollector implements IGarbageCollector {
|
|
|
123
131
|
* @param gcResult - The result of the GC run on the gcData.
|
|
124
132
|
* @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
|
|
125
133
|
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
126
|
-
* @returns
|
|
134
|
+
* @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
|
|
127
135
|
*/
|
|
128
136
|
private runMarkPhase;
|
|
129
137
|
/**
|
|
130
138
|
* Runs the GC Sweep phase. It does the following:
|
|
131
|
-
*
|
|
132
|
-
*
|
|
139
|
+
*
|
|
140
|
+
* 1. Marks tombstone-ready nodes as tombstones.
|
|
141
|
+
*
|
|
142
|
+
* 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
|
|
133
143
|
*
|
|
134
144
|
* @param gcResult - The result of the GC run on the gcData.
|
|
135
|
-
* @param
|
|
136
|
-
* @param
|
|
137
|
-
* @param logger - The logger to be used to log any telemetry.
|
|
138
|
-
* @returns A list of nodes that have been deleted.
|
|
145
|
+
* @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
|
|
146
|
+
* @param sweepReadyNodes - List of nodes that are sweep-ready.
|
|
139
147
|
*/
|
|
140
148
|
private runSweepPhase;
|
|
141
149
|
/**
|
|
@@ -165,6 +173,18 @@ export declare class GarbageCollector implements IGarbageCollector {
|
|
|
165
173
|
* Called to refresh the latest summary state. This happens when either a pending summary is acked.
|
|
166
174
|
*/
|
|
167
175
|
refreshLatestSummary(result: IRefreshSummaryResult): Promise<void>;
|
|
176
|
+
/**
|
|
177
|
+
* Process a GC message.
|
|
178
|
+
* @param message - The GC message from the container runtime.
|
|
179
|
+
* @param local - Whether it was send by this client.
|
|
180
|
+
*/
|
|
181
|
+
processMessage(message: ContainerRuntimeGCMessage, local: boolean): void;
|
|
182
|
+
/**
|
|
183
|
+
* Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
|
|
184
|
+
* tracking for nodes that are actually deleted by the runtime.
|
|
185
|
+
* @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
|
|
186
|
+
*/
|
|
187
|
+
private deleteSweepReadyNodes;
|
|
168
188
|
/**
|
|
169
189
|
* Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
|
|
170
190
|
* or throw an error if failing on incorrect usage is configured.
|
|
@@ -191,10 +211,18 @@ export declare class GarbageCollector implements IGarbageCollector {
|
|
|
191
211
|
isNodeDeleted(nodePath: string): boolean;
|
|
192
212
|
dispose(): void;
|
|
193
213
|
/**
|
|
194
|
-
* Generates the stats of a garbage collection
|
|
195
|
-
* @param gcResult - The result of
|
|
196
|
-
* @returns the
|
|
214
|
+
* Generates the stats of a garbage collection mark phase run.
|
|
215
|
+
* @param gcResult - The result of the current GC run.
|
|
216
|
+
* @returns the stats of the mark phase run.
|
|
217
|
+
*/
|
|
218
|
+
private getMarkPhaseStats;
|
|
219
|
+
/**
|
|
220
|
+
* Generates the stats of a garbage collection sweep phase run.
|
|
221
|
+
* @param deletedNodes - The nodes that have been deleted until this run.
|
|
222
|
+
* @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
|
|
223
|
+
* @param markPhaseStats - The stats of the mark phase run.
|
|
224
|
+
* @returns the stats of the sweep phase run.
|
|
197
225
|
*/
|
|
198
|
-
private
|
|
226
|
+
private getSweepPhaseStats;
|
|
199
227
|
}
|
|
200
228
|
//# sourceMappingURL=garbageCollection.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAwB,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEnD,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAMX,MAAM,iBAAiB,CAAC;AAazB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IACrE,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgB;IAEjD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IAsKjE;;;OAGG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjD;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA6EnC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAwEhC;;;;;;;;;;;;OAYG;YACW,KAAK;IA2CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAqDpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IAgErB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;OAIG;IACI,cAAc,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAyBxE;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAqC7B;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAyD/B;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAqBtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0DzB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;CAkD1B"}
|
|
@@ -11,6 +11,7 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
|
11
11
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
12
12
|
const containerRuntime_1 = require("../containerRuntime");
|
|
13
13
|
const error_1 = require("../error");
|
|
14
|
+
const messageTypes_1 = require("../messageTypes");
|
|
14
15
|
const gcConfigs_1 = require("./gcConfigs");
|
|
15
16
|
const gcDefinitions_1 = require("./gcDefinitions");
|
|
16
17
|
const gcHelpers_1 = require("./gcHelpers");
|
|
@@ -49,7 +50,7 @@ class GarbageCollector {
|
|
|
49
50
|
}
|
|
50
51
|
/** If false, loading or using a Tombstoned object should merely log, not fail */
|
|
51
52
|
get tombstoneEnforcementAllowed() {
|
|
52
|
-
return this.configs.
|
|
53
|
+
return this.configs.sweepEnabled;
|
|
53
54
|
}
|
|
54
55
|
/** If true, throw an error when a tombstone data store is retrieved */
|
|
55
56
|
get throwOnTombstoneLoad() {
|
|
@@ -83,6 +84,7 @@ class GarbageCollector {
|
|
|
83
84
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
84
85
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
85
86
|
this.activeConnection = createParams.activeConnection;
|
|
87
|
+
this.submitMessage = createParams.submitMessage;
|
|
86
88
|
const baseSnapshot = createParams.baseSnapshot;
|
|
87
89
|
const readAndParseBlob = createParams.readAndParseBlob;
|
|
88
90
|
this.mc = (0, telemetry_utils_1.createChildMonitoringContext)({
|
|
@@ -144,7 +146,7 @@ class GarbageCollector {
|
|
|
144
146
|
/**
|
|
145
147
|
* Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
|
|
146
148
|
* connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
|
|
147
|
-
* GC state and updates their inactive or sweep
|
|
149
|
+
* GC state and updates their inactive or sweep-ready state.
|
|
148
150
|
*/
|
|
149
151
|
this.initializeGCStateFromBaseSnapshotP = new core_utils_1.LazyPromise(async () => {
|
|
150
152
|
/**
|
|
@@ -200,6 +202,7 @@ class GarbageCollector {
|
|
|
200
202
|
eventName: "GarbageCollectorLoaded",
|
|
201
203
|
gcConfigs: JSON.stringify(this.configs),
|
|
202
204
|
gcOptions: JSON.stringify(createParams.gcOptions),
|
|
205
|
+
...createParams.createContainerMetadata,
|
|
203
206
|
});
|
|
204
207
|
}
|
|
205
208
|
/**
|
|
@@ -293,7 +296,7 @@ class GarbageCollector {
|
|
|
293
296
|
const gcNodes = {};
|
|
294
297
|
for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
|
|
295
298
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
296
|
-
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
299
|
+
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
|
|
297
300
|
}
|
|
298
301
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
299
302
|
}
|
|
@@ -301,7 +304,7 @@ class GarbageCollector {
|
|
|
301
304
|
}
|
|
302
305
|
/**
|
|
303
306
|
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
304
|
-
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep
|
|
307
|
+
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
|
|
305
308
|
* @param connected - Whether the runtime connected / disconnected.
|
|
306
309
|
* @param clientId - The clientId of this runtime.
|
|
307
310
|
*/
|
|
@@ -317,7 +320,7 @@ class GarbageCollector {
|
|
|
317
320
|
* the receiving summarizer client.
|
|
318
321
|
*
|
|
319
322
|
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
320
|
-
* sweep in phases and we want to track when inactive and sweep
|
|
323
|
+
* sweep in phases and we want to track when inactive and sweep-ready objects are used in any client.
|
|
321
324
|
*/
|
|
322
325
|
if (this.activeConnection() && this.configs.shouldRunGC) {
|
|
323
326
|
this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
|
|
@@ -375,7 +378,11 @@ class GarbageCollector {
|
|
|
375
378
|
await this.runtime.updateStateBeforeGC();
|
|
376
379
|
/** GC step */
|
|
377
380
|
const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
|
|
378
|
-
event.end({
|
|
381
|
+
event.end({
|
|
382
|
+
...gcStats,
|
|
383
|
+
timestamp: currentReferenceTimestampMs,
|
|
384
|
+
sweep: this.configs.shouldRunSweep,
|
|
385
|
+
});
|
|
379
386
|
/** Post-GC steps */
|
|
380
387
|
// Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
|
|
381
388
|
// updates its state so that we don't send false positives based on intermediate state. For example, we may get
|
|
@@ -390,10 +397,16 @@ class GarbageCollector {
|
|
|
390
397
|
}
|
|
391
398
|
/**
|
|
392
399
|
* Runs garbage collection. It does the following:
|
|
400
|
+
*
|
|
393
401
|
* 1. It generates / analyzes the runtime's reference graph.
|
|
394
|
-
*
|
|
402
|
+
*
|
|
403
|
+
* 2. Generates mark phase stats.
|
|
404
|
+
*
|
|
395
405
|
* 3. Runs Mark phase.
|
|
406
|
+
*
|
|
396
407
|
* 4. Runs Sweep phase.
|
|
408
|
+
*
|
|
409
|
+
* 5. Generates sweep phase stats.
|
|
397
410
|
*/
|
|
398
411
|
async runGC(fullGC, currentReferenceTimestampMs, logger) {
|
|
399
412
|
// 1. Generate / analyze the runtime's reference graph.
|
|
@@ -403,17 +416,20 @@ class GarbageCollector {
|
|
|
403
416
|
// Get all referenced nodes - References in this run + references between the previous and current runs.
|
|
404
417
|
const allReferencedNodeIds = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
|
|
405
418
|
gcResult.referencedNodeIds;
|
|
406
|
-
// 2.
|
|
407
|
-
//
|
|
408
|
-
const
|
|
419
|
+
// 2. Get the mark phase stats based on the previous / current GC state.
|
|
420
|
+
// This is done before running mark phase because we need the previous GC state before it is updated.
|
|
421
|
+
const markPhaseStats = this.getMarkPhaseStats(gcResult);
|
|
409
422
|
// 3. Run the Mark phase.
|
|
410
|
-
// It will mark nodes as referenced / unreferenced and return
|
|
411
|
-
const sweepReadyNodeIds = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
|
|
423
|
+
// It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
|
|
424
|
+
const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
|
|
412
425
|
// 4. Run the Sweep phase.
|
|
413
|
-
// It will
|
|
414
|
-
|
|
415
|
-
this.
|
|
416
|
-
|
|
426
|
+
// It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
|
|
427
|
+
// sweep op. All clients, including this one, will delete these nodes once it processes the op.
|
|
428
|
+
this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
|
|
429
|
+
this.gcDataFromLastRun = (0, gcHelpers_1.cloneGCData)(gcData);
|
|
430
|
+
// 5. Get the sweep phase stats.
|
|
431
|
+
const sweepPhaseStats = this.getSweepPhaseStats(this.deletedNodes, sweepReadyNodeIds, markPhaseStats);
|
|
432
|
+
return { ...markPhaseStats, ...sweepPhaseStats };
|
|
417
433
|
}
|
|
418
434
|
/**
|
|
419
435
|
* Runs the GC Mark phase. It does the following:
|
|
@@ -427,7 +443,7 @@ class GarbageCollector {
|
|
|
427
443
|
* @param gcResult - The result of the GC run on the gcData.
|
|
428
444
|
* @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
|
|
429
445
|
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
430
|
-
* @returns
|
|
446
|
+
* @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
|
|
431
447
|
*/
|
|
432
448
|
runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
|
|
433
449
|
// 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
|
|
@@ -441,83 +457,94 @@ class GarbageCollector {
|
|
|
441
457
|
}
|
|
442
458
|
}
|
|
443
459
|
// 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
|
|
444
|
-
const
|
|
460
|
+
const tombstoneReadyNodeIds = new Set();
|
|
461
|
+
const sweepReadyNodeIds = new Set();
|
|
445
462
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
446
463
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
447
464
|
if (nodeStateTracker === undefined) {
|
|
448
|
-
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
465
|
+
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
|
|
449
466
|
}
|
|
450
467
|
else {
|
|
451
468
|
// If a node was already unreferenced, update its tracking information. Since the current reference time
|
|
452
469
|
// is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
|
|
453
470
|
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
454
|
-
// If a node is sweep
|
|
471
|
+
// If a node is tombstone or sweep-ready, store it so it can be returned.
|
|
472
|
+
if (nodeStateTracker.state === gcDefinitions_1.UnreferencedState.TombstoneReady) {
|
|
473
|
+
tombstoneReadyNodeIds.add(nodeId);
|
|
474
|
+
}
|
|
455
475
|
if (nodeStateTracker.state === gcDefinitions_1.UnreferencedState.SweepReady) {
|
|
456
|
-
sweepReadyNodeIds.
|
|
476
|
+
sweepReadyNodeIds.add(nodeId);
|
|
457
477
|
}
|
|
458
478
|
}
|
|
459
479
|
}
|
|
460
480
|
// 3. Call the runtime to update referenced nodes in this run.
|
|
461
481
|
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
|
|
462
|
-
return sweepReadyNodeIds;
|
|
482
|
+
return { tombstoneReadyNodeIds, sweepReadyNodeIds };
|
|
463
483
|
}
|
|
464
484
|
/**
|
|
465
485
|
* Runs the GC Sweep phase. It does the following:
|
|
466
|
-
*
|
|
467
|
-
*
|
|
486
|
+
*
|
|
487
|
+
* 1. Marks tombstone-ready nodes as tombstones.
|
|
488
|
+
*
|
|
489
|
+
* 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
|
|
468
490
|
*
|
|
469
491
|
* @param gcResult - The result of the GC run on the gcData.
|
|
470
|
-
* @param
|
|
471
|
-
* @param
|
|
472
|
-
* @param logger - The logger to be used to log any telemetry.
|
|
473
|
-
* @returns A list of nodes that have been deleted.
|
|
492
|
+
* @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
|
|
493
|
+
* @param sweepReadyNodes - List of nodes that are sweep-ready.
|
|
474
494
|
*/
|
|
475
|
-
runSweepPhase(gcResult,
|
|
476
|
-
// Log events for objects that are ready to be deleted by sweep. This will give us data on sweep when
|
|
477
|
-
// its not enabled.
|
|
478
|
-
this.telemetryTracker.logSweepEvents(logger, currentReferenceTimestampMs, this.unreferencedNodesState, this.completedRuns, this.getLastSummaryTimestampMs());
|
|
495
|
+
runSweepPhase(gcResult, tombstoneReadyNodes, sweepReadyNodes) {
|
|
479
496
|
/**
|
|
480
|
-
*
|
|
481
|
-
* Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
|
|
482
|
-
* Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted.
|
|
483
|
-
* Sweep mode - Sweep ready modes are deleted.
|
|
497
|
+
* Under "Test Mode", unreferenced nodes are immediately deleted without waiting for them to be sweep-ready.
|
|
484
498
|
*
|
|
485
|
-
*
|
|
486
|
-
*
|
|
499
|
+
* Otherwise, depending on how long it's been since the node was unreferenced, it will either be
|
|
500
|
+
* marked as Tombstone, or deleted by Sweep.
|
|
487
501
|
*/
|
|
488
502
|
if (this.configs.testMode) {
|
|
489
503
|
// If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
|
|
490
504
|
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
|
|
491
|
-
return
|
|
505
|
+
return;
|
|
492
506
|
}
|
|
507
|
+
// If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
|
|
508
|
+
// This is important because a container may never load during a node's Sweep Grace Period,
|
|
509
|
+
// so that node would directly become sweep-ready skipping over tombstone-ready state,
|
|
510
|
+
// but should be Tombstoned since Sweep is disabled.
|
|
511
|
+
const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
|
|
512
|
+
? {
|
|
513
|
+
nodesToTombstone: [...tombstoneReadyNodes],
|
|
514
|
+
nodesToDelete: [...sweepReadyNodes],
|
|
515
|
+
}
|
|
516
|
+
: {
|
|
517
|
+
nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
|
|
518
|
+
nodesToDelete: [],
|
|
519
|
+
};
|
|
493
520
|
if (this.configs.tombstoneMode) {
|
|
494
|
-
this.tombstones =
|
|
495
|
-
// If we are running in GC tombstone mode, update tombstoned routes.
|
|
496
|
-
// involving access to "deleted" data without actually deleting the data from summaries.
|
|
521
|
+
this.tombstones = nodesToTombstone;
|
|
522
|
+
// If we are running in GC tombstone mode, update tombstoned routes.
|
|
497
523
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
498
|
-
return [];
|
|
499
|
-
}
|
|
500
|
-
if (!this.configs.shouldRunSweep) {
|
|
501
|
-
return [];
|
|
502
524
|
}
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
//
|
|
518
|
-
|
|
525
|
+
if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
|
|
526
|
+
// Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
|
|
527
|
+
// to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
|
|
528
|
+
// local state when processing the op.
|
|
529
|
+
const sweepReadyDSAndBlobs = nodesToDelete.filter((nodeId) => {
|
|
530
|
+
const nodeType = this.runtime.getNodeType(nodeId);
|
|
531
|
+
return nodeType === gcDefinitions_1.GCNodeType.DataStore || nodeType === gcDefinitions_1.GCNodeType.Blob;
|
|
532
|
+
});
|
|
533
|
+
const contents = {
|
|
534
|
+
type: gcDefinitions_1.GarbageCollectionMessageType.Sweep,
|
|
535
|
+
deletedNodeIds: sweepReadyDSAndBlobs,
|
|
536
|
+
};
|
|
537
|
+
// Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
|
|
538
|
+
// is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
|
|
539
|
+
// accidental usage. The clients will sync without the delete op too but it may take longer.
|
|
540
|
+
const containerGCMessage = {
|
|
541
|
+
type: messageTypes_1.ContainerMessageType.GC,
|
|
542
|
+
contents,
|
|
543
|
+
compatDetails: { behavior: "Ignore" },
|
|
544
|
+
};
|
|
545
|
+
this.submitMessage(containerGCMessage);
|
|
546
|
+
return;
|
|
519
547
|
}
|
|
520
|
-
return deletedNodeIds;
|
|
521
548
|
}
|
|
522
549
|
/**
|
|
523
550
|
* Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
|
|
@@ -627,6 +654,65 @@ class GarbageCollector {
|
|
|
627
654
|
async refreshLatestSummary(result) {
|
|
628
655
|
return this.summaryStateTracker.refreshLatestSummary(result);
|
|
629
656
|
}
|
|
657
|
+
/**
|
|
658
|
+
* Process a GC message.
|
|
659
|
+
* @param message - The GC message from the container runtime.
|
|
660
|
+
* @param local - Whether it was send by this client.
|
|
661
|
+
*/
|
|
662
|
+
processMessage(message, local) {
|
|
663
|
+
switch (message.contents.type) {
|
|
664
|
+
case "Sweep": {
|
|
665
|
+
// Delete the nodes whose ids are present in the contents.
|
|
666
|
+
this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
|
|
667
|
+
break;
|
|
668
|
+
}
|
|
669
|
+
default: {
|
|
670
|
+
if (!(0, gcHelpers_1.compatBehaviorAllowsGCMessageType)(message.contents.type, message.compatDetails?.behavior)) {
|
|
671
|
+
const error = telemetry_utils_1.DataProcessingError.create(`Garbage collection message of unknown type ${message.contents.type}`, "processMessage");
|
|
672
|
+
throw error;
|
|
673
|
+
}
|
|
674
|
+
break;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
|
|
680
|
+
* tracking for nodes that are actually deleted by the runtime.
|
|
681
|
+
* @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
|
|
682
|
+
*/
|
|
683
|
+
deleteSweepReadyNodes(sweepReadyNodeIds) {
|
|
684
|
+
// Use a set for lookup because its much faster than array or map.
|
|
685
|
+
const sweepReadyNodesSet = new Set(sweepReadyNodeIds);
|
|
686
|
+
// The ids in the sweep-ready nodes do not contain DDS node ids. This is an optimization to reduce the size
|
|
687
|
+
// of the GC op. Since GC applies to data store only, all its DDSes are deleted along with it. So, get the
|
|
688
|
+
// DDS nodes ID from the unreferenced nodes state.
|
|
689
|
+
const allSweepReadyNodeIds = Array.from(sweepReadyNodeIds);
|
|
690
|
+
for (const [id] of this.unreferencedNodesState) {
|
|
691
|
+
// Ignore data store nodes since they would already be in the list.
|
|
692
|
+
const pathParts = id.split("/");
|
|
693
|
+
if (pathParts.length <= 2) {
|
|
694
|
+
continue;
|
|
695
|
+
}
|
|
696
|
+
// Get the data store id part. Note that this may include blobs but that's okay since the part would just
|
|
697
|
+
// be "_blobs" and it won't be found.
|
|
698
|
+
const dsId = `/${pathParts[1]}`;
|
|
699
|
+
if (sweepReadyNodesSet.has(dsId)) {
|
|
700
|
+
allSweepReadyNodeIds.push(id);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
|
|
704
|
+
// Clear unreferenced state tracking for deleted nodes.
|
|
705
|
+
for (const nodeId of deletedNodeIds) {
|
|
706
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
707
|
+
if (nodeStateTracker !== undefined) {
|
|
708
|
+
// Stop tracking so as to clear out any running timers.
|
|
709
|
+
nodeStateTracker.stopTracking();
|
|
710
|
+
// Delete the node as we don't need to track it any more.
|
|
711
|
+
this.unreferencedNodesState.delete(nodeId);
|
|
712
|
+
}
|
|
713
|
+
this.deletedNodes.add(nodeId);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
630
716
|
/**
|
|
631
717
|
* Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
|
|
632
718
|
* or throw an error if failing on incorrect usage is configured.
|
|
@@ -715,12 +801,12 @@ class GarbageCollector {
|
|
|
715
801
|
this.sessionExpiryTimer = undefined;
|
|
716
802
|
}
|
|
717
803
|
/**
|
|
718
|
-
* Generates the stats of a garbage collection
|
|
719
|
-
* @param gcResult - The result of
|
|
720
|
-
* @returns the
|
|
804
|
+
* Generates the stats of a garbage collection mark phase run.
|
|
805
|
+
* @param gcResult - The result of the current GC run.
|
|
806
|
+
* @returns the stats of the mark phase run.
|
|
721
807
|
*/
|
|
722
|
-
|
|
723
|
-
const
|
|
808
|
+
getMarkPhaseStats(gcResult) {
|
|
809
|
+
const markPhaseStats = {
|
|
724
810
|
nodeCount: 0,
|
|
725
811
|
dataStoreCount: 0,
|
|
726
812
|
attachmentBlobCount: 0,
|
|
@@ -732,33 +818,33 @@ class GarbageCollector {
|
|
|
732
818
|
updatedAttachmentBlobCount: 0,
|
|
733
819
|
};
|
|
734
820
|
const updateNodeStats = (nodeId, referenced) => {
|
|
735
|
-
|
|
821
|
+
markPhaseStats.nodeCount++;
|
|
736
822
|
// If there is no previous GC data, every node's state is generated and is considered as updated.
|
|
737
823
|
// Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
|
|
738
824
|
const stateUpdated = this.gcDataFromLastRun === undefined ||
|
|
739
825
|
this.unreferencedNodesState.has(nodeId) === referenced;
|
|
740
826
|
if (stateUpdated) {
|
|
741
|
-
|
|
827
|
+
markPhaseStats.updatedNodeCount++;
|
|
742
828
|
}
|
|
743
829
|
if (!referenced) {
|
|
744
|
-
|
|
830
|
+
markPhaseStats.unrefNodeCount++;
|
|
745
831
|
}
|
|
746
832
|
if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.DataStore) {
|
|
747
|
-
|
|
833
|
+
markPhaseStats.dataStoreCount++;
|
|
748
834
|
if (stateUpdated) {
|
|
749
|
-
|
|
835
|
+
markPhaseStats.updatedDataStoreCount++;
|
|
750
836
|
}
|
|
751
837
|
if (!referenced) {
|
|
752
|
-
|
|
838
|
+
markPhaseStats.unrefDataStoreCount++;
|
|
753
839
|
}
|
|
754
840
|
}
|
|
755
841
|
if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.Blob) {
|
|
756
|
-
|
|
842
|
+
markPhaseStats.attachmentBlobCount++;
|
|
757
843
|
if (stateUpdated) {
|
|
758
|
-
|
|
844
|
+
markPhaseStats.updatedAttachmentBlobCount++;
|
|
759
845
|
}
|
|
760
846
|
if (!referenced) {
|
|
761
|
-
|
|
847
|
+
markPhaseStats.unrefAttachmentBlobCount++;
|
|
762
848
|
}
|
|
763
849
|
}
|
|
764
850
|
};
|
|
@@ -768,7 +854,58 @@ class GarbageCollector {
|
|
|
768
854
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
769
855
|
updateNodeStats(nodeId, false /* referenced */);
|
|
770
856
|
}
|
|
771
|
-
return
|
|
857
|
+
return markPhaseStats;
|
|
858
|
+
}
|
|
859
|
+
/**
|
|
860
|
+
* Generates the stats of a garbage collection sweep phase run.
|
|
861
|
+
* @param deletedNodes - The nodes that have been deleted until this run.
|
|
862
|
+
* @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
|
|
863
|
+
* @param markPhaseStats - The stats of the mark phase run.
|
|
864
|
+
* @returns the stats of the sweep phase run.
|
|
865
|
+
*/
|
|
866
|
+
getSweepPhaseStats(deletedNodes, sweepReadyNodes, markPhaseStats) {
|
|
867
|
+
// Initialize the life time node counts to the mark phase node counts. If sweep is not enabled,
|
|
868
|
+
// these will be the life time node count for this container.
|
|
869
|
+
const sweepPhaseStats = {
|
|
870
|
+
lifetimeNodeCount: markPhaseStats.nodeCount,
|
|
871
|
+
lifetimeDataStoreCount: markPhaseStats.dataStoreCount,
|
|
872
|
+
lifetimeAttachmentBlobCount: markPhaseStats.attachmentBlobCount,
|
|
873
|
+
deletedNodeCount: 0,
|
|
874
|
+
deletedDataStoreCount: 0,
|
|
875
|
+
deletedAttachmentBlobCount: 0,
|
|
876
|
+
};
|
|
877
|
+
for (const nodeId of deletedNodes) {
|
|
878
|
+
sweepPhaseStats.deletedNodeCount++;
|
|
879
|
+
const nodeType = this.runtime.getNodeType(nodeId);
|
|
880
|
+
if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
|
|
881
|
+
sweepPhaseStats.deletedDataStoreCount++;
|
|
882
|
+
}
|
|
883
|
+
else if (nodeType === gcDefinitions_1.GCNodeType.Blob) {
|
|
884
|
+
sweepPhaseStats.deletedAttachmentBlobCount++;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
// If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
|
|
888
|
+
// deleted in previous runs. So, add the deleted node counts to life time stats.
|
|
889
|
+
sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
|
|
890
|
+
sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
|
|
891
|
+
sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
|
|
892
|
+
if (this.configs.shouldRunSweep) {
|
|
893
|
+
return sweepPhaseStats;
|
|
894
|
+
}
|
|
895
|
+
// If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
|
|
896
|
+
// is the final state the node will be in.
|
|
897
|
+
// If sweep is enabled, this will happen in the run after the GC op round trips back.
|
|
898
|
+
for (const nodeId of sweepReadyNodes) {
|
|
899
|
+
sweepPhaseStats.deletedNodeCount++;
|
|
900
|
+
const nodeType = this.runtime.getNodeType(nodeId);
|
|
901
|
+
if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
|
|
902
|
+
sweepPhaseStats.deletedDataStoreCount++;
|
|
903
|
+
}
|
|
904
|
+
else if (nodeType === gcDefinitions_1.GCNodeType.Blob) {
|
|
905
|
+
sweepPhaseStats.deletedAttachmentBlobCount++;
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
return sweepPhaseStats;
|
|
772
909
|
}
|
|
773
910
|
}
|
|
774
911
|
exports.GarbageCollector = GarbageCollector;
|