@fluidframework/container-runtime 2.0.0-internal.3.2.1 → 2.0.0-internal.3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerRuntime.d.ts +32 -53
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +55 -21
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +8 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts +19 -0
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/dist/deltaManagerSummarizerProxy.js +40 -0
- package/dist/deltaManagerSummarizerProxy.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +2 -33
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +36 -181
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts +22 -0
- package/dist/gc/gcConfigs.d.ts.map +1 -0
- package/dist/gc/gcConfigs.js +138 -0
- package/dist/gc/gcConfigs.js.map +1 -0
- package/dist/gc/gcDefinitions.d.ts +101 -3
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +8 -3
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +12 -1
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +55 -1
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +1 -2
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +28 -37
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +3 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +2 -1
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +9 -0
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +19 -2
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +24 -10
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +4 -0
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +42 -4
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +14 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +35 -18
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +29 -21
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/storageServiceWithAttachBlobs.d.ts +17 -0
- package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/dist/storageServiceWithAttachBlobs.js +32 -0
- package/dist/storageServiceWithAttachBlobs.js.map +1 -0
- package/dist/summary/runWhileConnectedCoordinator.d.ts +3 -2
- package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js +5 -4
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +2 -0
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/lib/containerRuntime.d.ts +32 -53
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +56 -22
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +9 -4
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts +19 -0
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/lib/deltaManagerSummarizerProxy.js +36 -0
- package/lib/deltaManagerSummarizerProxy.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +2 -33
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +39 -184
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts +22 -0
- package/lib/gc/gcConfigs.d.ts.map +1 -0
- package/lib/gc/gcConfigs.js +134 -0
- package/lib/gc/gcConfigs.js.map +1 -0
- package/lib/gc/gcDefinitions.d.ts +101 -3
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +7 -2
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +12 -1
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +53 -0
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +1 -2
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +28 -37
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +3 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +1 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +9 -0
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +17 -1
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +25 -11
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +4 -0
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +42 -4
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +14 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +35 -18
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +30 -22
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/storageServiceWithAttachBlobs.d.ts +17 -0
- package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/lib/storageServiceWithAttachBlobs.js +28 -0
- package/lib/storageServiceWithAttachBlobs.js.map +1 -0
- package/lib/summary/runWhileConnectedCoordinator.d.ts +3 -2
- package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js +5 -4
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +2 -0
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/package.json +20 -31
- package/src/containerRuntime.ts +92 -76
- package/src/dataStores.ts +9 -4
- package/src/deltaManagerSummarizerProxy.ts +46 -0
- package/src/gc/garbageCollection.ts +50 -290
- package/src/gc/gcConfigs.ts +177 -0
- package/src/gc/gcDefinitions.ts +110 -4
- package/src/gc/gcHelpers.ts +78 -1
- package/src/gc/gcSummaryStateTracker.ts +35 -42
- package/src/gc/index.ts +8 -2
- package/src/index.ts +1 -2
- package/src/opLifecycle/README.md +2 -2
- package/src/opLifecycle/batchManager.ts +19 -1
- package/src/opLifecycle/index.ts +1 -1
- package/src/opLifecycle/opCompressor.ts +31 -12
- package/src/opLifecycle/opDecompressor.ts +49 -5
- package/src/opLifecycle/opSplitter.ts +44 -20
- package/src/opLifecycle/outbox.ts +36 -22
- package/src/packageVersion.ts +1 -1
- package/src/storageServiceWithAttachBlobs.ts +38 -0
- package/src/summary/runWhileConnectedCoordinator.ts +7 -7
- package/src/summary/summarizerTypes.ts +2 -0
|
@@ -5,11 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
7
|
import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
|
|
8
|
-
import {
|
|
9
|
-
ClientSessionExpiredError,
|
|
10
|
-
DataProcessingError,
|
|
11
|
-
UsageError,
|
|
12
|
-
} from "@fluidframework/container-utils";
|
|
8
|
+
import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
|
|
13
9
|
import { IRequestHeader } from "@fluidframework/core-interfaces";
|
|
14
10
|
import {
|
|
15
11
|
cloneGCData,
|
|
@@ -27,7 +23,6 @@ import {
|
|
|
27
23
|
IGarbageCollectionState,
|
|
28
24
|
ISummarizeResult,
|
|
29
25
|
ITelemetryContext,
|
|
30
|
-
IGarbageCollectionSummaryDetailsLegacy,
|
|
31
26
|
} from "@fluidframework/runtime-definitions";
|
|
32
27
|
import {
|
|
33
28
|
packagePathToTelemetryProperty,
|
|
@@ -43,35 +38,21 @@ import {
|
|
|
43
38
|
TelemetryDataTag,
|
|
44
39
|
} from "@fluidframework/telemetry-utils";
|
|
45
40
|
|
|
46
|
-
import {
|
|
47
|
-
import {
|
|
48
|
-
import {
|
|
49
|
-
ReadFluidDataStoreAttributes,
|
|
50
|
-
dataStoreAttributesBlobName,
|
|
51
|
-
ICreateContainerMetadata,
|
|
52
|
-
} from "../summary";
|
|
41
|
+
import { RuntimeHeaders } from "../containerRuntime";
|
|
42
|
+
import { ICreateContainerMetadata } from "../summary";
|
|
43
|
+
import { generateGCConfigs } from "./gcConfigs";
|
|
53
44
|
import {
|
|
54
|
-
defaultInactiveTimeoutMs,
|
|
55
|
-
defaultSessionExpiryDurationMs,
|
|
56
45
|
disableSweepLogKey,
|
|
57
|
-
disableTombstoneKey,
|
|
58
46
|
GCNodeType,
|
|
59
|
-
gcTestModeKey,
|
|
60
47
|
IGarbageCollector,
|
|
61
48
|
IGarbageCollectorCreateParams,
|
|
62
49
|
IGarbageCollectionRuntime,
|
|
63
50
|
IGCStats,
|
|
64
|
-
oneDayMs,
|
|
65
|
-
runGCKey,
|
|
66
|
-
runSessionExpiryKey,
|
|
67
|
-
runSweepKey,
|
|
68
|
-
trackGCStateKey,
|
|
69
|
-
gcTombstoneGenerationOptionName,
|
|
70
51
|
UnreferencedState,
|
|
71
|
-
GCFeatureMatrix,
|
|
72
52
|
IGCMetadata,
|
|
53
|
+
IGarbageCollectorConfigs,
|
|
73
54
|
} from "./gcDefinitions";
|
|
74
|
-
import {
|
|
55
|
+
import { getSnapshotDataFromOldSnapshotFormat, sendGCUnexpectedUsageEvent } from "./gcHelpers";
|
|
75
56
|
import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
|
|
76
57
|
import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
|
|
77
58
|
import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
|
|
@@ -118,36 +99,13 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
118
99
|
return new GarbageCollector(createParams);
|
|
119
100
|
}
|
|
120
101
|
|
|
121
|
-
/**
|
|
122
|
-
* Tracks if GC is enabled for this document. This is specified during document creation and doesn't change
|
|
123
|
-
* throughout its lifetime.
|
|
124
|
-
*/
|
|
125
|
-
private readonly gcEnabled: boolean;
|
|
126
|
-
/**
|
|
127
|
-
* Tracks if sweep phase is enabled for this document. This is specified during document creation and doesn't change
|
|
128
|
-
* throughout its lifetime.
|
|
129
|
-
*/
|
|
130
|
-
private readonly sweepEnabled: boolean;
|
|
131
|
-
|
|
132
|
-
/**
|
|
133
|
-
* Tracks if GC should run or not. Even if GC is enabled for a document (see gcEnabled), it can be explicitly
|
|
134
|
-
* disabled via runtime options or feature flags.
|
|
135
|
-
*/
|
|
136
|
-
public readonly shouldRunGC: boolean;
|
|
137
|
-
/**
|
|
138
|
-
* Tracks if sweep phase should run or not. Even if the sweep phase is enabled for a document (see sweepEnabled), it
|
|
139
|
-
* can be explicitly disabled via feature flags. It also won't run if session expiry is not enabled.
|
|
140
|
-
*/
|
|
141
|
-
private readonly shouldRunSweep: boolean;
|
|
142
|
-
|
|
143
|
-
public readonly trackGCState: boolean;
|
|
144
|
-
|
|
145
|
-
private readonly testMode: boolean;
|
|
146
|
-
private readonly tombstoneMode: boolean;
|
|
147
102
|
private readonly mc: MonitoringContext;
|
|
148
103
|
|
|
149
|
-
|
|
150
|
-
|
|
104
|
+
private readonly configs: IGarbageCollectorConfigs;
|
|
105
|
+
|
|
106
|
+
public get shouldRunGC(): boolean {
|
|
107
|
+
return this.configs.shouldRunGC;
|
|
108
|
+
}
|
|
151
109
|
|
|
152
110
|
// Keeps track of the GC state from the last run.
|
|
153
111
|
private gcDataFromLastRun: IGarbageCollectionData | undefined;
|
|
@@ -181,16 +139,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
181
139
|
|
|
182
140
|
private readonly runtime: IGarbageCollectionRuntime;
|
|
183
141
|
private readonly createContainerMetadata: ICreateContainerMetadata;
|
|
184
|
-
private readonly gcOptions: IGCRuntimeOptions;
|
|
185
142
|
private readonly isSummarizerClient: boolean;
|
|
186
143
|
|
|
187
|
-
/** The time in ms to expire a session for a client for gc. */
|
|
188
|
-
private readonly sessionExpiryTimeoutMs: number | undefined;
|
|
189
|
-
/** The time after which an unreferenced node is inactive. */
|
|
190
|
-
private readonly inactiveTimeoutMs: number;
|
|
191
|
-
/** The time after which an unreferenced node is ready to be swept. */
|
|
192
|
-
private readonly sweepTimeoutMs: number | undefined;
|
|
193
|
-
|
|
194
144
|
private readonly summaryStateTracker: GCSummaryStateTracker;
|
|
195
145
|
|
|
196
146
|
/** For a given node path, returns the node's package path. */
|
|
@@ -202,23 +152,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
202
152
|
/** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
|
|
203
153
|
private readonly activeConnection: () => boolean;
|
|
204
154
|
|
|
205
|
-
/** Returns a list of all the configurations for garbage collection. */
|
|
206
|
-
private get configs() {
|
|
207
|
-
return {
|
|
208
|
-
gcEnabled: this.gcEnabled,
|
|
209
|
-
sweepEnabled: this.sweepEnabled,
|
|
210
|
-
runGC: this.shouldRunGC,
|
|
211
|
-
runSweep: this.shouldRunSweep,
|
|
212
|
-
testMode: this.testMode,
|
|
213
|
-
tombstoneMode: this.tombstoneMode,
|
|
214
|
-
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
215
|
-
sweepTimeout: this.sweepTimeoutMs,
|
|
216
|
-
inactiveTimeout: this.inactiveTimeoutMs,
|
|
217
|
-
trackGCState: this.trackGCState,
|
|
218
|
-
...this.gcOptions,
|
|
219
|
-
};
|
|
220
|
-
}
|
|
221
|
-
|
|
222
155
|
public get summaryStateNeedsReset(): boolean {
|
|
223
156
|
return this.summaryStateTracker.doesSummaryStateNeedReset();
|
|
224
157
|
}
|
|
@@ -229,14 +162,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
229
162
|
protected constructor(createParams: IGarbageCollectorCreateParams) {
|
|
230
163
|
this.runtime = createParams.runtime;
|
|
231
164
|
this.isSummarizerClient = createParams.isSummarizerClient;
|
|
232
|
-
this.gcOptions = createParams.gcOptions;
|
|
233
165
|
this.createContainerMetadata = createParams.createContainerMetadata;
|
|
234
166
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
235
167
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
236
168
|
this.activeConnection = createParams.activeConnection;
|
|
237
169
|
|
|
238
170
|
const baseSnapshot = createParams.baseSnapshot;
|
|
239
|
-
const metadata = createParams.metadata;
|
|
240
171
|
const readAndParseBlob = createParams.readAndParseBlob;
|
|
241
172
|
|
|
242
173
|
this.mc = loggerToMonitoringContext(
|
|
@@ -251,83 +182,15 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
251
182
|
this.runtime.closeFn,
|
|
252
183
|
);
|
|
253
184
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
/**
|
|
257
|
-
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
258
|
-
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
|
|
259
|
-
*
|
|
260
|
-
* The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
|
|
261
|
-
* The buffer is added to account for any clock skew or other edge cases.
|
|
262
|
-
* We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
|
|
263
|
-
*/
|
|
264
|
-
function computeSweepTimeout(sessionExpiryTimeoutMs: number | undefined) {
|
|
265
|
-
const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
|
|
266
|
-
const bufferMs = oneDayMs;
|
|
267
|
-
return (
|
|
268
|
-
sessionExpiryTimeoutMs &&
|
|
269
|
-
sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs
|
|
270
|
-
);
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
/**
|
|
274
|
-
* The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
|
|
275
|
-
* 1. Whether running GC mark phase is allowed or not.
|
|
276
|
-
* 2. Whether running GC sweep phase is allowed or not.
|
|
277
|
-
* 3. Whether GC session expiry is enabled or not.
|
|
278
|
-
* For existing containers, we get this information from the metadata blob of its summary.
|
|
279
|
-
*/
|
|
280
|
-
if (createParams.existing) {
|
|
281
|
-
gcVersionInBaseSnapshot = getGCVersion(metadata);
|
|
282
|
-
// Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
|
|
283
|
-
// other existing documents, GC is enabled.
|
|
284
|
-
this.gcEnabled = gcVersionInBaseSnapshot > 0;
|
|
285
|
-
this.sweepEnabled = metadata?.sweepEnabled ?? false;
|
|
286
|
-
this.sessionExpiryTimeoutMs = metadata?.sessionExpiryTimeoutMs;
|
|
287
|
-
this.sweepTimeoutMs =
|
|
288
|
-
metadata?.sweepTimeoutMs ?? computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
|
|
289
|
-
this.persistedGcFeatureMatrix = metadata?.gcFeatureMatrix;
|
|
290
|
-
} else {
|
|
291
|
-
// Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
|
|
292
|
-
// scenario but explicitly failing makes it clearer and promotes correct usage.
|
|
293
|
-
if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
|
|
294
|
-
throw new UsageError(
|
|
295
|
-
"GC sweep phase cannot be enabled without enabling GC mark phase",
|
|
296
|
-
);
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
// This Test Override only applies for new containers
|
|
300
|
-
const testOverrideSweepTimeoutMs = this.mc.config.getNumber(
|
|
301
|
-
"Fluid.GarbageCollection.TestOverride.SweepTimeoutMs",
|
|
302
|
-
);
|
|
303
|
-
|
|
304
|
-
// For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
|
|
305
|
-
// flag in GC options to false.
|
|
306
|
-
this.gcEnabled = this.gcOptions.gcAllowed !== false;
|
|
307
|
-
// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
|
|
308
|
-
this.sweepEnabled = this.gcOptions.sweepAllowed === true;
|
|
309
|
-
|
|
310
|
-
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
311
|
-
if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
|
|
312
|
-
this.sessionExpiryTimeoutMs =
|
|
313
|
-
this.gcOptions.sessionExpiryTimeoutMs ?? defaultSessionExpiryDurationMs;
|
|
314
|
-
}
|
|
315
|
-
this.sweepTimeoutMs =
|
|
316
|
-
testOverrideSweepTimeoutMs ?? computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
317
|
-
if (this.gcOptions[gcTombstoneGenerationOptionName] !== undefined) {
|
|
318
|
-
this.persistedGcFeatureMatrix = {
|
|
319
|
-
tombstoneGeneration: this.gcOptions[gcTombstoneGenerationOptionName],
|
|
320
|
-
};
|
|
321
|
-
}
|
|
322
|
-
}
|
|
185
|
+
this.configs = generateGCConfigs(this.mc, createParams);
|
|
323
186
|
|
|
324
187
|
// If session expiry is enabled, we need to close the container when the session expiry timeout expires.
|
|
325
|
-
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
188
|
+
if (this.configs.sessionExpiryTimeoutMs !== undefined) {
|
|
326
189
|
// If Test Override config is set, override Session Expiry timeout.
|
|
327
190
|
const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber(
|
|
328
191
|
"Fluid.GarbageCollection.TestOverride.SessionExpiryMs",
|
|
329
192
|
);
|
|
330
|
-
const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.sessionExpiryTimeoutMs;
|
|
193
|
+
const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.configs.sessionExpiryTimeoutMs;
|
|
331
194
|
|
|
332
195
|
this.sessionExpiryTimer = new Timer(timeoutMs, () => {
|
|
333
196
|
this.runtime.closeFn(
|
|
@@ -337,65 +200,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
337
200
|
this.sessionExpiryTimer.start();
|
|
338
201
|
}
|
|
339
202
|
|
|
340
|
-
/**
|
|
341
|
-
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
342
|
-
*
|
|
343
|
-
* 1. GC should be enabled for this container.
|
|
344
|
-
*
|
|
345
|
-
* 2. GC should not be disabled via disableGC GC option.
|
|
346
|
-
*
|
|
347
|
-
* These conditions can be overridden via runGCKey feature flag.
|
|
348
|
-
*/
|
|
349
|
-
this.shouldRunGC =
|
|
350
|
-
this.mc.config.getBoolean(runGCKey) ??
|
|
351
|
-
// GC must be enabled for the document.
|
|
352
|
-
(this.gcEnabled &&
|
|
353
|
-
// GC must not be disabled via GC options.
|
|
354
|
-
!this.gcOptions.disableGC);
|
|
355
|
-
|
|
356
|
-
/**
|
|
357
|
-
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
358
|
-
*
|
|
359
|
-
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
360
|
-
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
361
|
-
* 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
|
|
362
|
-
* the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
|
|
363
|
-
* 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
364
|
-
* feature flag.
|
|
365
|
-
*/
|
|
366
|
-
this.shouldRunSweep =
|
|
367
|
-
this.shouldRunGC &&
|
|
368
|
-
this.sweepTimeoutMs !== undefined &&
|
|
369
|
-
(this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
|
|
370
|
-
|
|
371
|
-
this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
|
|
372
|
-
|
|
373
|
-
// Override inactive timeout if test config or gc options to override it is set.
|
|
374
|
-
this.inactiveTimeoutMs =
|
|
375
|
-
this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs") ??
|
|
376
|
-
this.gcOptions.inactiveTimeoutMs ??
|
|
377
|
-
defaultInactiveTimeoutMs;
|
|
378
|
-
|
|
379
|
-
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
380
|
-
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
381
|
-
throw new UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
385
|
-
this.testMode =
|
|
386
|
-
this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
|
|
387
|
-
// Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
|
|
388
|
-
// via feature flags.
|
|
389
|
-
this.tombstoneMode =
|
|
390
|
-
!this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
|
|
391
|
-
|
|
392
203
|
this.summaryStateTracker = new GCSummaryStateTracker(
|
|
393
204
|
this.shouldRunGC,
|
|
394
|
-
this.
|
|
395
|
-
this.tombstoneMode,
|
|
205
|
+
this.configs.tombstoneMode,
|
|
396
206
|
this.mc,
|
|
397
207
|
baseSnapshot?.trees[gcTreeKey] !== undefined /* wasGCRunInBaseSnapshot */,
|
|
398
|
-
gcVersionInBaseSnapshot,
|
|
208
|
+
this.configs.gcVersionInBaseSnapshot,
|
|
399
209
|
);
|
|
400
210
|
|
|
401
211
|
// Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
|
|
@@ -413,71 +223,20 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
413
223
|
return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
414
224
|
}
|
|
415
225
|
|
|
416
|
-
// back-compat - Older documents will have the GC blobs in each data store's
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
};
|
|
422
|
-
const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
|
|
423
|
-
assert(
|
|
424
|
-
dataStoreSnapshotTree !== undefined,
|
|
425
|
-
0x2a8 /* "Expected data store snapshot tree in base snapshot" */,
|
|
226
|
+
// back-compat - Older documents will have the GC blobs in each data store's snapshot tree.
|
|
227
|
+
return getSnapshotDataFromOldSnapshotFormat(
|
|
228
|
+
baseSnapshot,
|
|
229
|
+
createParams.metadata,
|
|
230
|
+
readAndParseBlob,
|
|
426
231
|
);
|
|
427
|
-
for (const [dsId, dsSnapshotTree] of Object.entries(
|
|
428
|
-
dataStoreSnapshotTree.trees,
|
|
429
|
-
)) {
|
|
430
|
-
const blobId = dsSnapshotTree.blobs[gcTreeKey];
|
|
431
|
-
if (blobId === undefined) {
|
|
432
|
-
continue;
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
const gcSummaryDetails =
|
|
436
|
-
await readAndParseBlob<IGarbageCollectionSummaryDetailsLegacy>(blobId);
|
|
437
|
-
// If there are no nodes for this data store, skip it.
|
|
438
|
-
if (gcSummaryDetails.gcData?.gcNodes === undefined) {
|
|
439
|
-
continue;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
const dsRootId = `/${dsId}`;
|
|
443
|
-
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
444
|
-
// Construct that entry by adding root data store ids to its outbound routes.
|
|
445
|
-
const initialSnapshotDetails =
|
|
446
|
-
await readAndParseBlob<ReadFluidDataStoreAttributes>(
|
|
447
|
-
dsSnapshotTree.blobs[dataStoreAttributesBlobName],
|
|
448
|
-
);
|
|
449
|
-
if (initialSnapshotDetails.isRootDataStore) {
|
|
450
|
-
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
for (const [id, outboundRoutes] of Object.entries(
|
|
454
|
-
gcSummaryDetails.gcData.gcNodes,
|
|
455
|
-
)) {
|
|
456
|
-
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
457
|
-
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
458
|
-
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
459
|
-
gcState.gcNodes[rootId] = {
|
|
460
|
-
outboundRoutes: Array.from(outboundRoutes),
|
|
461
|
-
};
|
|
462
|
-
}
|
|
463
|
-
assert(
|
|
464
|
-
gcState.gcNodes[dsRootId] !== undefined,
|
|
465
|
-
0x2a9 /* GC nodes for data store not in GC blob */,
|
|
466
|
-
);
|
|
467
|
-
gcState.gcNodes[dsRootId].unreferencedTimestampMs =
|
|
468
|
-
gcSummaryDetails.unrefTimestamp;
|
|
469
|
-
}
|
|
470
|
-
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
471
|
-
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
472
|
-
return Object.keys(gcState.gcNodes).length === 1
|
|
473
|
-
? undefined
|
|
474
|
-
: { gcState, tombstones: undefined, deletedNodes: undefined };
|
|
475
232
|
} catch (error) {
|
|
476
233
|
const dpe = DataProcessingError.wrapIfUnrecognized(
|
|
477
234
|
error,
|
|
478
235
|
"FailedToInitializeGC",
|
|
479
236
|
);
|
|
480
|
-
dpe.addTelemetryProperties({
|
|
237
|
+
dpe.addTelemetryProperties({
|
|
238
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
239
|
+
});
|
|
481
240
|
throw dpe;
|
|
482
241
|
}
|
|
483
242
|
},
|
|
@@ -545,6 +304,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
545
304
|
this.mc.logger.sendTelemetryEvent({
|
|
546
305
|
eventName: "GarbageCollectorLoaded",
|
|
547
306
|
gcConfigs: JSON.stringify(this.configs),
|
|
307
|
+
gcOptions: JSON.stringify(createParams.gcOptions),
|
|
548
308
|
});
|
|
549
309
|
}
|
|
550
310
|
}
|
|
@@ -573,7 +333,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
573
333
|
|
|
574
334
|
// If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
|
|
575
335
|
// tombstone routes.
|
|
576
|
-
if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
336
|
+
if (this.configs.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
577
337
|
// Create a copy since we are writing from a source we don't control
|
|
578
338
|
this.tombstones = Array.from(baseSnapshotData.tombstones);
|
|
579
339
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
@@ -614,7 +374,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
614
374
|
// tombstones.
|
|
615
375
|
// If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
|
|
616
376
|
// in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
|
|
617
|
-
if (this.shouldRunSweep) {
|
|
377
|
+
if (this.configs.shouldRunSweep) {
|
|
618
378
|
const snapshotDeletedNodes = snapshotData?.deletedNodes
|
|
619
379
|
? new Set(snapshotData.deletedNodes)
|
|
620
380
|
: undefined;
|
|
@@ -631,7 +391,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
631
391
|
// Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
|
|
632
392
|
}
|
|
633
393
|
}
|
|
634
|
-
} else if (this.tombstoneMode) {
|
|
394
|
+
} else if (this.configs.tombstoneMode) {
|
|
635
395
|
// The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
|
|
636
396
|
// notify the runtime to update its state as well.
|
|
637
397
|
this.tombstones = snapshotData?.tombstones ? Array.from(snapshotData.tombstones) : [];
|
|
@@ -656,9 +416,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
656
416
|
nodeId,
|
|
657
417
|
new UnreferencedStateTracker(
|
|
658
418
|
nodeData.unreferencedTimestampMs,
|
|
659
|
-
this.inactiveTimeoutMs,
|
|
419
|
+
this.configs.inactiveTimeoutMs,
|
|
660
420
|
currentReferenceTimestampMs,
|
|
661
|
-
this.sweepTimeoutMs,
|
|
421
|
+
this.configs.sweepTimeoutMs,
|
|
662
422
|
),
|
|
663
423
|
);
|
|
664
424
|
}
|
|
@@ -687,7 +447,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
687
447
|
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
688
448
|
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
689
449
|
*/
|
|
690
|
-
if (this.activeConnection() && this.shouldRunGC) {
|
|
450
|
+
if (this.activeConnection() && this.configs.shouldRunGC) {
|
|
691
451
|
this.initializeGCStateFromBaseSnapshotP.catch((error) => {});
|
|
692
452
|
}
|
|
693
453
|
}
|
|
@@ -709,7 +469,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
709
469
|
): Promise<IGCStats | undefined> {
|
|
710
470
|
const fullGC =
|
|
711
471
|
options.fullGC ??
|
|
712
|
-
(this.
|
|
472
|
+
(this.configs.runFullGC === true ||
|
|
713
473
|
this.summaryStateTracker.doesSummaryStateNeedReset());
|
|
714
474
|
const logger = options.logger
|
|
715
475
|
? ChildLogger.create(options.logger, undefined, {
|
|
@@ -797,13 +557,13 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
797
557
|
|
|
798
558
|
let updatedGCData: IGarbageCollectionData = gcData;
|
|
799
559
|
|
|
800
|
-
if (this.shouldRunSweep) {
|
|
560
|
+
if (this.configs.shouldRunSweep) {
|
|
801
561
|
updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
|
|
802
|
-
} else if (this.testMode) {
|
|
562
|
+
} else if (this.configs.testMode) {
|
|
803
563
|
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
804
564
|
// involving access to deleted data.
|
|
805
565
|
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
|
|
806
|
-
} else if (this.tombstoneMode) {
|
|
566
|
+
} else if (this.configs.tombstoneMode) {
|
|
807
567
|
this.tombstones = sweepReadyNodes;
|
|
808
568
|
// If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
|
|
809
569
|
// involving access to "deleted" data without actually deleting the data from summaries.
|
|
@@ -831,7 +591,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
831
591
|
trackState: boolean,
|
|
832
592
|
telemetryContext?: ITelemetryContext,
|
|
833
593
|
): ISummarizeResult | undefined {
|
|
834
|
-
if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
594
|
+
if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
835
595
|
return;
|
|
836
596
|
}
|
|
837
597
|
|
|
@@ -859,11 +619,11 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
859
619
|
* If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
|
|
860
620
|
* into the metadata blob. If GC is disabled, the gcFeature is 0.
|
|
861
621
|
*/
|
|
862
|
-
gcFeature: this.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
863
|
-
gcFeatureMatrix: this.persistedGcFeatureMatrix,
|
|
864
|
-
sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
|
|
865
|
-
sweepEnabled: this.sweepEnabled,
|
|
866
|
-
sweepTimeoutMs: this.sweepTimeoutMs,
|
|
622
|
+
gcFeature: this.configs.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
623
|
+
gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
|
|
624
|
+
sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
|
|
625
|
+
sweepEnabled: this.configs.sweepEnabled,
|
|
626
|
+
sweepTimeoutMs: this.configs.sweepTimeoutMs,
|
|
867
627
|
};
|
|
868
628
|
}
|
|
869
629
|
|
|
@@ -904,7 +664,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
904
664
|
{
|
|
905
665
|
proposalHandle,
|
|
906
666
|
summaryRefSeq: result.summaryRefSeq,
|
|
907
|
-
|
|
667
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
908
668
|
},
|
|
909
669
|
);
|
|
910
670
|
}
|
|
@@ -927,7 +687,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
927
687
|
packagePath?: readonly string[],
|
|
928
688
|
requestHeaders?: IRequestHeader,
|
|
929
689
|
) {
|
|
930
|
-
if (!this.shouldRunGC) {
|
|
690
|
+
if (!this.configs.shouldRunGC) {
|
|
931
691
|
return;
|
|
932
692
|
}
|
|
933
693
|
|
|
@@ -953,7 +713,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
953
713
|
* @param toNodePath - The node to which the reference is added.
|
|
954
714
|
*/
|
|
955
715
|
public addedOutboundReference(fromNodePath: string, toNodePath: string) {
|
|
956
|
-
if (!this.shouldRunGC) {
|
|
716
|
+
if (!this.configs.shouldRunGC) {
|
|
957
717
|
return;
|
|
958
718
|
}
|
|
959
719
|
|
|
@@ -1052,9 +812,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1052
812
|
nodeId,
|
|
1053
813
|
new UnreferencedStateTracker(
|
|
1054
814
|
currentReferenceTimestampMs,
|
|
1055
|
-
this.inactiveTimeoutMs,
|
|
815
|
+
this.configs.inactiveTimeoutMs,
|
|
1056
816
|
currentReferenceTimestampMs,
|
|
1057
|
-
this.sweepTimeoutMs,
|
|
817
|
+
this.configs.sweepTimeoutMs,
|
|
1058
818
|
),
|
|
1059
819
|
);
|
|
1060
820
|
} else {
|
|
@@ -1334,7 +1094,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1334
1094
|
private logSweepEvents(logger: ITelemetryLogger, currentReferenceTimestampMs: number) {
|
|
1335
1095
|
if (
|
|
1336
1096
|
this.mc.config.getBoolean(disableSweepLogKey) === true ||
|
|
1337
|
-
this.sweepTimeoutMs === undefined
|
|
1097
|
+
this.configs.sweepTimeoutMs === undefined
|
|
1338
1098
|
) {
|
|
1339
1099
|
return;
|
|
1340
1100
|
}
|
|
@@ -1360,7 +1120,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1360
1120
|
id: nodeId,
|
|
1361
1121
|
type: nodeType,
|
|
1362
1122
|
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
1363
|
-
timeout: this.sweepTimeoutMs,
|
|
1123
|
+
timeout: this.configs.sweepTimeoutMs,
|
|
1364
1124
|
completedGCRuns: this.completedRuns,
|
|
1365
1125
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1366
1126
|
});
|
|
@@ -1410,8 +1170,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1410
1170
|
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
1411
1171
|
timeout:
|
|
1412
1172
|
nodeStateTracker.state === UnreferencedState.Inactive
|
|
1413
|
-
? this.inactiveTimeoutMs
|
|
1414
|
-
: this.sweepTimeoutMs,
|
|
1173
|
+
? this.configs.inactiveTimeoutMs
|
|
1174
|
+
: this.configs.sweepTimeoutMs,
|
|
1415
1175
|
completedGCRuns: this.completedRuns,
|
|
1416
1176
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1417
1177
|
...this.createContainerMetadata,
|