@fluidframework/container-runtime 2.0.0-internal.3.2.1 → 2.0.0-internal.3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerRuntime.d.ts +32 -53
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +55 -21
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +8 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts +19 -0
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/dist/deltaManagerSummarizerProxy.js +40 -0
- package/dist/deltaManagerSummarizerProxy.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +2 -33
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +36 -181
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts +22 -0
- package/dist/gc/gcConfigs.d.ts.map +1 -0
- package/dist/gc/gcConfigs.js +138 -0
- package/dist/gc/gcConfigs.js.map +1 -0
- package/dist/gc/gcDefinitions.d.ts +101 -3
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +8 -3
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +12 -1
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +55 -1
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +1 -2
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +28 -37
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +3 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +2 -1
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +9 -0
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +19 -2
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +24 -10
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +4 -0
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +42 -4
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +14 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +35 -18
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +29 -21
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/storageServiceWithAttachBlobs.d.ts +17 -0
- package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/dist/storageServiceWithAttachBlobs.js +32 -0
- package/dist/storageServiceWithAttachBlobs.js.map +1 -0
- package/dist/summary/runWhileConnectedCoordinator.d.ts +3 -2
- package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js +5 -4
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +2 -0
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/lib/containerRuntime.d.ts +32 -53
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +56 -22
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +9 -4
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts +19 -0
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/lib/deltaManagerSummarizerProxy.js +36 -0
- package/lib/deltaManagerSummarizerProxy.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +2 -33
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +39 -184
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts +22 -0
- package/lib/gc/gcConfigs.d.ts.map +1 -0
- package/lib/gc/gcConfigs.js +134 -0
- package/lib/gc/gcConfigs.js.map +1 -0
- package/lib/gc/gcDefinitions.d.ts +101 -3
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +7 -2
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +12 -1
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +53 -0
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +1 -2
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +28 -37
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +3 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +1 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +9 -0
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +17 -1
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +25 -11
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +4 -0
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +42 -4
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +14 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +35 -18
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +30 -22
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/storageServiceWithAttachBlobs.d.ts +17 -0
- package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/lib/storageServiceWithAttachBlobs.js +28 -0
- package/lib/storageServiceWithAttachBlobs.js.map +1 -0
- package/lib/summary/runWhileConnectedCoordinator.d.ts +3 -2
- package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js +5 -4
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +2 -0
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/package.json +20 -31
- package/src/containerRuntime.ts +92 -76
- package/src/dataStores.ts +9 -4
- package/src/deltaManagerSummarizerProxy.ts +46 -0
- package/src/gc/garbageCollection.ts +50 -290
- package/src/gc/gcConfigs.ts +177 -0
- package/src/gc/gcDefinitions.ts +110 -4
- package/src/gc/gcHelpers.ts +78 -1
- package/src/gc/gcSummaryStateTracker.ts +35 -42
- package/src/gc/index.ts +8 -2
- package/src/index.ts +1 -2
- package/src/opLifecycle/README.md +2 -2
- package/src/opLifecycle/batchManager.ts +19 -1
- package/src/opLifecycle/index.ts +1 -1
- package/src/opLifecycle/opCompressor.ts +31 -12
- package/src/opLifecycle/opDecompressor.ts +49 -5
- package/src/opLifecycle/opSplitter.ts +44 -20
- package/src/opLifecycle/outbox.ts +36 -22
- package/src/packageVersion.ts +1 -1
- package/src/storageServiceWithAttachBlobs.ts +38 -0
- package/src/summary/runWhileConnectedCoordinator.ts +7 -7
- package/src/summary/summarizerTypes.ts +2 -0
|
@@ -23,8 +23,7 @@ const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
|
|
|
23
23
|
const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
24
24
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
25
25
|
const containerRuntime_1 = require("../containerRuntime");
|
|
26
|
-
const
|
|
27
|
-
const summary_1 = require("../summary");
|
|
26
|
+
const gcConfigs_1 = require("./gcConfigs");
|
|
28
27
|
const gcDefinitions_1 = require("./gcDefinitions");
|
|
29
28
|
const gcHelpers_1 = require("./gcHelpers");
|
|
30
29
|
const gcSummaryStateTracker_1 = require("./gcSummaryStateTracker");
|
|
@@ -54,7 +53,6 @@ const gcUnreferencedStateTracker_1 = require("./gcUnreferencedStateTracker");
|
|
|
54
53
|
*/
|
|
55
54
|
class GarbageCollector {
|
|
56
55
|
constructor(createParams) {
|
|
57
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
58
56
|
// Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
|
|
59
57
|
// outbound routes from that node.
|
|
60
58
|
this.newReferencesSinceLastRun = new Map();
|
|
@@ -73,136 +71,31 @@ class GarbageCollector {
|
|
|
73
71
|
this.completedRuns = 0;
|
|
74
72
|
this.runtime = createParams.runtime;
|
|
75
73
|
this.isSummarizerClient = createParams.isSummarizerClient;
|
|
76
|
-
this.gcOptions = createParams.gcOptions;
|
|
77
74
|
this.createContainerMetadata = createParams.createContainerMetadata;
|
|
78
75
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
79
76
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
80
77
|
this.activeConnection = createParams.activeConnection;
|
|
81
78
|
const baseSnapshot = createParams.baseSnapshot;
|
|
82
|
-
const metadata = createParams.metadata;
|
|
83
79
|
const readAndParseBlob = createParams.readAndParseBlob;
|
|
84
80
|
this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
|
|
85
81
|
all: { completedGCRuns: () => this.completedRuns },
|
|
86
82
|
}));
|
|
87
83
|
this.sweepReadyUsageHandler = new gcSweepReadyUsageDetection_1.SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
91
|
-
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
|
|
92
|
-
*
|
|
93
|
-
* The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
|
|
94
|
-
* The buffer is added to account for any clock skew or other edge cases.
|
|
95
|
-
* We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
|
|
96
|
-
*/
|
|
97
|
-
function computeSweepTimeout(sessionExpiryTimeoutMs) {
|
|
98
|
-
const maxSnapshotCacheExpiryMs = 5 * gcDefinitions_1.oneDayMs;
|
|
99
|
-
const bufferMs = gcDefinitions_1.oneDayMs;
|
|
100
|
-
return (sessionExpiryTimeoutMs &&
|
|
101
|
-
sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
|
|
102
|
-
}
|
|
103
|
-
/**
|
|
104
|
-
* The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
|
|
105
|
-
* 1. Whether running GC mark phase is allowed or not.
|
|
106
|
-
* 2. Whether running GC sweep phase is allowed or not.
|
|
107
|
-
* 3. Whether GC session expiry is enabled or not.
|
|
108
|
-
* For existing containers, we get this information from the metadata blob of its summary.
|
|
109
|
-
*/
|
|
110
|
-
if (createParams.existing) {
|
|
111
|
-
gcVersionInBaseSnapshot = (0, gcHelpers_1.getGCVersion)(metadata);
|
|
112
|
-
// Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
|
|
113
|
-
// other existing documents, GC is enabled.
|
|
114
|
-
this.gcEnabled = gcVersionInBaseSnapshot > 0;
|
|
115
|
-
this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
|
|
116
|
-
this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
|
|
117
|
-
this.sweepTimeoutMs =
|
|
118
|
-
(_b = metadata === null || metadata === void 0 ? void 0 : metadata.sweepTimeoutMs) !== null && _b !== void 0 ? _b : computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
|
|
119
|
-
this.persistedGcFeatureMatrix = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix;
|
|
120
|
-
}
|
|
121
|
-
else {
|
|
122
|
-
// Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
|
|
123
|
-
// scenario but explicitly failing makes it clearer and promotes correct usage.
|
|
124
|
-
if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
|
|
125
|
-
throw new container_utils_1.UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
|
|
126
|
-
}
|
|
127
|
-
// This Test Override only applies for new containers
|
|
128
|
-
const testOverrideSweepTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
|
|
129
|
-
// For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
|
|
130
|
-
// flag in GC options to false.
|
|
131
|
-
this.gcEnabled = this.gcOptions.gcAllowed !== false;
|
|
132
|
-
// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
|
|
133
|
-
this.sweepEnabled = this.gcOptions.sweepAllowed === true;
|
|
134
|
-
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
135
|
-
if (this.mc.config.getBoolean(gcDefinitions_1.runSessionExpiryKey) && this.gcEnabled) {
|
|
136
|
-
this.sessionExpiryTimeoutMs =
|
|
137
|
-
(_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : gcDefinitions_1.defaultSessionExpiryDurationMs;
|
|
138
|
-
}
|
|
139
|
-
this.sweepTimeoutMs =
|
|
140
|
-
testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
141
|
-
if (this.gcOptions[gcDefinitions_1.gcTombstoneGenerationOptionName] !== undefined) {
|
|
142
|
-
this.persistedGcFeatureMatrix = {
|
|
143
|
-
tombstoneGeneration: this.gcOptions[gcDefinitions_1.gcTombstoneGenerationOptionName],
|
|
144
|
-
};
|
|
145
|
-
}
|
|
146
|
-
}
|
|
84
|
+
this.configs = (0, gcConfigs_1.generateGCConfigs)(this.mc, createParams);
|
|
147
85
|
// If session expiry is enabled, we need to close the container when the session expiry timeout expires.
|
|
148
|
-
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
86
|
+
if (this.configs.sessionExpiryTimeoutMs !== undefined) {
|
|
149
87
|
// If Test Override config is set, override Session Expiry timeout.
|
|
150
88
|
const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
|
|
151
|
-
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
|
|
89
|
+
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.configs.sessionExpiryTimeoutMs;
|
|
152
90
|
this.sessionExpiryTimer = new common_utils_1.Timer(timeoutMs, () => {
|
|
153
91
|
this.runtime.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
154
92
|
});
|
|
155
93
|
this.sessionExpiryTimer.start();
|
|
156
94
|
}
|
|
157
|
-
|
|
158
|
-
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
159
|
-
*
|
|
160
|
-
* 1. GC should be enabled for this container.
|
|
161
|
-
*
|
|
162
|
-
* 2. GC should not be disabled via disableGC GC option.
|
|
163
|
-
*
|
|
164
|
-
* These conditions can be overridden via runGCKey feature flag.
|
|
165
|
-
*/
|
|
166
|
-
this.shouldRunGC =
|
|
167
|
-
(_d = this.mc.config.getBoolean(gcDefinitions_1.runGCKey)) !== null && _d !== void 0 ? _d :
|
|
168
|
-
// GC must be enabled for the document.
|
|
169
|
-
(this.gcEnabled &&
|
|
170
|
-
// GC must not be disabled via GC options.
|
|
171
|
-
!this.gcOptions.disableGC);
|
|
172
|
-
/**
|
|
173
|
-
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
174
|
-
*
|
|
175
|
-
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
176
|
-
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
177
|
-
* 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
|
|
178
|
-
* the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
|
|
179
|
-
* 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
180
|
-
* feature flag.
|
|
181
|
-
*/
|
|
182
|
-
this.shouldRunSweep =
|
|
183
|
-
this.shouldRunGC &&
|
|
184
|
-
this.sweepTimeoutMs !== undefined &&
|
|
185
|
-
((_e = this.mc.config.getBoolean(gcDefinitions_1.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
|
|
186
|
-
this.trackGCState = this.mc.config.getBoolean(gcDefinitions_1.trackGCStateKey) === true;
|
|
187
|
-
// Override inactive timeout if test config or gc options to override it is set.
|
|
188
|
-
this.inactiveTimeoutMs =
|
|
189
|
-
(_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : gcDefinitions_1.defaultInactiveTimeoutMs;
|
|
190
|
-
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
191
|
-
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
192
|
-
throw new container_utils_1.UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
193
|
-
}
|
|
194
|
-
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
195
|
-
this.testMode =
|
|
196
|
-
(_h = this.mc.config.getBoolean(gcDefinitions_1.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
|
|
197
|
-
// Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
|
|
198
|
-
// via feature flags.
|
|
199
|
-
this.tombstoneMode =
|
|
200
|
-
!this.shouldRunSweep && this.mc.config.getBoolean(gcDefinitions_1.disableTombstoneKey) !== true;
|
|
201
|
-
this.summaryStateTracker = new gcSummaryStateTracker_1.GCSummaryStateTracker(this.shouldRunGC, this.trackGCState, this.tombstoneMode, this.mc, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[runtime_definitions_1.gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */, gcVersionInBaseSnapshot);
|
|
95
|
+
this.summaryStateTracker = new gcSummaryStateTracker_1.GCSummaryStateTracker(this.shouldRunGC, this.configs.tombstoneMode, this.mc, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[runtime_definitions_1.gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */, this.configs.gcVersionInBaseSnapshot);
|
|
202
96
|
// Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
|
|
203
97
|
// it involves fetching blobs from storage which is expensive.
|
|
204
98
|
this.baseSnapshotDataP = new common_utils_1.LazyPromise(async () => {
|
|
205
|
-
var _a;
|
|
206
99
|
if (baseSnapshot === undefined) {
|
|
207
100
|
return undefined;
|
|
208
101
|
}
|
|
@@ -212,52 +105,14 @@ class GarbageCollector {
|
|
|
212
105
|
if (gcSnapshotTree !== undefined) {
|
|
213
106
|
return (0, garbage_collector_1.getGCDataFromSnapshot)(gcSnapshotTree, readAndParseBlob);
|
|
214
107
|
}
|
|
215
|
-
// back-compat - Older documents will have the GC blobs in each data store's
|
|
216
|
-
|
|
217
|
-
// Add a node for the root node that is not present in older snapshot format.
|
|
218
|
-
const gcState = {
|
|
219
|
-
gcNodes: { "/": { outboundRoutes: [] } },
|
|
220
|
-
};
|
|
221
|
-
const dataStoreSnapshotTree = (0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata);
|
|
222
|
-
(0, common_utils_1.assert)(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
|
|
223
|
-
for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
|
|
224
|
-
const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcTreeKey];
|
|
225
|
-
if (blobId === undefined) {
|
|
226
|
-
continue;
|
|
227
|
-
}
|
|
228
|
-
const gcSummaryDetails = await readAndParseBlob(blobId);
|
|
229
|
-
// If there are no nodes for this data store, skip it.
|
|
230
|
-
if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
|
|
231
|
-
continue;
|
|
232
|
-
}
|
|
233
|
-
const dsRootId = `/${dsId}`;
|
|
234
|
-
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
235
|
-
// Construct that entry by adding root data store ids to its outbound routes.
|
|
236
|
-
const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[summary_1.dataStoreAttributesBlobName]);
|
|
237
|
-
if (initialSnapshotDetails.isRootDataStore) {
|
|
238
|
-
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
239
|
-
}
|
|
240
|
-
for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
|
|
241
|
-
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
242
|
-
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
243
|
-
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
244
|
-
gcState.gcNodes[rootId] = {
|
|
245
|
-
outboundRoutes: Array.from(outboundRoutes),
|
|
246
|
-
};
|
|
247
|
-
}
|
|
248
|
-
(0, common_utils_1.assert)(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
|
|
249
|
-
gcState.gcNodes[dsRootId].unreferencedTimestampMs =
|
|
250
|
-
gcSummaryDetails.unrefTimestamp;
|
|
251
|
-
}
|
|
252
|
-
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
253
|
-
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
254
|
-
return Object.keys(gcState.gcNodes).length === 1
|
|
255
|
-
? undefined
|
|
256
|
-
: { gcState, tombstones: undefined, deletedNodes: undefined };
|
|
108
|
+
// back-compat - Older documents will have the GC blobs in each data store's snapshot tree.
|
|
109
|
+
return (0, gcHelpers_1.getSnapshotDataFromOldSnapshotFormat)(baseSnapshot, createParams.metadata, readAndParseBlob);
|
|
257
110
|
}
|
|
258
111
|
catch (error) {
|
|
259
112
|
const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
|
|
260
|
-
dpe.addTelemetryProperties({
|
|
113
|
+
dpe.addTelemetryProperties({
|
|
114
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
115
|
+
});
|
|
261
116
|
throw dpe;
|
|
262
117
|
}
|
|
263
118
|
});
|
|
@@ -319,15 +174,15 @@ class GarbageCollector {
|
|
|
319
174
|
this.mc.logger.sendTelemetryEvent({
|
|
320
175
|
eventName: "GarbageCollectorLoaded",
|
|
321
176
|
gcConfigs: JSON.stringify(this.configs),
|
|
177
|
+
gcOptions: JSON.stringify(createParams.gcOptions),
|
|
322
178
|
});
|
|
323
179
|
}
|
|
324
180
|
}
|
|
325
181
|
static create(createParams) {
|
|
326
182
|
return new GarbageCollector(createParams);
|
|
327
183
|
}
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
|
|
184
|
+
get shouldRunGC() {
|
|
185
|
+
return this.configs.shouldRunGC;
|
|
331
186
|
}
|
|
332
187
|
get summaryStateNeedsReset() {
|
|
333
188
|
return this.summaryStateTracker.doesSummaryStateNeedReset();
|
|
@@ -354,7 +209,7 @@ class GarbageCollector {
|
|
|
354
209
|
}
|
|
355
210
|
// If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
|
|
356
211
|
// tombstone routes.
|
|
357
|
-
if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
212
|
+
if (this.configs.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
358
213
|
// Create a copy since we are writing from a source we don't control
|
|
359
214
|
this.tombstones = Array.from(baseSnapshotData.tombstones);
|
|
360
215
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
@@ -389,7 +244,7 @@ class GarbageCollector {
|
|
|
389
244
|
// tombstones.
|
|
390
245
|
// If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
|
|
391
246
|
// in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
|
|
392
|
-
if (this.shouldRunSweep) {
|
|
247
|
+
if (this.configs.shouldRunSweep) {
|
|
393
248
|
const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.deletedNodes)
|
|
394
249
|
? new Set(snapshotData.deletedNodes)
|
|
395
250
|
: undefined;
|
|
@@ -407,7 +262,7 @@ class GarbageCollector {
|
|
|
407
262
|
}
|
|
408
263
|
}
|
|
409
264
|
}
|
|
410
|
-
else if (this.tombstoneMode) {
|
|
265
|
+
else if (this.configs.tombstoneMode) {
|
|
411
266
|
// The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
|
|
412
267
|
// notify the runtime to update its state as well.
|
|
413
268
|
this.tombstones = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? Array.from(snapshotData.tombstones) : [];
|
|
@@ -425,7 +280,7 @@ class GarbageCollector {
|
|
|
425
280
|
const gcNodes = {};
|
|
426
281
|
for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
|
|
427
282
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
428
|
-
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
283
|
+
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
429
284
|
}
|
|
430
285
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
431
286
|
}
|
|
@@ -451,7 +306,7 @@ class GarbageCollector {
|
|
|
451
306
|
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
452
307
|
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
453
308
|
*/
|
|
454
|
-
if (this.activeConnection() && this.shouldRunGC) {
|
|
309
|
+
if (this.activeConnection() && this.configs.shouldRunGC) {
|
|
455
310
|
this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
|
|
456
311
|
}
|
|
457
312
|
}
|
|
@@ -461,7 +316,7 @@ class GarbageCollector {
|
|
|
461
316
|
*/
|
|
462
317
|
async collectGarbage(options, telemetryContext) {
|
|
463
318
|
var _a;
|
|
464
|
-
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.
|
|
319
|
+
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.configs.runFullGC === true ||
|
|
465
320
|
this.summaryStateTracker.doesSummaryStateNeedReset());
|
|
466
321
|
const logger = options.logger
|
|
467
322
|
? telemetry_utils_1.ChildLogger.create(options.logger, undefined, {
|
|
@@ -518,15 +373,15 @@ class GarbageCollector {
|
|
|
518
373
|
// delete these objects here instead.
|
|
519
374
|
this.logSweepEvents(logger, currentReferenceTimestampMs);
|
|
520
375
|
let updatedGCData = gcData;
|
|
521
|
-
if (this.shouldRunSweep) {
|
|
376
|
+
if (this.configs.shouldRunSweep) {
|
|
522
377
|
updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
|
|
523
378
|
}
|
|
524
|
-
else if (this.testMode) {
|
|
379
|
+
else if (this.configs.testMode) {
|
|
525
380
|
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
526
381
|
// involving access to deleted data.
|
|
527
382
|
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
|
|
528
383
|
}
|
|
529
|
-
else if (this.tombstoneMode) {
|
|
384
|
+
else if (this.configs.tombstoneMode) {
|
|
530
385
|
this.tombstones = sweepReadyNodes;
|
|
531
386
|
// If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
|
|
532
387
|
// involving access to "deleted" data without actually deleting the data from summaries.
|
|
@@ -547,7 +402,7 @@ class GarbageCollector {
|
|
|
547
402
|
*/
|
|
548
403
|
summarize(fullTree, trackState, telemetryContext) {
|
|
549
404
|
var _a;
|
|
550
|
-
if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
405
|
+
if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
551
406
|
return;
|
|
552
407
|
}
|
|
553
408
|
const gcState = { gcNodes: {} };
|
|
@@ -565,11 +420,11 @@ class GarbageCollector {
|
|
|
565
420
|
* If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
|
|
566
421
|
* into the metadata blob. If GC is disabled, the gcFeature is 0.
|
|
567
422
|
*/
|
|
568
|
-
gcFeature: this.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
569
|
-
gcFeatureMatrix: this.persistedGcFeatureMatrix,
|
|
570
|
-
sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
|
|
571
|
-
sweepEnabled: this.sweepEnabled,
|
|
572
|
-
sweepTimeoutMs: this.sweepTimeoutMs,
|
|
423
|
+
gcFeature: this.configs.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
424
|
+
gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
|
|
425
|
+
sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
|
|
426
|
+
sweepEnabled: this.configs.sweepEnabled,
|
|
427
|
+
sweepTimeoutMs: this.configs.sweepTimeoutMs,
|
|
573
428
|
};
|
|
574
429
|
}
|
|
575
430
|
/**
|
|
@@ -595,7 +450,7 @@ class GarbageCollector {
|
|
|
595
450
|
throw container_utils_1.DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
|
|
596
451
|
proposalHandle,
|
|
597
452
|
summaryRefSeq: result.summaryRefSeq,
|
|
598
|
-
|
|
453
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
599
454
|
});
|
|
600
455
|
}
|
|
601
456
|
this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
|
|
@@ -610,7 +465,7 @@ class GarbageCollector {
|
|
|
610
465
|
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
611
466
|
*/
|
|
612
467
|
nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
|
|
613
|
-
if (!this.shouldRunGC) {
|
|
468
|
+
if (!this.configs.shouldRunGC) {
|
|
614
469
|
return;
|
|
615
470
|
}
|
|
616
471
|
const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
|
|
@@ -627,7 +482,7 @@ class GarbageCollector {
|
|
|
627
482
|
*/
|
|
628
483
|
addedOutboundReference(fromNodePath, toNodePath) {
|
|
629
484
|
var _a;
|
|
630
|
-
if (!this.shouldRunGC) {
|
|
485
|
+
if (!this.configs.shouldRunGC) {
|
|
631
486
|
return;
|
|
632
487
|
}
|
|
633
488
|
const outboundRoutes = (_a = this.newReferencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
|
|
@@ -704,7 +559,7 @@ class GarbageCollector {
|
|
|
704
559
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
705
560
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
706
561
|
if (nodeStateTracker === undefined) {
|
|
707
|
-
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
562
|
+
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
708
563
|
}
|
|
709
564
|
else {
|
|
710
565
|
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
@@ -939,7 +794,7 @@ class GarbageCollector {
|
|
|
939
794
|
*/
|
|
940
795
|
logSweepEvents(logger, currentReferenceTimestampMs) {
|
|
941
796
|
if (this.mc.config.getBoolean(gcDefinitions_1.disableSweepLogKey) === true ||
|
|
942
|
-
this.sweepTimeoutMs === undefined) {
|
|
797
|
+
this.configs.sweepTimeoutMs === undefined) {
|
|
943
798
|
return;
|
|
944
799
|
}
|
|
945
800
|
this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
|
|
@@ -961,7 +816,7 @@ class GarbageCollector {
|
|
|
961
816
|
id: nodeId,
|
|
962
817
|
type: nodeType,
|
|
963
818
|
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
964
|
-
timeout: this.sweepTimeoutMs,
|
|
819
|
+
timeout: this.configs.sweepTimeoutMs,
|
|
965
820
|
completedGCRuns: this.completedRuns,
|
|
966
821
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
967
822
|
});
|
|
@@ -991,8 +846,8 @@ class GarbageCollector {
|
|
|
991
846
|
}
|
|
992
847
|
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
993
848
|
const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === gcDefinitions_1.UnreferencedState.Inactive
|
|
994
|
-
? this.inactiveTimeoutMs
|
|
995
|
-
: this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
849
|
+
? this.configs.inactiveTimeoutMs
|
|
850
|
+
: this.configs.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
996
851
|
// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
|
|
997
852
|
// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
|
|
998
853
|
// but it's a good signal nonetheless and we can consume it with a grain of salt.
|