@fluidframework/container-runtime 2.0.0-internal.3.2.1 → 2.0.0-internal.3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerRuntime.d.ts +32 -53
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +55 -21
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +8 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts +19 -0
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/dist/deltaManagerSummarizerProxy.js +40 -0
- package/dist/deltaManagerSummarizerProxy.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +2 -33
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +36 -181
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts +22 -0
- package/dist/gc/gcConfigs.d.ts.map +1 -0
- package/dist/gc/gcConfigs.js +138 -0
- package/dist/gc/gcConfigs.js.map +1 -0
- package/dist/gc/gcDefinitions.d.ts +101 -3
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +8 -3
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +12 -1
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +55 -1
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +1 -2
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +28 -37
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +3 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +2 -1
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +9 -0
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +19 -2
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +24 -10
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +4 -0
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +42 -4
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +14 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +35 -18
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +29 -21
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/storageServiceWithAttachBlobs.d.ts +17 -0
- package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/dist/storageServiceWithAttachBlobs.js +32 -0
- package/dist/storageServiceWithAttachBlobs.js.map +1 -0
- package/dist/summary/runWhileConnectedCoordinator.d.ts +3 -2
- package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js +5 -4
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +2 -0
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/lib/containerRuntime.d.ts +32 -53
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +56 -22
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +9 -4
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts +19 -0
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/lib/deltaManagerSummarizerProxy.js +36 -0
- package/lib/deltaManagerSummarizerProxy.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +2 -33
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +39 -184
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts +22 -0
- package/lib/gc/gcConfigs.d.ts.map +1 -0
- package/lib/gc/gcConfigs.js +134 -0
- package/lib/gc/gcConfigs.js.map +1 -0
- package/lib/gc/gcDefinitions.d.ts +101 -3
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +7 -2
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +12 -1
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +53 -0
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +1 -2
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +28 -37
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +3 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +1 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +9 -0
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +17 -1
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +25 -11
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +4 -0
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +42 -4
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +14 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +35 -18
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +30 -22
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/storageServiceWithAttachBlobs.d.ts +17 -0
- package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/lib/storageServiceWithAttachBlobs.js +28 -0
- package/lib/storageServiceWithAttachBlobs.js.map +1 -0
- package/lib/summary/runWhileConnectedCoordinator.d.ts +3 -2
- package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js +5 -4
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +2 -0
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/package.json +20 -31
- package/src/containerRuntime.ts +92 -76
- package/src/dataStores.ts +9 -4
- package/src/deltaManagerSummarizerProxy.ts +46 -0
- package/src/gc/garbageCollection.ts +50 -290
- package/src/gc/gcConfigs.ts +177 -0
- package/src/gc/gcDefinitions.ts +110 -4
- package/src/gc/gcHelpers.ts +78 -1
- package/src/gc/gcSummaryStateTracker.ts +35 -42
- package/src/gc/index.ts +8 -2
- package/src/index.ts +1 -2
- package/src/opLifecycle/README.md +2 -2
- package/src/opLifecycle/batchManager.ts +19 -1
- package/src/opLifecycle/index.ts +1 -1
- package/src/opLifecycle/opCompressor.ts +31 -12
- package/src/opLifecycle/opDecompressor.ts +49 -5
- package/src/opLifecycle/opSplitter.ts +44 -20
- package/src/opLifecycle/outbox.ts +36 -22
- package/src/packageVersion.ts +1 -1
- package/src/storageServiceWithAttachBlobs.ts +38 -0
- package/src/summary/runWhileConnectedCoordinator.ts +7 -7
- package/src/summary/summarizerTypes.ts +2 -0
|
@@ -14,16 +14,15 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
14
14
|
return t;
|
|
15
15
|
};
|
|
16
16
|
import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
|
|
17
|
-
import { ClientSessionExpiredError, DataProcessingError
|
|
17
|
+
import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
|
|
18
18
|
import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot, runGarbageCollection, trimLeadingSlashes, } from "@fluidframework/garbage-collector";
|
|
19
19
|
import { gcTreeKey, } from "@fluidframework/runtime-definitions";
|
|
20
20
|
import { packagePathToTelemetryProperty, } from "@fluidframework/runtime-utils";
|
|
21
21
|
import { ChildLogger, generateStack, loggerToMonitoringContext, PerformanceEvent, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
|
|
22
22
|
import { RuntimeHeaders } from "../containerRuntime";
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import { getGCVersion, sendGCUnexpectedUsageEvent } from "./gcHelpers";
|
|
23
|
+
import { generateGCConfigs } from "./gcConfigs";
|
|
24
|
+
import { disableSweepLogKey, GCNodeType, UnreferencedState, } from "./gcDefinitions";
|
|
25
|
+
import { getSnapshotDataFromOldSnapshotFormat, sendGCUnexpectedUsageEvent } from "./gcHelpers";
|
|
27
26
|
import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
|
|
28
27
|
import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
|
|
29
28
|
import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
|
|
@@ -51,7 +50,6 @@ import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
|
|
|
51
50
|
*/
|
|
52
51
|
export class GarbageCollector {
|
|
53
52
|
constructor(createParams) {
|
|
54
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
55
53
|
// Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
|
|
56
54
|
// outbound routes from that node.
|
|
57
55
|
this.newReferencesSinceLastRun = new Map();
|
|
@@ -70,136 +68,31 @@ export class GarbageCollector {
|
|
|
70
68
|
this.completedRuns = 0;
|
|
71
69
|
this.runtime = createParams.runtime;
|
|
72
70
|
this.isSummarizerClient = createParams.isSummarizerClient;
|
|
73
|
-
this.gcOptions = createParams.gcOptions;
|
|
74
71
|
this.createContainerMetadata = createParams.createContainerMetadata;
|
|
75
72
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
76
73
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
77
74
|
this.activeConnection = createParams.activeConnection;
|
|
78
75
|
const baseSnapshot = createParams.baseSnapshot;
|
|
79
|
-
const metadata = createParams.metadata;
|
|
80
76
|
const readAndParseBlob = createParams.readAndParseBlob;
|
|
81
77
|
this.mc = loggerToMonitoringContext(ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
|
|
82
78
|
all: { completedGCRuns: () => this.completedRuns },
|
|
83
79
|
}));
|
|
84
80
|
this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
88
|
-
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
|
|
89
|
-
*
|
|
90
|
-
* The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
|
|
91
|
-
* The buffer is added to account for any clock skew or other edge cases.
|
|
92
|
-
* We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
|
|
93
|
-
*/
|
|
94
|
-
function computeSweepTimeout(sessionExpiryTimeoutMs) {
|
|
95
|
-
const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
|
|
96
|
-
const bufferMs = oneDayMs;
|
|
97
|
-
return (sessionExpiryTimeoutMs &&
|
|
98
|
-
sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
|
|
102
|
-
* 1. Whether running GC mark phase is allowed or not.
|
|
103
|
-
* 2. Whether running GC sweep phase is allowed or not.
|
|
104
|
-
* 3. Whether GC session expiry is enabled or not.
|
|
105
|
-
* For existing containers, we get this information from the metadata blob of its summary.
|
|
106
|
-
*/
|
|
107
|
-
if (createParams.existing) {
|
|
108
|
-
gcVersionInBaseSnapshot = getGCVersion(metadata);
|
|
109
|
-
// Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
|
|
110
|
-
// other existing documents, GC is enabled.
|
|
111
|
-
this.gcEnabled = gcVersionInBaseSnapshot > 0;
|
|
112
|
-
this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
|
|
113
|
-
this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
|
|
114
|
-
this.sweepTimeoutMs =
|
|
115
|
-
(_b = metadata === null || metadata === void 0 ? void 0 : metadata.sweepTimeoutMs) !== null && _b !== void 0 ? _b : computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
|
|
116
|
-
this.persistedGcFeatureMatrix = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix;
|
|
117
|
-
}
|
|
118
|
-
else {
|
|
119
|
-
// Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
|
|
120
|
-
// scenario but explicitly failing makes it clearer and promotes correct usage.
|
|
121
|
-
if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
|
|
122
|
-
throw new UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
|
|
123
|
-
}
|
|
124
|
-
// This Test Override only applies for new containers
|
|
125
|
-
const testOverrideSweepTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
|
|
126
|
-
// For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
|
|
127
|
-
// flag in GC options to false.
|
|
128
|
-
this.gcEnabled = this.gcOptions.gcAllowed !== false;
|
|
129
|
-
// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
|
|
130
|
-
this.sweepEnabled = this.gcOptions.sweepAllowed === true;
|
|
131
|
-
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
132
|
-
if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
|
|
133
|
-
this.sessionExpiryTimeoutMs =
|
|
134
|
-
(_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : defaultSessionExpiryDurationMs;
|
|
135
|
-
}
|
|
136
|
-
this.sweepTimeoutMs =
|
|
137
|
-
testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
138
|
-
if (this.gcOptions[gcTombstoneGenerationOptionName] !== undefined) {
|
|
139
|
-
this.persistedGcFeatureMatrix = {
|
|
140
|
-
tombstoneGeneration: this.gcOptions[gcTombstoneGenerationOptionName],
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
}
|
|
81
|
+
this.configs = generateGCConfigs(this.mc, createParams);
|
|
144
82
|
// If session expiry is enabled, we need to close the container when the session expiry timeout expires.
|
|
145
|
-
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
83
|
+
if (this.configs.sessionExpiryTimeoutMs !== undefined) {
|
|
146
84
|
// If Test Override config is set, override Session Expiry timeout.
|
|
147
85
|
const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
|
|
148
|
-
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
|
|
86
|
+
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.configs.sessionExpiryTimeoutMs;
|
|
149
87
|
this.sessionExpiryTimer = new Timer(timeoutMs, () => {
|
|
150
88
|
this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
151
89
|
});
|
|
152
90
|
this.sessionExpiryTimer.start();
|
|
153
91
|
}
|
|
154
|
-
|
|
155
|
-
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
156
|
-
*
|
|
157
|
-
* 1. GC should be enabled for this container.
|
|
158
|
-
*
|
|
159
|
-
* 2. GC should not be disabled via disableGC GC option.
|
|
160
|
-
*
|
|
161
|
-
* These conditions can be overridden via runGCKey feature flag.
|
|
162
|
-
*/
|
|
163
|
-
this.shouldRunGC =
|
|
164
|
-
(_d = this.mc.config.getBoolean(runGCKey)) !== null && _d !== void 0 ? _d :
|
|
165
|
-
// GC must be enabled for the document.
|
|
166
|
-
(this.gcEnabled &&
|
|
167
|
-
// GC must not be disabled via GC options.
|
|
168
|
-
!this.gcOptions.disableGC);
|
|
169
|
-
/**
|
|
170
|
-
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
171
|
-
*
|
|
172
|
-
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
173
|
-
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
174
|
-
* 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
|
|
175
|
-
* the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
|
|
176
|
-
* 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
177
|
-
* feature flag.
|
|
178
|
-
*/
|
|
179
|
-
this.shouldRunSweep =
|
|
180
|
-
this.shouldRunGC &&
|
|
181
|
-
this.sweepTimeoutMs !== undefined &&
|
|
182
|
-
((_e = this.mc.config.getBoolean(runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
|
|
183
|
-
this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
|
|
184
|
-
// Override inactive timeout if test config or gc options to override it is set.
|
|
185
|
-
this.inactiveTimeoutMs =
|
|
186
|
-
(_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : defaultInactiveTimeoutMs;
|
|
187
|
-
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
188
|
-
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
189
|
-
throw new UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
190
|
-
}
|
|
191
|
-
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
192
|
-
this.testMode =
|
|
193
|
-
(_h = this.mc.config.getBoolean(gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
|
|
194
|
-
// Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
|
|
195
|
-
// via feature flags.
|
|
196
|
-
this.tombstoneMode =
|
|
197
|
-
!this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
|
|
198
|
-
this.summaryStateTracker = new GCSummaryStateTracker(this.shouldRunGC, this.trackGCState, this.tombstoneMode, this.mc, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */, gcVersionInBaseSnapshot);
|
|
92
|
+
this.summaryStateTracker = new GCSummaryStateTracker(this.shouldRunGC, this.configs.tombstoneMode, this.mc, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */, this.configs.gcVersionInBaseSnapshot);
|
|
199
93
|
// Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
|
|
200
94
|
// it involves fetching blobs from storage which is expensive.
|
|
201
95
|
this.baseSnapshotDataP = new LazyPromise(async () => {
|
|
202
|
-
var _a;
|
|
203
96
|
if (baseSnapshot === undefined) {
|
|
204
97
|
return undefined;
|
|
205
98
|
}
|
|
@@ -209,52 +102,14 @@ export class GarbageCollector {
|
|
|
209
102
|
if (gcSnapshotTree !== undefined) {
|
|
210
103
|
return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
211
104
|
}
|
|
212
|
-
// back-compat - Older documents will have the GC blobs in each data store's
|
|
213
|
-
|
|
214
|
-
// Add a node for the root node that is not present in older snapshot format.
|
|
215
|
-
const gcState = {
|
|
216
|
-
gcNodes: { "/": { outboundRoutes: [] } },
|
|
217
|
-
};
|
|
218
|
-
const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
|
|
219
|
-
assert(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
|
|
220
|
-
for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
|
|
221
|
-
const blobId = dsSnapshotTree.blobs[gcTreeKey];
|
|
222
|
-
if (blobId === undefined) {
|
|
223
|
-
continue;
|
|
224
|
-
}
|
|
225
|
-
const gcSummaryDetails = await readAndParseBlob(blobId);
|
|
226
|
-
// If there are no nodes for this data store, skip it.
|
|
227
|
-
if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
|
|
228
|
-
continue;
|
|
229
|
-
}
|
|
230
|
-
const dsRootId = `/${dsId}`;
|
|
231
|
-
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
232
|
-
// Construct that entry by adding root data store ids to its outbound routes.
|
|
233
|
-
const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[dataStoreAttributesBlobName]);
|
|
234
|
-
if (initialSnapshotDetails.isRootDataStore) {
|
|
235
|
-
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
236
|
-
}
|
|
237
|
-
for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
|
|
238
|
-
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
239
|
-
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
240
|
-
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
241
|
-
gcState.gcNodes[rootId] = {
|
|
242
|
-
outboundRoutes: Array.from(outboundRoutes),
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
assert(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
|
|
246
|
-
gcState.gcNodes[dsRootId].unreferencedTimestampMs =
|
|
247
|
-
gcSummaryDetails.unrefTimestamp;
|
|
248
|
-
}
|
|
249
|
-
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
250
|
-
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
251
|
-
return Object.keys(gcState.gcNodes).length === 1
|
|
252
|
-
? undefined
|
|
253
|
-
: { gcState, tombstones: undefined, deletedNodes: undefined };
|
|
105
|
+
// back-compat - Older documents will have the GC blobs in each data store's snapshot tree.
|
|
106
|
+
return getSnapshotDataFromOldSnapshotFormat(baseSnapshot, createParams.metadata, readAndParseBlob);
|
|
254
107
|
}
|
|
255
108
|
catch (error) {
|
|
256
109
|
const dpe = DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
|
|
257
|
-
dpe.addTelemetryProperties({
|
|
110
|
+
dpe.addTelemetryProperties({
|
|
111
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
112
|
+
});
|
|
258
113
|
throw dpe;
|
|
259
114
|
}
|
|
260
115
|
});
|
|
@@ -316,15 +171,15 @@ export class GarbageCollector {
|
|
|
316
171
|
this.mc.logger.sendTelemetryEvent({
|
|
317
172
|
eventName: "GarbageCollectorLoaded",
|
|
318
173
|
gcConfigs: JSON.stringify(this.configs),
|
|
174
|
+
gcOptions: JSON.stringify(createParams.gcOptions),
|
|
319
175
|
});
|
|
320
176
|
}
|
|
321
177
|
}
|
|
322
178
|
static create(createParams) {
|
|
323
179
|
return new GarbageCollector(createParams);
|
|
324
180
|
}
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
|
|
181
|
+
get shouldRunGC() {
|
|
182
|
+
return this.configs.shouldRunGC;
|
|
328
183
|
}
|
|
329
184
|
get summaryStateNeedsReset() {
|
|
330
185
|
return this.summaryStateTracker.doesSummaryStateNeedReset();
|
|
@@ -351,7 +206,7 @@ export class GarbageCollector {
|
|
|
351
206
|
}
|
|
352
207
|
// If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
|
|
353
208
|
// tombstone routes.
|
|
354
|
-
if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
209
|
+
if (this.configs.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
355
210
|
// Create a copy since we are writing from a source we don't control
|
|
356
211
|
this.tombstones = Array.from(baseSnapshotData.tombstones);
|
|
357
212
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
@@ -386,7 +241,7 @@ export class GarbageCollector {
|
|
|
386
241
|
// tombstones.
|
|
387
242
|
// If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
|
|
388
243
|
// in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
|
|
389
|
-
if (this.shouldRunSweep) {
|
|
244
|
+
if (this.configs.shouldRunSweep) {
|
|
390
245
|
const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.deletedNodes)
|
|
391
246
|
? new Set(snapshotData.deletedNodes)
|
|
392
247
|
: undefined;
|
|
@@ -404,7 +259,7 @@ export class GarbageCollector {
|
|
|
404
259
|
}
|
|
405
260
|
}
|
|
406
261
|
}
|
|
407
|
-
else if (this.tombstoneMode) {
|
|
262
|
+
else if (this.configs.tombstoneMode) {
|
|
408
263
|
// The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
|
|
409
264
|
// notify the runtime to update its state as well.
|
|
410
265
|
this.tombstones = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? Array.from(snapshotData.tombstones) : [];
|
|
@@ -422,7 +277,7 @@ export class GarbageCollector {
|
|
|
422
277
|
const gcNodes = {};
|
|
423
278
|
for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
|
|
424
279
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
425
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
280
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
426
281
|
}
|
|
427
282
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
428
283
|
}
|
|
@@ -448,7 +303,7 @@ export class GarbageCollector {
|
|
|
448
303
|
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
449
304
|
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
450
305
|
*/
|
|
451
|
-
if (this.activeConnection() && this.shouldRunGC) {
|
|
306
|
+
if (this.activeConnection() && this.configs.shouldRunGC) {
|
|
452
307
|
this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
|
|
453
308
|
}
|
|
454
309
|
}
|
|
@@ -458,7 +313,7 @@ export class GarbageCollector {
|
|
|
458
313
|
*/
|
|
459
314
|
async collectGarbage(options, telemetryContext) {
|
|
460
315
|
var _a;
|
|
461
|
-
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.
|
|
316
|
+
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.configs.runFullGC === true ||
|
|
462
317
|
this.summaryStateTracker.doesSummaryStateNeedReset());
|
|
463
318
|
const logger = options.logger
|
|
464
319
|
? ChildLogger.create(options.logger, undefined, {
|
|
@@ -515,15 +370,15 @@ export class GarbageCollector {
|
|
|
515
370
|
// delete these objects here instead.
|
|
516
371
|
this.logSweepEvents(logger, currentReferenceTimestampMs);
|
|
517
372
|
let updatedGCData = gcData;
|
|
518
|
-
if (this.shouldRunSweep) {
|
|
373
|
+
if (this.configs.shouldRunSweep) {
|
|
519
374
|
updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
|
|
520
375
|
}
|
|
521
|
-
else if (this.testMode) {
|
|
376
|
+
else if (this.configs.testMode) {
|
|
522
377
|
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
523
378
|
// involving access to deleted data.
|
|
524
379
|
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
|
|
525
380
|
}
|
|
526
|
-
else if (this.tombstoneMode) {
|
|
381
|
+
else if (this.configs.tombstoneMode) {
|
|
527
382
|
this.tombstones = sweepReadyNodes;
|
|
528
383
|
// If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
|
|
529
384
|
// involving access to "deleted" data without actually deleting the data from summaries.
|
|
@@ -544,7 +399,7 @@ export class GarbageCollector {
|
|
|
544
399
|
*/
|
|
545
400
|
summarize(fullTree, trackState, telemetryContext) {
|
|
546
401
|
var _a;
|
|
547
|
-
if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
402
|
+
if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
548
403
|
return;
|
|
549
404
|
}
|
|
550
405
|
const gcState = { gcNodes: {} };
|
|
@@ -562,11 +417,11 @@ export class GarbageCollector {
|
|
|
562
417
|
* If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
|
|
563
418
|
* into the metadata blob. If GC is disabled, the gcFeature is 0.
|
|
564
419
|
*/
|
|
565
|
-
gcFeature: this.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
566
|
-
gcFeatureMatrix: this.persistedGcFeatureMatrix,
|
|
567
|
-
sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
|
|
568
|
-
sweepEnabled: this.sweepEnabled,
|
|
569
|
-
sweepTimeoutMs: this.sweepTimeoutMs,
|
|
420
|
+
gcFeature: this.configs.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
421
|
+
gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
|
|
422
|
+
sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
|
|
423
|
+
sweepEnabled: this.configs.sweepEnabled,
|
|
424
|
+
sweepTimeoutMs: this.configs.sweepTimeoutMs,
|
|
570
425
|
};
|
|
571
426
|
}
|
|
572
427
|
/**
|
|
@@ -592,7 +447,7 @@ export class GarbageCollector {
|
|
|
592
447
|
throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
|
|
593
448
|
proposalHandle,
|
|
594
449
|
summaryRefSeq: result.summaryRefSeq,
|
|
595
|
-
|
|
450
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
596
451
|
});
|
|
597
452
|
}
|
|
598
453
|
this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
|
|
@@ -607,7 +462,7 @@ export class GarbageCollector {
|
|
|
607
462
|
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
608
463
|
*/
|
|
609
464
|
nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
|
|
610
|
-
if (!this.shouldRunGC) {
|
|
465
|
+
if (!this.configs.shouldRunGC) {
|
|
611
466
|
return;
|
|
612
467
|
}
|
|
613
468
|
const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
|
|
@@ -624,7 +479,7 @@ export class GarbageCollector {
|
|
|
624
479
|
*/
|
|
625
480
|
addedOutboundReference(fromNodePath, toNodePath) {
|
|
626
481
|
var _a;
|
|
627
|
-
if (!this.shouldRunGC) {
|
|
482
|
+
if (!this.configs.shouldRunGC) {
|
|
628
483
|
return;
|
|
629
484
|
}
|
|
630
485
|
const outboundRoutes = (_a = this.newReferencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
|
|
@@ -701,7 +556,7 @@ export class GarbageCollector {
|
|
|
701
556
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
702
557
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
703
558
|
if (nodeStateTracker === undefined) {
|
|
704
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
559
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
705
560
|
}
|
|
706
561
|
else {
|
|
707
562
|
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
@@ -936,7 +791,7 @@ export class GarbageCollector {
|
|
|
936
791
|
*/
|
|
937
792
|
logSweepEvents(logger, currentReferenceTimestampMs) {
|
|
938
793
|
if (this.mc.config.getBoolean(disableSweepLogKey) === true ||
|
|
939
|
-
this.sweepTimeoutMs === undefined) {
|
|
794
|
+
this.configs.sweepTimeoutMs === undefined) {
|
|
940
795
|
return;
|
|
941
796
|
}
|
|
942
797
|
this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
|
|
@@ -958,7 +813,7 @@ export class GarbageCollector {
|
|
|
958
813
|
id: nodeId,
|
|
959
814
|
type: nodeType,
|
|
960
815
|
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
961
|
-
timeout: this.sweepTimeoutMs,
|
|
816
|
+
timeout: this.configs.sweepTimeoutMs,
|
|
962
817
|
completedGCRuns: this.completedRuns,
|
|
963
818
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
964
819
|
});
|
|
@@ -988,8 +843,8 @@ export class GarbageCollector {
|
|
|
988
843
|
}
|
|
989
844
|
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
990
845
|
const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === UnreferencedState.Inactive
|
|
991
|
-
? this.inactiveTimeoutMs
|
|
992
|
-
: this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
846
|
+
? this.configs.inactiveTimeoutMs
|
|
847
|
+
: this.configs.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
993
848
|
// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
|
|
994
849
|
// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
|
|
995
850
|
// but it's a good signal nonetheless and we can consume it with a grain of salt.
|