@fluidframework/container-runtime 2.0.0-internal.2.1.1 → 2.0.0-internal.2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +1 -1
- package/dist/blobManager.d.ts +20 -5
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +57 -15
- package/dist/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +39 -40
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +115 -278
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +3 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +21 -3
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +8 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +26 -13
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +15 -17
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +92 -106
- package/dist/garbageCollection.js.map +1 -1
- package/dist/garbageCollectionConstants.d.ts +19 -0
- package/dist/garbageCollectionConstants.d.ts.map +1 -0
- package/dist/garbageCollectionConstants.js +34 -0
- package/dist/garbageCollectionConstants.js.map +1 -0
- package/dist/gcSweepReadyUsageDetection.js +2 -2
- package/dist/gcSweepReadyUsageDetection.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -6
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +30 -0
- package/dist/opLifecycle/batchManager.d.ts.map +1 -0
- package/dist/{batchManager.js → opLifecycle/batchManager.js} +25 -10
- package/dist/opLifecycle/batchManager.js.map +1 -0
- package/dist/opLifecycle/definitions.d.ts +40 -0
- package/dist/opLifecycle/definitions.d.ts.map +1 -0
- package/dist/opLifecycle/definitions.js +7 -0
- package/dist/opLifecycle/definitions.js.map +1 -0
- package/dist/opLifecycle/index.d.ts +12 -0
- package/dist/opLifecycle/index.d.ts.map +1 -0
- package/dist/opLifecycle/index.js +21 -0
- package/dist/opLifecycle/index.js.map +1 -0
- package/dist/opLifecycle/opCompressor.d.ts +18 -0
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -0
- package/dist/opLifecycle/opCompressor.js +53 -0
- package/dist/opLifecycle/opCompressor.js.map +1 -0
- package/dist/opLifecycle/opDecompressor.d.ts +20 -0
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -0
- package/dist/opLifecycle/opDecompressor.js +72 -0
- package/dist/opLifecycle/opDecompressor.js.map +1 -0
- package/dist/opLifecycle/opSplitter.d.ts +17 -0
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -0
- package/dist/opLifecycle/opSplitter.js +61 -0
- package/dist/opLifecycle/opSplitter.js.map +1 -0
- package/dist/opLifecycle/outbox.d.ts +47 -0
- package/dist/opLifecycle/outbox.d.ts.map +1 -0
- package/dist/opLifecycle/outbox.js +153 -0
- package/dist/opLifecycle/outbox.js.map +1 -0
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +26 -0
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
- package/dist/opLifecycle/remoteMessageProcessor.js +81 -0
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -0
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/summaryFormat.js +2 -2
- package/dist/summaryFormat.js.map +1 -1
- package/lib/blobManager.d.ts +20 -5
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +59 -17
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +39 -40
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +113 -275
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +3 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +23 -5
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +8 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +28 -15
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +15 -17
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +72 -86
- package/lib/garbageCollection.js.map +1 -1
- package/lib/garbageCollectionConstants.d.ts +19 -0
- package/lib/garbageCollectionConstants.d.ts.map +1 -0
- package/lib/garbageCollectionConstants.js +31 -0
- package/lib/garbageCollectionConstants.js.map +1 -0
- package/lib/gcSweepReadyUsageDetection.js +1 -1
- package/lib/gcSweepReadyUsageDetection.js.map +1 -1
- package/lib/index.d.ts +4 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +3 -2
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +30 -0
- package/lib/opLifecycle/batchManager.d.ts.map +1 -0
- package/lib/{batchManager.js → opLifecycle/batchManager.js} +25 -10
- package/lib/opLifecycle/batchManager.js.map +1 -0
- package/lib/opLifecycle/definitions.d.ts +40 -0
- package/lib/opLifecycle/definitions.d.ts.map +1 -0
- package/lib/opLifecycle/definitions.js +6 -0
- package/lib/opLifecycle/definitions.js.map +1 -0
- package/lib/opLifecycle/index.d.ts +12 -0
- package/lib/opLifecycle/index.d.ts.map +1 -0
- package/lib/opLifecycle/index.js +11 -0
- package/lib/opLifecycle/index.js.map +1 -0
- package/lib/opLifecycle/opCompressor.d.ts +18 -0
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -0
- package/lib/opLifecycle/opCompressor.js +49 -0
- package/lib/opLifecycle/opCompressor.js.map +1 -0
- package/lib/opLifecycle/opDecompressor.d.ts +20 -0
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -0
- package/lib/opLifecycle/opDecompressor.js +68 -0
- package/lib/opLifecycle/opDecompressor.js.map +1 -0
- package/lib/opLifecycle/opSplitter.d.ts +17 -0
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -0
- package/lib/opLifecycle/opSplitter.js +57 -0
- package/lib/opLifecycle/opSplitter.js.map +1 -0
- package/lib/opLifecycle/outbox.d.ts +47 -0
- package/lib/opLifecycle/outbox.d.ts.map +1 -0
- package/lib/opLifecycle/outbox.js +149 -0
- package/lib/opLifecycle/outbox.js.map +1 -0
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +26 -0
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
- package/lib/opLifecycle/remoteMessageProcessor.js +76 -0
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -0
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/summaryFormat.js +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/package.json +35 -21
- package/prettier.config.cjs +8 -0
- package/src/blobManager.ts +74 -19
- package/src/containerRuntime.ts +144 -341
- package/src/dataStoreContext.ts +33 -5
- package/src/dataStores.ts +32 -16
- package/src/garbageCollection.ts +106 -82
- package/src/garbageCollectionConstants.ts +35 -0
- package/src/gcSweepReadyUsageDetection.ts +1 -1
- package/src/index.ts +6 -4
- package/src/{batchManager.ts → opLifecycle/batchManager.ts} +41 -23
- package/src/opLifecycle/definitions.ts +44 -0
- package/src/opLifecycle/index.ts +17 -0
- package/src/opLifecycle/opCompressor.ts +64 -0
- package/src/opLifecycle/opDecompressor.ts +84 -0
- package/src/opLifecycle/opSplitter.ts +78 -0
- package/src/opLifecycle/outbox.ts +204 -0
- package/src/opLifecycle/remoteMessageProcessor.ts +90 -0
- package/src/packageVersion.ts +1 -1
- package/src/summaryFormat.ts +1 -1
- package/dist/batchManager.d.ts +0 -36
- package/dist/batchManager.d.ts.map +0 -1
- package/dist/batchManager.js.map +0 -1
- package/lib/batchManager.d.ts +0 -36
- package/lib/batchManager.d.ts.map +0 -1
- package/lib/batchManager.js.map +0 -1
|
@@ -15,7 +15,7 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
15
15
|
return t;
|
|
16
16
|
};
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
-
exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType =
|
|
18
|
+
exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = void 0;
|
|
19
19
|
const common_utils_1 = require("@fluidframework/common-utils");
|
|
20
20
|
const container_utils_1 = require("@fluidframework/container-utils");
|
|
21
21
|
const garbage_collector_1 = require("@fluidframework/garbage-collector");
|
|
@@ -25,34 +25,11 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
|
25
25
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
26
26
|
const containerRuntime_1 = require("./containerRuntime");
|
|
27
27
|
const dataStores_1 = require("./dataStores");
|
|
28
|
+
const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
|
|
28
29
|
const gcSweepReadyUsageDetection_1 = require("./gcSweepReadyUsageDetection");
|
|
29
30
|
const summaryFormat_1 = require("./summaryFormat");
|
|
30
31
|
/** This is the current version of garbage collection. */
|
|
31
32
|
const GCVersion = 1;
|
|
32
|
-
// The key for the GC tree in summary.
|
|
33
|
-
exports.gcTreeKey = "gc";
|
|
34
|
-
// They prefix for GC blobs in the GC tree in summary.
|
|
35
|
-
exports.gcBlobPrefix = "__gc";
|
|
36
|
-
// The key for tombstone blob in the GC tree in summary.
|
|
37
|
-
exports.gcTombstoneBlobKey = "__tombstones";
|
|
38
|
-
// Feature gate key to turn GC on / off.
|
|
39
|
-
exports.runGCKey = "Fluid.GarbageCollection.RunGC";
|
|
40
|
-
// Feature gate key to turn GC sweep on / off.
|
|
41
|
-
exports.runSweepKey = "Fluid.GarbageCollection.RunSweep";
|
|
42
|
-
// Feature gate key to turn GC test mode on / off.
|
|
43
|
-
exports.gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
44
|
-
// Feature gate key to expire a session after a set period of time.
|
|
45
|
-
exports.runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
|
|
46
|
-
// Feature gate key to write the gc blob as a handle if the data is the same.
|
|
47
|
-
exports.trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
|
|
48
|
-
// Feature gate key to turn GC sweep log off.
|
|
49
|
-
exports.disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
|
|
50
|
-
// Feature gate key to tombstone datastores.
|
|
51
|
-
exports.testTombstoneKey = "Fluid.GarbageCollection.Test.Tombstone";
|
|
52
|
-
// One day in milliseconds.
|
|
53
|
-
exports.oneDayMs = 1 * 24 * 60 * 60 * 1000;
|
|
54
|
-
exports.defaultInactiveTimeoutMs = 7 * exports.oneDayMs; // 7 days
|
|
55
|
-
exports.defaultSessionExpiryDurationMs = 30 * exports.oneDayMs; // 30 days
|
|
56
33
|
/** The types of GC nodes in the GC reference graph. */
|
|
57
34
|
exports.GCNodeType = {
|
|
58
35
|
// Nodes that are for data stores.
|
|
@@ -165,7 +142,7 @@ exports.UnreferencedStateTracker = UnreferencedStateTracker;
|
|
|
165
142
|
*/
|
|
166
143
|
class GarbageCollector {
|
|
167
144
|
constructor(createParams) {
|
|
168
|
-
var _a, _b, _c, _d, _e, _f, _g, _h
|
|
145
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
169
146
|
/**
|
|
170
147
|
* Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
|
|
171
148
|
*
|
|
@@ -197,6 +174,7 @@ class GarbageCollector {
|
|
|
197
174
|
this.runtime = createParams.runtime;
|
|
198
175
|
this.isSummarizerClient = createParams.isSummarizerClient;
|
|
199
176
|
this.gcOptions = createParams.gcOptions;
|
|
177
|
+
this.createContainerMetadata = createParams.createContainerMetadata;
|
|
200
178
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
201
179
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
202
180
|
this.activeConnection = createParams.activeConnection;
|
|
@@ -215,8 +193,8 @@ class GarbageCollector {
|
|
|
215
193
|
* We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
|
|
216
194
|
*/
|
|
217
195
|
function computeSweepTimeout(sessionExpiryTimeoutMs) {
|
|
218
|
-
const maxSnapshotCacheExpiryMs = 5 *
|
|
219
|
-
const bufferMs =
|
|
196
|
+
const maxSnapshotCacheExpiryMs = 5 * garbageCollectionConstants_1.oneDayMs;
|
|
197
|
+
const bufferMs = garbageCollectionConstants_1.oneDayMs;
|
|
220
198
|
return sessionExpiryTimeoutMs &&
|
|
221
199
|
(sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
|
|
222
200
|
}
|
|
@@ -252,8 +230,8 @@ class GarbageCollector {
|
|
|
252
230
|
// ...unless we're using the TestOverride
|
|
253
231
|
this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
|
|
254
232
|
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
255
|
-
if (this.mc.config.getBoolean(
|
|
256
|
-
this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c :
|
|
233
|
+
if (this.mc.config.getBoolean(garbageCollectionConstants_1.runSessionExpiryKey) && this.gcEnabled) {
|
|
234
|
+
this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : garbageCollectionConstants_1.defaultSessionExpiryDurationMs;
|
|
257
235
|
}
|
|
258
236
|
this.sweepTimeoutMs =
|
|
259
237
|
testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
@@ -278,7 +256,7 @@ class GarbageCollector {
|
|
|
278
256
|
*
|
|
279
257
|
* These conditions can be overridden via runGCKey feature flag.
|
|
280
258
|
*/
|
|
281
|
-
this.shouldRunGC = (_d = this.mc.config.getBoolean(
|
|
259
|
+
this.shouldRunGC = (_d = this.mc.config.getBoolean(garbageCollectionConstants_1.runGCKey)) !== null && _d !== void 0 ? _d : (
|
|
282
260
|
// GC must be enabled for the document.
|
|
283
261
|
this.gcEnabled
|
|
284
262
|
// GC must not be disabled via GC options.
|
|
@@ -296,43 +274,34 @@ class GarbageCollector {
|
|
|
296
274
|
this.shouldRunSweep =
|
|
297
275
|
this.shouldRunGC
|
|
298
276
|
&& this.sweepTimeoutMs !== undefined
|
|
299
|
-
&& ((_e = this.mc.config.getBoolean(
|
|
300
|
-
this.trackGCState = this.mc.config.getBoolean(
|
|
277
|
+
&& ((_e = this.mc.config.getBoolean(garbageCollectionConstants_1.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
|
|
278
|
+
this.trackGCState = this.mc.config.getBoolean(garbageCollectionConstants_1.trackGCStateKey) === true;
|
|
301
279
|
// Override inactive timeout if test config or gc options to override it is set.
|
|
302
|
-
this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g :
|
|
280
|
+
this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : garbageCollectionConstants_1.defaultInactiveTimeoutMs;
|
|
303
281
|
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
304
282
|
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
305
283
|
throw new container_utils_1.UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
306
284
|
}
|
|
307
285
|
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
308
|
-
this.testMode = (_h = this.mc.config.getBoolean(
|
|
309
|
-
|
|
286
|
+
this.testMode = (_h = this.mc.config.getBoolean(garbageCollectionConstants_1.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
|
|
287
|
+
// Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
|
|
288
|
+
this.tombstoneMode = this.mc.config.getBoolean(garbageCollectionConstants_1.disableTombstoneKey) !== true;
|
|
310
289
|
// The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
|
|
311
290
|
// contain GC tree and GC is enabled.
|
|
312
|
-
const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[
|
|
291
|
+
const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey]) !== undefined;
|
|
313
292
|
this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
|
|
314
|
-
// Get the GC
|
|
315
|
-
//
|
|
316
|
-
|
|
293
|
+
// Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
|
|
294
|
+
// it involves fetching blobs from storage which is expensive.
|
|
295
|
+
this.baseSnapshotDataP = new common_utils_1.LazyPromise(async () => {
|
|
317
296
|
var _a;
|
|
318
297
|
if (baseSnapshot === undefined) {
|
|
319
298
|
return undefined;
|
|
320
299
|
}
|
|
321
300
|
try {
|
|
322
301
|
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
323
|
-
const gcSnapshotTree = baseSnapshot.trees[
|
|
302
|
+
const gcSnapshotTree = baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey];
|
|
324
303
|
if (gcSnapshotTree !== undefined) {
|
|
325
|
-
|
|
326
|
-
if (baseGCData.tombstones !== undefined && this.tombstoneMode) {
|
|
327
|
-
this.tombstones = baseGCData.tombstones;
|
|
328
|
-
}
|
|
329
|
-
if (this.trackGCState) {
|
|
330
|
-
this.latestSummaryData = {
|
|
331
|
-
serializedGCState: JSON.stringify(generateSortedGCState(baseGCData.gcState)),
|
|
332
|
-
serializedTombstones: JSON.stringify(baseGCData.tombstones),
|
|
333
|
-
};
|
|
334
|
-
}
|
|
335
|
-
return baseGCData.gcState;
|
|
304
|
+
return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
336
305
|
}
|
|
337
306
|
// back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
|
|
338
307
|
// consolidate into IGarbageCollectionState format.
|
|
@@ -368,7 +337,7 @@ class GarbageCollector {
|
|
|
368
337
|
}
|
|
369
338
|
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
370
339
|
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
371
|
-
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
340
|
+
return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
|
|
372
341
|
}
|
|
373
342
|
catch (error) {
|
|
374
343
|
const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
|
|
@@ -377,11 +346,11 @@ class GarbageCollector {
|
|
|
377
346
|
}
|
|
378
347
|
});
|
|
379
348
|
/**
|
|
380
|
-
* Set up the initializer which initializes the
|
|
381
|
-
*
|
|
382
|
-
*
|
|
349
|
+
* Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
|
|
350
|
+
* connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
|
|
351
|
+
* GC state and updates their inactive or sweep ready state.
|
|
383
352
|
*/
|
|
384
|
-
this.
|
|
353
|
+
this.initializeGCStateFromBaseSnapshotP = new common_utils_1.LazyPromise(async () => {
|
|
385
354
|
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
386
355
|
/**
|
|
387
356
|
* If there is no current reference timestamp, skip initialization. We need the current timestamp to track
|
|
@@ -399,34 +368,41 @@ class GarbageCollector {
|
|
|
399
368
|
});
|
|
400
369
|
return;
|
|
401
370
|
}
|
|
402
|
-
const
|
|
371
|
+
const baseSnapshotData = await this.baseSnapshotDataP;
|
|
403
372
|
/**
|
|
404
|
-
* The base
|
|
373
|
+
* The base snapshot data will not be present if the container is loaded from:
|
|
405
374
|
* 1. The first summary created by the detached container.
|
|
406
375
|
* 2. A summary that was generated with GC disabled.
|
|
407
376
|
* 3. A summary that was generated before GC even existed.
|
|
408
377
|
*/
|
|
409
|
-
if (
|
|
378
|
+
if (baseSnapshotData === undefined) {
|
|
410
379
|
return;
|
|
411
380
|
}
|
|
412
381
|
const gcNodes = {};
|
|
413
|
-
for (const [nodeId, nodeData] of Object.entries(
|
|
382
|
+
for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
|
|
414
383
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
415
384
|
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
416
385
|
}
|
|
417
386
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
418
387
|
}
|
|
419
388
|
this.previousGCDataFromLastRun = { gcNodes };
|
|
389
|
+
// If tracking state across summaries, update latest summary data from the base snapshot's GC data.
|
|
390
|
+
if (this.trackGCState) {
|
|
391
|
+
this.latestSummaryData = {
|
|
392
|
+
serializedGCState: JSON.stringify(generateSortedGCState(baseSnapshotData.gcState)),
|
|
393
|
+
serializedTombstones: JSON.stringify(baseSnapshotData.tombstones),
|
|
394
|
+
};
|
|
395
|
+
}
|
|
420
396
|
});
|
|
421
397
|
// Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
|
|
422
398
|
// which the caller uses to initialize each node's GC state.
|
|
423
399
|
this.baseGCDetailsP = new common_utils_1.LazyPromise(async () => {
|
|
424
|
-
const
|
|
425
|
-
if (
|
|
400
|
+
const baseSnapshotData = await this.baseSnapshotDataP;
|
|
401
|
+
if (baseSnapshotData === undefined) {
|
|
426
402
|
return new Map();
|
|
427
403
|
}
|
|
428
404
|
const gcNodes = {};
|
|
429
|
-
for (const [nodeId, nodeData] of Object.entries(
|
|
405
|
+
for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
|
|
430
406
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
431
407
|
}
|
|
432
408
|
// Run GC on the nodes in the base summary to get the routes used in each node in the container.
|
|
@@ -436,7 +412,7 @@ class GarbageCollector {
|
|
|
436
412
|
const baseGCDetailsMap = (0, garbage_collector_1.unpackChildNodesGCDetails)({ gcData: { gcNodes }, usedRoutes });
|
|
437
413
|
// Currently, the nodes may write the GC data. So, we need to update its base GC details with the
|
|
438
414
|
// unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
|
|
439
|
-
for (const [nodeId, nodeData] of Object.entries(
|
|
415
|
+
for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
|
|
440
416
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
441
417
|
const dataStoreGCDetails = baseGCDetailsMap.get(nodeId.slice(1));
|
|
442
418
|
if (dataStoreGCDetails !== undefined) {
|
|
@@ -479,6 +455,26 @@ class GarbageCollector {
|
|
|
479
455
|
get configs() {
|
|
480
456
|
return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
|
|
481
457
|
}
|
|
458
|
+
/**
|
|
459
|
+
* Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
|
|
460
|
+
* before they are loaded or used. This is important to get accurate information of whether tombstoned object are
|
|
461
|
+
* in use or not.
|
|
462
|
+
*/
|
|
463
|
+
async initializeBaseState() {
|
|
464
|
+
const baseSnapshotData = await this.baseSnapshotDataP;
|
|
465
|
+
/**
|
|
466
|
+
* The base snapshot data or tombstone state will not be present if the container is loaded from:
|
|
467
|
+
* 1. The first summary created by the detached container.
|
|
468
|
+
* 2. A summary that was generated with GC disabled.
|
|
469
|
+
* 3. A summary that was generated before GC even existed.
|
|
470
|
+
* 4. A summary that was generated with tombstone feature disabled.
|
|
471
|
+
*/
|
|
472
|
+
if (!this.tombstoneMode || (baseSnapshotData === null || baseSnapshotData === void 0 ? void 0 : baseSnapshotData.tombstones) === undefined) {
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
this.tombstones = baseSnapshotData.tombstones;
|
|
476
|
+
this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
|
|
477
|
+
}
|
|
482
478
|
/**
|
|
483
479
|
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
484
480
|
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
|
|
@@ -500,7 +496,7 @@ class GarbageCollector {
|
|
|
500
496
|
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
501
497
|
*/
|
|
502
498
|
if (this.activeConnection() && this.shouldRunGC) {
|
|
503
|
-
this.
|
|
499
|
+
this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
|
|
504
500
|
}
|
|
505
501
|
}
|
|
506
502
|
/**
|
|
@@ -542,8 +538,8 @@ class GarbageCollector {
|
|
|
542
538
|
}, { end: true, cancel: "error" });
|
|
543
539
|
}
|
|
544
540
|
async runPreGCSteps() {
|
|
545
|
-
// Ensure that
|
|
546
|
-
await this.
|
|
541
|
+
// Ensure that state has been initialized from the base snapshot data.
|
|
542
|
+
await this.initializeGCStateFromBaseSnapshotP;
|
|
547
543
|
// Let the runtime update its pending state before GC runs.
|
|
548
544
|
await this.runtime.updateStateBeforeGC();
|
|
549
545
|
}
|
|
@@ -563,23 +559,13 @@ class GarbageCollector {
|
|
|
563
559
|
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
564
560
|
// involving access to deleted data.
|
|
565
561
|
if (this.testMode) {
|
|
566
|
-
this.runtime.
|
|
562
|
+
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds, false /* tombstone */);
|
|
567
563
|
}
|
|
568
|
-
else {
|
|
564
|
+
else if (this.tombstoneMode) {
|
|
569
565
|
// If we are running in GC tombstone mode, tombstone objects for unused routes. This enables testing
|
|
570
566
|
// scenarios involving access to "deleted" data without actually deleting the data from summaries.
|
|
571
567
|
// Note: we will not tombstone in test mode
|
|
572
|
-
|
|
573
|
-
const tombstoneRoutes = [];
|
|
574
|
-
// Currently only tombstone datastores
|
|
575
|
-
for (const [key, value] of this.unreferencedNodesState.entries()) {
|
|
576
|
-
if (value.state === exports.UnreferencedState.SweepReady &&
|
|
577
|
-
this.runtime.getNodeType(key) === exports.GCNodeType.DataStore) {
|
|
578
|
-
tombstoneRoutes.push(key);
|
|
579
|
-
}
|
|
580
|
-
}
|
|
581
|
-
this.runtime.deleteUnusedRoutes(tombstoneRoutes);
|
|
582
|
-
}
|
|
568
|
+
this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
|
|
583
569
|
}
|
|
584
570
|
// Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
|
|
585
571
|
// updates its state so that we don't send false positives based on intermediate state. For example, we may get
|
|
@@ -605,7 +591,9 @@ class GarbageCollector {
|
|
|
605
591
|
};
|
|
606
592
|
}
|
|
607
593
|
const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
|
|
608
|
-
const serializedTombstones = this.
|
|
594
|
+
const serializedTombstones = this.tombstoneMode
|
|
595
|
+
? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
|
|
596
|
+
: undefined;
|
|
609
597
|
/**
|
|
610
598
|
* Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
|
|
611
599
|
* summary, send summary handles for them. Otherwise, send the data in summary blobs.
|
|
@@ -621,7 +609,7 @@ class GarbageCollector {
|
|
|
621
609
|
return {
|
|
622
610
|
summary: {
|
|
623
611
|
type: protocol_definitions_1.SummaryType.Handle,
|
|
624
|
-
handle: `/${
|
|
612
|
+
handle: `/${garbageCollectionConstants_1.gcTreeKey}`,
|
|
625
613
|
handleType: protocol_definitions_1.SummaryType.Tree,
|
|
626
614
|
},
|
|
627
615
|
stats,
|
|
@@ -645,11 +633,11 @@ class GarbageCollector {
|
|
|
645
633
|
*/
|
|
646
634
|
buildGCSummaryTree(serializedGCState, serializedTombstones, trackState) {
|
|
647
635
|
var _a, _b;
|
|
648
|
-
const gcStateBlobKey = `${
|
|
636
|
+
const gcStateBlobKey = `${garbageCollectionConstants_1.gcBlobPrefix}_root`;
|
|
649
637
|
const builder = new runtime_utils_1.SummaryTreeBuilder();
|
|
650
638
|
// If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
|
|
651
639
|
if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
|
|
652
|
-
builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${
|
|
640
|
+
builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${gcStateBlobKey}`);
|
|
653
641
|
}
|
|
654
642
|
else {
|
|
655
643
|
builder.addBlob(gcStateBlobKey, serializedGCState);
|
|
@@ -660,10 +648,10 @@ class GarbageCollector {
|
|
|
660
648
|
}
|
|
661
649
|
// If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
|
|
662
650
|
if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones && trackState) {
|
|
663
|
-
builder.addHandle(
|
|
651
|
+
builder.addHandle(garbageCollectionConstants_1.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${garbageCollectionConstants_1.gcTombstoneBlobKey}`);
|
|
664
652
|
}
|
|
665
653
|
else {
|
|
666
|
-
builder.addBlob(
|
|
654
|
+
builder.addBlob(garbageCollectionConstants_1.gcTombstoneBlobKey, serializedTombstones);
|
|
667
655
|
}
|
|
668
656
|
return builder.getSummaryTree();
|
|
669
657
|
}
|
|
@@ -713,7 +701,7 @@ class GarbageCollector {
|
|
|
713
701
|
const metadata = await readAndParseBlob(metadataBlobId);
|
|
714
702
|
this.latestSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
|
|
715
703
|
}
|
|
716
|
-
const gcSnapshotTree = snapshot.trees[
|
|
704
|
+
const gcSnapshotTree = snapshot.trees[garbageCollectionConstants_1.gcTreeKey];
|
|
717
705
|
if (gcSnapshotTree !== undefined && this.trackGCState) {
|
|
718
706
|
const latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
719
707
|
this.latestSummaryData = {
|
|
@@ -988,7 +976,7 @@ class GarbageCollector {
|
|
|
988
976
|
* this will give us a view into how much deleted content a container has.
|
|
989
977
|
*/
|
|
990
978
|
logSweepEvents(logger, currentReferenceTimestampMs) {
|
|
991
|
-
if (this.mc.config.getBoolean(
|
|
979
|
+
if (this.mc.config.getBoolean(garbageCollectionConstants_1.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
|
|
992
980
|
return;
|
|
993
981
|
}
|
|
994
982
|
this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
|
|
@@ -1038,31 +1026,24 @@ class GarbageCollector {
|
|
|
1038
1026
|
return;
|
|
1039
1027
|
}
|
|
1040
1028
|
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
1041
|
-
const propsToLog = {
|
|
1042
|
-
id: nodeId,
|
|
1043
|
-
type: nodeType,
|
|
1044
|
-
unrefTime: nodeStateTracker.unreferencedTimestampMs,
|
|
1045
|
-
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
1046
|
-
timeout: nodeStateTracker.state === exports.UnreferencedState.Inactive
|
|
1029
|
+
const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === exports.UnreferencedState.Inactive
|
|
1047
1030
|
? this.inactiveTimeoutMs
|
|
1048
|
-
: this.sweepTimeoutMs,
|
|
1049
|
-
completedGCRuns: this.completedRuns,
|
|
1050
|
-
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1051
|
-
externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.externalRequest],
|
|
1052
|
-
viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle],
|
|
1053
|
-
fromId: fromNodeId,
|
|
1054
|
-
};
|
|
1031
|
+
: this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.externalRequest], viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
1055
1032
|
// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
|
|
1056
1033
|
// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
|
|
1057
1034
|
// but it's a good signal nonetheless and we can consume it with a grain of salt.
|
|
1035
|
+
// Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
|
|
1036
|
+
// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
|
|
1058
1037
|
if (this.isSummarizerClient) {
|
|
1059
1038
|
this.pendingEventsQueue.push(Object.assign(Object.assign({}, propsToLog), { usageType, state }));
|
|
1060
1039
|
}
|
|
1061
1040
|
else {
|
|
1062
1041
|
// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
|
|
1063
1042
|
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
|
|
1043
|
+
// Events generated:
|
|
1044
|
+
// InactiveObject_Loaded, SweepReadyObject_Loaded
|
|
1064
1045
|
if (usageType === "Loaded") {
|
|
1065
|
-
this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg:
|
|
1046
|
+
this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(packagePath), stack: (0, telemetry_utils_1.generateStack)() }));
|
|
1066
1047
|
}
|
|
1067
1048
|
// If SweepReady Usage Detection is enabed, the handler may close the interactive container.
|
|
1068
1049
|
// Once Sweep is fully implemented, this will be removed since the objects will be gone
|
|
@@ -1073,6 +1054,11 @@ class GarbageCollector {
|
|
|
1073
1054
|
}
|
|
1074
1055
|
}
|
|
1075
1056
|
async logUnreferencedEvents(logger) {
|
|
1057
|
+
// Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
|
|
1058
|
+
// summary time they are then logged.
|
|
1059
|
+
// Events generated:
|
|
1060
|
+
// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
|
|
1061
|
+
// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
|
|
1076
1062
|
for (const eventProps of this.pendingEventsQueue) {
|
|
1077
1063
|
const { usageType, state } = eventProps, propsToLog = __rest(eventProps, ["usageType", "state"]);
|
|
1078
1064
|
/**
|
|
@@ -1094,19 +1080,19 @@ class GarbageCollector {
|
|
|
1094
1080
|
}
|
|
1095
1081
|
exports.GarbageCollector = GarbageCollector;
|
|
1096
1082
|
/**
|
|
1097
|
-
* Gets the garbage collection
|
|
1083
|
+
* Gets the base garbage collection state from the given snapshot tree. It contains GC state and tombstone state.
|
|
1098
1084
|
* The GC state may be written into multiple blobs. Merge the GC state from all such blobs into one.
|
|
1099
1085
|
*/
|
|
1100
1086
|
async function getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob) {
|
|
1101
1087
|
let rootGCState = { gcNodes: {} };
|
|
1102
1088
|
let tombstones;
|
|
1103
1089
|
for (const key of Object.keys(gcSnapshotTree.blobs)) {
|
|
1104
|
-
if (key ===
|
|
1090
|
+
if (key === garbageCollectionConstants_1.gcTombstoneBlobKey) {
|
|
1105
1091
|
tombstones = await readAndParseBlob(gcSnapshotTree.blobs[key]);
|
|
1106
1092
|
continue;
|
|
1107
1093
|
}
|
|
1108
1094
|
// Skip blobs that do not start with the GC prefix.
|
|
1109
|
-
if (!key.startsWith(
|
|
1095
|
+
if (!key.startsWith(garbageCollectionConstants_1.gcBlobPrefix)) {
|
|
1110
1096
|
continue;
|
|
1111
1097
|
}
|
|
1112
1098
|
const blobId = gcSnapshotTree.blobs[key];
|