@fluidframework/container-runtime 2.0.0-internal.1.1.3 → 2.0.0-internal.1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/dist/batchManager.d.ts +37 -0
  2. package/dist/batchManager.d.ts.map +1 -0
  3. package/dist/batchManager.js +73 -0
  4. package/dist/batchManager.js.map +1 -0
  5. package/dist/batchTracker.d.ts +1 -2
  6. package/dist/batchTracker.d.ts.map +1 -1
  7. package/dist/batchTracker.js +1 -2
  8. package/dist/batchTracker.js.map +1 -1
  9. package/dist/containerRuntime.d.ts +52 -20
  10. package/dist/containerRuntime.d.ts.map +1 -1
  11. package/dist/containerRuntime.js +240 -119
  12. package/dist/containerRuntime.js.map +1 -1
  13. package/dist/dataStoreContext.d.ts +12 -6
  14. package/dist/dataStoreContext.d.ts.map +1 -1
  15. package/dist/dataStoreContext.js +16 -13
  16. package/dist/dataStoreContext.js.map +1 -1
  17. package/dist/dataStores.d.ts +6 -2
  18. package/dist/dataStores.d.ts.map +1 -1
  19. package/dist/dataStores.js +7 -9
  20. package/dist/dataStores.js.map +1 -1
  21. package/dist/deltaScheduler.d.ts +6 -4
  22. package/dist/deltaScheduler.d.ts.map +1 -1
  23. package/dist/deltaScheduler.js +6 -4
  24. package/dist/deltaScheduler.js.map +1 -1
  25. package/dist/garbageCollection.d.ts +41 -12
  26. package/dist/garbageCollection.d.ts.map +1 -1
  27. package/dist/garbageCollection.js +176 -98
  28. package/dist/garbageCollection.js.map +1 -1
  29. package/dist/gcSweepReadyUsageDetection.d.ts +53 -0
  30. package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -0
  31. package/dist/gcSweepReadyUsageDetection.js +135 -0
  32. package/dist/gcSweepReadyUsageDetection.js.map +1 -0
  33. package/dist/orderedClientElection.d.ts +28 -10
  34. package/dist/orderedClientElection.d.ts.map +1 -1
  35. package/dist/orderedClientElection.js +14 -4
  36. package/dist/orderedClientElection.js.map +1 -1
  37. package/dist/packageVersion.d.ts +1 -1
  38. package/dist/packageVersion.js +1 -1
  39. package/dist/packageVersion.js.map +1 -1
  40. package/dist/pendingStateManager.d.ts +0 -11
  41. package/dist/pendingStateManager.d.ts.map +1 -1
  42. package/dist/pendingStateManager.js +9 -44
  43. package/dist/pendingStateManager.js.map +1 -1
  44. package/dist/runningSummarizer.js +1 -1
  45. package/dist/runningSummarizer.js.map +1 -1
  46. package/dist/scheduleManager.d.ts +6 -3
  47. package/dist/scheduleManager.d.ts.map +1 -1
  48. package/dist/scheduleManager.js +22 -14
  49. package/dist/scheduleManager.js.map +1 -1
  50. package/dist/summarizerTypes.d.ts +16 -9
  51. package/dist/summarizerTypes.d.ts.map +1 -1
  52. package/dist/summarizerTypes.js +1 -1
  53. package/dist/summarizerTypes.js.map +1 -1
  54. package/dist/summaryCollection.d.ts +1 -0
  55. package/dist/summaryCollection.d.ts.map +1 -1
  56. package/dist/summaryCollection.js +29 -13
  57. package/dist/summaryCollection.js.map +1 -1
  58. package/dist/summaryManager.d.ts +2 -2
  59. package/dist/summaryManager.js +2 -2
  60. package/dist/summaryManager.js.map +1 -1
  61. package/lib/batchManager.d.ts +37 -0
  62. package/lib/batchManager.d.ts.map +1 -0
  63. package/lib/batchManager.js +69 -0
  64. package/lib/batchManager.js.map +1 -0
  65. package/lib/batchTracker.d.ts +1 -2
  66. package/lib/batchTracker.d.ts.map +1 -1
  67. package/lib/batchTracker.js +1 -2
  68. package/lib/batchTracker.js.map +1 -1
  69. package/lib/containerRuntime.d.ts +52 -20
  70. package/lib/containerRuntime.d.ts.map +1 -1
  71. package/lib/containerRuntime.js +243 -122
  72. package/lib/containerRuntime.js.map +1 -1
  73. package/lib/dataStoreContext.d.ts +12 -6
  74. package/lib/dataStoreContext.d.ts.map +1 -1
  75. package/lib/dataStoreContext.js +16 -13
  76. package/lib/dataStoreContext.js.map +1 -1
  77. package/lib/dataStores.d.ts +6 -2
  78. package/lib/dataStores.d.ts.map +1 -1
  79. package/lib/dataStores.js +7 -9
  80. package/lib/dataStores.js.map +1 -1
  81. package/lib/deltaScheduler.d.ts +6 -4
  82. package/lib/deltaScheduler.d.ts.map +1 -1
  83. package/lib/deltaScheduler.js +6 -4
  84. package/lib/deltaScheduler.js.map +1 -1
  85. package/lib/garbageCollection.d.ts +41 -12
  86. package/lib/garbageCollection.d.ts.map +1 -1
  87. package/lib/garbageCollection.js +175 -97
  88. package/lib/garbageCollection.js.map +1 -1
  89. package/lib/gcSweepReadyUsageDetection.d.ts +53 -0
  90. package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -0
  91. package/lib/gcSweepReadyUsageDetection.js +130 -0
  92. package/lib/gcSweepReadyUsageDetection.js.map +1 -0
  93. package/lib/orderedClientElection.d.ts +28 -10
  94. package/lib/orderedClientElection.d.ts.map +1 -1
  95. package/lib/orderedClientElection.js +14 -4
  96. package/lib/orderedClientElection.js.map +1 -1
  97. package/lib/packageVersion.d.ts +1 -1
  98. package/lib/packageVersion.js +1 -1
  99. package/lib/packageVersion.js.map +1 -1
  100. package/lib/pendingStateManager.d.ts +0 -11
  101. package/lib/pendingStateManager.d.ts.map +1 -1
  102. package/lib/pendingStateManager.js +9 -44
  103. package/lib/pendingStateManager.js.map +1 -1
  104. package/lib/runningSummarizer.js +1 -1
  105. package/lib/runningSummarizer.js.map +1 -1
  106. package/lib/scheduleManager.d.ts +6 -3
  107. package/lib/scheduleManager.d.ts.map +1 -1
  108. package/lib/scheduleManager.js +24 -16
  109. package/lib/scheduleManager.js.map +1 -1
  110. package/lib/summarizerTypes.d.ts +16 -9
  111. package/lib/summarizerTypes.d.ts.map +1 -1
  112. package/lib/summarizerTypes.js +1 -1
  113. package/lib/summarizerTypes.js.map +1 -1
  114. package/lib/summaryCollection.d.ts +1 -0
  115. package/lib/summaryCollection.d.ts.map +1 -1
  116. package/lib/summaryCollection.js +29 -13
  117. package/lib/summaryCollection.js.map +1 -1
  118. package/lib/summaryManager.d.ts +2 -2
  119. package/lib/summaryManager.js +2 -2
  120. package/lib/summaryManager.js.map +1 -1
  121. package/package.json +21 -18
  122. package/src/batchManager.ts +91 -0
  123. package/src/batchTracker.ts +1 -2
  124. package/src/containerRuntime.ts +336 -185
  125. package/src/dataStoreContext.ts +18 -14
  126. package/src/dataStores.ts +7 -8
  127. package/src/deltaScheduler.ts +6 -4
  128. package/src/garbageCollection.ts +224 -134
  129. package/src/gcSweepReadyUsageDetection.ts +147 -0
  130. package/src/orderedClientElection.ts +31 -10
  131. package/src/packageVersion.ts +1 -1
  132. package/src/pendingStateManager.ts +9 -57
  133. package/src/runningSummarizer.ts +1 -1
  134. package/src/scheduleManager.ts +32 -12
  135. package/src/summarizerTypes.ts +17 -9
  136. package/src/summaryCollection.ts +31 -16
  137. package/src/summaryManager.ts +2 -2
@@ -15,7 +15,7 @@ var __rest = (this && this.__rest) || function (s, e) {
15
15
  return t;
16
16
  };
17
17
  Object.defineProperty(exports, "__esModule", { value: true });
18
- exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = exports.defaultSessionExpiryDurationMs = exports.defaultInactiveTimeoutMs = exports.oneDayMs = exports.trackGCStateKey = exports.disableSessionExpiryKey = exports.runSessionExpiryKey = exports.gcTestModeKey = exports.runSweepKey = exports.runGCKey = exports.gcBlobPrefix = exports.gcTreeKey = void 0;
18
+ exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = exports.defaultSessionExpiryDurationMs = exports.defaultInactiveTimeoutMs = exports.oneDayMs = exports.disableSweepLogKey = exports.trackGCStateKey = exports.disableSessionExpiryKey = exports.runSessionExpiryKey = exports.gcTestModeKey = exports.runSweepKey = exports.runGCKey = exports.gcBlobPrefix = exports.gcTreeKey = void 0;
19
19
  const common_utils_1 = require("@fluidframework/common-utils");
20
20
  const container_utils_1 = require("@fluidframework/container-utils");
21
21
  const garbage_collector_1 = require("@fluidframework/garbage-collector");
@@ -25,6 +25,7 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
25
25
  const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
26
26
  const containerRuntime_1 = require("./containerRuntime");
27
27
  const dataStores_1 = require("./dataStores");
28
+ const gcSweepReadyUsageDetection_1 = require("./gcSweepReadyUsageDetection");
28
29
  const summaryFormat_1 = require("./summaryFormat");
29
30
  /** This is the current version of garbage collection. */
30
31
  const GCVersion = 1;
@@ -42,12 +43,12 @@ exports.gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
42
43
  const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
43
44
  // Feature gate key to expire a session after a set period of time.
44
45
  exports.runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
45
- // Feature gate key to disable expiring session after a set period of time, even if expiry value is present
46
+ // Feature gate key to disable expiring session after a set period of time, even if expiry value is present.
46
47
  exports.disableSessionExpiryKey = "Fluid.GarbageCollection.DisableSessionExpiry";
47
48
  // Feature gate key to write the gc blob as a handle if the data is the same.
48
49
  exports.trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
49
50
  // Feature gate key to turn GC sweep log off.
50
- const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
51
+ exports.disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
51
52
  // One day in milliseconds.
52
53
  exports.oneDayMs = 1 * 24 * 60 * 60 * 1000;
53
54
  exports.defaultInactiveTimeoutMs = 7 * exports.oneDayMs; // 7 days
@@ -80,10 +81,10 @@ class UnreferencedStateTracker {
80
81
  constructor(unreferencedTimestampMs,
81
82
  /** The time after which node transitions to Inactive state. */
82
83
  inactiveTimeoutMs,
84
+ /** The current reference timestamp used to track how long this node has been unreferenced for. */
85
+ currentReferenceTimestampMs,
83
86
  /** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
84
- sweepTimeoutMs,
85
- /** The current reference timestamp; undefined if no ops have ever been processed which can happen in tests. */
86
- currentReferenceTimestampMs) {
87
+ sweepTimeoutMs) {
87
88
  this.unreferencedTimestampMs = unreferencedTimestampMs;
88
89
  this.inactiveTimeoutMs = inactiveTimeoutMs;
89
90
  this.sweepTimeoutMs = sweepTimeoutMs;
@@ -102,11 +103,7 @@ class UnreferencedStateTracker {
102
103
  this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
103
104
  }
104
105
  });
105
- // If there is no current reference timestamp, can't track the node's unreferenced state at this time.
106
- // This will happen later when updateTracking is called with a reference timestamp.
107
- if (currentReferenceTimestampMs !== undefined) {
108
- this.updateTracking(currentReferenceTimestampMs);
109
- }
106
+ this.updateTracking(currentReferenceTimestampMs);
110
107
  }
111
108
  get state() {
112
109
  return this._state;
@@ -149,8 +146,12 @@ exports.UnreferencedStateTracker = UnreferencedStateTracker;
149
146
  * its state across summaries.
150
147
  *
151
148
  * Node - represented as nodeId, it's a node on the GC graph
149
+ *
152
150
  * Outbound Route - a path from one node to another node, think `nodeA` -\> `nodeB`
151
+ *
153
152
  * Graph - all nodes with their respective routes
153
+ *
154
+ * ```
154
155
  * GC Graph
155
156
  *
156
157
  * Node
@@ -160,6 +161,7 @@ exports.UnreferencedStateTracker = UnreferencedStateTracker;
160
161
  * / \\
161
162
  * Node Node
162
163
  * NodeId = "dds1" NodeId = "dds2"
164
+ * ```
163
165
  */
164
166
  class GarbageCollector {
165
167
  constructor(createParams) {
@@ -170,10 +172,12 @@ class GarbageCollector {
170
172
  this._writeDataAtRoot = true;
171
173
  /**
172
174
  * Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
175
+ *
173
176
  * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
174
- * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
177
+ * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
178
+ *
175
179
  * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
176
- * a document and the first time GC is enabled after is was disabled before.
180
+ * a document and the first time GC is enabled after is was disabled before.
177
181
  *
178
182
  * Note that the state needs reset only for the very first time summary is generated by this client. After that, the
179
183
  * state will be up-to-date and this flag will be reset.
@@ -198,10 +202,12 @@ class GarbageCollector {
198
202
  this.gcOptions = createParams.gcOptions;
199
203
  this.getNodePackagePath = createParams.getNodePackagePath;
200
204
  this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
205
+ this.activeConnection = createParams.activeConnection;
201
206
  const baseSnapshot = createParams.baseSnapshot;
202
207
  const metadata = createParams.metadata;
203
208
  const readAndParseBlob = createParams.readAndParseBlob;
204
209
  this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } }));
210
+ this.sweepReadyUsageHandler = new gcSweepReadyUsageDetection_1.SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
205
211
  let prevSummaryGCVersion;
206
212
  /**
207
213
  * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
@@ -260,8 +266,11 @@ class GarbageCollector {
260
266
  this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
261
267
  /**
262
268
  * Whether GC should run or not. The following conditions have to be met to run sweep:
269
+ *
263
270
  * 1. GC should be enabled for this container.
271
+ *
264
272
  * 2. GC should not be disabled via disableGC GC option.
273
+ *
265
274
  * These conditions can be overridden via runGCKey feature flag.
266
275
  */
267
276
  this.shouldRunGC = (_d = this.mc.config.getBoolean(exports.runGCKey)) !== null && _d !== void 0 ? _d : (
@@ -271,10 +280,13 @@ class GarbageCollector {
271
280
  && !this.gcOptions.disableGC);
272
281
  /**
273
282
  * Whether sweep should run or not. The following conditions have to be met to run sweep:
283
+ *
274
284
  * 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
285
+ *
275
286
  * 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
287
+ *
276
288
  * 3. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
277
- * feature flag.
289
+ * feature flag.
278
290
  */
279
291
  this.shouldRunSweep = false; // disable while TEMPORARY measure hardcoding snapshotCacheExpiryMs is here
280
292
  // this.shouldRunGC
@@ -304,52 +316,59 @@ class GarbageCollector {
304
316
  if (baseSnapshot === undefined) {
305
317
  return undefined;
306
318
  }
307
- // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
308
- const gcSnapshotTree = baseSnapshot.trees[exports.gcTreeKey];
309
- if (gcSnapshotTree !== undefined) {
310
- // If the GC tree is written at root, we should also do the same.
311
- this._writeDataAtRoot = true;
312
- const baseGCState = await getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);
313
- if (this.trackGCState) {
314
- this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(baseGCState));
315
- }
316
- return baseGCState;
317
- }
318
- // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
319
- // consolidate into IGarbageCollectionState format.
320
- // Add a node for the root node that is not present in older snapshot format.
321
- const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
322
- const dataStoreSnapshotTree = (0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata);
323
- (0, common_utils_1.assert)(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
324
- for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
325
- const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcBlobKey];
326
- if (blobId === undefined) {
327
- continue;
328
- }
329
- const gcSummaryDetails = await readAndParseBlob(blobId);
330
- // If there are no nodes for this data store, skip it.
331
- if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
332
- continue;
333
- }
334
- const dsRootId = `/${dsId}`;
335
- // Since we used to write GC data at data store level, we won't have an entry for the root ("/").
336
- // Construct that entry by adding root data store ids to its outbound routes.
337
- const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[summaryFormat_1.dataStoreAttributesBlobName]);
338
- if (initialSnapshotDetails.isRootDataStore) {
339
- gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
319
+ try {
320
+ // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
321
+ const gcSnapshotTree = baseSnapshot.trees[exports.gcTreeKey];
322
+ if (gcSnapshotTree !== undefined) {
323
+ // If the GC tree is written at root, we should also do the same.
324
+ this._writeDataAtRoot = true;
325
+ const baseGCState = await getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);
326
+ if (this.trackGCState) {
327
+ this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(baseGCState));
328
+ }
329
+ return baseGCState;
340
330
  }
341
- for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
342
- // Prefix the data store id to the GC node ids to make them relative to the root from being
343
- // relative to the data store. Similar to how its done in DataStore::getGCData.
344
- const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
345
- gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
331
+ // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
332
+ // consolidate into IGarbageCollectionState format.
333
+ // Add a node for the root node that is not present in older snapshot format.
334
+ const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
335
+ const dataStoreSnapshotTree = (0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata);
336
+ (0, common_utils_1.assert)(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
337
+ for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
338
+ const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcBlobKey];
339
+ if (blobId === undefined) {
340
+ continue;
341
+ }
342
+ const gcSummaryDetails = await readAndParseBlob(blobId);
343
+ // If there are no nodes for this data store, skip it.
344
+ if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
345
+ continue;
346
+ }
347
+ const dsRootId = `/${dsId}`;
348
+ // Since we used to write GC data at data store level, we won't have an entry for the root ("/").
349
+ // Construct that entry by adding root data store ids to its outbound routes.
350
+ const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[summaryFormat_1.dataStoreAttributesBlobName]);
351
+ if (initialSnapshotDetails.isRootDataStore) {
352
+ gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
353
+ }
354
+ for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
355
+ // Prefix the data store id to the GC node ids to make them relative to the root from being
356
+ // relative to the data store. Similar to how its done in DataStore::getGCData.
357
+ const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
358
+ gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
359
+ }
360
+ (0, common_utils_1.assert)(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
361
+ gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
346
362
  }
347
- (0, common_utils_1.assert)(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
348
- gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
363
+ // If there is only one node (root node just added above), either GC is disabled or we are loading from
364
+ // the first summary generated by detached container. In both cases, GC was not run - return undefined.
365
+ return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
366
+ }
367
+ catch (error) {
368
+ const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
369
+ dpe.addTelemetryProperties({ gcConfigs: JSON.stringify(this.configs) });
370
+ throw dpe;
349
371
  }
350
- // If there is only one node (root node just added above), either GC is disabled or we are loading from the
351
- // very first summary generated by detached container. In both cases, GC was not run - return undefined.
352
- return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
353
372
  });
354
373
  /**
355
374
  * Set up the initializer which initializes the base GC state from the base snapshot. Note that the reference
@@ -358,14 +377,36 @@ class GarbageCollector {
358
377
  */
359
378
  this.initializeBaseStateP = new common_utils_1.LazyPromise(async () => {
360
379
  const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
380
+ /**
381
+ * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
382
+ * how long objects have been unreferenced and if they can be deleted.
383
+ *
384
+ * Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
385
+ * for this container and it is in read mode. In this scenario, there is no point in running GC anyway
386
+ * because references in the container do not change without any ops, i.e., there is nothing to collect.
387
+ */
388
+ if (currentReferenceTimestampMs === undefined) {
389
+ // Log an event so we can evaluate how often we run into this scenario.
390
+ this.mc.logger.sendErrorEvent({
391
+ eventName: "GarbageCollectorInitializedWithoutTimestamp",
392
+ gcConfigs: JSON.stringify(this.configs),
393
+ });
394
+ return;
395
+ }
361
396
  const baseState = await baseSummaryStateP;
397
+ /**
398
+ * The base state will not be present if the container is loaded from:
399
+ * 1. The first summary created by the detached container.
400
+ * 2. A summary that was generated with GC disabled.
401
+ * 3. A summary that was generated before GC even existed.
402
+ */
362
403
  if (baseState === undefined) {
363
404
  return;
364
405
  }
365
406
  const gcNodes = {};
366
407
  for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
367
408
  if (nodeData.unreferencedTimestampMs !== undefined) {
368
- this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, this.sweepTimeoutMs, currentReferenceTimestampMs));
409
+ this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
369
410
  }
370
411
  gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
371
412
  }
@@ -401,19 +442,10 @@ class GarbageCollector {
401
442
  });
402
443
  // Log all the GC options and the state determined by the garbage collector. This is interesting only for the
403
444
  // summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
404
- const gcConfigProps = JSON.stringify(Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, writeAtRoot: this._writeDataAtRoot, testMode: this.testMode, sessionExpiry: this.sessionExpiryTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, existing: createParams.existing, trackGCState: this.trackGCState }, this.gcOptions));
405
445
  if (this.isSummarizerClient) {
406
446
  this.mc.logger.sendTelemetryEvent({
407
447
  eventName: "GarbageCollectorLoaded",
408
- gcConfigs: gcConfigProps,
409
- });
410
- }
411
- // Initialize the base state that is used to detect when inactive objects are used.
412
- if (this.shouldRunGC) {
413
- this.initializeBaseStateP.catch((error) => {
414
- const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
415
- dpe.addTelemetryProperties({ gcConfigs: gcConfigProps });
416
- throw dpe;
448
+ gcConfigs: JSON.stringify(this.configs),
417
449
  });
418
450
  }
419
451
  }
@@ -422,11 +454,16 @@ class GarbageCollector {
422
454
  }
423
455
  /**
424
456
  * Tells whether the GC state needs to be reset in the next summary. We need to do this if:
457
+ *
425
458
  * 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
459
+ *
426
460
  * 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
461
+ *
427
462
  * 3. The GC version in the latest summary is different from the current GC version. This can happen if:
428
- * 3.1. The summary this client loaded with has data from a different GC version.
429
- * 3.2. This client's latest summary was updated from a snapshot that has a different GC version.
463
+ *
464
+ * 3.1. The summary this client loaded with has data from a different GC version.
465
+ *
466
+ * 3.2. This client's latest summary was updated from a snapshot that has a different GC version.
430
467
  */
431
468
  get summaryStateNeedsReset() {
432
469
  return this.initialStateNeedsReset ||
@@ -435,21 +472,63 @@ class GarbageCollector {
435
472
  get writeDataAtRoot() {
436
473
  return this._writeDataAtRoot;
437
474
  }
475
+ /** Returns a list of all the configurations for garbage collection. */
476
+ get configs() {
477
+ return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, writeAtRoot: this._writeDataAtRoot, testMode: this.testMode, sessionExpiry: this.sessionExpiryTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
478
+ }
479
+ /**
480
+ * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
481
+ * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
482
+ * @param connected - Whether the runtime connected / disconnected.
483
+ * @param clientId - The clientId of this runtime.
484
+ */
485
+ setConnectionState(connected, clientId) {
486
+ /**
487
+ * For non-summarizer clients, initialize the base state when the container becomes active, i.e., it transitions
488
+ * to "write" mode. This will ensure that the container's own join op is processed and there is a recent
489
+ * reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
490
+ * could affect the GC state will have been processed.
491
+ *
492
+ * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
493
+ * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
494
+ */
495
+ if (this.activeConnection() && !this.isSummarizerClient && this.shouldRunGC) {
496
+ this.initializeBaseStateP.catch((error) => { });
497
+ }
498
+ }
438
499
  /**
439
500
  * Runs garbage collection and updates the reference / used state of the nodes in the container.
440
- * @returns the number of data stores that have been marked as unreferenced.
501
+ * @returns stats of the GC run or undefined if GC did not run.
441
502
  */
442
503
  async collectGarbage(options) {
443
- const { fullGC = this.gcOptions.runFullGC === true || this.summaryStateNeedsReset, } = options;
504
+ var _a;
505
+ const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
444
506
  const logger = options.logger
445
507
  ? telemetry_utils_1.ChildLogger.create(options.logger, undefined, { all: { completedGCRuns: () => this.completedRuns } })
446
508
  : this.mc.logger;
509
+ /**
510
+ * If there is no current reference timestamp, skip running GC. We need the current timestamp to track
511
+ * how long objects have been unreferenced and if they should be deleted.
512
+ *
513
+ * Note that the only scenario where GC is called and there is no reference timestamp is when no ops have ever
514
+ * been processed for this container and it is in read mode. In this scenario, there is no point in running GC
515
+ * anyway because references in the container do not change without any ops, i.e., there is nothing to collect.
516
+ */
517
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
518
+ if (currentReferenceTimestampMs === undefined) {
519
+ // Log an event so we can evaluate how often we run into this scenario.
520
+ logger.sendErrorEvent({
521
+ eventName: "CollectGarbageCalledWithoutTimestamp",
522
+ gcConfigs: JSON.stringify(this.configs),
523
+ });
524
+ return undefined;
525
+ }
447
526
  return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
448
527
  await this.runPreGCSteps();
449
528
  // Get the runtime's GC data and run GC on the reference graph in it.
450
529
  const gcData = await this.runtime.getGCData(fullGC);
451
530
  const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcData.gcNodes, ["/"]);
452
- const gcStats = await this.runPostGCSteps(gcData, gcResult, logger);
531
+ const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
453
532
  event.end(Object.assign({}, gcStats));
454
533
  this.completedRuns++;
455
534
  return gcStats;
@@ -461,7 +540,7 @@ class GarbageCollector {
461
540
  // Let the runtime update its pending state before GC runs.
462
541
  await this.runtime.updateStateBeforeGC();
463
542
  }
464
- async runPostGCSteps(gcData, gcResult, logger) {
543
+ async runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs) {
465
544
  // Generate statistics from the current run. This is done before updating the current state because it
466
545
  // generates some of its data based on previous state of the system.
467
546
  const gcStats = this.generateStats(gcResult);
@@ -469,7 +548,6 @@ class GarbageCollector {
469
548
  // the current run. We need to identify than and update their unreferenced state if needed.
470
549
  this.updateStateSinceLastRun(gcData, logger);
471
550
  // Update the current state and update the runtime of all routes or ids that used as per the GC run.
472
- const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
473
551
  this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
474
552
  this.runtime.updateUsedRoutes(gcResult.referencedNodeIds, currentReferenceTimestampMs);
475
553
  // Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
@@ -649,14 +727,6 @@ class GarbageCollector {
649
727
  this.unreferencedNodesState.delete(nodeId);
650
728
  }
651
729
  }
652
- /**
653
- * If there is no current reference time, skip tracking when a node becomes unreferenced. This would happen
654
- * if no ops have been processed ever and we still try to run GC. If so, there is nothing interesting to track
655
- * anyway.
656
- */
657
- if (currentReferenceTimestampMs === undefined) {
658
- return;
659
- }
660
730
  /**
661
731
  * If a node became unreferenced in this run, start tracking it.
662
732
  * If a node was already unreferenced, update its tracking information. Since the current reference time is
@@ -665,7 +735,7 @@ class GarbageCollector {
665
735
  for (const nodeId of gcResult.deletedNodeIds) {
666
736
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
667
737
  if (nodeStateTracker === undefined) {
668
- this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.inactiveTimeoutMs, this.sweepTimeoutMs, currentReferenceTimestampMs));
738
+ this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
669
739
  }
670
740
  else {
671
741
  nodeStateTracker.updateTracking(currentReferenceTimestampMs);
@@ -708,14 +778,19 @@ class GarbageCollector {
708
778
  * run, and then add the references since last run.
709
779
  *
710
780
  * Note on why we need to combine the data from previous run, current run and all references in between -
781
+ *
711
782
  * 1. We need data from last run because some of its references may have been deleted since then. If those
712
- * references added new outbound references before getting deleted, we need to detect them.
783
+ * references added new outbound references before getting deleted, we need to detect them.
784
+ *
713
785
  * 2. We need new outbound references since last run because some of them may have been deleted later. If those
714
- * references added new outbound references before getting deleted, we need to detect them.
786
+ * references added new outbound references before getting deleted, we need to detect them.
787
+ *
715
788
  * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
716
- * - We don't require DDSes handles to be stored in a referenced DDS. For this, we need GC at DDS level
717
- * which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
718
- * - A new data store may have "root" DDSes already created and we don't detect them today.
789
+ *
790
+ * - We don't require DDSes handles to be stored in a referenced DDS. For this, we need GC at DDS level
791
+ * which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
792
+ *
793
+ * - A new data store may have "root" DDSes already created and we don't detect them today.
719
794
  */
720
795
  const gcDataSuperSet = (0, garbage_collector_1.concatGarbageCollectionData)(this.previousGCDataFromLastRun, currentGCData);
721
796
  this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
@@ -843,9 +918,7 @@ class GarbageCollector {
843
918
  * this will give us a view into how much deleted content a container has.
844
919
  */
845
920
  logSweepEvents(logger, currentReferenceTimestampMs) {
846
- if (this.mc.config.getBoolean(disableSweepLogKey) === true
847
- || currentReferenceTimestampMs === undefined
848
- || this.sweepTimeoutMs === undefined) {
921
+ if (this.mc.config.getBoolean(exports.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
849
922
  return;
850
923
  }
851
924
  this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
@@ -883,11 +956,6 @@ class GarbageCollector {
883
956
  if (currentReferenceTimestampMs === undefined || nodeStateTracker.state === exports.UnreferencedState.Active) {
884
957
  return;
885
958
  }
886
- // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
887
- // summarizer clients if they are based off of user actions (such as scrolling to content for these objects).
888
- if (!this.isSummarizerClient && usageType !== "Loaded") {
889
- return;
890
- }
891
959
  // We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
892
960
  // as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
893
961
  const nodeType = this.runtime.getNodeType(nodeId);
@@ -921,7 +989,17 @@ class GarbageCollector {
921
989
  this.pendingEventsQueue.push(Object.assign(Object.assign({}, propsToLog), { usageType, state }));
922
990
  }
923
991
  else {
924
- this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: packagePath ? { value: packagePath.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined }));
992
+ // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
993
+ // summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
994
+ if (usageType === "Loaded") {
995
+ this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: packagePath ? { value: packagePath.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined }));
996
+ }
997
+ // If SweepReady Usage Detection is enabed, the handler may close the interactive container.
998
+ // Once Sweep is fully implemented, this will be removed since the objects will be gone
999
+ // and errors will arise elsewhere in the runtime
1000
+ if (state === exports.UnreferencedState.SweepReady) {
1001
+ this.sweepReadyUsageHandler.usageDetectedInInteractiveClient(Object.assign(Object.assign({}, propsToLog), { usageType }));
1002
+ }
925
1003
  }
926
1004
  }
927
1005
  async logUnreferencedEvents(logger) {