@fluidframework/container-runtime 2.0.0-internal.2.1.1 → 2.0.0-internal.2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/.eslintrc.js +1 -1
  2. package/dist/blobManager.d.ts +20 -5
  3. package/dist/blobManager.d.ts.map +1 -1
  4. package/dist/blobManager.js +57 -15
  5. package/dist/blobManager.js.map +1 -1
  6. package/dist/containerRuntime.d.ts +39 -40
  7. package/dist/containerRuntime.d.ts.map +1 -1
  8. package/dist/containerRuntime.js +115 -278
  9. package/dist/containerRuntime.js.map +1 -1
  10. package/dist/dataStoreContext.d.ts +3 -1
  11. package/dist/dataStoreContext.d.ts.map +1 -1
  12. package/dist/dataStoreContext.js +21 -3
  13. package/dist/dataStoreContext.js.map +1 -1
  14. package/dist/dataStores.d.ts +8 -5
  15. package/dist/dataStores.d.ts.map +1 -1
  16. package/dist/dataStores.js +26 -13
  17. package/dist/dataStores.js.map +1 -1
  18. package/dist/garbageCollection.d.ts +15 -17
  19. package/dist/garbageCollection.d.ts.map +1 -1
  20. package/dist/garbageCollection.js +92 -106
  21. package/dist/garbageCollection.js.map +1 -1
  22. package/dist/garbageCollectionConstants.d.ts +19 -0
  23. package/dist/garbageCollectionConstants.d.ts.map +1 -0
  24. package/dist/garbageCollectionConstants.js +34 -0
  25. package/dist/garbageCollectionConstants.js.map +1 -0
  26. package/dist/gcSweepReadyUsageDetection.js +2 -2
  27. package/dist/gcSweepReadyUsageDetection.js.map +1 -1
  28. package/dist/index.d.ts +4 -2
  29. package/dist/index.d.ts.map +1 -1
  30. package/dist/index.js +8 -6
  31. package/dist/index.js.map +1 -1
  32. package/dist/opLifecycle/batchManager.d.ts +30 -0
  33. package/dist/opLifecycle/batchManager.d.ts.map +1 -0
  34. package/dist/{batchManager.js → opLifecycle/batchManager.js} +25 -10
  35. package/dist/opLifecycle/batchManager.js.map +1 -0
  36. package/dist/opLifecycle/definitions.d.ts +40 -0
  37. package/dist/opLifecycle/definitions.d.ts.map +1 -0
  38. package/dist/opLifecycle/definitions.js +7 -0
  39. package/dist/opLifecycle/definitions.js.map +1 -0
  40. package/dist/opLifecycle/index.d.ts +12 -0
  41. package/dist/opLifecycle/index.d.ts.map +1 -0
  42. package/dist/opLifecycle/index.js +21 -0
  43. package/dist/opLifecycle/index.js.map +1 -0
  44. package/dist/opLifecycle/opCompressor.d.ts +18 -0
  45. package/dist/opLifecycle/opCompressor.d.ts.map +1 -0
  46. package/dist/opLifecycle/opCompressor.js +53 -0
  47. package/dist/opLifecycle/opCompressor.js.map +1 -0
  48. package/dist/opLifecycle/opDecompressor.d.ts +20 -0
  49. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -0
  50. package/dist/opLifecycle/opDecompressor.js +72 -0
  51. package/dist/opLifecycle/opDecompressor.js.map +1 -0
  52. package/dist/opLifecycle/opSplitter.d.ts +17 -0
  53. package/dist/opLifecycle/opSplitter.d.ts.map +1 -0
  54. package/dist/opLifecycle/opSplitter.js +61 -0
  55. package/dist/opLifecycle/opSplitter.js.map +1 -0
  56. package/dist/opLifecycle/outbox.d.ts +47 -0
  57. package/dist/opLifecycle/outbox.d.ts.map +1 -0
  58. package/dist/opLifecycle/outbox.js +153 -0
  59. package/dist/opLifecycle/outbox.js.map +1 -0
  60. package/dist/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  61. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  62. package/dist/opLifecycle/remoteMessageProcessor.js +81 -0
  63. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -0
  64. package/dist/packageVersion.d.ts +1 -1
  65. package/dist/packageVersion.js +1 -1
  66. package/dist/packageVersion.js.map +1 -1
  67. package/dist/summaryFormat.js +2 -2
  68. package/dist/summaryFormat.js.map +1 -1
  69. package/lib/blobManager.d.ts +20 -5
  70. package/lib/blobManager.d.ts.map +1 -1
  71. package/lib/blobManager.js +59 -17
  72. package/lib/blobManager.js.map +1 -1
  73. package/lib/containerRuntime.d.ts +39 -40
  74. package/lib/containerRuntime.d.ts.map +1 -1
  75. package/lib/containerRuntime.js +113 -275
  76. package/lib/containerRuntime.js.map +1 -1
  77. package/lib/dataStoreContext.d.ts +3 -1
  78. package/lib/dataStoreContext.d.ts.map +1 -1
  79. package/lib/dataStoreContext.js +23 -5
  80. package/lib/dataStoreContext.js.map +1 -1
  81. package/lib/dataStores.d.ts +8 -5
  82. package/lib/dataStores.d.ts.map +1 -1
  83. package/lib/dataStores.js +28 -15
  84. package/lib/dataStores.js.map +1 -1
  85. package/lib/garbageCollection.d.ts +15 -17
  86. package/lib/garbageCollection.d.ts.map +1 -1
  87. package/lib/garbageCollection.js +72 -86
  88. package/lib/garbageCollection.js.map +1 -1
  89. package/lib/garbageCollectionConstants.d.ts +19 -0
  90. package/lib/garbageCollectionConstants.d.ts.map +1 -0
  91. package/lib/garbageCollectionConstants.js +31 -0
  92. package/lib/garbageCollectionConstants.js.map +1 -0
  93. package/lib/gcSweepReadyUsageDetection.js +1 -1
  94. package/lib/gcSweepReadyUsageDetection.js.map +1 -1
  95. package/lib/index.d.ts +4 -2
  96. package/lib/index.d.ts.map +1 -1
  97. package/lib/index.js +3 -2
  98. package/lib/index.js.map +1 -1
  99. package/lib/opLifecycle/batchManager.d.ts +30 -0
  100. package/lib/opLifecycle/batchManager.d.ts.map +1 -0
  101. package/lib/{batchManager.js → opLifecycle/batchManager.js} +25 -10
  102. package/lib/opLifecycle/batchManager.js.map +1 -0
  103. package/lib/opLifecycle/definitions.d.ts +40 -0
  104. package/lib/opLifecycle/definitions.d.ts.map +1 -0
  105. package/lib/opLifecycle/definitions.js +6 -0
  106. package/lib/opLifecycle/definitions.js.map +1 -0
  107. package/lib/opLifecycle/index.d.ts +12 -0
  108. package/lib/opLifecycle/index.d.ts.map +1 -0
  109. package/lib/opLifecycle/index.js +11 -0
  110. package/lib/opLifecycle/index.js.map +1 -0
  111. package/lib/opLifecycle/opCompressor.d.ts +18 -0
  112. package/lib/opLifecycle/opCompressor.d.ts.map +1 -0
  113. package/lib/opLifecycle/opCompressor.js +49 -0
  114. package/lib/opLifecycle/opCompressor.js.map +1 -0
  115. package/lib/opLifecycle/opDecompressor.d.ts +20 -0
  116. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -0
  117. package/lib/opLifecycle/opDecompressor.js +68 -0
  118. package/lib/opLifecycle/opDecompressor.js.map +1 -0
  119. package/lib/opLifecycle/opSplitter.d.ts +17 -0
  120. package/lib/opLifecycle/opSplitter.d.ts.map +1 -0
  121. package/lib/opLifecycle/opSplitter.js +57 -0
  122. package/lib/opLifecycle/opSplitter.js.map +1 -0
  123. package/lib/opLifecycle/outbox.d.ts +47 -0
  124. package/lib/opLifecycle/outbox.d.ts.map +1 -0
  125. package/lib/opLifecycle/outbox.js +149 -0
  126. package/lib/opLifecycle/outbox.js.map +1 -0
  127. package/lib/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  128. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  129. package/lib/opLifecycle/remoteMessageProcessor.js +76 -0
  130. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -0
  131. package/lib/packageVersion.d.ts +1 -1
  132. package/lib/packageVersion.js +1 -1
  133. package/lib/packageVersion.js.map +1 -1
  134. package/lib/summaryFormat.js +1 -1
  135. package/lib/summaryFormat.js.map +1 -1
  136. package/package.json +35 -21
  137. package/prettier.config.cjs +8 -0
  138. package/src/blobManager.ts +74 -19
  139. package/src/containerRuntime.ts +144 -341
  140. package/src/dataStoreContext.ts +33 -5
  141. package/src/dataStores.ts +32 -16
  142. package/src/garbageCollection.ts +106 -82
  143. package/src/garbageCollectionConstants.ts +35 -0
  144. package/src/gcSweepReadyUsageDetection.ts +1 -1
  145. package/src/index.ts +6 -4
  146. package/src/{batchManager.ts → opLifecycle/batchManager.ts} +41 -23
  147. package/src/opLifecycle/definitions.ts +44 -0
  148. package/src/opLifecycle/index.ts +17 -0
  149. package/src/opLifecycle/opCompressor.ts +64 -0
  150. package/src/opLifecycle/opDecompressor.ts +84 -0
  151. package/src/opLifecycle/opSplitter.ts +78 -0
  152. package/src/opLifecycle/outbox.ts +204 -0
  153. package/src/opLifecycle/remoteMessageProcessor.ts +90 -0
  154. package/src/packageVersion.ts +1 -1
  155. package/src/summaryFormat.ts +1 -1
  156. package/dist/batchManager.d.ts +0 -36
  157. package/dist/batchManager.d.ts.map +0 -1
  158. package/dist/batchManager.js.map +0 -1
  159. package/lib/batchManager.d.ts +0 -36
  160. package/lib/batchManager.d.ts.map +0 -1
  161. package/lib/batchManager.js.map +0 -1
@@ -15,7 +15,7 @@ var __rest = (this && this.__rest) || function (s, e) {
15
15
  return t;
16
16
  };
17
17
  Object.defineProperty(exports, "__esModule", { value: true });
18
- exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = exports.defaultSessionExpiryDurationMs = exports.defaultInactiveTimeoutMs = exports.oneDayMs = exports.testTombstoneKey = exports.disableSweepLogKey = exports.trackGCStateKey = exports.runSessionExpiryKey = exports.gcTestModeKey = exports.runSweepKey = exports.runGCKey = exports.gcTombstoneBlobKey = exports.gcBlobPrefix = exports.gcTreeKey = void 0;
18
+ exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = void 0;
19
19
  const common_utils_1 = require("@fluidframework/common-utils");
20
20
  const container_utils_1 = require("@fluidframework/container-utils");
21
21
  const garbage_collector_1 = require("@fluidframework/garbage-collector");
@@ -25,34 +25,11 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
25
25
  const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
26
26
  const containerRuntime_1 = require("./containerRuntime");
27
27
  const dataStores_1 = require("./dataStores");
28
+ const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
28
29
  const gcSweepReadyUsageDetection_1 = require("./gcSweepReadyUsageDetection");
29
30
  const summaryFormat_1 = require("./summaryFormat");
30
31
  /** This is the current version of garbage collection. */
31
32
  const GCVersion = 1;
32
- // The key for the GC tree in summary.
33
- exports.gcTreeKey = "gc";
34
- // They prefix for GC blobs in the GC tree in summary.
35
- exports.gcBlobPrefix = "__gc";
36
- // The key for tombstone blob in the GC tree in summary.
37
- exports.gcTombstoneBlobKey = "__tombstones";
38
- // Feature gate key to turn GC on / off.
39
- exports.runGCKey = "Fluid.GarbageCollection.RunGC";
40
- // Feature gate key to turn GC sweep on / off.
41
- exports.runSweepKey = "Fluid.GarbageCollection.RunSweep";
42
- // Feature gate key to turn GC test mode on / off.
43
- exports.gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
44
- // Feature gate key to expire a session after a set period of time.
45
- exports.runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
46
- // Feature gate key to write the gc blob as a handle if the data is the same.
47
- exports.trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
48
- // Feature gate key to turn GC sweep log off.
49
- exports.disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
50
- // Feature gate key to tombstone datastores.
51
- exports.testTombstoneKey = "Fluid.GarbageCollection.Test.Tombstone";
52
- // One day in milliseconds.
53
- exports.oneDayMs = 1 * 24 * 60 * 60 * 1000;
54
- exports.defaultInactiveTimeoutMs = 7 * exports.oneDayMs; // 7 days
55
- exports.defaultSessionExpiryDurationMs = 30 * exports.oneDayMs; // 30 days
56
33
  /** The types of GC nodes in the GC reference graph. */
57
34
  exports.GCNodeType = {
58
35
  // Nodes that are for data stores.
@@ -165,7 +142,7 @@ exports.UnreferencedStateTracker = UnreferencedStateTracker;
165
142
  */
166
143
  class GarbageCollector {
167
144
  constructor(createParams) {
168
- var _a, _b, _c, _d, _e, _f, _g, _h, _j;
145
+ var _a, _b, _c, _d, _e, _f, _g, _h;
169
146
  /**
170
147
  * Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
171
148
  *
@@ -197,6 +174,7 @@ class GarbageCollector {
197
174
  this.runtime = createParams.runtime;
198
175
  this.isSummarizerClient = createParams.isSummarizerClient;
199
176
  this.gcOptions = createParams.gcOptions;
177
+ this.createContainerMetadata = createParams.createContainerMetadata;
200
178
  this.getNodePackagePath = createParams.getNodePackagePath;
201
179
  this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
202
180
  this.activeConnection = createParams.activeConnection;
@@ -215,8 +193,8 @@ class GarbageCollector {
215
193
  * We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
216
194
  */
217
195
  function computeSweepTimeout(sessionExpiryTimeoutMs) {
218
- const maxSnapshotCacheExpiryMs = 5 * exports.oneDayMs;
219
- const bufferMs = exports.oneDayMs;
196
+ const maxSnapshotCacheExpiryMs = 5 * garbageCollectionConstants_1.oneDayMs;
197
+ const bufferMs = garbageCollectionConstants_1.oneDayMs;
220
198
  return sessionExpiryTimeoutMs &&
221
199
  (sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
222
200
  }
@@ -252,8 +230,8 @@ class GarbageCollector {
252
230
  // ...unless we're using the TestOverride
253
231
  this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
254
232
  // Set the Session Expiry only if the flag is enabled and GC is enabled.
255
- if (this.mc.config.getBoolean(exports.runSessionExpiryKey) && this.gcEnabled) {
256
- this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : exports.defaultSessionExpiryDurationMs;
233
+ if (this.mc.config.getBoolean(garbageCollectionConstants_1.runSessionExpiryKey) && this.gcEnabled) {
234
+ this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : garbageCollectionConstants_1.defaultSessionExpiryDurationMs;
257
235
  }
258
236
  this.sweepTimeoutMs =
259
237
  testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
@@ -278,7 +256,7 @@ class GarbageCollector {
278
256
  *
279
257
  * These conditions can be overridden via runGCKey feature flag.
280
258
  */
281
- this.shouldRunGC = (_d = this.mc.config.getBoolean(exports.runGCKey)) !== null && _d !== void 0 ? _d : (
259
+ this.shouldRunGC = (_d = this.mc.config.getBoolean(garbageCollectionConstants_1.runGCKey)) !== null && _d !== void 0 ? _d : (
282
260
  // GC must be enabled for the document.
283
261
  this.gcEnabled
284
262
  // GC must not be disabled via GC options.
@@ -296,43 +274,34 @@ class GarbageCollector {
296
274
  this.shouldRunSweep =
297
275
  this.shouldRunGC
298
276
  && this.sweepTimeoutMs !== undefined
299
- && ((_e = this.mc.config.getBoolean(exports.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
300
- this.trackGCState = this.mc.config.getBoolean(exports.trackGCStateKey) === true;
277
+ && ((_e = this.mc.config.getBoolean(garbageCollectionConstants_1.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
278
+ this.trackGCState = this.mc.config.getBoolean(garbageCollectionConstants_1.trackGCStateKey) === true;
301
279
  // Override inactive timeout if test config or gc options to override it is set.
302
- this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : exports.defaultInactiveTimeoutMs;
280
+ this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : garbageCollectionConstants_1.defaultInactiveTimeoutMs;
303
281
  // Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
304
282
  if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
305
283
  throw new container_utils_1.UsageError("inactive timeout should not be greater than the sweep timeout");
306
284
  }
307
285
  // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
308
- this.testMode = (_h = this.mc.config.getBoolean(exports.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
309
- this.tombstoneMode = (_j = this.mc.config.getBoolean(exports.testTombstoneKey)) !== null && _j !== void 0 ? _j : false;
286
+ this.testMode = (_h = this.mc.config.getBoolean(garbageCollectionConstants_1.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
287
+ // Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
288
+ this.tombstoneMode = this.mc.config.getBoolean(garbageCollectionConstants_1.disableTombstoneKey) !== true;
310
289
  // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
311
290
  // contain GC tree and GC is enabled.
312
- const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[exports.gcTreeKey]) !== undefined;
291
+ const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey]) !== undefined;
313
292
  this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
314
- // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
315
- // this once since it involves fetching blobs from storage which is expensive.
316
- const baseSummaryStateP = new common_utils_1.LazyPromise(async () => {
293
+ // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
294
+ // it involves fetching blobs from storage which is expensive.
295
+ this.baseSnapshotDataP = new common_utils_1.LazyPromise(async () => {
317
296
  var _a;
318
297
  if (baseSnapshot === undefined) {
319
298
  return undefined;
320
299
  }
321
300
  try {
322
301
  // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
323
- const gcSnapshotTree = baseSnapshot.trees[exports.gcTreeKey];
302
+ const gcSnapshotTree = baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey];
324
303
  if (gcSnapshotTree !== undefined) {
325
- const baseGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
326
- if (baseGCData.tombstones !== undefined && this.tombstoneMode) {
327
- this.tombstones = baseGCData.tombstones;
328
- }
329
- if (this.trackGCState) {
330
- this.latestSummaryData = {
331
- serializedGCState: JSON.stringify(generateSortedGCState(baseGCData.gcState)),
332
- serializedTombstones: JSON.stringify(baseGCData.tombstones),
333
- };
334
- }
335
- return baseGCData.gcState;
304
+ return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
336
305
  }
337
306
  // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
338
307
  // consolidate into IGarbageCollectionState format.
@@ -368,7 +337,7 @@ class GarbageCollector {
368
337
  }
369
338
  // If there is only one node (root node just added above), either GC is disabled or we are loading from
370
339
  // the first summary generated by detached container. In both cases, GC was not run - return undefined.
371
- return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
340
+ return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
372
341
  }
373
342
  catch (error) {
374
343
  const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
@@ -377,11 +346,11 @@ class GarbageCollector {
377
346
  }
378
347
  });
379
348
  /**
380
- * Set up the initializer which initializes the base GC state from the base snapshot. Note that the reference
381
- * timestamp maybe from old ops which were not summarized and stored in the file. So, the unreferenced state
382
- * may be out of date. This is fine because the state is updated every time GC runs based on the time then.
349
+ * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
350
+ * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
351
+ * GC state and updates their inactive or sweep ready state.
383
352
  */
384
- this.initializeBaseStateP = new common_utils_1.LazyPromise(async () => {
353
+ this.initializeGCStateFromBaseSnapshotP = new common_utils_1.LazyPromise(async () => {
385
354
  const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
386
355
  /**
387
356
  * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
@@ -399,34 +368,41 @@ class GarbageCollector {
399
368
  });
400
369
  return;
401
370
  }
402
- const baseState = await baseSummaryStateP;
371
+ const baseSnapshotData = await this.baseSnapshotDataP;
403
372
  /**
404
- * The base state will not be present if the container is loaded from:
373
+ * The base snapshot data will not be present if the container is loaded from:
405
374
  * 1. The first summary created by the detached container.
406
375
  * 2. A summary that was generated with GC disabled.
407
376
  * 3. A summary that was generated before GC even existed.
408
377
  */
409
- if (baseState === undefined) {
378
+ if (baseSnapshotData === undefined) {
410
379
  return;
411
380
  }
412
381
  const gcNodes = {};
413
- for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
382
+ for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
414
383
  if (nodeData.unreferencedTimestampMs !== undefined) {
415
384
  this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
416
385
  }
417
386
  gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
418
387
  }
419
388
  this.previousGCDataFromLastRun = { gcNodes };
389
+ // If tracking state across summaries, update latest summary data from the base snapshot's GC data.
390
+ if (this.trackGCState) {
391
+ this.latestSummaryData = {
392
+ serializedGCState: JSON.stringify(generateSortedGCState(baseSnapshotData.gcState)),
393
+ serializedTombstones: JSON.stringify(baseSnapshotData.tombstones),
394
+ };
395
+ }
420
396
  });
421
397
  // Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
422
398
  // which the caller uses to initialize each node's GC state.
423
399
  this.baseGCDetailsP = new common_utils_1.LazyPromise(async () => {
424
- const baseState = await baseSummaryStateP;
425
- if (baseState === undefined) {
400
+ const baseSnapshotData = await this.baseSnapshotDataP;
401
+ if (baseSnapshotData === undefined) {
426
402
  return new Map();
427
403
  }
428
404
  const gcNodes = {};
429
- for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
405
+ for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
430
406
  gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
431
407
  }
432
408
  // Run GC on the nodes in the base summary to get the routes used in each node in the container.
@@ -436,7 +412,7 @@ class GarbageCollector {
436
412
  const baseGCDetailsMap = (0, garbage_collector_1.unpackChildNodesGCDetails)({ gcData: { gcNodes }, usedRoutes });
437
413
  // Currently, the nodes may write the GC data. So, we need to update its base GC details with the
438
414
  // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
439
- for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
415
+ for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
440
416
  if (nodeData.unreferencedTimestampMs !== undefined) {
441
417
  const dataStoreGCDetails = baseGCDetailsMap.get(nodeId.slice(1));
442
418
  if (dataStoreGCDetails !== undefined) {
@@ -479,6 +455,26 @@ class GarbageCollector {
479
455
  get configs() {
480
456
  return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
481
457
  }
458
+ /**
459
+ * Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
460
+ * before they are loaded or used. This is important to get accurate information of whether tombstoned object are
461
+ * in use or not.
462
+ */
463
+ async initializeBaseState() {
464
+ const baseSnapshotData = await this.baseSnapshotDataP;
465
+ /**
466
+ * The base snapshot data or tombstone state will not be present if the container is loaded from:
467
+ * 1. The first summary created by the detached container.
468
+ * 2. A summary that was generated with GC disabled.
469
+ * 3. A summary that was generated before GC even existed.
470
+ * 4. A summary that was generated with tombstone feature disabled.
471
+ */
472
+ if (!this.tombstoneMode || (baseSnapshotData === null || baseSnapshotData === void 0 ? void 0 : baseSnapshotData.tombstones) === undefined) {
473
+ return;
474
+ }
475
+ this.tombstones = baseSnapshotData.tombstones;
476
+ this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
477
+ }
482
478
  /**
483
479
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
484
480
  * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
@@ -500,7 +496,7 @@ class GarbageCollector {
500
496
  * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
501
497
  */
502
498
  if (this.activeConnection() && this.shouldRunGC) {
503
- this.initializeBaseStateP.catch((error) => { });
499
+ this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
504
500
  }
505
501
  }
506
502
  /**
@@ -542,8 +538,8 @@ class GarbageCollector {
542
538
  }, { end: true, cancel: "error" });
543
539
  }
544
540
  async runPreGCSteps() {
545
- // Ensure that base state has been initialized.
546
- await this.initializeBaseStateP;
541
+ // Ensure that state has been initialized from the base snapshot data.
542
+ await this.initializeGCStateFromBaseSnapshotP;
547
543
  // Let the runtime update its pending state before GC runs.
548
544
  await this.runtime.updateStateBeforeGC();
549
545
  }
@@ -563,23 +559,13 @@ class GarbageCollector {
563
559
  // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
564
560
  // involving access to deleted data.
565
561
  if (this.testMode) {
566
- this.runtime.deleteUnusedRoutes(gcResult.deletedNodeIds);
562
+ this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds, false /* tombstone */);
567
563
  }
568
- else {
564
+ else if (this.tombstoneMode) {
569
565
  // If we are running in GC tombstone mode, tombstone objects for unused routes. This enables testing
570
566
  // scenarios involving access to "deleted" data without actually deleting the data from summaries.
571
567
  // Note: we will not tombstone in test mode
572
- if (this.tombstoneMode) {
573
- const tombstoneRoutes = [];
574
- // Currently only tombstone datastores
575
- for (const [key, value] of this.unreferencedNodesState.entries()) {
576
- if (value.state === exports.UnreferencedState.SweepReady &&
577
- this.runtime.getNodeType(key) === exports.GCNodeType.DataStore) {
578
- tombstoneRoutes.push(key);
579
- }
580
- }
581
- this.runtime.deleteUnusedRoutes(tombstoneRoutes);
582
- }
568
+ this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
583
569
  }
584
570
  // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
585
571
  // updates its state so that we don't send false positives based on intermediate state. For example, we may get
@@ -605,7 +591,9 @@ class GarbageCollector {
605
591
  };
606
592
  }
607
593
  const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
608
- const serializedTombstones = this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined;
594
+ const serializedTombstones = this.tombstoneMode
595
+ ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
596
+ : undefined;
609
597
  /**
610
598
  * Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
611
599
  * summary, send summary handles for them. Otherwise, send the data in summary blobs.
@@ -621,7 +609,7 @@ class GarbageCollector {
621
609
  return {
622
610
  summary: {
623
611
  type: protocol_definitions_1.SummaryType.Handle,
624
- handle: `/${exports.gcTreeKey}`,
612
+ handle: `/${garbageCollectionConstants_1.gcTreeKey}`,
625
613
  handleType: protocol_definitions_1.SummaryType.Tree,
626
614
  },
627
615
  stats,
@@ -645,11 +633,11 @@ class GarbageCollector {
645
633
  */
646
634
  buildGCSummaryTree(serializedGCState, serializedTombstones, trackState) {
647
635
  var _a, _b;
648
- const gcStateBlobKey = `${exports.gcBlobPrefix}_root`;
636
+ const gcStateBlobKey = `${garbageCollectionConstants_1.gcBlobPrefix}_root`;
649
637
  const builder = new runtime_utils_1.SummaryTreeBuilder();
650
638
  // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
651
639
  if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
652
- builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${exports.gcTreeKey}/${gcStateBlobKey}`);
640
+ builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${gcStateBlobKey}`);
653
641
  }
654
642
  else {
655
643
  builder.addBlob(gcStateBlobKey, serializedGCState);
@@ -660,10 +648,10 @@ class GarbageCollector {
660
648
  }
661
649
  // If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
662
650
  if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones && trackState) {
663
- builder.addHandle(exports.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${exports.gcTreeKey}/${exports.gcTombstoneBlobKey}`);
651
+ builder.addHandle(garbageCollectionConstants_1.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${garbageCollectionConstants_1.gcTombstoneBlobKey}`);
664
652
  }
665
653
  else {
666
- builder.addBlob(exports.gcTombstoneBlobKey, serializedTombstones);
654
+ builder.addBlob(garbageCollectionConstants_1.gcTombstoneBlobKey, serializedTombstones);
667
655
  }
668
656
  return builder.getSummaryTree();
669
657
  }
@@ -713,7 +701,7 @@ class GarbageCollector {
713
701
  const metadata = await readAndParseBlob(metadataBlobId);
714
702
  this.latestSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
715
703
  }
716
- const gcSnapshotTree = snapshot.trees[exports.gcTreeKey];
704
+ const gcSnapshotTree = snapshot.trees[garbageCollectionConstants_1.gcTreeKey];
717
705
  if (gcSnapshotTree !== undefined && this.trackGCState) {
718
706
  const latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
719
707
  this.latestSummaryData = {
@@ -988,7 +976,7 @@ class GarbageCollector {
988
976
  * this will give us a view into how much deleted content a container has.
989
977
  */
990
978
  logSweepEvents(logger, currentReferenceTimestampMs) {
991
- if (this.mc.config.getBoolean(exports.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
979
+ if (this.mc.config.getBoolean(garbageCollectionConstants_1.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
992
980
  return;
993
981
  }
994
982
  this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
@@ -1038,31 +1026,24 @@ class GarbageCollector {
1038
1026
  return;
1039
1027
  }
1040
1028
  this.loggedUnreferencedEvents.add(uniqueEventId);
1041
- const propsToLog = {
1042
- id: nodeId,
1043
- type: nodeType,
1044
- unrefTime: nodeStateTracker.unreferencedTimestampMs,
1045
- age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
1046
- timeout: nodeStateTracker.state === exports.UnreferencedState.Inactive
1029
+ const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === exports.UnreferencedState.Inactive
1047
1030
  ? this.inactiveTimeoutMs
1048
- : this.sweepTimeoutMs,
1049
- completedGCRuns: this.completedRuns,
1050
- lastSummaryTime: this.getLastSummaryTimestampMs(),
1051
- externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.externalRequest],
1052
- viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle],
1053
- fromId: fromNodeId,
1054
- };
1031
+ : this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.externalRequest], viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle], fromId: fromNodeId });
1055
1032
  // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
1056
1033
  // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
1057
1034
  // but it's a good signal nonetheless and we can consume it with a grain of salt.
1035
+ // Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
1036
+ // SweepReady errors are usages of Objects that will be deleted by GC Sweep!
1058
1037
  if (this.isSummarizerClient) {
1059
1038
  this.pendingEventsQueue.push(Object.assign(Object.assign({}, propsToLog), { usageType, state }));
1060
1039
  }
1061
1040
  else {
1062
1041
  // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
1063
1042
  // summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
1043
+ // Events generated:
1044
+ // InactiveObject_Loaded, SweepReadyObject_Loaded
1064
1045
  if (usageType === "Loaded") {
1065
- this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: packagePath ? { value: packagePath.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined, stack: (0, telemetry_utils_1.generateStack)() }));
1046
+ this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(packagePath), stack: (0, telemetry_utils_1.generateStack)() }));
1066
1047
  }
1067
1048
  // If SweepReady Usage Detection is enabed, the handler may close the interactive container.
1068
1049
  // Once Sweep is fully implemented, this will be removed since the objects will be gone
@@ -1073,6 +1054,11 @@ class GarbageCollector {
1073
1054
  }
1074
1055
  }
1075
1056
  async logUnreferencedEvents(logger) {
1057
+ // Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
1058
+ // summary time they are then logged.
1059
+ // Events generated:
1060
+ // InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
1061
+ // SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
1076
1062
  for (const eventProps of this.pendingEventsQueue) {
1077
1063
  const { usageType, state } = eventProps, propsToLog = __rest(eventProps, ["usageType", "state"]);
1078
1064
  /**
@@ -1094,19 +1080,19 @@ class GarbageCollector {
1094
1080
  }
1095
1081
  exports.GarbageCollector = GarbageCollector;
1096
1082
  /**
1097
- * Gets the garbage collection data from the given snapshot tree. It contains GC state and tombstone state.
1083
+ * Gets the base garbage collection state from the given snapshot tree. It contains GC state and tombstone state.
1098
1084
  * The GC state may be written into multiple blobs. Merge the GC state from all such blobs into one.
1099
1085
  */
1100
1086
  async function getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob) {
1101
1087
  let rootGCState = { gcNodes: {} };
1102
1088
  let tombstones;
1103
1089
  for (const key of Object.keys(gcSnapshotTree.blobs)) {
1104
- if (key === exports.gcTombstoneBlobKey) {
1090
+ if (key === garbageCollectionConstants_1.gcTombstoneBlobKey) {
1105
1091
  tombstones = await readAndParseBlob(gcSnapshotTree.blobs[key]);
1106
1092
  continue;
1107
1093
  }
1108
1094
  // Skip blobs that do not start with the GC prefix.
1109
- if (!key.startsWith(exports.gcBlobPrefix)) {
1095
+ if (!key.startsWith(garbageCollectionConstants_1.gcBlobPrefix)) {
1110
1096
  continue;
1111
1097
  }
1112
1098
  const blobId = gcSnapshotTree.blobs[key];