@fluidframework/container-runtime 2.0.0-internal.7.3.0 → 2.0.0-internal.7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/api-extractor-lint.json +13 -0
  3. package/api-extractor.json +9 -1
  4. package/api-report/container-runtime.api.md +124 -107
  5. package/dist/blobManager.d.ts +4 -4
  6. package/dist/blobManager.d.ts.map +1 -1
  7. package/dist/blobManager.js.map +1 -1
  8. package/dist/container-runtime-alpha.d.ts +1473 -0
  9. package/dist/container-runtime-beta.d.ts +300 -0
  10. package/dist/container-runtime-public.d.ts +300 -0
  11. package/dist/container-runtime-untrimmed.d.ts +1836 -0
  12. package/dist/containerRuntime.d.ts +30 -30
  13. package/dist/containerRuntime.d.ts.map +1 -1
  14. package/dist/containerRuntime.js +62 -40
  15. package/dist/containerRuntime.js.map +1 -1
  16. package/dist/dataStoreRegistry.d.ts +1 -1
  17. package/dist/dataStoreRegistry.js +1 -1
  18. package/dist/dataStoreRegistry.js.map +1 -1
  19. package/dist/dataStores.d.ts +10 -15
  20. package/dist/dataStores.d.ts.map +1 -1
  21. package/dist/dataStores.js +77 -40
  22. package/dist/dataStores.js.map +1 -1
  23. package/dist/gc/garbageCollection.d.ts +41 -13
  24. package/dist/gc/garbageCollection.d.ts.map +1 -1
  25. package/dist/gc/garbageCollection.js +215 -78
  26. package/dist/gc/garbageCollection.js.map +1 -1
  27. package/dist/gc/gcConfigs.d.ts.map +1 -1
  28. package/dist/gc/gcConfigs.js +34 -37
  29. package/dist/gc/gcConfigs.js.map +1 -1
  30. package/dist/gc/gcDefinitions.d.ts +121 -46
  31. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  32. package/dist/gc/gcDefinitions.js +26 -18
  33. package/dist/gc/gcDefinitions.js.map +1 -1
  34. package/dist/gc/gcHelpers.d.ts +18 -25
  35. package/dist/gc/gcHelpers.d.ts.map +1 -1
  36. package/dist/gc/gcHelpers.js +29 -45
  37. package/dist/gc/gcHelpers.js.map +1 -1
  38. package/dist/gc/gcTelemetry.d.ts +0 -5
  39. package/dist/gc/gcTelemetry.d.ts.map +1 -1
  40. package/dist/gc/gcTelemetry.js +14 -42
  41. package/dist/gc/gcTelemetry.js.map +1 -1
  42. package/dist/gc/gcUnreferencedStateTracker.d.ts +11 -5
  43. package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
  44. package/dist/gc/gcUnreferencedStateTracker.js +43 -19
  45. package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
  46. package/dist/gc/index.d.ts +1 -1
  47. package/dist/gc/index.d.ts.map +1 -1
  48. package/dist/gc/index.js +4 -5
  49. package/dist/gc/index.js.map +1 -1
  50. package/dist/index.d.ts +14 -2
  51. package/dist/index.d.ts.map +1 -1
  52. package/dist/index.js +16 -5
  53. package/dist/index.js.map +1 -1
  54. package/dist/messageTypes.d.ts +15 -7
  55. package/dist/messageTypes.d.ts.map +1 -1
  56. package/dist/messageTypes.js +6 -1
  57. package/dist/messageTypes.js.map +1 -1
  58. package/dist/opLifecycle/definitions.d.ts +1 -1
  59. package/dist/opLifecycle/definitions.js.map +1 -1
  60. package/dist/packageVersion.d.ts +1 -1
  61. package/dist/packageVersion.js +1 -1
  62. package/dist/packageVersion.js.map +1 -1
  63. package/dist/pendingStateManager.d.ts +1 -0
  64. package/dist/pendingStateManager.d.ts.map +1 -1
  65. package/dist/pendingStateManager.js +1 -0
  66. package/dist/pendingStateManager.js.map +1 -1
  67. package/dist/summary/orderedClientElection.d.ts +1 -1
  68. package/dist/summary/orderedClientElection.js.map +1 -1
  69. package/dist/summary/runWhileConnectedCoordinator.d.ts +2 -2
  70. package/dist/summary/runWhileConnectedCoordinator.js +1 -1
  71. package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
  72. package/dist/summary/summarizer.d.ts +1 -1
  73. package/dist/summary/summarizer.js +1 -1
  74. package/dist/summary/summarizer.js.map +1 -1
  75. package/dist/summary/summarizerTypes.d.ts +30 -30
  76. package/dist/summary/summarizerTypes.js.map +1 -1
  77. package/dist/summary/summaryCollection.d.ts +10 -10
  78. package/dist/summary/summaryCollection.js +1 -1
  79. package/dist/summary/summaryCollection.js.map +1 -1
  80. package/dist/summary/summaryFormat.d.ts +3 -3
  81. package/dist/summary/summaryFormat.js.map +1 -1
  82. package/lib/blobManager.d.ts +4 -4
  83. package/lib/blobManager.d.ts.map +1 -1
  84. package/lib/blobManager.js.map +1 -1
  85. package/lib/container-runtime-alpha.d.ts +1473 -0
  86. package/lib/container-runtime-beta.d.ts +300 -0
  87. package/lib/container-runtime-public.d.ts +300 -0
  88. package/lib/container-runtime-untrimmed.d.ts +1836 -0
  89. package/lib/containerRuntime.d.ts +30 -30
  90. package/lib/containerRuntime.d.ts.map +1 -1
  91. package/lib/containerRuntime.js +64 -42
  92. package/lib/containerRuntime.js.map +1 -1
  93. package/lib/dataStoreRegistry.d.ts +1 -1
  94. package/lib/dataStoreRegistry.js +1 -1
  95. package/lib/dataStoreRegistry.js.map +1 -1
  96. package/lib/dataStores.d.ts +10 -15
  97. package/lib/dataStores.d.ts.map +1 -1
  98. package/lib/dataStores.js +80 -43
  99. package/lib/dataStores.js.map +1 -1
  100. package/lib/gc/garbageCollection.d.ts +41 -13
  101. package/lib/gc/garbageCollection.d.ts.map +1 -1
  102. package/lib/gc/garbageCollection.js +217 -80
  103. package/lib/gc/garbageCollection.js.map +1 -1
  104. package/lib/gc/gcConfigs.d.ts.map +1 -1
  105. package/lib/gc/gcConfigs.js +37 -40
  106. package/lib/gc/gcConfigs.js.map +1 -1
  107. package/lib/gc/gcDefinitions.d.ts +121 -46
  108. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  109. package/lib/gc/gcDefinitions.js +25 -17
  110. package/lib/gc/gcDefinitions.js.map +1 -1
  111. package/lib/gc/gcHelpers.d.ts +18 -25
  112. package/lib/gc/gcHelpers.d.ts.map +1 -1
  113. package/lib/gc/gcHelpers.js +27 -43
  114. package/lib/gc/gcHelpers.js.map +1 -1
  115. package/lib/gc/gcTelemetry.d.ts +0 -5
  116. package/lib/gc/gcTelemetry.d.ts.map +1 -1
  117. package/lib/gc/gcTelemetry.js +15 -43
  118. package/lib/gc/gcTelemetry.js.map +1 -1
  119. package/lib/gc/gcUnreferencedStateTracker.d.ts +11 -5
  120. package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
  121. package/lib/gc/gcUnreferencedStateTracker.js +43 -19
  122. package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
  123. package/lib/gc/index.d.ts +1 -1
  124. package/lib/gc/index.d.ts.map +1 -1
  125. package/lib/gc/index.js +1 -1
  126. package/lib/gc/index.js.map +1 -1
  127. package/lib/index.d.ts +14 -2
  128. package/lib/index.d.ts.map +1 -1
  129. package/lib/index.js +15 -1
  130. package/lib/index.js.map +1 -1
  131. package/lib/messageTypes.d.ts +15 -7
  132. package/lib/messageTypes.d.ts.map +1 -1
  133. package/lib/messageTypes.js +6 -1
  134. package/lib/messageTypes.js.map +1 -1
  135. package/lib/opLifecycle/definitions.d.ts +1 -1
  136. package/lib/opLifecycle/definitions.js.map +1 -1
  137. package/lib/packageVersion.d.ts +1 -1
  138. package/lib/packageVersion.js +1 -1
  139. package/lib/packageVersion.js.map +1 -1
  140. package/lib/pendingStateManager.d.ts +1 -0
  141. package/lib/pendingStateManager.d.ts.map +1 -1
  142. package/lib/pendingStateManager.js +1 -0
  143. package/lib/pendingStateManager.js.map +1 -1
  144. package/lib/summary/orderedClientElection.d.ts +1 -1
  145. package/lib/summary/orderedClientElection.js.map +1 -1
  146. package/lib/summary/runWhileConnectedCoordinator.d.ts +2 -2
  147. package/lib/summary/runWhileConnectedCoordinator.js +1 -1
  148. package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
  149. package/lib/summary/summarizer.d.ts +1 -1
  150. package/lib/summary/summarizer.js +1 -1
  151. package/lib/summary/summarizer.js.map +1 -1
  152. package/lib/summary/summarizerTypes.d.ts +30 -30
  153. package/lib/summary/summarizerTypes.js.map +1 -1
  154. package/lib/summary/summaryCollection.d.ts +10 -10
  155. package/lib/summary/summaryCollection.js +1 -1
  156. package/lib/summary/summaryCollection.js.map +1 -1
  157. package/lib/summary/summaryFormat.d.ts +3 -3
  158. package/lib/summary/summaryFormat.js.map +1 -1
  159. package/package.json +42 -19
  160. package/src/blobManager.ts +5 -5
  161. package/src/containerRuntime.ts +86 -56
  162. package/src/dataStoreRegistry.ts +1 -1
  163. package/src/dataStores.ts +140 -69
  164. package/src/gc/garbageCollection.md +14 -15
  165. package/src/gc/garbageCollection.ts +256 -96
  166. package/src/gc/gcConfigs.ts +50 -52
  167. package/src/gc/gcDefinitions.ts +137 -52
  168. package/src/gc/gcHelpers.ts +31 -52
  169. package/src/gc/gcTelemetry.ts +16 -57
  170. package/src/gc/gcUnreferencedStateTracker.ts +61 -22
  171. package/src/gc/index.ts +6 -4
  172. package/src/index.ts +19 -1
  173. package/src/messageTypes.ts +19 -4
  174. package/src/opLifecycle/definitions.ts +1 -1
  175. package/src/packageVersion.ts +1 -1
  176. package/src/pendingStateManager.ts +1 -0
  177. package/src/summary/orderedClientElection.ts +1 -1
  178. package/src/summary/runWhileConnectedCoordinator.ts +2 -2
  179. package/src/summary/summarizer.ts +1 -1
  180. package/src/summary/summarizerTypes.ts +30 -30
  181. package/src/summary/summaryCollection.ts +10 -10
  182. package/src/summary/summaryFormat.ts +3 -3
  183. package/dist/id-compressor/appendOnlySortedMap.d.ts +0 -124
  184. package/dist/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
  185. package/dist/id-compressor/appendOnlySortedMap.js +0 -318
  186. package/dist/id-compressor/appendOnlySortedMap.js.map +0 -1
  187. package/dist/id-compressor/finalSpace.d.ts +0 -29
  188. package/dist/id-compressor/finalSpace.d.ts.map +0 -1
  189. package/dist/id-compressor/finalSpace.js +0 -62
  190. package/dist/id-compressor/finalSpace.js.map +0 -1
  191. package/dist/id-compressor/idCompressor.d.ts +0 -54
  192. package/dist/id-compressor/idCompressor.d.ts.map +0 -1
  193. package/dist/id-compressor/idCompressor.js +0 -495
  194. package/dist/id-compressor/idCompressor.js.map +0 -1
  195. package/dist/id-compressor/identifiers.d.ts +0 -32
  196. package/dist/id-compressor/identifiers.d.ts.map +0 -1
  197. package/dist/id-compressor/identifiers.js +0 -15
  198. package/dist/id-compressor/identifiers.js.map +0 -1
  199. package/dist/id-compressor/index.d.ts +0 -13
  200. package/dist/id-compressor/index.d.ts.map +0 -1
  201. package/dist/id-compressor/index.js +0 -32
  202. package/dist/id-compressor/index.js.map +0 -1
  203. package/dist/id-compressor/persistanceUtilities.d.ts +0 -22
  204. package/dist/id-compressor/persistanceUtilities.d.ts.map +0 -1
  205. package/dist/id-compressor/persistanceUtilities.js +0 -43
  206. package/dist/id-compressor/persistanceUtilities.js.map +0 -1
  207. package/dist/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
  208. package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
  209. package/dist/id-compressor/sessionSpaceNormalizer.js +0 -80
  210. package/dist/id-compressor/sessionSpaceNormalizer.js.map +0 -1
  211. package/dist/id-compressor/sessions.d.ts +0 -115
  212. package/dist/id-compressor/sessions.d.ts.map +0 -1
  213. package/dist/id-compressor/sessions.js +0 -305
  214. package/dist/id-compressor/sessions.js.map +0 -1
  215. package/dist/id-compressor/utilities.d.ts +0 -52
  216. package/dist/id-compressor/utilities.d.ts.map +0 -1
  217. package/dist/id-compressor/utilities.js +0 -169
  218. package/dist/id-compressor/utilities.js.map +0 -1
  219. package/lib/id-compressor/appendOnlySortedMap.d.ts +0 -124
  220. package/lib/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
  221. package/lib/id-compressor/appendOnlySortedMap.js +0 -314
  222. package/lib/id-compressor/appendOnlySortedMap.js.map +0 -1
  223. package/lib/id-compressor/finalSpace.d.ts +0 -29
  224. package/lib/id-compressor/finalSpace.d.ts.map +0 -1
  225. package/lib/id-compressor/finalSpace.js +0 -58
  226. package/lib/id-compressor/finalSpace.js.map +0 -1
  227. package/lib/id-compressor/idCompressor.d.ts +0 -54
  228. package/lib/id-compressor/idCompressor.d.ts.map +0 -1
  229. package/lib/id-compressor/idCompressor.js +0 -491
  230. package/lib/id-compressor/idCompressor.js.map +0 -1
  231. package/lib/id-compressor/identifiers.d.ts +0 -32
  232. package/lib/id-compressor/identifiers.d.ts.map +0 -1
  233. package/lib/id-compressor/identifiers.js +0 -11
  234. package/lib/id-compressor/identifiers.js.map +0 -1
  235. package/lib/id-compressor/index.d.ts +0 -13
  236. package/lib/id-compressor/index.d.ts.map +0 -1
  237. package/lib/id-compressor/index.js +0 -13
  238. package/lib/id-compressor/index.js.map +0 -1
  239. package/lib/id-compressor/persistanceUtilities.d.ts +0 -22
  240. package/lib/id-compressor/persistanceUtilities.d.ts.map +0 -1
  241. package/lib/id-compressor/persistanceUtilities.js +0 -34
  242. package/lib/id-compressor/persistanceUtilities.js.map +0 -1
  243. package/lib/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
  244. package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
  245. package/lib/id-compressor/sessionSpaceNormalizer.js +0 -76
  246. package/lib/id-compressor/sessionSpaceNormalizer.js.map +0 -1
  247. package/lib/id-compressor/sessions.d.ts +0 -115
  248. package/lib/id-compressor/sessions.d.ts.map +0 -1
  249. package/lib/id-compressor/sessions.js +0 -290
  250. package/lib/id-compressor/sessions.js.map +0 -1
  251. package/lib/id-compressor/utilities.d.ts +0 -52
  252. package/lib/id-compressor/utilities.d.ts.map +0 -1
  253. package/lib/id-compressor/utilities.js +0 -151
  254. package/lib/id-compressor/utilities.js.map +0 -1
  255. package/src/id-compressor/README.md +0 -3
  256. package/src/id-compressor/appendOnlySortedMap.ts +0 -366
  257. package/src/id-compressor/finalSpace.ts +0 -67
  258. package/src/id-compressor/idCompressor.ts +0 -630
  259. package/src/id-compressor/identifiers.ts +0 -42
  260. package/src/id-compressor/index.ts +0 -26
  261. package/src/id-compressor/persistanceUtilities.ts +0 -58
  262. package/src/id-compressor/sessionSpaceNormalizer.ts +0 -83
  263. package/src/id-compressor/sessions.ts +0 -405
  264. package/src/id-compressor/utilities.ts +0 -190
@@ -21,13 +21,13 @@ import {
21
21
  MonitoringContext,
22
22
  PerformanceEvent,
23
23
  } from "@fluidframework/telemetry-utils";
24
-
25
24
  import {
26
25
  InactiveResponseHeaderKey,
27
26
  RuntimeHeaderData,
28
27
  TombstoneResponseHeaderKey,
29
28
  } from "../containerRuntime";
30
29
  import { ClientSessionExpiredError } from "../error";
30
+ import { ContainerMessageType, ContainerRuntimeGCMessage } from "../messageTypes";
31
31
  import { IRefreshSummaryResult } from "../summary";
32
32
  import { generateGCConfigs } from "./gcConfigs";
33
33
  import {
@@ -40,8 +40,17 @@ import {
40
40
  UnreferencedState,
41
41
  IGCMetadata,
42
42
  IGarbageCollectorConfigs,
43
+ IMarkPhaseStats,
44
+ ISweepPhaseStats,
45
+ GarbageCollectionMessage,
46
+ GarbageCollectionMessageType,
43
47
  } from "./gcDefinitions";
44
- import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot } from "./gcHelpers";
48
+ import {
49
+ cloneGCData,
50
+ compatBehaviorAllowsGCMessageType,
51
+ concatGarbageCollectionData,
52
+ getGCDataFromSnapshot,
53
+ } from "./gcHelpers";
45
54
  import { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
46
55
  import { IGarbageCollectionSnapshotData, IGarbageCollectionState } from "./gcSummaryDefinitions";
47
56
  import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
@@ -115,7 +124,7 @@ export class GarbageCollector implements IGarbageCollector {
115
124
 
116
125
  /** If false, loading or using a Tombstoned object should merely log, not fail */
117
126
  public get tombstoneEnforcementAllowed(): boolean {
118
- return this.configs.tombstoneEnforcementAllowed;
127
+ return this.configs.sweepEnabled;
119
128
  }
120
129
  /** If true, throw an error when a tombstone data store is retrieved */
121
130
  public get throwOnTombstoneLoad(): boolean {
@@ -135,6 +144,8 @@ export class GarbageCollector implements IGarbageCollector {
135
144
  /** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
136
145
  private readonly activeConnection: () => boolean;
137
146
 
147
+ private readonly submitMessage: (message: ContainerRuntimeGCMessage) => void;
148
+
138
149
  public get summaryStateNeedsReset(): boolean {
139
150
  return this.summaryStateTracker.doesSummaryStateNeedReset;
140
151
  }
@@ -150,6 +161,7 @@ export class GarbageCollector implements IGarbageCollector {
150
161
  this.getNodePackagePath = createParams.getNodePackagePath;
151
162
  this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
152
163
  this.activeConnection = createParams.activeConnection;
164
+ this.submitMessage = createParams.submitMessage;
153
165
 
154
166
  const baseSnapshot = createParams.baseSnapshot;
155
167
  const readAndParseBlob = createParams.readAndParseBlob;
@@ -245,7 +257,7 @@ export class GarbageCollector implements IGarbageCollector {
245
257
  /**
246
258
  * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
247
259
  * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
248
- * GC state and updates their inactive or sweep ready state.
260
+ * GC state and updates their inactive or sweep-ready state.
249
261
  */
250
262
  this.initializeGCStateFromBaseSnapshotP = new LazyPromise<void>(async () => {
251
263
  /**
@@ -305,6 +317,7 @@ export class GarbageCollector implements IGarbageCollector {
305
317
  eventName: "GarbageCollectorLoaded",
306
318
  gcConfigs: JSON.stringify(this.configs),
307
319
  gcOptions: JSON.stringify(createParams.gcOptions),
320
+ ...createParams.createContainerMetadata,
308
321
  });
309
322
  }
310
323
 
@@ -415,6 +428,7 @@ export class GarbageCollector implements IGarbageCollector {
415
428
  this.configs.inactiveTimeoutMs,
416
429
  currentReferenceTimestampMs,
417
430
  this.configs.sweepTimeoutMs,
431
+ this.configs.sweepGracePeriodMs,
418
432
  ),
419
433
  );
420
434
  }
@@ -425,7 +439,7 @@ export class GarbageCollector implements IGarbageCollector {
425
439
 
426
440
  /**
427
441
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
428
- * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
442
+ * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
429
443
  * @param connected - Whether the runtime connected / disconnected.
430
444
  * @param clientId - The clientId of this runtime.
431
445
  */
@@ -441,7 +455,7 @@ export class GarbageCollector implements IGarbageCollector {
441
455
  * the receiving summarizer client.
442
456
  *
443
457
  * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
444
- * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
458
+ * sweep in phases and we want to track when inactive and sweep-ready objects are used in any client.
445
459
  */
446
460
  if (this.activeConnection() && this.configs.shouldRunGC) {
447
461
  this.initializeGCStateFromBaseSnapshotP.catch((error) => {});
@@ -520,7 +534,11 @@ export class GarbageCollector implements IGarbageCollector {
520
534
 
521
535
  /** GC step */
522
536
  const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
523
- event.end({ ...gcStats, timestamp: currentReferenceTimestampMs });
537
+ event.end({
538
+ ...gcStats,
539
+ timestamp: currentReferenceTimestampMs,
540
+ sweep: this.configs.shouldRunSweep,
541
+ });
524
542
 
525
543
  /** Post-GC steps */
526
544
  // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
@@ -540,10 +558,16 @@ export class GarbageCollector implements IGarbageCollector {
540
558
 
541
559
  /**
542
560
  * Runs garbage collection. It does the following:
561
+ *
543
562
  * 1. It generates / analyzes the runtime's reference graph.
544
- * 2. Generates stats for the GC run based on previous / current GC state.
563
+ *
564
+ * 2. Generates mark phase stats.
565
+ *
545
566
  * 3. Runs Mark phase.
567
+ *
546
568
  * 4. Runs Sweep phase.
569
+ *
570
+ * 5. Generates sweep phase stats.
547
571
  */
548
572
  private async runGC(
549
573
  fullGC: boolean,
@@ -559,32 +583,33 @@ export class GarbageCollector implements IGarbageCollector {
559
583
  this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
560
584
  gcResult.referencedNodeIds;
561
585
 
562
- // 2. Generate stats based on the previous / current GC state.
563
- // Must happen before running Mark / Sweep phase because previous GC state will be updated in these stages.
564
- const gcStats = this.generateStats(gcResult);
586
+ // 2. Get the mark phase stats based on the previous / current GC state.
587
+ // This is done before running mark phase because we need the previous GC state before it is updated.
588
+ const markPhaseStats = this.getMarkPhaseStats(gcResult);
565
589
 
566
590
  // 3. Run the Mark phase.
567
- // It will mark nodes as referenced / unreferenced and return a list of node ids that are ready to be swept.
568
- const sweepReadyNodeIds = this.runMarkPhase(
591
+ // It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
592
+ const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(
569
593
  gcResult,
570
594
  allReferencedNodeIds,
571
595
  currentReferenceTimestampMs,
572
596
  );
573
597
 
574
598
  // 4. Run the Sweep phase.
575
- // It will delete sweep ready nodes and return a list of deleted node ids.
576
- const deletedNodeIds = this.runSweepPhase(
577
- gcResult,
599
+ // It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
600
+ // sweep op. All clients, including this one, will delete these nodes once it processes the op.
601
+ this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
602
+
603
+ this.gcDataFromLastRun = cloneGCData(gcData);
604
+
605
+ // 5. Get the sweep phase stats.
606
+ const sweepPhaseStats = this.getSweepPhaseStats(
607
+ this.deletedNodes,
578
608
  sweepReadyNodeIds,
579
- currentReferenceTimestampMs,
580
- logger,
609
+ markPhaseStats,
581
610
  );
582
611
 
583
- this.gcDataFromLastRun = cloneGCData(
584
- gcData,
585
- (id: string) => deletedNodeIds.includes(id) /* filter out deleted nodes */,
586
- );
587
- return gcStats;
612
+ return { ...markPhaseStats, ...sweepPhaseStats };
588
613
  }
589
614
 
590
615
  /**
@@ -599,13 +624,13 @@ export class GarbageCollector implements IGarbageCollector {
599
624
  * @param gcResult - The result of the GC run on the gcData.
600
625
  * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
601
626
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
602
- * @returns A list of sweep ready nodes, i.e., nodes that ready to be deleted.
627
+ * @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
603
628
  */
604
629
  private runMarkPhase(
605
630
  gcResult: IGCResult,
606
631
  allReferencedNodeIds: string[],
607
632
  currentReferenceTimestampMs: number,
608
- ): string[] {
633
+ ): { tombstoneReadyNodeIds: Set<string>; sweepReadyNodeIds: Set<string> } {
609
634
  // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
610
635
  for (const nodeId of allReferencedNodeIds) {
611
636
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
@@ -618,7 +643,8 @@ export class GarbageCollector implements IGarbageCollector {
618
643
  }
619
644
 
620
645
  // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
621
- const sweepReadyNodeIds: string[] = [];
646
+ const tombstoneReadyNodeIds: Set<string> = new Set();
647
+ const sweepReadyNodeIds: Set<string> = new Set();
622
648
  for (const nodeId of gcResult.deletedNodeIds) {
623
649
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
624
650
  if (nodeStateTracker === undefined) {
@@ -629,6 +655,7 @@ export class GarbageCollector implements IGarbageCollector {
629
655
  this.configs.inactiveTimeoutMs,
630
656
  currentReferenceTimestampMs,
631
657
  this.configs.sweepTimeoutMs,
658
+ this.configs.sweepGracePeriodMs,
632
659
  ),
633
660
  );
634
661
  } else {
@@ -636,9 +663,12 @@ export class GarbageCollector implements IGarbageCollector {
636
663
  // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
637
664
  nodeStateTracker.updateTracking(currentReferenceTimestampMs);
638
665
 
639
- // If a node is sweep ready, store it so it can be returned.
666
+ // If a node is tombstone or sweep-ready, store it so it can be returned.
667
+ if (nodeStateTracker.state === UnreferencedState.TombstoneReady) {
668
+ tombstoneReadyNodeIds.add(nodeId);
669
+ }
640
670
  if (nodeStateTracker.state === UnreferencedState.SweepReady) {
641
- sweepReadyNodeIds.push(nodeId);
671
+ sweepReadyNodeIds.add(nodeId);
642
672
  }
643
673
  }
644
674
  }
@@ -646,82 +676,82 @@ export class GarbageCollector implements IGarbageCollector {
646
676
  // 3. Call the runtime to update referenced nodes in this run.
647
677
  this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
648
678
 
649
- return sweepReadyNodeIds;
679
+ return { tombstoneReadyNodeIds, sweepReadyNodeIds };
650
680
  }
651
681
 
652
682
  /**
653
683
  * Runs the GC Sweep phase. It does the following:
654
- * 1. Calls the runtime to delete nodes that are sweep ready.
655
- * 2. Clears tracking for deleted nodes.
684
+ *
685
+ * 1. Marks tombstone-ready nodes as tombstones.
686
+ *
687
+ * 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
656
688
  *
657
689
  * @param gcResult - The result of the GC run on the gcData.
658
- * @param sweepReadyNodes - List of nodes that are sweep ready.
659
- * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
660
- * @param logger - The logger to be used to log any telemetry.
661
- * @returns A list of nodes that have been deleted.
690
+ * @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
691
+ * @param sweepReadyNodes - List of nodes that are sweep-ready.
662
692
  */
663
693
  private runSweepPhase(
664
694
  gcResult: IGCResult,
665
- sweepReadyNodes: string[],
666
- currentReferenceTimestampMs: number,
667
- logger: ITelemetryLoggerExt,
668
- ): string[] {
669
- // Log events for objects that are ready to be deleted by sweep. This will give us data on sweep when
670
- // its not enabled.
671
- this.telemetryTracker.logSweepEvents(
672
- logger,
673
- currentReferenceTimestampMs,
674
- this.unreferencedNodesState,
675
- this.completedRuns,
676
- this.getLastSummaryTimestampMs(),
677
- );
678
-
695
+ tombstoneReadyNodes: Set<string>,
696
+ sweepReadyNodes: Set<string>,
697
+ ) {
679
698
  /**
680
- * Currently, there are 3 modes for sweep:
681
- * Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
682
- * Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted.
683
- * Sweep mode - Sweep ready modes are deleted.
699
+ * Under "Test Mode", unreferenced nodes are immediately deleted without waiting for them to be sweep-ready.
684
700
  *
685
- * These modes serve as staging for applications that want to enable sweep by providing an incremental
686
- * way to test and validate sweep works as expected.
701
+ * Otherwise, depending on how long it's been since the node was unreferenced, it will either be
702
+ * marked as Tombstone, or deleted by Sweep.
687
703
  */
704
+
688
705
  if (this.configs.testMode) {
689
706
  // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
690
707
  this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
691
- return [];
708
+ return;
692
709
  }
693
710
 
711
+ // If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
712
+ // This is important because a container may never load during a node's Sweep Grace Period,
713
+ // so that node would directly become sweep-ready skipping over tombstone-ready state,
714
+ // but should be Tombstoned since Sweep is disabled.
715
+ const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
716
+ ? {
717
+ nodesToTombstone: [...tombstoneReadyNodes],
718
+ nodesToDelete: [...sweepReadyNodes],
719
+ }
720
+ : {
721
+ nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
722
+ nodesToDelete: [],
723
+ };
724
+
694
725
  if (this.configs.tombstoneMode) {
695
- this.tombstones = sweepReadyNodes;
696
- // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
697
- // involving access to "deleted" data without actually deleting the data from summaries.
726
+ this.tombstones = nodesToTombstone;
727
+ // If we are running in GC tombstone mode, update tombstoned routes.
698
728
  this.runtime.updateTombstonedRoutes(this.tombstones);
699
- return [];
700
729
  }
701
730
 
702
- if (!this.configs.shouldRunSweep) {
703
- return [];
704
- }
705
-
706
- // 1. Call the runtime to delete sweep ready nodes. The runtime returns a list of nodes it deleted.
707
- // TODO: GC:Validation - validate that removed routes are not double delete and that the child routes of
708
- // removed routes are deleted as well.
709
- const deletedNodeIds = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
731
+ if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
732
+ // Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
733
+ // to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
734
+ // local state when processing the op.
735
+ const sweepReadyDSAndBlobs = nodesToDelete.filter((nodeId) => {
736
+ const nodeType = this.runtime.getNodeType(nodeId);
737
+ return nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob;
738
+ });
739
+ const contents: GarbageCollectionMessage = {
740
+ type: GarbageCollectionMessageType.Sweep,
741
+ deletedNodeIds: sweepReadyDSAndBlobs,
742
+ };
710
743
 
711
- // 2. Clear unreferenced state tracking for deleted nodes.
712
- for (const nodeId of deletedNodeIds) {
713
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
714
- // TODO: GC:Validation - assert that the nodeStateTracker is defined
715
- if (nodeStateTracker !== undefined) {
716
- // Stop tracking so as to clear out any running timers.
717
- nodeStateTracker.stopTracking();
718
- // Delete the node as we don't need to track it any more.
719
- this.unreferencedNodesState.delete(nodeId);
720
- }
721
- // TODO: GC:Validation - assert that the deleted node is not a duplicate
722
- this.deletedNodes.add(nodeId);
744
+ // Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
745
+ // is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
746
+ // accidental usage. The clients will sync without the delete op too but it may take longer.
747
+ const containerGCMessage: ContainerRuntimeGCMessage = {
748
+ type: ContainerMessageType.GC,
749
+ contents,
750
+ compatDetails: { behavior: "Ignore" },
751
+ };
752
+ this.submitMessage(containerGCMessage);
753
+ return;
723
754
  }
724
- return deletedNodeIds;
725
755
  }
726
756
 
727
757
  /**
@@ -861,6 +891,78 @@ export class GarbageCollector implements IGarbageCollector {
861
891
  return this.summaryStateTracker.refreshLatestSummary(result);
862
892
  }
863
893
 
894
+ /**
895
+ * Process a GC message.
896
+ * @param message - The GC message from the container runtime.
897
+ * @param local - Whether it was send by this client.
898
+ */
899
+ public processMessage(message: ContainerRuntimeGCMessage, local: boolean) {
900
+ switch (message.contents.type) {
901
+ case "Sweep": {
902
+ // Delete the nodes whose ids are present in the contents.
903
+ this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
904
+ break;
905
+ }
906
+ default: {
907
+ if (
908
+ !compatBehaviorAllowsGCMessageType(
909
+ message.contents.type,
910
+ message.compatDetails?.behavior,
911
+ )
912
+ ) {
913
+ const error = DataProcessingError.create(
914
+ `Garbage collection message of unknown type ${message.contents.type}`,
915
+ "processMessage",
916
+ );
917
+ throw error;
918
+ }
919
+ break;
920
+ }
921
+ }
922
+ }
923
+
924
+ /**
925
+ * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
926
+ * tracking for nodes that are actually deleted by the runtime.
927
+ * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
928
+ */
929
+ private deleteSweepReadyNodes(sweepReadyNodeIds: readonly string[]) {
930
+ // Use a set for lookup because its much faster than array or map.
931
+ const sweepReadyNodesSet: Set<string> = new Set(sweepReadyNodeIds);
932
+
933
+ // The ids in the sweep-ready nodes do not contain DDS node ids. This is an optimization to reduce the size
934
+ // of the GC op. Since GC applies to data store only, all its DDSes are deleted along with it. So, get the
935
+ // DDS nodes ID from the unreferenced nodes state.
936
+ const allSweepReadyNodeIds = Array.from(sweepReadyNodeIds);
937
+ for (const [id] of this.unreferencedNodesState) {
938
+ // Ignore data store nodes since they would already be in the list.
939
+ const pathParts = id.split("/");
940
+ if (pathParts.length <= 2) {
941
+ continue;
942
+ }
943
+
944
+ // Get the data store id part. Note that this may include blobs but that's okay since the part would just
945
+ // be "_blobs" and it won't be found.
946
+ const dsId = `/${pathParts[1]}`;
947
+ if (sweepReadyNodesSet.has(dsId)) {
948
+ allSweepReadyNodeIds.push(id);
949
+ }
950
+ }
951
+ const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
952
+
953
+ // Clear unreferenced state tracking for deleted nodes.
954
+ for (const nodeId of deletedNodeIds) {
955
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
956
+ if (nodeStateTracker !== undefined) {
957
+ // Stop tracking so as to clear out any running timers.
958
+ nodeStateTracker.stopTracking();
959
+ // Delete the node as we don't need to track it any more.
960
+ this.unreferencedNodesState.delete(nodeId);
961
+ }
962
+ this.deletedNodes.add(nodeId);
963
+ }
964
+ }
965
+
864
966
  /**
865
967
  * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
866
968
  * or throw an error if failing on incorrect usage is configured.
@@ -976,12 +1078,12 @@ export class GarbageCollector implements IGarbageCollector {
976
1078
  }
977
1079
 
978
1080
  /**
979
- * Generates the stats of a garbage collection run from the given results of the run.
980
- * @param gcResult - The result of a GC run.
981
- * @returns the GC stats of the GC run.
1081
+ * Generates the stats of a garbage collection mark phase run.
1082
+ * @param gcResult - The result of the current GC run.
1083
+ * @returns the stats of the mark phase run.
982
1084
  */
983
- private generateStats(gcResult: IGCResult): IGCStats {
984
- const gcStats: IGCStats = {
1085
+ private getMarkPhaseStats(gcResult: IGCResult): IMarkPhaseStats {
1086
+ const markPhaseStats: IMarkPhaseStats = {
985
1087
  nodeCount: 0,
986
1088
  dataStoreCount: 0,
987
1089
  attachmentBlobCount: 0,
@@ -994,35 +1096,35 @@ export class GarbageCollector implements IGarbageCollector {
994
1096
  };
995
1097
 
996
1098
  const updateNodeStats = (nodeId: string, referenced: boolean) => {
997
- gcStats.nodeCount++;
1099
+ markPhaseStats.nodeCount++;
998
1100
  // If there is no previous GC data, every node's state is generated and is considered as updated.
999
1101
  // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
1000
1102
  const stateUpdated =
1001
1103
  this.gcDataFromLastRun === undefined ||
1002
1104
  this.unreferencedNodesState.has(nodeId) === referenced;
1003
1105
  if (stateUpdated) {
1004
- gcStats.updatedNodeCount++;
1106
+ markPhaseStats.updatedNodeCount++;
1005
1107
  }
1006
1108
  if (!referenced) {
1007
- gcStats.unrefNodeCount++;
1109
+ markPhaseStats.unrefNodeCount++;
1008
1110
  }
1009
1111
 
1010
1112
  if (this.runtime.getNodeType(nodeId) === GCNodeType.DataStore) {
1011
- gcStats.dataStoreCount++;
1113
+ markPhaseStats.dataStoreCount++;
1012
1114
  if (stateUpdated) {
1013
- gcStats.updatedDataStoreCount++;
1115
+ markPhaseStats.updatedDataStoreCount++;
1014
1116
  }
1015
1117
  if (!referenced) {
1016
- gcStats.unrefDataStoreCount++;
1118
+ markPhaseStats.unrefDataStoreCount++;
1017
1119
  }
1018
1120
  }
1019
1121
  if (this.runtime.getNodeType(nodeId) === GCNodeType.Blob) {
1020
- gcStats.attachmentBlobCount++;
1122
+ markPhaseStats.attachmentBlobCount++;
1021
1123
  if (stateUpdated) {
1022
- gcStats.updatedAttachmentBlobCount++;
1124
+ markPhaseStats.updatedAttachmentBlobCount++;
1023
1125
  }
1024
1126
  if (!referenced) {
1025
- gcStats.unrefAttachmentBlobCount++;
1127
+ markPhaseStats.unrefAttachmentBlobCount++;
1026
1128
  }
1027
1129
  }
1028
1130
  };
@@ -1035,6 +1137,64 @@ export class GarbageCollector implements IGarbageCollector {
1035
1137
  updateNodeStats(nodeId, false /* referenced */);
1036
1138
  }
1037
1139
 
1038
- return gcStats;
1140
+ return markPhaseStats;
1141
+ }
1142
+
1143
+ /**
1144
+ * Generates the stats of a garbage collection sweep phase run.
1145
+ * @param deletedNodes - The nodes that have been deleted until this run.
1146
+ * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
1147
+ * @param markPhaseStats - The stats of the mark phase run.
1148
+ * @returns the stats of the sweep phase run.
1149
+ */
1150
+ private getSweepPhaseStats(
1151
+ deletedNodes: Set<string>,
1152
+ sweepReadyNodes: Set<string>,
1153
+ markPhaseStats: IMarkPhaseStats,
1154
+ ): ISweepPhaseStats {
1155
+ // Initialize the life time node counts to the mark phase node counts. If sweep is not enabled,
1156
+ // these will be the life time node count for this container.
1157
+ const sweepPhaseStats: ISweepPhaseStats = {
1158
+ lifetimeNodeCount: markPhaseStats.nodeCount,
1159
+ lifetimeDataStoreCount: markPhaseStats.dataStoreCount,
1160
+ lifetimeAttachmentBlobCount: markPhaseStats.attachmentBlobCount,
1161
+ deletedNodeCount: 0,
1162
+ deletedDataStoreCount: 0,
1163
+ deletedAttachmentBlobCount: 0,
1164
+ };
1165
+
1166
+ for (const nodeId of deletedNodes) {
1167
+ sweepPhaseStats.deletedNodeCount++;
1168
+ const nodeType = this.runtime.getNodeType(nodeId);
1169
+ if (nodeType === GCNodeType.DataStore) {
1170
+ sweepPhaseStats.deletedDataStoreCount++;
1171
+ } else if (nodeType === GCNodeType.Blob) {
1172
+ sweepPhaseStats.deletedAttachmentBlobCount++;
1173
+ }
1174
+ }
1175
+
1176
+ // If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
1177
+ // deleted in previous runs. So, add the deleted node counts to life time stats.
1178
+ sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
1179
+ sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
1180
+ sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
1181
+
1182
+ if (this.configs.shouldRunSweep) {
1183
+ return sweepPhaseStats;
1184
+ }
1185
+
1186
+ // If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
1187
+ // is the final state the node will be in.
1188
+ // If sweep is enabled, this will happen in the run after the GC op round trips back.
1189
+ for (const nodeId of sweepReadyNodes) {
1190
+ sweepPhaseStats.deletedNodeCount++;
1191
+ const nodeType = this.runtime.getNodeType(nodeId);
1192
+ if (nodeType === GCNodeType.DataStore) {
1193
+ sweepPhaseStats.deletedDataStoreCount++;
1194
+ } else if (nodeType === GCNodeType.Blob) {
1195
+ sweepPhaseStats.deletedAttachmentBlobCount++;
1196
+ }
1197
+ }
1198
+ return sweepPhaseStats;
1039
1199
  }
1040
1200
  }