@fluidframework/container-runtime 2.0.0-internal.3.0.2 → 2.0.0-internal.3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (474) hide show
  1. package/.eslintrc.js +19 -19
  2. package/.mocharc.js +2 -2
  3. package/api-extractor.json +2 -2
  4. package/dist/batchTracker.d.ts.map +1 -1
  5. package/dist/batchTracker.js +2 -1
  6. package/dist/batchTracker.js.map +1 -1
  7. package/dist/blobManager.d.ts +15 -2
  8. package/dist/blobManager.d.ts.map +1 -1
  9. package/dist/blobManager.js +109 -37
  10. package/dist/blobManager.js.map +1 -1
  11. package/dist/connectionTelemetry.d.ts.map +1 -1
  12. package/dist/connectionTelemetry.js +11 -9
  13. package/dist/connectionTelemetry.js.map +1 -1
  14. package/dist/containerHandleContext.d.ts.map +1 -1
  15. package/dist/containerHandleContext.js +3 -1
  16. package/dist/containerHandleContext.js.map +1 -1
  17. package/dist/containerRuntime.d.ts +23 -11
  18. package/dist/containerRuntime.d.ts.map +1 -1
  19. package/dist/containerRuntime.js +225 -132
  20. package/dist/containerRuntime.js.map +1 -1
  21. package/dist/dataStore.d.ts.map +1 -1
  22. package/dist/dataStore.js +11 -9
  23. package/dist/dataStore.js.map +1 -1
  24. package/dist/dataStoreContext.d.ts +27 -13
  25. package/dist/dataStoreContext.d.ts.map +1 -1
  26. package/dist/dataStoreContext.js +95 -56
  27. package/dist/dataStoreContext.js.map +1 -1
  28. package/dist/dataStoreContexts.d.ts.map +1 -1
  29. package/dist/dataStoreContexts.js +7 -3
  30. package/dist/dataStoreContexts.js.map +1 -1
  31. package/dist/dataStoreRegistry.d.ts.map +1 -1
  32. package/dist/dataStoreRegistry.js +3 -1
  33. package/dist/dataStoreRegistry.js.map +1 -1
  34. package/dist/dataStores.d.ts +28 -4
  35. package/dist/dataStores.d.ts.map +1 -1
  36. package/dist/dataStores.js +122 -44
  37. package/dist/dataStores.js.map +1 -1
  38. package/dist/deltaScheduler.d.ts.map +1 -1
  39. package/dist/deltaScheduler.js +8 -3
  40. package/dist/deltaScheduler.js.map +1 -1
  41. package/dist/{garbageCollection.d.ts → gc/garbageCollection.d.ts} +27 -203
  42. package/dist/gc/garbageCollection.d.ts.map +1 -0
  43. package/dist/{garbageCollection.js → gc/garbageCollection.js} +210 -400
  44. package/dist/gc/garbageCollection.js.map +1 -0
  45. package/dist/gc/gcDefinitions.d.ts +189 -0
  46. package/dist/gc/gcDefinitions.d.ts.map +1 -0
  47. package/dist/{garbageCollectionConstants.js → gc/gcDefinitions.js} +27 -2
  48. package/dist/gc/gcDefinitions.js.map +1 -0
  49. package/dist/gc/gcHelpers.d.ts +30 -0
  50. package/dist/gc/gcHelpers.d.ts.map +1 -0
  51. package/dist/gc/gcHelpers.js +65 -0
  52. package/dist/gc/gcHelpers.js.map +1 -0
  53. package/dist/gc/gcSummaryStateTracker.d.ts +86 -0
  54. package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -0
  55. package/dist/gc/gcSummaryStateTracker.js +246 -0
  56. package/dist/gc/gcSummaryStateTracker.js.map +1 -0
  57. package/{lib → dist/gc}/gcSweepReadyUsageDetection.d.ts +5 -5
  58. package/dist/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
  59. package/dist/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +15 -11
  60. package/dist/gc/gcSweepReadyUsageDetection.js.map +1 -0
  61. package/dist/gc/gcUnreferencedStateTracker.d.ts +34 -0
  62. package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
  63. package/dist/gc/gcUnreferencedStateTracker.js +94 -0
  64. package/dist/gc/gcUnreferencedStateTracker.js.map +1 -0
  65. package/dist/gc/index.d.ts +11 -0
  66. package/dist/gc/index.d.ts.map +1 -0
  67. package/dist/gc/index.js +40 -0
  68. package/dist/gc/index.js.map +1 -0
  69. package/dist/index.d.ts +2 -5
  70. package/dist/index.d.ts.map +1 -1
  71. package/dist/index.js +6 -9
  72. package/dist/index.js.map +1 -1
  73. package/dist/opLifecycle/batchManager.d.ts +2 -13
  74. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  75. package/dist/opLifecycle/batchManager.js +19 -41
  76. package/dist/opLifecycle/batchManager.js.map +1 -1
  77. package/dist/opLifecycle/definitions.d.ts +4 -0
  78. package/dist/opLifecycle/definitions.d.ts.map +1 -1
  79. package/dist/opLifecycle/definitions.js.map +1 -1
  80. package/dist/opLifecycle/index.d.ts.map +1 -1
  81. package/dist/opLifecycle/index.js.map +1 -1
  82. package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
  83. package/dist/opLifecycle/opCompressor.js +1 -0
  84. package/dist/opLifecycle/opCompressor.js.map +1 -1
  85. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
  86. package/dist/opLifecycle/opDecompressor.js +5 -2
  87. package/dist/opLifecycle/opDecompressor.js.map +1 -1
  88. package/dist/opLifecycle/opSplitter.d.ts +1 -1
  89. package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
  90. package/dist/opLifecycle/opSplitter.js +24 -13
  91. package/dist/opLifecycle/opSplitter.js.map +1 -1
  92. package/dist/opLifecycle/outbox.d.ts +19 -3
  93. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  94. package/dist/opLifecycle/outbox.js +78 -45
  95. package/dist/opLifecycle/outbox.js.map +1 -1
  96. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  97. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
  98. package/dist/opProperties.d.ts.map +1 -1
  99. package/dist/opProperties.js +1 -3
  100. package/dist/opProperties.js.map +1 -1
  101. package/dist/packageVersion.d.ts +1 -1
  102. package/dist/packageVersion.js +1 -1
  103. package/dist/packageVersion.js.map +1 -1
  104. package/dist/pendingStateManager.d.ts +8 -2
  105. package/dist/pendingStateManager.d.ts.map +1 -1
  106. package/dist/pendingStateManager.js +21 -13
  107. package/dist/pendingStateManager.js.map +1 -1
  108. package/dist/scheduleManager.d.ts.map +1 -1
  109. package/dist/scheduleManager.js +3 -2
  110. package/dist/scheduleManager.js.map +1 -1
  111. package/dist/serializedSnapshotStorage.d.ts +2 -2
  112. package/dist/serializedSnapshotStorage.d.ts.map +1 -1
  113. package/dist/serializedSnapshotStorage.js +5 -3
  114. package/dist/serializedSnapshotStorage.js.map +1 -1
  115. package/dist/summary/index.d.ts +17 -0
  116. package/dist/summary/index.d.ts.map +1 -0
  117. package/dist/summary/index.js +47 -0
  118. package/dist/summary/index.js.map +1 -0
  119. package/dist/summary/orderedClientElection.d.ts.map +1 -0
  120. package/dist/{orderedClientElection.js → summary/orderedClientElection.js} +10 -4
  121. package/dist/summary/orderedClientElection.js.map +1 -0
  122. package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
  123. package/dist/summary/runWhileConnectedCoordinator.js.map +1 -0
  124. package/{lib → dist/summary}/runningSummarizer.d.ts +19 -18
  125. package/dist/summary/runningSummarizer.d.ts.map +1 -0
  126. package/dist/{runningSummarizer.js → summary/runningSummarizer.js} +191 -76
  127. package/dist/summary/runningSummarizer.js.map +1 -0
  128. package/dist/{summarizer.d.ts → summary/summarizer.d.ts} +4 -6
  129. package/dist/summary/summarizer.d.ts.map +1 -0
  130. package/dist/{summarizer.js → summary/summarizer.js} +31 -71
  131. package/dist/summary/summarizer.js.map +1 -0
  132. package/dist/summary/summarizerClientElection.d.ts.map +1 -0
  133. package/dist/summary/summarizerClientElection.js.map +1 -0
  134. package/dist/summary/summarizerHandle.d.ts.map +1 -0
  135. package/dist/summary/summarizerHandle.js.map +1 -0
  136. package/{lib → dist/summary}/summarizerHeuristics.d.ts +1 -1
  137. package/dist/summary/summarizerHeuristics.d.ts.map +1 -0
  138. package/dist/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +6 -9
  139. package/dist/summary/summarizerHeuristics.js.map +1 -0
  140. package/{lib → dist/summary}/summarizerTypes.d.ts +22 -22
  141. package/dist/summary/summarizerTypes.d.ts.map +1 -0
  142. package/dist/summary/summarizerTypes.js.map +1 -0
  143. package/dist/summary/summaryCollection.d.ts.map +1 -0
  144. package/dist/{summaryCollection.js → summary/summaryCollection.js} +18 -8
  145. package/dist/summary/summaryCollection.js.map +1 -0
  146. package/{lib → dist/summary}/summaryFormat.d.ts +1 -40
  147. package/dist/summary/summaryFormat.d.ts.map +1 -0
  148. package/dist/{summaryFormat.js → summary/summaryFormat.js} +19 -20
  149. package/dist/summary/summaryFormat.js.map +1 -0
  150. package/dist/summary/summaryGenerator.d.ts.map +1 -0
  151. package/dist/{summaryGenerator.js → summary/summaryGenerator.js} +33 -15
  152. package/dist/summary/summaryGenerator.js.map +1 -0
  153. package/dist/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
  154. package/dist/summary/summaryManager.d.ts.map +1 -0
  155. package/dist/{summaryManager.js → summary/summaryManager.js} +21 -9
  156. package/dist/summary/summaryManager.js.map +1 -0
  157. package/dist/throttler.d.ts +2 -2
  158. package/dist/throttler.d.ts.map +1 -1
  159. package/dist/throttler.js +4 -4
  160. package/dist/throttler.js.map +1 -1
  161. package/lib/batchTracker.d.ts.map +1 -1
  162. package/lib/batchTracker.js +2 -1
  163. package/lib/batchTracker.js.map +1 -1
  164. package/lib/blobManager.d.ts +15 -2
  165. package/lib/blobManager.d.ts.map +1 -1
  166. package/lib/blobManager.js +109 -37
  167. package/lib/blobManager.js.map +1 -1
  168. package/lib/connectionTelemetry.d.ts.map +1 -1
  169. package/lib/connectionTelemetry.js +11 -9
  170. package/lib/connectionTelemetry.js.map +1 -1
  171. package/lib/containerHandleContext.d.ts.map +1 -1
  172. package/lib/containerHandleContext.js +3 -1
  173. package/lib/containerHandleContext.js.map +1 -1
  174. package/lib/containerRuntime.d.ts +23 -11
  175. package/lib/containerRuntime.d.ts.map +1 -1
  176. package/lib/containerRuntime.js +201 -108
  177. package/lib/containerRuntime.js.map +1 -1
  178. package/lib/dataStore.d.ts.map +1 -1
  179. package/lib/dataStore.js +11 -9
  180. package/lib/dataStore.js.map +1 -1
  181. package/lib/dataStoreContext.d.ts +27 -13
  182. package/lib/dataStoreContext.d.ts.map +1 -1
  183. package/lib/dataStoreContext.js +84 -45
  184. package/lib/dataStoreContext.js.map +1 -1
  185. package/lib/dataStoreContexts.d.ts.map +1 -1
  186. package/lib/dataStoreContexts.js +7 -3
  187. package/lib/dataStoreContexts.js.map +1 -1
  188. package/lib/dataStoreRegistry.d.ts.map +1 -1
  189. package/lib/dataStoreRegistry.js +3 -1
  190. package/lib/dataStoreRegistry.js.map +1 -1
  191. package/lib/dataStores.d.ts +28 -4
  192. package/lib/dataStores.d.ts.map +1 -1
  193. package/lib/dataStores.js +120 -42
  194. package/lib/dataStores.js.map +1 -1
  195. package/lib/deltaScheduler.d.ts.map +1 -1
  196. package/lib/deltaScheduler.js +9 -4
  197. package/lib/deltaScheduler.js.map +1 -1
  198. package/lib/{garbageCollection.d.ts → gc/garbageCollection.d.ts} +27 -203
  199. package/lib/gc/garbageCollection.d.ts.map +1 -0
  200. package/lib/{garbageCollection.js → gc/garbageCollection.js} +190 -379
  201. package/lib/gc/garbageCollection.js.map +1 -0
  202. package/lib/gc/gcDefinitions.d.ts +189 -0
  203. package/lib/gc/gcDefinitions.d.ts.map +1 -0
  204. package/lib/{garbageCollectionConstants.js → gc/gcDefinitions.js} +26 -1
  205. package/lib/gc/gcDefinitions.js.map +1 -0
  206. package/lib/gc/gcHelpers.d.ts +30 -0
  207. package/lib/gc/gcHelpers.d.ts.map +1 -0
  208. package/lib/gc/gcHelpers.js +58 -0
  209. package/lib/gc/gcHelpers.js.map +1 -0
  210. package/lib/gc/gcSummaryStateTracker.d.ts +86 -0
  211. package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -0
  212. package/lib/gc/gcSummaryStateTracker.js +242 -0
  213. package/lib/gc/gcSummaryStateTracker.js.map +1 -0
  214. package/{dist → lib/gc}/gcSweepReadyUsageDetection.d.ts +5 -5
  215. package/lib/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
  216. package/lib/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +15 -11
  217. package/lib/gc/gcSweepReadyUsageDetection.js.map +1 -0
  218. package/lib/gc/gcUnreferencedStateTracker.d.ts +34 -0
  219. package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
  220. package/lib/gc/gcUnreferencedStateTracker.js +90 -0
  221. package/lib/gc/gcUnreferencedStateTracker.js.map +1 -0
  222. package/lib/gc/index.d.ts +11 -0
  223. package/lib/gc/index.d.ts.map +1 -0
  224. package/lib/gc/index.js +11 -0
  225. package/lib/gc/index.js.map +1 -0
  226. package/lib/index.d.ts +2 -5
  227. package/lib/index.d.ts.map +1 -1
  228. package/lib/index.js +1 -4
  229. package/lib/index.js.map +1 -1
  230. package/lib/opLifecycle/batchManager.d.ts +2 -13
  231. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  232. package/lib/opLifecycle/batchManager.js +19 -41
  233. package/lib/opLifecycle/batchManager.js.map +1 -1
  234. package/lib/opLifecycle/definitions.d.ts +4 -0
  235. package/lib/opLifecycle/definitions.d.ts.map +1 -1
  236. package/lib/opLifecycle/definitions.js.map +1 -1
  237. package/lib/opLifecycle/index.d.ts.map +1 -1
  238. package/lib/opLifecycle/index.js.map +1 -1
  239. package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
  240. package/lib/opLifecycle/opCompressor.js +1 -0
  241. package/lib/opLifecycle/opCompressor.js.map +1 -1
  242. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
  243. package/lib/opLifecycle/opDecompressor.js +5 -2
  244. package/lib/opLifecycle/opDecompressor.js.map +1 -1
  245. package/lib/opLifecycle/opSplitter.d.ts +1 -1
  246. package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
  247. package/lib/opLifecycle/opSplitter.js +25 -14
  248. package/lib/opLifecycle/opSplitter.js.map +1 -1
  249. package/lib/opLifecycle/outbox.d.ts +19 -3
  250. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  251. package/lib/opLifecycle/outbox.js +79 -46
  252. package/lib/opLifecycle/outbox.js.map +1 -1
  253. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  254. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
  255. package/lib/opProperties.d.ts.map +1 -1
  256. package/lib/opProperties.js +1 -3
  257. package/lib/opProperties.js.map +1 -1
  258. package/lib/packageVersion.d.ts +1 -1
  259. package/lib/packageVersion.js +1 -1
  260. package/lib/packageVersion.js.map +1 -1
  261. package/lib/pendingStateManager.d.ts +8 -2
  262. package/lib/pendingStateManager.d.ts.map +1 -1
  263. package/lib/pendingStateManager.js +21 -13
  264. package/lib/pendingStateManager.js.map +1 -1
  265. package/lib/scheduleManager.d.ts.map +1 -1
  266. package/lib/scheduleManager.js +3 -2
  267. package/lib/scheduleManager.js.map +1 -1
  268. package/lib/serializedSnapshotStorage.d.ts +2 -2
  269. package/lib/serializedSnapshotStorage.d.ts.map +1 -1
  270. package/lib/serializedSnapshotStorage.js +5 -3
  271. package/lib/serializedSnapshotStorage.js.map +1 -1
  272. package/lib/summary/index.d.ts +17 -0
  273. package/lib/summary/index.d.ts.map +1 -0
  274. package/lib/summary/index.js +16 -0
  275. package/lib/summary/index.js.map +1 -0
  276. package/lib/summary/orderedClientElection.d.ts.map +1 -0
  277. package/lib/{orderedClientElection.js → summary/orderedClientElection.js} +10 -4
  278. package/lib/summary/orderedClientElection.js.map +1 -0
  279. package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
  280. package/lib/summary/runWhileConnectedCoordinator.js.map +1 -0
  281. package/{dist → lib/summary}/runningSummarizer.d.ts +19 -18
  282. package/lib/summary/runningSummarizer.d.ts.map +1 -0
  283. package/lib/{runningSummarizer.js → summary/runningSummarizer.js} +193 -78
  284. package/lib/summary/runningSummarizer.js.map +1 -0
  285. package/lib/{summarizer.d.ts → summary/summarizer.d.ts} +4 -6
  286. package/lib/summary/summarizer.d.ts.map +1 -0
  287. package/lib/{summarizer.js → summary/summarizer.js} +33 -73
  288. package/lib/summary/summarizer.js.map +1 -0
  289. package/lib/summary/summarizerClientElection.d.ts.map +1 -0
  290. package/lib/summary/summarizerClientElection.js.map +1 -0
  291. package/lib/summary/summarizerHandle.d.ts.map +1 -0
  292. package/lib/summary/summarizerHandle.js.map +1 -0
  293. package/{dist → lib/summary}/summarizerHeuristics.d.ts +1 -1
  294. package/lib/summary/summarizerHeuristics.d.ts.map +1 -0
  295. package/lib/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +6 -9
  296. package/lib/summary/summarizerHeuristics.js.map +1 -0
  297. package/{dist → lib/summary}/summarizerTypes.d.ts +22 -22
  298. package/lib/summary/summarizerTypes.d.ts.map +1 -0
  299. package/lib/summary/summarizerTypes.js.map +1 -0
  300. package/lib/summary/summaryCollection.d.ts.map +1 -0
  301. package/lib/{summaryCollection.js → summary/summaryCollection.js} +18 -8
  302. package/lib/summary/summaryCollection.js.map +1 -0
  303. package/{dist → lib/summary}/summaryFormat.d.ts +1 -40
  304. package/lib/summary/summaryFormat.d.ts.map +1 -0
  305. package/lib/{summaryFormat.js → summary/summaryFormat.js} +20 -20
  306. package/lib/summary/summaryFormat.js.map +1 -0
  307. package/lib/summary/summaryGenerator.d.ts.map +1 -0
  308. package/lib/{summaryGenerator.js → summary/summaryGenerator.js} +33 -15
  309. package/lib/summary/summaryGenerator.js.map +1 -0
  310. package/lib/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
  311. package/lib/summary/summaryManager.d.ts.map +1 -0
  312. package/lib/{summaryManager.js → summary/summaryManager.js} +21 -9
  313. package/lib/summary/summaryManager.js.map +1 -0
  314. package/lib/throttler.d.ts +2 -2
  315. package/lib/throttler.d.ts.map +1 -1
  316. package/lib/throttler.js +4 -4
  317. package/lib/throttler.js.map +1 -1
  318. package/package.json +60 -51
  319. package/prettier.config.cjs +1 -1
  320. package/src/batchTracker.ts +54 -49
  321. package/src/blobManager.ts +825 -674
  322. package/src/connectionTelemetry.ts +280 -249
  323. package/src/containerHandleContext.ts +27 -29
  324. package/src/containerRuntime.ts +3249 -2978
  325. package/src/dataStore.ts +172 -159
  326. package/src/dataStoreContext.ts +1141 -1057
  327. package/src/dataStoreContexts.ts +178 -161
  328. package/src/dataStoreRegistry.ts +25 -20
  329. package/src/dataStores.ts +880 -731
  330. package/src/deltaScheduler.ts +158 -150
  331. package/{garbageCollection.md → src/gc/garbageCollection.md} +16 -3
  332. package/src/gc/garbageCollection.ts +1506 -0
  333. package/src/gc/gcDefinitions.ts +244 -0
  334. package/src/gc/gcHelpers.ts +86 -0
  335. package/src/gc/gcSummaryStateTracker.ts +339 -0
  336. package/src/gc/gcSweepReadyUsageDetection.ts +145 -0
  337. package/src/gc/gcUnreferencedStateTracker.ts +114 -0
  338. package/src/gc/index.ts +40 -0
  339. package/src/index.ts +67 -70
  340. package/src/opLifecycle/README.md +152 -0
  341. package/src/opLifecycle/batchManager.ts +101 -144
  342. package/src/opLifecycle/definitions.ts +33 -29
  343. package/src/opLifecycle/index.ts +5 -5
  344. package/src/opLifecycle/opCompressor.ts +55 -53
  345. package/src/opLifecycle/opDecompressor.ts +100 -81
  346. package/src/opLifecycle/opSplitter.ts +233 -188
  347. package/src/opLifecycle/outbox.ts +251 -195
  348. package/src/opLifecycle/remoteMessageProcessor.ts +62 -62
  349. package/src/opProperties.ts +11 -9
  350. package/src/packageVersion.ts +1 -1
  351. package/src/pendingStateManager.ts +396 -338
  352. package/src/scheduleManager.ts +299 -269
  353. package/src/serializedSnapshotStorage.ts +126 -112
  354. package/src/summary/index.ts +99 -0
  355. package/src/summary/orderedClientElection.ts +564 -0
  356. package/src/summary/runWhileConnectedCoordinator.ts +113 -0
  357. package/src/summary/runningSummarizer.ts +788 -0
  358. package/src/summary/summarizer.ts +386 -0
  359. package/src/summary/summarizerClientElection.ts +139 -0
  360. package/src/{summarizerHandle.ts → summary/summarizerHandle.ts} +11 -9
  361. package/src/summary/summarizerHeuristics.ts +219 -0
  362. package/src/summary/summarizerTypes.ts +521 -0
  363. package/src/summary/summaryCollection.ts +450 -0
  364. package/src/summary/summaryFormat.ts +226 -0
  365. package/src/summary/summaryGenerator.ts +505 -0
  366. package/src/summary/summaryManager.ts +423 -0
  367. package/src/throttler.ts +131 -122
  368. package/tsconfig.esnext.json +6 -6
  369. package/tsconfig.json +9 -13
  370. package/dist/garbageCollection.d.ts.map +0 -1
  371. package/dist/garbageCollection.js.map +0 -1
  372. package/dist/garbageCollectionConstants.d.ts +0 -25
  373. package/dist/garbageCollectionConstants.d.ts.map +0 -1
  374. package/dist/garbageCollectionConstants.js.map +0 -1
  375. package/dist/garbageCollectionTombstoneUtils.d.ts +0 -14
  376. package/dist/garbageCollectionTombstoneUtils.d.ts.map +0 -1
  377. package/dist/garbageCollectionTombstoneUtils.js +0 -23
  378. package/dist/garbageCollectionTombstoneUtils.js.map +0 -1
  379. package/dist/gcSweepReadyUsageDetection.d.ts.map +0 -1
  380. package/dist/gcSweepReadyUsageDetection.js.map +0 -1
  381. package/dist/orderedClientElection.d.ts.map +0 -1
  382. package/dist/orderedClientElection.js.map +0 -1
  383. package/dist/runWhileConnectedCoordinator.d.ts.map +0 -1
  384. package/dist/runWhileConnectedCoordinator.js.map +0 -1
  385. package/dist/runningSummarizer.d.ts.map +0 -1
  386. package/dist/runningSummarizer.js.map +0 -1
  387. package/dist/summarizer.d.ts.map +0 -1
  388. package/dist/summarizer.js.map +0 -1
  389. package/dist/summarizerClientElection.d.ts.map +0 -1
  390. package/dist/summarizerClientElection.js.map +0 -1
  391. package/dist/summarizerHandle.d.ts.map +0 -1
  392. package/dist/summarizerHandle.js.map +0 -1
  393. package/dist/summarizerHeuristics.d.ts.map +0 -1
  394. package/dist/summarizerHeuristics.js.map +0 -1
  395. package/dist/summarizerTypes.d.ts.map +0 -1
  396. package/dist/summarizerTypes.js.map +0 -1
  397. package/dist/summaryCollection.d.ts.map +0 -1
  398. package/dist/summaryCollection.js.map +0 -1
  399. package/dist/summaryFormat.d.ts.map +0 -1
  400. package/dist/summaryFormat.js.map +0 -1
  401. package/dist/summaryGenerator.d.ts.map +0 -1
  402. package/dist/summaryGenerator.js.map +0 -1
  403. package/dist/summaryManager.d.ts.map +0 -1
  404. package/dist/summaryManager.js.map +0 -1
  405. package/lib/garbageCollection.d.ts.map +0 -1
  406. package/lib/garbageCollection.js.map +0 -1
  407. package/lib/garbageCollectionConstants.d.ts +0 -25
  408. package/lib/garbageCollectionConstants.d.ts.map +0 -1
  409. package/lib/garbageCollectionConstants.js.map +0 -1
  410. package/lib/garbageCollectionTombstoneUtils.d.ts +0 -14
  411. package/lib/garbageCollectionTombstoneUtils.d.ts.map +0 -1
  412. package/lib/garbageCollectionTombstoneUtils.js +0 -19
  413. package/lib/garbageCollectionTombstoneUtils.js.map +0 -1
  414. package/lib/gcSweepReadyUsageDetection.d.ts.map +0 -1
  415. package/lib/gcSweepReadyUsageDetection.js.map +0 -1
  416. package/lib/orderedClientElection.d.ts.map +0 -1
  417. package/lib/orderedClientElection.js.map +0 -1
  418. package/lib/runWhileConnectedCoordinator.d.ts.map +0 -1
  419. package/lib/runWhileConnectedCoordinator.js.map +0 -1
  420. package/lib/runningSummarizer.d.ts.map +0 -1
  421. package/lib/runningSummarizer.js.map +0 -1
  422. package/lib/summarizer.d.ts.map +0 -1
  423. package/lib/summarizer.js.map +0 -1
  424. package/lib/summarizerClientElection.d.ts.map +0 -1
  425. package/lib/summarizerClientElection.js.map +0 -1
  426. package/lib/summarizerHandle.d.ts.map +0 -1
  427. package/lib/summarizerHandle.js.map +0 -1
  428. package/lib/summarizerHeuristics.d.ts.map +0 -1
  429. package/lib/summarizerHeuristics.js.map +0 -1
  430. package/lib/summarizerTypes.d.ts.map +0 -1
  431. package/lib/summarizerTypes.js.map +0 -1
  432. package/lib/summaryCollection.d.ts.map +0 -1
  433. package/lib/summaryCollection.js.map +0 -1
  434. package/lib/summaryFormat.d.ts.map +0 -1
  435. package/lib/summaryFormat.js.map +0 -1
  436. package/lib/summaryGenerator.d.ts.map +0 -1
  437. package/lib/summaryGenerator.js.map +0 -1
  438. package/lib/summaryManager.d.ts.map +0 -1
  439. package/lib/summaryManager.js.map +0 -1
  440. package/src/garbageCollection.ts +0 -1800
  441. package/src/garbageCollectionConstants.ts +0 -41
  442. package/src/garbageCollectionTombstoneUtils.ts +0 -28
  443. package/src/gcSweepReadyUsageDetection.ts +0 -139
  444. package/src/orderedClientElection.ts +0 -532
  445. package/src/runWhileConnectedCoordinator.ts +0 -106
  446. package/src/runningSummarizer.ts +0 -610
  447. package/src/summarizer.ts +0 -421
  448. package/src/summarizerClientElection.ts +0 -132
  449. package/src/summarizerHeuristics.ts +0 -222
  450. package/src/summarizerTypes.ts +0 -507
  451. package/src/summaryCollection.ts +0 -421
  452. package/src/summaryFormat.ts +0 -256
  453. package/src/summaryGenerator.ts +0 -450
  454. package/src/summaryManager.ts +0 -394
  455. /package/dist/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
  456. /package/dist/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +0 -0
  457. /package/dist/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +0 -0
  458. /package/dist/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
  459. /package/dist/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
  460. /package/dist/{summarizerHandle.d.ts → summary/summarizerHandle.d.ts} +0 -0
  461. /package/dist/{summarizerHandle.js → summary/summarizerHandle.js} +0 -0
  462. /package/dist/{summarizerTypes.js → summary/summarizerTypes.js} +0 -0
  463. /package/dist/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
  464. /package/dist/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
  465. /package/lib/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
  466. /package/lib/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +0 -0
  467. /package/lib/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +0 -0
  468. /package/lib/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
  469. /package/lib/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
  470. /package/lib/{summarizerHandle.d.ts → summary/summarizerHandle.d.ts} +0 -0
  471. /package/lib/{summarizerHandle.js → summary/summarizerHandle.js} +0 -0
  472. /package/lib/{summarizerTypes.js → summary/summarizerTypes.js} +0 -0
  473. /package/lib/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
  474. /package/lib/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
@@ -14,105 +14,19 @@ var __rest = (this && this.__rest) || function (s, e) {
14
14
  return t;
15
15
  };
16
16
  import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
17
- import { ClientSessionExpiredError, DataProcessingError, UsageError } from "@fluidframework/container-utils";
17
+ import { ClientSessionExpiredError, DataProcessingError, UsageError, } from "@fluidframework/container-utils";
18
18
  import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot, runGarbageCollection, trimLeadingSlashes, } from "@fluidframework/garbage-collector";
19
- import { SummaryType } from "@fluidframework/protocol-definitions";
20
- import { gcTreeKey, gcBlobPrefix, gcTombstoneBlobKey, gcDeletedBlobKey, } from "@fluidframework/runtime-definitions";
21
- import { mergeStats, packagePathToTelemetryProperty, SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
19
+ import { gcTreeKey, } from "@fluidframework/runtime-definitions";
20
+ import { packagePathToTelemetryProperty, } from "@fluidframework/runtime-utils";
22
21
  import { ChildLogger, generateStack, loggerToMonitoringContext, PerformanceEvent, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
23
- import { RuntimeHeaders } from "./containerRuntime";
24
- import { getSummaryForDatastores } from "./dataStores";
25
- import { currentGCVersion, defaultInactiveTimeoutMs, defaultSessionExpiryDurationMs, disableSweepLogKey, disableTombstoneKey, gcVersionUpgradeToV2Key, gcTestModeKey, oneDayMs, runGCKey, runSessionExpiryKey, runSweepKey, stableGCVersion, trackGCStateKey, gcTombstoneGenerationOptionName } from "./garbageCollectionConstants";
26
- import { sendGCTombstoneEvent } from "./garbageCollectionTombstoneUtils";
22
+ import { RuntimeHeaders } from "../containerRuntime";
23
+ import { getSummaryForDatastores } from "../dataStores";
24
+ import { dataStoreAttributesBlobName, } from "../summary";
25
+ import { defaultInactiveTimeoutMs, defaultSessionExpiryDurationMs, disableSweepLogKey, disableTombstoneKey, GCNodeType, gcTestModeKey, oneDayMs, runGCKey, runSessionExpiryKey, runSweepKey, trackGCStateKey, gcTombstoneGenerationOptionName, UnreferencedState, } from "./gcDefinitions";
26
+ import { getGCVersion, sendGCUnexpectedUsageEvent } from "./gcHelpers";
27
+ import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
27
28
  import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
28
- import { getGCVersion, metadataBlobName, dataStoreAttributesBlobName, } from "./summaryFormat";
29
- /** The types of GC nodes in the GC reference graph. */
30
- export const GCNodeType = {
31
- // Nodes that are for data stores.
32
- DataStore: "DataStore",
33
- // Nodes that are within a data store. For example, DDS nodes.
34
- SubDataStore: "SubDataStore",
35
- // Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
36
- Blob: "Blob",
37
- // Nodes that are neither of the above. For example, root node.
38
- Other: "Other",
39
- };
40
- /** The state of node that is unreferenced. */
41
- export const UnreferencedState = {
42
- /** The node is active, i.e., it can become referenced again. */
43
- Active: "Active",
44
- /** The node is inactive, i.e., it should not become referenced. */
45
- Inactive: "Inactive",
46
- /** The node is ready to be deleted by the sweep phase. */
47
- SweepReady: "SweepReady",
48
- };
49
- /**
50
- * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
51
- * be deleted by the sweep phase.
52
- */
53
- export class UnreferencedStateTracker {
54
- constructor(unreferencedTimestampMs,
55
- /** The time after which node transitions to Inactive state. */
56
- inactiveTimeoutMs,
57
- /** The current reference timestamp used to track how long this node has been unreferenced for. */
58
- currentReferenceTimestampMs,
59
- /** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
60
- sweepTimeoutMs) {
61
- this.unreferencedTimestampMs = unreferencedTimestampMs;
62
- this.inactiveTimeoutMs = inactiveTimeoutMs;
63
- this.sweepTimeoutMs = sweepTimeoutMs;
64
- this._state = UnreferencedState.Active;
65
- if (this.sweepTimeoutMs !== undefined) {
66
- assert(this.inactiveTimeoutMs <= this.sweepTimeoutMs, 0x3b0 /* inactive timeout must not be greater than the sweep timeout */);
67
- }
68
- this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
69
- this._state = UnreferencedState.SweepReady;
70
- assert(!this.inactiveTimer.hasTimer, 0x3b1 /* inactiveTimer still running after sweepTimer fired! */);
71
- });
72
- this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
73
- this._state = UnreferencedState.Inactive;
74
- // After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
75
- if (this.sweepTimeoutMs !== undefined) {
76
- this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
77
- }
78
- });
79
- this.updateTracking(currentReferenceTimestampMs);
80
- }
81
- get state() {
82
- return this._state;
83
- }
84
- /* Updates the unreferenced state based on the provided timestamp. */
85
- updateTracking(currentReferenceTimestampMs) {
86
- const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
87
- // If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
88
- if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
89
- this._state = UnreferencedState.SweepReady;
90
- this.clearTimers();
91
- return;
92
- }
93
- // If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
94
- // Also, start a timer for the sweep timeout.
95
- if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
96
- this._state = UnreferencedState.Inactive;
97
- this.inactiveTimer.clear();
98
- if (this.sweepTimeoutMs !== undefined) {
99
- this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
100
- }
101
- return;
102
- }
103
- // The node is still active. Ensure the inactive timer is running with the proper remaining duration.
104
- this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
105
- }
106
- clearTimers() {
107
- this.inactiveTimer.clear();
108
- this.sweepTimer.clear();
109
- }
110
- /** Stop tracking this node. Reset the unreferenced timers and state, if any. */
111
- stopTracking() {
112
- this.clearTimers();
113
- this._state = UnreferencedState.Active;
114
- }
115
- }
29
+ import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
116
30
  /**
117
31
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
118
32
  * its state across summaries.
@@ -124,15 +38,15 @@ export class UnreferencedStateTracker {
124
38
  * Graph - all nodes with their respective routes
125
39
  *
126
40
  * ```
127
- * GC Graph
41
+ * GC Graph
128
42
  *
129
- * Node
130
- * NodeId = "datastore1"
131
- * / \\
132
- * OutboundRoute OutboundRoute
133
- * / \\
134
- * Node Node
135
- * NodeId = "dds1" NodeId = "dds2"
43
+ * Node
44
+ * NodeId = "datastore1"
45
+ * / \\
46
+ * OutboundRoute OutboundRoute
47
+ * / \\
48
+ * Node Node
49
+ * NodeId = "dds1" NodeId = "dds2"
136
50
  * ```
137
51
  */
138
52
  export class GarbageCollector {
@@ -164,12 +78,11 @@ export class GarbageCollector {
164
78
  const baseSnapshot = createParams.baseSnapshot;
165
79
  const metadata = createParams.metadata;
166
80
  const readAndParseBlob = createParams.readAndParseBlob;
167
- this.mc = loggerToMonitoringContext(ChildLogger.create(createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } }));
168
- // If version upgrade is not enabled, fall back to the stable GC version.
169
- this.currentGCVersion =
170
- this.mc.config.getBoolean(gcVersionUpgradeToV2Key) === true ? currentGCVersion : stableGCVersion;
81
+ this.mc = loggerToMonitoringContext(ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
82
+ all: { completedGCRuns: () => this.completedRuns },
83
+ }));
171
84
  this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
172
- let prevSummaryGCVersion;
85
+ let gcVersionInBaseSnapshot;
173
86
  /**
174
87
  * Sweep timeout is the time after which unreferenced content can be swept.
175
88
  * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
@@ -181,8 +94,8 @@ export class GarbageCollector {
181
94
  function computeSweepTimeout(sessionExpiryTimeoutMs) {
182
95
  const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
183
96
  const bufferMs = oneDayMs;
184
- return sessionExpiryTimeoutMs &&
185
- (sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
97
+ return (sessionExpiryTimeoutMs &&
98
+ sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
186
99
  }
187
100
  /**
188
101
  * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
@@ -192,10 +105,10 @@ export class GarbageCollector {
192
105
  * For existing containers, we get this information from the metadata blob of its summary.
193
106
  */
194
107
  if (createParams.existing) {
195
- prevSummaryGCVersion = getGCVersion(metadata);
108
+ gcVersionInBaseSnapshot = getGCVersion(metadata);
196
109
  // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
197
110
  // other existing documents, GC is enabled.
198
- this.gcEnabled = prevSummaryGCVersion > 0;
111
+ this.gcEnabled = gcVersionInBaseSnapshot > 0;
199
112
  this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
200
113
  this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
201
114
  this.sweepTimeoutMs =
@@ -217,7 +130,8 @@ export class GarbageCollector {
217
130
  this.sweepEnabled = this.gcOptions.sweepAllowed === true;
218
131
  // Set the Session Expiry only if the flag is enabled and GC is enabled.
219
132
  if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
220
- this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : defaultSessionExpiryDurationMs;
133
+ this.sessionExpiryTimeoutMs =
134
+ (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : defaultSessionExpiryDurationMs;
221
135
  }
222
136
  this.sweepTimeoutMs =
223
137
  testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
@@ -232,12 +146,11 @@ export class GarbageCollector {
232
146
  // If Test Override config is set, override Session Expiry timeout.
233
147
  const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
234
148
  const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
235
- this.sessionExpiryTimer = new Timer(timeoutMs, () => { this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs)); });
149
+ this.sessionExpiryTimer = new Timer(timeoutMs, () => {
150
+ this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
151
+ });
236
152
  this.sessionExpiryTimer.start();
237
153
  }
238
- // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
239
- // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
240
- this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
241
154
  /**
242
155
  * Whether GC should run or not. The following conditions have to be met to run sweep:
243
156
  *
@@ -247,11 +160,12 @@ export class GarbageCollector {
247
160
  *
248
161
  * These conditions can be overridden via runGCKey feature flag.
249
162
  */
250
- this.shouldRunGC = (_d = this.mc.config.getBoolean(runGCKey)) !== null && _d !== void 0 ? _d : (
251
- // GC must be enabled for the document.
252
- this.gcEnabled
253
- // GC must not be disabled via GC options.
254
- && !this.gcOptions.disableGC);
163
+ this.shouldRunGC =
164
+ (_d = this.mc.config.getBoolean(runGCKey)) !== null && _d !== void 0 ? _d :
165
+ // GC must be enabled for the document.
166
+ (this.gcEnabled &&
167
+ // GC must not be disabled via GC options.
168
+ !this.gcOptions.disableGC);
255
169
  /**
256
170
  * Whether sweep should run or not. The following conditions have to be met to run sweep:
257
171
  *
@@ -263,23 +177,25 @@ export class GarbageCollector {
263
177
  * feature flag.
264
178
  */
265
179
  this.shouldRunSweep =
266
- this.shouldRunGC
267
- && this.sweepTimeoutMs !== undefined
268
- && ((_e = this.mc.config.getBoolean(runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
180
+ this.shouldRunGC &&
181
+ this.sweepTimeoutMs !== undefined &&
182
+ ((_e = this.mc.config.getBoolean(runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
269
183
  this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
270
184
  // Override inactive timeout if test config or gc options to override it is set.
271
- this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : defaultInactiveTimeoutMs;
185
+ this.inactiveTimeoutMs =
186
+ (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : defaultInactiveTimeoutMs;
272
187
  // Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
273
188
  if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
274
189
  throw new UsageError("inactive timeout should not be greater than the sweep timeout");
275
190
  }
276
191
  // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
277
- this.testMode = (_h = this.mc.config.getBoolean(gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
192
+ this.testMode =
193
+ (_h = this.mc.config.getBoolean(gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
278
194
  // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
279
195
  // via feature flags.
280
- this.tombstoneMode = !this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
281
- // If GC ran in the container that generated the base snapshot, it will have a GC tree.
282
- this.wasGCRunInLatestSummary = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined;
196
+ this.tombstoneMode =
197
+ !this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
198
+ this.summaryStateTracker = new GCSummaryStateTracker(this.shouldRunGC, this.trackGCState, this.tombstoneMode, this.mc, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */, gcVersionInBaseSnapshot);
283
199
  // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
284
200
  // it involves fetching blobs from storage which is expensive.
285
201
  this.baseSnapshotDataP = new LazyPromise(async () => {
@@ -296,7 +212,9 @@ export class GarbageCollector {
296
212
  // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
297
213
  // consolidate into IGarbageCollectionState format.
298
214
  // Add a node for the root node that is not present in older snapshot format.
299
- const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
215
+ const gcState = {
216
+ gcNodes: { "/": { outboundRoutes: [] } },
217
+ };
300
218
  const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
301
219
  assert(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
302
220
  for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
@@ -320,10 +238,13 @@ export class GarbageCollector {
320
238
  // Prefix the data store id to the GC node ids to make them relative to the root from being
321
239
  // relative to the data store. Similar to how its done in DataStore::getGCData.
322
240
  const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
323
- gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
241
+ gcState.gcNodes[rootId] = {
242
+ outboundRoutes: Array.from(outboundRoutes),
243
+ };
324
244
  }
325
245
  assert(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
326
- gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
246
+ gcState.gcNodes[dsRootId].unreferencedTimestampMs =
247
+ gcSummaryDetails.unrefTimestamp;
327
248
  }
328
249
  // If there is only one node (root node just added above), either GC is disabled or we are loading from
329
250
  // the first summary generated by detached container. In both cases, GC was not run - return undefined.
@@ -401,46 +322,13 @@ export class GarbageCollector {
401
322
  static create(createParams) {
402
323
  return new GarbageCollector(createParams);
403
324
  }
404
- /**
405
- * Tells whether the GC state needs to be reset in the next summary. We need to do this if:
406
- *
407
- * 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
408
- *
409
- * 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
410
- *
411
- * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
412
- *
413
- * 4. The GC version in the latest summary is different from the current GC version. This can happen if:
414
- *
415
- * 4.1. The summary this client loaded with has data from a different GC version.
416
- *
417
- * 4.2. This client's latest summary was updated from a snapshot that has a different GC version.
418
- */
419
- get summaryStateNeedsReset() {
420
- return this.gcStateNeedsReset ||
421
- (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion);
422
- }
423
- /**
424
- * Tells whether the GC state needs to be reset. This can happen under 3 conditions:
425
- *
426
- * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
427
- * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
428
- *
429
- * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
430
- * a document and the first time GC is enabled after is was disabled before.
431
- *
432
- * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
433
- *
434
- * Note that the state will be reset only once for the first summary generated after this returns true. After that,
435
- * this will return false.
436
- */
437
- get gcStateNeedsReset() {
438
- return this.wasGCRunInLatestSummary !== this.shouldRunGC;
439
- }
440
325
  /** Returns a list of all the configurations for garbage collection. */
441
326
  get configs() {
442
327
  return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
443
328
  }
329
+ get summaryStateNeedsReset() {
330
+ return this.summaryStateTracker.doesSummaryStateNeedReset();
331
+ }
444
332
  /**
445
333
  * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
446
334
  * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
@@ -464,6 +352,7 @@ export class GarbageCollector {
464
352
  // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
465
353
  // tombstone routes.
466
354
  if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
355
+ // Create a copy since we are writing from a source we don't control
467
356
  this.tombstones = Array.from(baseSnapshotData.tombstones);
468
357
  this.runtime.updateTombstonedRoutes(this.tombstones);
469
358
  }
@@ -491,7 +380,6 @@ export class GarbageCollector {
491
380
  for (const [, nodeStateTracker] of this.unreferencedNodesState) {
492
381
  nodeStateTracker.stopTracking();
493
382
  }
494
- ;
495
383
  this.unreferencedNodesState.clear();
496
384
  // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
497
385
  // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
@@ -499,7 +387,9 @@ export class GarbageCollector {
499
387
  // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
500
388
  // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
501
389
  if (this.shouldRunSweep) {
502
- const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? new Set(snapshotData.tombstones) : undefined;
390
+ const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.deletedNodes)
391
+ ? new Set(snapshotData.deletedNodes)
392
+ : undefined;
503
393
  // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
504
394
  // delete them.
505
395
  if (snapshotDeletedNodes !== undefined) {
@@ -520,10 +410,11 @@ export class GarbageCollector {
520
410
  this.tombstones = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? Array.from(snapshotData.tombstones) : [];
521
411
  this.runtime.updateTombstonedRoutes(this.tombstones);
522
412
  }
413
+ // Update the summary state tracker's state from this snapshot.
414
+ this.summaryStateTracker.updateStateFromSnapshotData(snapshotData);
523
415
  // If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
524
416
  if (snapshotData === undefined) {
525
417
  this.gcDataFromLastRun = undefined;
526
- this.latestSummaryData = undefined;
527
418
  return;
528
419
  }
529
420
  // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
@@ -536,14 +427,6 @@ export class GarbageCollector {
536
427
  gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
537
428
  }
538
429
  this.gcDataFromLastRun = { gcNodes };
539
- // If tracking state across summaries, update latest summary data from the snapshot's GC data.
540
- if (this.trackGCState) {
541
- this.latestSummaryData = {
542
- serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
543
- serializedTombstones: JSON.stringify(snapshotData.tombstones),
544
- serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
545
- };
546
- }
547
430
  }
548
431
  /**
549
432
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
@@ -575,9 +458,12 @@ export class GarbageCollector {
575
458
  */
576
459
  async collectGarbage(options, telemetryContext) {
577
460
  var _a;
578
- const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
461
+ const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true ||
462
+ this.summaryStateTracker.doesSummaryStateNeedReset());
579
463
  const logger = options.logger
580
- ? ChildLogger.create(options.logger, undefined, { all: { completedGCRuns: () => this.completedRuns } })
464
+ ? ChildLogger.create(options.logger, undefined, {
465
+ all: { completedGCRuns: () => this.completedRuns },
466
+ })
581
467
  : this.mc.logger;
582
468
  /**
583
469
  * If there is no current reference timestamp, skip running GC. We need the current timestamp to track
@@ -597,7 +483,10 @@ export class GarbageCollector {
597
483
  return undefined;
598
484
  }
599
485
  // Add the options that are used to run GC to the telemetry context.
600
- telemetryContext === null || telemetryContext === void 0 ? void 0 : telemetryContext.setAll("fluid_GC", "Options", { fullGC, runSweep: options.runSweep });
486
+ telemetryContext === null || telemetryContext === void 0 ? void 0 : telemetryContext.setMultiple("fluid_GC", "Options", {
487
+ fullGC,
488
+ runSweep: options.runSweep,
489
+ });
601
490
  return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
602
491
  await this.runPreGCSteps();
603
492
  // Get the runtime's GC data and run GC on the reference graph in it.
@@ -619,26 +508,29 @@ export class GarbageCollector {
619
508
  // Generate statistics from the current run. This is done before updating the current state because it
620
509
  // generates some of its data based on previous state of the system.
621
510
  const gcStats = this.generateStats(gcResult);
622
- // Update the state since the last GC run. There can be nodes that were referenced between the last and
623
- // the current run. We need to identify than and update their unreferenced state if needed.
624
- this.updateStateSinceLastRun(gcData, logger);
625
- // Update the current state and update the runtime of all routes or ids that used as per the GC run.
626
- this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
511
+ // Update the current mark state and update the runtime of all used routes or ids that used as per the GC run.
512
+ const sweepReadyNodes = this.updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger);
627
513
  this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
628
514
  // Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
629
515
  // delete these objects here instead.
630
516
  this.logSweepEvents(logger, currentReferenceTimestampMs);
631
- // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
632
- // involving access to deleted data.
633
- if (this.testMode) {
517
+ let updatedGCData = gcData;
518
+ if (this.shouldRunSweep) {
519
+ updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
520
+ }
521
+ else if (this.testMode) {
522
+ // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
523
+ // involving access to deleted data.
634
524
  this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
635
525
  }
636
526
  else if (this.tombstoneMode) {
527
+ this.tombstones = sweepReadyNodes;
637
528
  // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
638
529
  // involving access to "deleted" data without actually deleting the data from summaries.
639
530
  // Note: we will not tombstone in test mode.
640
531
  this.runtime.updateTombstonedRoutes(this.tombstones);
641
532
  }
533
+ this.gcDataFromLastRun = cloneGCData(updatedGCData);
642
534
  // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
643
535
  // updates its state so that we don't send false positives based on intermediate state. For example, we may get
644
536
  // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
@@ -662,89 +554,7 @@ export class GarbageCollector {
662
554
  unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
663
555
  };
664
556
  }
665
- const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
666
- // Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
667
- // to identify deleted nodes' usage.
668
- const serializedDeletedNodes = this.deletedNodes.size > 0
669
- ? JSON.stringify(Array.from(this.deletedNodes).sort())
670
- : undefined;
671
- // If running in tombstone mode, serialize and write tombstones, if any.
672
- const serializedTombstones = this.tombstoneMode
673
- ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
674
- : undefined;
675
- /**
676
- * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
677
- * write summary handle instead of summary tree for GC.
678
- * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
679
- * for each of these that did not change, write a summary handle.
680
- */
681
- if (this.trackGCState) {
682
- this.pendingSummaryData = { serializedGCState, serializedTombstones, serializedDeletedNodes };
683
- if (trackState && !fullTree && this.latestSummaryData !== undefined) {
684
- // If nothing changed since last summary, send a summary handle for the entire GC data.
685
- if (this.latestSummaryData.serializedGCState === serializedGCState
686
- && this.latestSummaryData.serializedTombstones === serializedTombstones) {
687
- const stats = mergeStats();
688
- stats.handleNodeCount++;
689
- return {
690
- summary: {
691
- type: SummaryType.Handle,
692
- handle: `/${gcTreeKey}`,
693
- handleType: SummaryType.Tree,
694
- },
695
- stats,
696
- };
697
- }
698
- // If some state changed, build a GC summary tree.
699
- return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
700
- }
701
- }
702
- // If not tracking GC state, build a GC summary tree without any summary handles.
703
- return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
704
- }
705
- /**
706
- * Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
707
- * If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
708
- * If trackState is true, only states that changed are written. Rest are written as handles.
709
- * @param serializedGCState - The GC state serialized as string.
710
- * @param serializedTombstones - The tombstone state serialized as string.
711
- * @param serializedDeletedNodes - Deleted nodes serialized as string.
712
- * @param trackState - Whether we are tracking GC state across summaries.
713
- * @returns the GC summary tree.
714
- */
715
- buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, trackState) {
716
- var _a, _b, _c;
717
- const gcStateBlobKey = `${gcBlobPrefix}_root`;
718
- const builder = new SummaryTreeBuilder();
719
- // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
720
- if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
721
- builder.addHandle(gcStateBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcStateBlobKey}`);
722
- }
723
- else {
724
- builder.addBlob(gcStateBlobKey, serializedGCState);
725
- }
726
- // If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
727
- // summary blob.
728
- if (serializedTombstones !== undefined) {
729
- if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones && trackState) {
730
- builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
731
- }
732
- else {
733
- builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
734
- }
735
- }
736
- // If there are no deleted nodes, return the summary tree.
737
- if (serializedDeletedNodes === undefined) {
738
- return builder.getSummaryTree();
739
- }
740
- // If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
741
- if (((_c = this.latestSummaryData) === null || _c === void 0 ? void 0 : _c.serializedDeletedNodes) === serializedDeletedNodes && trackState) {
742
- builder.addHandle(gcDeletedBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcDeletedBlobKey}`);
743
- }
744
- else {
745
- builder.addBlob(gcDeletedBlobKey, serializedDeletedNodes);
746
- }
747
- return builder.getSummaryTree();
557
+ return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
748
558
  }
749
559
  getMetadata() {
750
560
  return {
@@ -752,7 +562,7 @@ export class GarbageCollector {
752
562
  * If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
753
563
  * into the metadata blob. If GC is disabled, the gcFeature is 0.
754
564
  */
755
- gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
565
+ gcFeature: this.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
756
566
  gcFeatureMatrix: this.persistedGcFeatureMatrix,
757
567
  sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
758
568
  sweepEnabled: this.sweepEnabled,
@@ -771,48 +581,22 @@ export class GarbageCollector {
771
581
  * is downloaded and should be used to update the state.
772
582
  */
773
583
  async refreshLatestSummary(proposalHandle, result, readAndParseBlob) {
774
- // If the latest summary was updated and the summary was tracked, this client is the one that generated this
775
- // summary. So, update wasGCRunInLatestSummary.
776
- // Note that this has to be updated if GC did not run too. Otherwise, `gcStateNeedsReset` will always return
777
- // true in scenarios where GC is disabled but enabled in the snapshot we loaded from.
778
- if (result.latestSummaryUpdated && result.wasSummaryTracked) {
779
- this.wasGCRunInLatestSummary = this.shouldRunGC;
780
- }
781
- if (!result.latestSummaryUpdated || !this.shouldRunGC) {
782
- return;
783
- }
784
- // If the summary was tracked by this client, it was the one that generated the summary in the first place.
785
- // Update latest state from pending.
786
- if (result.wasSummaryTracked) {
787
- this.latestSummaryGCVersion = this.currentGCVersion;
788
- if (this.trackGCState) {
789
- this.latestSummaryData = this.pendingSummaryData;
790
- this.pendingSummaryData = undefined;
584
+ const latestSnapshotData = await this.summaryStateTracker.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
585
+ // If the latest summary was updated but it was not tracked by this client, our state needs to be updated from
586
+ // this snapshot data.
587
+ if (this.shouldRunGC && result.latestSummaryUpdated && !result.wasSummaryTracked) {
588
+ // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
589
+ // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
590
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
591
+ if (currentReferenceTimestampMs === undefined) {
592
+ throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
593
+ proposalHandle,
594
+ summaryRefSeq: result.summaryRefSeq,
595
+ details: JSON.stringify(this.configs),
596
+ });
791
597
  }
792
- return;
793
- }
794
- // If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
795
- const snapshotTree = result.snapshotTree;
796
- const metadataBlobId = snapshotTree.blobs[metadataBlobName];
797
- if (metadataBlobId) {
798
- const metadata = await readAndParseBlob(metadataBlobId);
799
- this.latestSummaryGCVersion = getGCVersion(metadata);
800
- }
801
- // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
802
- // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
803
- const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
804
- if (currentReferenceTimestampMs === undefined) {
805
- throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, { proposalHandle, summaryRefSeq: result.summaryRefSeq, details: JSON.stringify(this.configs) });
598
+ this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
806
599
  }
807
- const gcSnapshotTree = snapshotTree.trees[gcTreeKey];
808
- // If GC ran in the container that generated this snapshot, it will have a GC tree.
809
- this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
810
- let latestGCData;
811
- if (gcSnapshotTree !== undefined) {
812
- latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
813
- }
814
- this.updateStateFromSnapshotData(latestGCData, currentReferenceTimestampMs);
815
- this.pendingSummaryData = undefined;
816
600
  }
817
601
  /**
818
602
  * Called when a node with the given id is updated. If the node is inactive, log an error.
@@ -859,12 +643,12 @@ export class GarbageCollector {
859
643
  else if (nodeType === GCNodeType.Blob) {
860
644
  eventName = "GC_Tombstone_Blob_Revived";
861
645
  }
862
- sendGCTombstoneEvent(this.mc, {
646
+ sendGCUnexpectedUsageEvent(this.mc, {
863
647
  eventName,
864
648
  category: "generic",
865
- isSummarizerClient: this.isSummarizerClient,
866
649
  url: trimLeadingSlashes(toNodePath),
867
650
  nodeType,
651
+ gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
868
652
  }, undefined /* packagePath */);
869
653
  }
870
654
  }
@@ -888,13 +672,16 @@ export class GarbageCollector {
888
672
  * @param gcData - The data representing the reference graph on which GC is run.
889
673
  * @param gcResult - The result of the GC run on the gcData.
890
674
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
675
+ * @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
891
676
  */
892
- updateCurrentState(gcData, gcResult, currentReferenceTimestampMs) {
893
- this.gcDataFromLastRun = cloneGCData(gcData);
894
- this.tombstones = [];
677
+ updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger) {
678
+ var _a;
679
+ // Get references from the current GC run + references between previous and current run and then update each
680
+ // node's state
681
+ const allNodesReferencedBetweenGCs = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
895
682
  this.newReferencesSinceLastRun.clear();
896
683
  // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
897
- for (const nodeId of gcResult.referencedNodeIds) {
684
+ for (const nodeId of allNodesReferencedBetweenGCs) {
898
685
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
899
686
  if (nodeStateTracker !== undefined) {
900
687
  // Stop tracking so as to clear out any running timers.
@@ -907,7 +694,10 @@ export class GarbageCollector {
907
694
  * If a node became unreferenced in this run, start tracking it.
908
695
  * If a node was already unreferenced, update its tracking information. Since the current reference time is
909
696
  * from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
697
+ *
698
+ * If a node is sweep ready, store and then return it.
910
699
  */
700
+ const sweepReadyNodes = [];
911
701
  for (const nodeId of gcResult.deletedNodeIds) {
912
702
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
913
703
  if (nodeStateTracker === undefined) {
@@ -915,14 +705,51 @@ export class GarbageCollector {
915
705
  }
916
706
  else {
917
707
  nodeStateTracker.updateTracking(currentReferenceTimestampMs);
918
- if (this.tombstoneMode && nodeStateTracker.state === UnreferencedState.SweepReady) {
919
- const nodeType = this.runtime.getNodeType(nodeId);
920
- if (nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) {
921
- this.tombstones.push(nodeId);
922
- }
708
+ if (nodeStateTracker.state === UnreferencedState.SweepReady) {
709
+ sweepReadyNodes.push(nodeId);
923
710
  }
924
711
  }
925
712
  }
713
+ return sweepReadyNodes;
714
+ }
715
+ /**
716
+ * Deletes nodes from both the runtime and garbage collection
717
+ * @param sweepReadyNodes - nodes that are ready to be deleted
718
+ */
719
+ runSweepPhase(sweepReadyNodes, gcData) {
720
+ // TODO: GC:Validation - validate that removed routes are not double deleted
721
+ // TODO: GC:Validation - validate that the child routes of removed routes are deleted as well
722
+ const sweptRoutes = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
723
+ const updatedGCData = this.deleteSweptRoutes(sweptRoutes, gcData);
724
+ for (const nodeId of sweptRoutes) {
725
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
726
+ // TODO: GC:Validation - assert that the nodeStateTracker is defined
727
+ if (nodeStateTracker !== undefined) {
728
+ // Stop tracking so as to clear out any running timers.
729
+ nodeStateTracker.stopTracking();
730
+ // Delete the node as we don't need to track it any more.
731
+ this.unreferencedNodesState.delete(nodeId);
732
+ }
733
+ // TODO: GC:Validation - assert that the deleted node is not a duplicate
734
+ this.deletedNodes.add(nodeId);
735
+ }
736
+ return updatedGCData;
737
+ }
738
+ /**
739
+ * @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
740
+ */
741
+ deleteSweptRoutes(sweptRoutes, gcData) {
742
+ const sweptRoutesSet = new Set(sweptRoutes);
743
+ const gcNodes = {};
744
+ for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
745
+ if (!sweptRoutesSet.has(id)) {
746
+ gcNodes[id] = Array.from(outboundRoutes);
747
+ }
748
+ }
749
+ // TODO: GC:Validation - assert that the nodeId is in gcData
750
+ return {
751
+ gcNodes,
752
+ };
926
753
  }
927
754
  /**
928
755
  * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
@@ -934,16 +761,19 @@ export class GarbageCollector {
934
761
  * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
935
762
  * unreferenced, they could have been accessed and in-memory reference to them added.
936
763
  *
937
- * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
764
+ * This function identifies nodes that were referenced since the last run.
938
765
  * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
766
+ *
767
+ * @returns - a list of all nodes referenced from the last local summary until now.
939
768
  */
940
- updateStateSinceLastRun(currentGCData, logger) {
769
+ findAllNodesReferencedBetweenGCs(currentGCData, previousGCData, logger) {
941
770
  // If we haven't run GC before there is nothing to do.
942
- if (this.gcDataFromLastRun === undefined) {
943
- return;
771
+ // No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
772
+ if (previousGCData === undefined) {
773
+ return undefined;
944
774
  }
945
775
  // Find any references that haven't been identified correctly.
946
- const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, this.gcDataFromLastRun, this.newReferencesSinceLastRun);
776
+ const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun);
947
777
  if (missingExplicitReferences.length > 0) {
948
778
  missingExplicitReferences.forEach((missingExplicitReference) => {
949
779
  logger.sendErrorEvent({
@@ -954,9 +784,9 @@ export class GarbageCollector {
954
784
  });
955
785
  }
956
786
  // No references were added since the last run so we don't have to update reference states of any unreferenced
957
- // nodes
787
+ // nodes. There is no in between state at this point.
958
788
  if (this.newReferencesSinceLastRun.size === 0) {
959
- return;
789
+ return undefined;
960
790
  }
961
791
  /**
962
792
  * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
@@ -974,7 +804,7 @@ export class GarbageCollector {
974
804
  * - We don't require DDSes handles to be stored in a referenced DDS.
975
805
  * - A new data store may have "root" DDSes already created and we don't detect them today.
976
806
  */
977
- const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
807
+ const gcDataSuperSet = concatGarbageCollectionData(previousGCData, currentGCData);
978
808
  const newOutboundRoutesSinceLastRun = [];
979
809
  this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
980
810
  if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
@@ -992,16 +822,11 @@ export class GarbageCollector {
992
822
  * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
993
823
  * unreferenced and add unreferenced state.
994
824
  */
995
- const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/", ...newOutboundRoutesSinceLastRun]);
996
- for (const nodeId of gcResult.referencedNodeIds) {
997
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
998
- if (nodeStateTracker !== undefined) {
999
- // Stop tracking so as to clear out any running timers.
1000
- nodeStateTracker.stopTracking();
1001
- // Delete the unreferenced state as we don't need to track it any more.
1002
- this.unreferencedNodesState.delete(nodeId);
1003
- }
1004
- }
825
+ const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [
826
+ "/",
827
+ ...newOutboundRoutesSinceLastRun,
828
+ ]);
829
+ return gcResult.referencedNodeIds;
1005
830
  }
1006
831
  /**
1007
832
  * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
@@ -1017,7 +842,7 @@ export class GarbageCollector {
1017
842
  * @returns - a list of missing explicit references
1018
843
  */
1019
844
  findMissingExplicitReferences(currentGCData, previousGCData, explicitReferences) {
1020
- assert(previousGCData !== undefined, 0x2b7);
845
+ assert(previousGCData !== undefined, 0x2b7 /* "Can't validate correctness without GC data from last run" */);
1021
846
  const currentGraph = Object.entries(currentGCData.gcNodes);
1022
847
  const missingExplicitReferences = [];
1023
848
  currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
@@ -1035,9 +860,10 @@ export class GarbageCollector {
1035
860
  */
1036
861
  currentOutboundRoutes.forEach((route) => {
1037
862
  const nodeType = this.runtime.getNodeType(route);
1038
- if ((nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob)
1039
- && !nodeId.startsWith(route)
1040
- && (!previousRoutes.includes(route) && !explicitRoutes.includes(route))) {
863
+ if ((nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) &&
864
+ !nodeId.startsWith(route) &&
865
+ !previousRoutes.includes(route) &&
866
+ !explicitRoutes.includes(route)) {
1041
867
  missingExplicitRoutes.push(route);
1042
868
  }
1043
869
  });
@@ -1109,7 +935,8 @@ export class GarbageCollector {
1109
935
  * this will give us a view into how much deleted content a container has.
1110
936
  */
1111
937
  logSweepEvents(logger, currentReferenceTimestampMs) {
1112
- if (this.mc.config.getBoolean(disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
938
+ if (this.mc.config.getBoolean(disableSweepLogKey) === true ||
939
+ this.sweepTimeoutMs === undefined) {
1113
940
  return;
1114
941
  }
1115
942
  this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
@@ -1144,7 +971,8 @@ export class GarbageCollector {
1144
971
  // If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
1145
972
  // logging as nothing interesting would have happened worth logging.
1146
973
  // If the node is active, skip logging.
1147
- if (currentReferenceTimestampMs === undefined || nodeStateTracker.state === UnreferencedState.Active) {
974
+ if (currentReferenceTimestampMs === undefined ||
975
+ nodeStateTracker.state === UnreferencedState.Active) {
1148
976
  return;
1149
977
  }
1150
978
  // We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
@@ -1161,7 +989,7 @@ export class GarbageCollector {
1161
989
  this.loggedUnreferencedEvents.add(uniqueEventId);
1162
990
  const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === UnreferencedState.Inactive
1163
991
  ? this.inactiveTimeoutMs
1164
- : this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.externalRequest], viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
992
+ : this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
1165
993
  // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
1166
994
  // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
1167
995
  // but it's a good signal nonetheless and we can consume it with a grain of salt.
@@ -1209,11 +1037,18 @@ export class GarbageCollector {
1209
1037
  * revived and a Revived event will be logged for it.
1210
1038
  */
1211
1039
  const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
1212
- const active = nodeStateTracker === undefined || nodeStateTracker.state === UnreferencedState.Active;
1040
+ const active = nodeStateTracker === undefined ||
1041
+ nodeStateTracker.state === UnreferencedState.Active;
1213
1042
  if ((usageType === "Revived") === active) {
1214
1043
  const pkg = await this.getNodePackagePath(eventProps.id);
1215
- const fromPkg = eventProps.fromId ? await this.getNodePackagePath(eventProps.fromId) : undefined;
1216
- const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: pkg ? { value: pkg.join("/"), tag: TelemetryDataTag.CodeArtifact } : undefined, fromPkg: fromPkg ? { value: fromPkg.join("/"), tag: TelemetryDataTag.CodeArtifact } : undefined });
1044
+ const fromPkg = eventProps.fromId
1045
+ ? await this.getNodePackagePath(eventProps.fromId)
1046
+ : undefined;
1047
+ const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: pkg
1048
+ ? { value: pkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
1049
+ : undefined, fromPkg: fromPkg
1050
+ ? { value: fromPkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
1051
+ : undefined });
1217
1052
  if (state === UnreferencedState.Inactive) {
1218
1053
  logger.sendTelemetryEvent(event);
1219
1054
  }
@@ -1225,28 +1060,4 @@ export class GarbageCollector {
1225
1060
  this.pendingEventsQueue = [];
1226
1061
  }
1227
1062
  }
1228
- function generateSortedGCState(gcState) {
1229
- const sortableArray = Object.entries(gcState.gcNodes);
1230
- sortableArray.sort(([a], [b]) => a.localeCompare(b));
1231
- const sortedGCState = { gcNodes: {} };
1232
- for (const [nodeId, nodeData] of sortableArray) {
1233
- nodeData.outboundRoutes.sort();
1234
- sortedGCState.gcNodes[nodeId] = nodeData;
1235
- }
1236
- return sortedGCState;
1237
- }
1238
- /** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
1239
- class TimerWithNoDefaultTimeout extends Timer {
1240
- constructor(callback) {
1241
- // The default timeout/handlers will never be used since start/restart pass overrides below
1242
- super(0, () => { throw new Error("DefaultHandler should not be used"); });
1243
- this.callback = callback;
1244
- }
1245
- start(timeoutMs) {
1246
- super.start(timeoutMs, this.callback);
1247
- }
1248
- restart(timeoutMs) {
1249
- super.restart(timeoutMs, this.callback);
1250
- }
1251
- }
1252
1063
  //# sourceMappingURL=garbageCollection.js.map