@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.1.0.148229

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (465) hide show
  1. package/dist/blobManager.d.ts +24 -11
  2. package/dist/blobManager.d.ts.map +1 -1
  3. package/dist/blobManager.js +112 -55
  4. package/dist/blobManager.js.map +1 -1
  5. package/dist/containerRuntime.d.ts +60 -75
  6. package/dist/containerRuntime.d.ts.map +1 -1
  7. package/dist/containerRuntime.js +295 -256
  8. package/dist/containerRuntime.js.map +1 -1
  9. package/dist/dataStoreContext.d.ts +39 -13
  10. package/dist/dataStoreContext.d.ts.map +1 -1
  11. package/dist/dataStoreContext.js +112 -49
  12. package/dist/dataStoreContext.js.map +1 -1
  13. package/dist/dataStores.d.ts +28 -4
  14. package/dist/dataStores.d.ts.map +1 -1
  15. package/dist/dataStores.js +107 -41
  16. package/dist/dataStores.js.map +1 -1
  17. package/dist/deltaManagerSummarizerProxy.d.ts +19 -0
  18. package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -0
  19. package/dist/deltaManagerSummarizerProxy.js +40 -0
  20. package/dist/deltaManagerSummarizerProxy.js.map +1 -0
  21. package/dist/gc/garbageCollection.d.ts +204 -0
  22. package/dist/gc/garbageCollection.d.ts.map +1 -0
  23. package/dist/{garbageCollection.js → gc/garbageCollection.js} +190 -554
  24. package/dist/gc/garbageCollection.js.map +1 -0
  25. package/dist/gc/gcConfigs.d.ts +22 -0
  26. package/dist/gc/gcConfigs.d.ts.map +1 -0
  27. package/dist/gc/gcConfigs.js +143 -0
  28. package/dist/gc/gcConfigs.js.map +1 -0
  29. package/dist/gc/gcDefinitions.d.ts +320 -0
  30. package/dist/gc/gcDefinitions.d.ts.map +1 -0
  31. package/dist/gc/gcDefinitions.js +81 -0
  32. package/dist/gc/gcDefinitions.js.map +1 -0
  33. package/dist/gc/gcHelpers.d.ts +86 -0
  34. package/dist/gc/gcHelpers.d.ts.map +1 -0
  35. package/dist/gc/gcHelpers.js +268 -0
  36. package/dist/gc/gcHelpers.js.map +1 -0
  37. package/dist/gc/gcReferenceGraphAlgorithm.d.ts +16 -0
  38. package/dist/gc/gcReferenceGraphAlgorithm.d.ts.map +1 -0
  39. package/dist/gc/gcReferenceGraphAlgorithm.js +49 -0
  40. package/dist/gc/gcReferenceGraphAlgorithm.js.map +1 -0
  41. package/dist/gc/gcSummaryDefinitions.d.ts +52 -0
  42. package/dist/gc/gcSummaryDefinitions.d.ts.map +1 -0
  43. package/dist/gc/gcSummaryDefinitions.js +7 -0
  44. package/dist/gc/gcSummaryDefinitions.js.map +1 -0
  45. package/dist/gc/gcSummaryStateTracker.d.ts +93 -0
  46. package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -0
  47. package/dist/gc/gcSummaryStateTracker.js +239 -0
  48. package/dist/gc/gcSummaryStateTracker.js.map +1 -0
  49. package/dist/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
  50. package/dist/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +2 -2
  51. package/dist/gc/gcSweepReadyUsageDetection.js.map +1 -0
  52. package/dist/gc/gcUnreferencedStateTracker.d.ts +34 -0
  53. package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
  54. package/dist/gc/gcUnreferencedStateTracker.js +94 -0
  55. package/dist/gc/gcUnreferencedStateTracker.js.map +1 -0
  56. package/dist/gc/index.d.ts +13 -0
  57. package/dist/gc/index.d.ts.map +1 -0
  58. package/dist/gc/index.js +50 -0
  59. package/dist/gc/index.js.map +1 -0
  60. package/dist/index.d.ts +3 -7
  61. package/dist/index.d.ts.map +1 -1
  62. package/dist/index.js +5 -9
  63. package/dist/index.js.map +1 -1
  64. package/dist/opLifecycle/batchManager.d.ts +11 -13
  65. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  66. package/dist/opLifecycle/batchManager.js +26 -38
  67. package/dist/opLifecycle/batchManager.js.map +1 -1
  68. package/dist/opLifecycle/definitions.d.ts +4 -0
  69. package/dist/opLifecycle/definitions.d.ts.map +1 -1
  70. package/dist/opLifecycle/definitions.js.map +1 -1
  71. package/dist/opLifecycle/index.d.ts +1 -1
  72. package/dist/opLifecycle/index.d.ts.map +1 -1
  73. package/dist/opLifecycle/index.js +2 -1
  74. package/dist/opLifecycle/index.js.map +1 -1
  75. package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
  76. package/dist/opLifecycle/opCompressor.js +25 -10
  77. package/dist/opLifecycle/opCompressor.js.map +1 -1
  78. package/dist/opLifecycle/opDecompressor.d.ts +4 -0
  79. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
  80. package/dist/opLifecycle/opDecompressor.js +42 -4
  81. package/dist/opLifecycle/opDecompressor.js.map +1 -1
  82. package/dist/opLifecycle/opSplitter.d.ts +15 -3
  83. package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
  84. package/dist/opLifecycle/opSplitter.js +35 -10
  85. package/dist/opLifecycle/opSplitter.js.map +1 -1
  86. package/dist/opLifecycle/outbox.d.ts +19 -3
  87. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  88. package/dist/opLifecycle/outbox.js +88 -49
  89. package/dist/opLifecycle/outbox.js.map +1 -1
  90. package/dist/packageVersion.d.ts +1 -1
  91. package/dist/packageVersion.js +1 -1
  92. package/dist/packageVersion.js.map +1 -1
  93. package/dist/pendingStateManager.d.ts +3 -3
  94. package/dist/pendingStateManager.d.ts.map +1 -1
  95. package/dist/pendingStateManager.js +20 -21
  96. package/dist/pendingStateManager.js.map +1 -1
  97. package/dist/storageServiceWithAttachBlobs.d.ts +17 -0
  98. package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -0
  99. package/dist/storageServiceWithAttachBlobs.js +32 -0
  100. package/dist/storageServiceWithAttachBlobs.js.map +1 -0
  101. package/dist/summary/index.d.ts +17 -0
  102. package/dist/summary/index.d.ts.map +1 -0
  103. package/dist/summary/index.js +46 -0
  104. package/dist/summary/index.js.map +1 -0
  105. package/dist/summary/orderedClientElection.d.ts.map +1 -0
  106. package/dist/summary/orderedClientElection.js.map +1 -0
  107. package/dist/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +3 -2
  108. package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
  109. package/dist/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +5 -4
  110. package/dist/summary/runWhileConnectedCoordinator.js.map +1 -0
  111. package/{lib → dist/summary}/runningSummarizer.d.ts +19 -18
  112. package/dist/summary/runningSummarizer.d.ts.map +1 -0
  113. package/dist/{runningSummarizer.js → summary/runningSummarizer.js} +158 -56
  114. package/dist/summary/runningSummarizer.js.map +1 -0
  115. package/dist/{summarizer.d.ts → summary/summarizer.d.ts} +4 -9
  116. package/dist/summary/summarizer.d.ts.map +1 -0
  117. package/dist/{summarizer.js → summary/summarizer.js} +9 -74
  118. package/dist/summary/summarizer.js.map +1 -0
  119. package/dist/summary/summarizerClientElection.d.ts.map +1 -0
  120. package/dist/summary/summarizerClientElection.js.map +1 -0
  121. package/{lib → dist/summary}/summarizerHeuristics.d.ts +1 -1
  122. package/dist/summary/summarizerHeuristics.d.ts.map +1 -0
  123. package/dist/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +3 -3
  124. package/dist/summary/summarizerHeuristics.js.map +1 -0
  125. package/dist/summary/summarizerNode/index.d.ts +8 -0
  126. package/dist/summary/summarizerNode/index.d.ts.map +1 -0
  127. package/dist/summary/summarizerNode/index.js +12 -0
  128. package/dist/summary/summarizerNode/index.js.map +1 -0
  129. package/dist/summary/summarizerNode/summarizerNode.d.ts +149 -0
  130. package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -0
  131. package/dist/summary/summarizerNode/summarizerNode.js +531 -0
  132. package/dist/summary/summarizerNode/summarizerNode.js.map +1 -0
  133. package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts +125 -0
  134. package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -0
  135. package/dist/summary/summarizerNode/summarizerNodeUtils.js +132 -0
  136. package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -0
  137. package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +22 -0
  138. package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -0
  139. package/dist/summary/summarizerNode/summarizerNodeWithGc.js +423 -0
  140. package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -0
  141. package/{lib → dist/summary}/summarizerTypes.d.ts +7 -17
  142. package/dist/summary/summarizerTypes.d.ts.map +1 -0
  143. package/dist/{summarizerTypes.js → summary/summarizerTypes.js} +0 -5
  144. package/dist/summary/summarizerTypes.js.map +1 -0
  145. package/dist/summary/summaryCollection.d.ts.map +1 -0
  146. package/dist/summary/summaryCollection.js.map +1 -0
  147. package/{lib → dist/summary}/summaryFormat.d.ts +3 -21
  148. package/dist/summary/summaryFormat.d.ts.map +1 -0
  149. package/dist/{summaryFormat.js → summary/summaryFormat.js} +1 -10
  150. package/dist/summary/summaryFormat.js.map +1 -0
  151. package/dist/summary/summaryGenerator.d.ts.map +1 -0
  152. package/dist/{summaryGenerator.js → summary/summaryGenerator.js} +4 -4
  153. package/dist/summary/summaryGenerator.js.map +1 -0
  154. package/dist/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
  155. package/dist/summary/summaryManager.d.ts.map +1 -0
  156. package/dist/summary/summaryManager.js.map +1 -0
  157. package/lib/blobManager.d.ts +24 -11
  158. package/lib/blobManager.d.ts.map +1 -1
  159. package/lib/blobManager.js +109 -52
  160. package/lib/blobManager.js.map +1 -1
  161. package/lib/containerRuntime.d.ts +60 -75
  162. package/lib/containerRuntime.d.ts.map +1 -1
  163. package/lib/containerRuntime.js +267 -228
  164. package/lib/containerRuntime.js.map +1 -1
  165. package/lib/dataStoreContext.d.ts +39 -13
  166. package/lib/dataStoreContext.d.ts.map +1 -1
  167. package/lib/dataStoreContext.js +101 -38
  168. package/lib/dataStoreContext.js.map +1 -1
  169. package/lib/dataStores.d.ts +28 -4
  170. package/lib/dataStores.d.ts.map +1 -1
  171. package/lib/dataStores.js +100 -34
  172. package/lib/dataStores.js.map +1 -1
  173. package/lib/deltaManagerSummarizerProxy.d.ts +19 -0
  174. package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -0
  175. package/lib/deltaManagerSummarizerProxy.js +36 -0
  176. package/lib/deltaManagerSummarizerProxy.js.map +1 -0
  177. package/lib/gc/garbageCollection.d.ts +204 -0
  178. package/lib/gc/garbageCollection.d.ts.map +1 -0
  179. package/lib/{garbageCollection.js → gc/garbageCollection.js} +172 -535
  180. package/lib/gc/garbageCollection.js.map +1 -0
  181. package/lib/gc/gcConfigs.d.ts +22 -0
  182. package/lib/gc/gcConfigs.d.ts.map +1 -0
  183. package/lib/gc/gcConfigs.js +139 -0
  184. package/lib/gc/gcConfigs.js.map +1 -0
  185. package/lib/gc/gcDefinitions.d.ts +320 -0
  186. package/lib/gc/gcDefinitions.d.ts.map +1 -0
  187. package/lib/gc/gcDefinitions.js +78 -0
  188. package/lib/gc/gcDefinitions.js.map +1 -0
  189. package/lib/gc/gcHelpers.d.ts +86 -0
  190. package/lib/gc/gcHelpers.d.ts.map +1 -0
  191. package/lib/gc/gcHelpers.js +254 -0
  192. package/lib/gc/gcHelpers.js.map +1 -0
  193. package/lib/gc/gcReferenceGraphAlgorithm.d.ts +16 -0
  194. package/lib/gc/gcReferenceGraphAlgorithm.d.ts.map +1 -0
  195. package/lib/gc/gcReferenceGraphAlgorithm.js +45 -0
  196. package/lib/gc/gcReferenceGraphAlgorithm.js.map +1 -0
  197. package/lib/gc/gcSummaryDefinitions.d.ts +52 -0
  198. package/lib/gc/gcSummaryDefinitions.d.ts.map +1 -0
  199. package/lib/gc/gcSummaryDefinitions.js +6 -0
  200. package/lib/gc/gcSummaryDefinitions.js.map +1 -0
  201. package/lib/gc/gcSummaryStateTracker.d.ts +93 -0
  202. package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -0
  203. package/lib/gc/gcSummaryStateTracker.js +235 -0
  204. package/lib/gc/gcSummaryStateTracker.js.map +1 -0
  205. package/lib/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
  206. package/lib/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +1 -1
  207. package/lib/gc/gcSweepReadyUsageDetection.js.map +1 -0
  208. package/lib/gc/gcUnreferencedStateTracker.d.ts +34 -0
  209. package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
  210. package/lib/gc/gcUnreferencedStateTracker.js +90 -0
  211. package/lib/gc/gcUnreferencedStateTracker.js.map +1 -0
  212. package/lib/gc/index.d.ts +13 -0
  213. package/lib/gc/index.d.ts.map +1 -0
  214. package/lib/gc/index.js +12 -0
  215. package/lib/gc/index.js.map +1 -0
  216. package/lib/index.d.ts +3 -7
  217. package/lib/index.d.ts.map +1 -1
  218. package/lib/index.js +1 -4
  219. package/lib/index.js.map +1 -1
  220. package/lib/opLifecycle/batchManager.d.ts +11 -13
  221. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  222. package/lib/opLifecycle/batchManager.js +24 -37
  223. package/lib/opLifecycle/batchManager.js.map +1 -1
  224. package/lib/opLifecycle/definitions.d.ts +4 -0
  225. package/lib/opLifecycle/definitions.d.ts.map +1 -1
  226. package/lib/opLifecycle/definitions.js.map +1 -1
  227. package/lib/opLifecycle/index.d.ts +1 -1
  228. package/lib/opLifecycle/index.d.ts.map +1 -1
  229. package/lib/opLifecycle/index.js +1 -1
  230. package/lib/opLifecycle/index.js.map +1 -1
  231. package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
  232. package/lib/opLifecycle/opCompressor.js +26 -11
  233. package/lib/opLifecycle/opCompressor.js.map +1 -1
  234. package/lib/opLifecycle/opDecompressor.d.ts +4 -0
  235. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
  236. package/lib/opLifecycle/opDecompressor.js +42 -4
  237. package/lib/opLifecycle/opDecompressor.js.map +1 -1
  238. package/lib/opLifecycle/opSplitter.d.ts +15 -3
  239. package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
  240. package/lib/opLifecycle/opSplitter.js +35 -10
  241. package/lib/opLifecycle/opSplitter.js.map +1 -1
  242. package/lib/opLifecycle/outbox.d.ts +19 -3
  243. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  244. package/lib/opLifecycle/outbox.js +90 -51
  245. package/lib/opLifecycle/outbox.js.map +1 -1
  246. package/lib/packageVersion.d.ts +1 -1
  247. package/lib/packageVersion.js +1 -1
  248. package/lib/packageVersion.js.map +1 -1
  249. package/lib/pendingStateManager.d.ts +3 -3
  250. package/lib/pendingStateManager.d.ts.map +1 -1
  251. package/lib/pendingStateManager.js +20 -21
  252. package/lib/pendingStateManager.js.map +1 -1
  253. package/lib/storageServiceWithAttachBlobs.d.ts +17 -0
  254. package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -0
  255. package/lib/storageServiceWithAttachBlobs.js +28 -0
  256. package/lib/storageServiceWithAttachBlobs.js.map +1 -0
  257. package/lib/summary/index.d.ts +17 -0
  258. package/lib/summary/index.d.ts.map +1 -0
  259. package/lib/summary/index.js +15 -0
  260. package/lib/summary/index.js.map +1 -0
  261. package/lib/summary/orderedClientElection.d.ts.map +1 -0
  262. package/lib/summary/orderedClientElection.js.map +1 -0
  263. package/lib/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +3 -2
  264. package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
  265. package/lib/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +5 -4
  266. package/lib/summary/runWhileConnectedCoordinator.js.map +1 -0
  267. package/{dist → lib/summary}/runningSummarizer.d.ts +19 -18
  268. package/lib/summary/runningSummarizer.d.ts.map +1 -0
  269. package/lib/{runningSummarizer.js → summary/runningSummarizer.js} +159 -57
  270. package/lib/summary/runningSummarizer.js.map +1 -0
  271. package/lib/{summarizer.d.ts → summary/summarizer.d.ts} +4 -9
  272. package/lib/summary/summarizer.d.ts.map +1 -0
  273. package/lib/{summarizer.js → summary/summarizer.js} +11 -76
  274. package/lib/summary/summarizer.js.map +1 -0
  275. package/lib/summary/summarizerClientElection.d.ts.map +1 -0
  276. package/lib/summary/summarizerClientElection.js.map +1 -0
  277. package/{dist → lib/summary}/summarizerHeuristics.d.ts +1 -1
  278. package/lib/summary/summarizerHeuristics.d.ts.map +1 -0
  279. package/lib/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +3 -3
  280. package/lib/summary/summarizerHeuristics.js.map +1 -0
  281. package/lib/summary/summarizerNode/index.d.ts +8 -0
  282. package/lib/summary/summarizerNode/index.d.ts.map +1 -0
  283. package/lib/summary/summarizerNode/index.js +7 -0
  284. package/lib/summary/summarizerNode/index.js.map +1 -0
  285. package/lib/summary/summarizerNode/summarizerNode.d.ts +149 -0
  286. package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -0
  287. package/lib/summary/summarizerNode/summarizerNode.js +526 -0
  288. package/lib/summary/summarizerNode/summarizerNode.js.map +1 -0
  289. package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts +125 -0
  290. package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -0
  291. package/lib/summary/summarizerNode/summarizerNodeUtils.js +125 -0
  292. package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -0
  293. package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +22 -0
  294. package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -0
  295. package/lib/summary/summarizerNode/summarizerNodeWithGc.js +419 -0
  296. package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -0
  297. package/{dist → lib/summary}/summarizerTypes.d.ts +7 -17
  298. package/lib/summary/summarizerTypes.d.ts.map +1 -0
  299. package/lib/summary/summarizerTypes.js +6 -0
  300. package/{dist → lib/summary}/summarizerTypes.js.map +1 -1
  301. package/lib/summary/summaryCollection.d.ts.map +1 -0
  302. package/lib/summary/summaryCollection.js.map +1 -0
  303. package/{dist → lib/summary}/summaryFormat.d.ts +3 -21
  304. package/lib/summary/summaryFormat.d.ts.map +1 -0
  305. package/lib/{summaryFormat.js → summary/summaryFormat.js} +0 -8
  306. package/lib/summary/summaryFormat.js.map +1 -0
  307. package/lib/summary/summaryGenerator.d.ts.map +1 -0
  308. package/lib/{summaryGenerator.js → summary/summaryGenerator.js} +4 -4
  309. package/lib/summary/summaryGenerator.js.map +1 -0
  310. package/lib/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
  311. package/lib/summary/summaryManager.d.ts.map +1 -0
  312. package/lib/summary/summaryManager.js.map +1 -0
  313. package/package.json +63 -60
  314. package/src/blobManager.ts +132 -69
  315. package/src/containerRuntime.ts +421 -382
  316. package/src/dataStoreContext.ts +140 -49
  317. package/src/dataStores.ts +139 -41
  318. package/src/deltaManagerSummarizerProxy.ts +46 -0
  319. package/{garbageCollection.md → src/gc/garbageCollection.md} +2 -2
  320. package/src/{garbageCollection.ts → gc/garbageCollection.ts} +245 -890
  321. package/src/gc/gcConfigs.ts +193 -0
  322. package/src/gc/gcDefinitions.ts +387 -0
  323. package/src/gc/gcHelpers.ts +332 -0
  324. package/src/gc/gcReferenceGraphAlgorithm.ts +52 -0
  325. package/src/gc/gcSummaryDefinitions.ts +54 -0
  326. package/src/gc/gcSummaryStateTracker.ts +329 -0
  327. package/src/{gcSweepReadyUsageDetection.ts → gc/gcSweepReadyUsageDetection.ts} +1 -1
  328. package/src/gc/gcUnreferencedStateTracker.ts +114 -0
  329. package/src/gc/index.ts +65 -0
  330. package/src/index.ts +10 -22
  331. package/src/opLifecycle/README.md +157 -0
  332. package/src/opLifecycle/batchManager.ts +26 -55
  333. package/src/opLifecycle/definitions.ts +4 -0
  334. package/src/opLifecycle/index.ts +1 -1
  335. package/src/opLifecycle/opCompressor.ts +32 -12
  336. package/src/opLifecycle/opDecompressor.ts +49 -5
  337. package/src/opLifecycle/opSplitter.ts +55 -12
  338. package/src/opLifecycle/outbox.ts +120 -60
  339. package/src/packageVersion.ts +1 -1
  340. package/src/pendingStateManager.ts +34 -27
  341. package/src/storageServiceWithAttachBlobs.ts +38 -0
  342. package/src/summary/index.ts +105 -0
  343. package/src/{runWhileConnectedCoordinator.ts → summary/runWhileConnectedCoordinator.ts} +7 -7
  344. package/src/{runningSummarizer.ts → summary/runningSummarizer.ts} +279 -139
  345. package/src/{summarizer.ts → summary/summarizer.ts} +12 -97
  346. package/src/{summarizerHeuristics.ts → summary/summarizerHeuristics.ts} +9 -4
  347. package/src/summary/summarizerNode/index.ts +12 -0
  348. package/src/summary/summarizerNode/summarizerNode.ts +766 -0
  349. package/src/summary/summarizerNode/summarizerNodeUtils.ts +214 -0
  350. package/src/summary/summarizerNode/summarizerNodeWithGc.ts +644 -0
  351. package/src/{summarizerTypes.ts → summary/summarizerTypes.ts} +8 -22
  352. package/src/{summaryFormat.ts → summary/summaryFormat.ts} +3 -29
  353. package/src/{summaryGenerator.ts → summary/summaryGenerator.ts} +12 -11
  354. package/src/{summaryManager.ts → summary/summaryManager.ts} +1 -1
  355. package/dist/garbageCollection.d.ts +0 -411
  356. package/dist/garbageCollection.d.ts.map +0 -1
  357. package/dist/garbageCollection.js.map +0 -1
  358. package/dist/garbageCollectionConstants.d.ts +0 -23
  359. package/dist/garbageCollectionConstants.d.ts.map +0 -1
  360. package/dist/garbageCollectionConstants.js +0 -36
  361. package/dist/garbageCollectionConstants.js.map +0 -1
  362. package/dist/garbageCollectionHelpers.d.ts +0 -15
  363. package/dist/garbageCollectionHelpers.d.ts.map +0 -1
  364. package/dist/garbageCollectionHelpers.js +0 -27
  365. package/dist/garbageCollectionHelpers.js.map +0 -1
  366. package/dist/gcSweepReadyUsageDetection.d.ts.map +0 -1
  367. package/dist/gcSweepReadyUsageDetection.js.map +0 -1
  368. package/dist/orderedClientElection.d.ts.map +0 -1
  369. package/dist/orderedClientElection.js.map +0 -1
  370. package/dist/runWhileConnectedCoordinator.d.ts.map +0 -1
  371. package/dist/runWhileConnectedCoordinator.js.map +0 -1
  372. package/dist/runningSummarizer.d.ts.map +0 -1
  373. package/dist/runningSummarizer.js.map +0 -1
  374. package/dist/serializedSnapshotStorage.d.ts +0 -58
  375. package/dist/serializedSnapshotStorage.d.ts.map +0 -1
  376. package/dist/serializedSnapshotStorage.js +0 -110
  377. package/dist/serializedSnapshotStorage.js.map +0 -1
  378. package/dist/summarizer.d.ts.map +0 -1
  379. package/dist/summarizer.js.map +0 -1
  380. package/dist/summarizerClientElection.d.ts.map +0 -1
  381. package/dist/summarizerClientElection.js.map +0 -1
  382. package/dist/summarizerHandle.d.ts +0 -12
  383. package/dist/summarizerHandle.d.ts.map +0 -1
  384. package/dist/summarizerHandle.js +0 -22
  385. package/dist/summarizerHandle.js.map +0 -1
  386. package/dist/summarizerHeuristics.d.ts.map +0 -1
  387. package/dist/summarizerHeuristics.js.map +0 -1
  388. package/dist/summarizerTypes.d.ts.map +0 -1
  389. package/dist/summaryCollection.d.ts.map +0 -1
  390. package/dist/summaryCollection.js.map +0 -1
  391. package/dist/summaryFormat.d.ts.map +0 -1
  392. package/dist/summaryFormat.js.map +0 -1
  393. package/dist/summaryGenerator.d.ts.map +0 -1
  394. package/dist/summaryGenerator.js.map +0 -1
  395. package/dist/summaryManager.d.ts.map +0 -1
  396. package/dist/summaryManager.js.map +0 -1
  397. package/lib/garbageCollection.d.ts +0 -411
  398. package/lib/garbageCollection.d.ts.map +0 -1
  399. package/lib/garbageCollection.js.map +0 -1
  400. package/lib/garbageCollectionConstants.d.ts +0 -23
  401. package/lib/garbageCollectionConstants.d.ts.map +0 -1
  402. package/lib/garbageCollectionConstants.js +0 -33
  403. package/lib/garbageCollectionConstants.js.map +0 -1
  404. package/lib/garbageCollectionHelpers.d.ts +0 -15
  405. package/lib/garbageCollectionHelpers.d.ts.map +0 -1
  406. package/lib/garbageCollectionHelpers.js +0 -23
  407. package/lib/garbageCollectionHelpers.js.map +0 -1
  408. package/lib/gcSweepReadyUsageDetection.d.ts.map +0 -1
  409. package/lib/gcSweepReadyUsageDetection.js.map +0 -1
  410. package/lib/orderedClientElection.d.ts.map +0 -1
  411. package/lib/orderedClientElection.js.map +0 -1
  412. package/lib/runWhileConnectedCoordinator.d.ts.map +0 -1
  413. package/lib/runWhileConnectedCoordinator.js.map +0 -1
  414. package/lib/runningSummarizer.d.ts.map +0 -1
  415. package/lib/runningSummarizer.js.map +0 -1
  416. package/lib/serializedSnapshotStorage.d.ts +0 -58
  417. package/lib/serializedSnapshotStorage.d.ts.map +0 -1
  418. package/lib/serializedSnapshotStorage.js +0 -106
  419. package/lib/serializedSnapshotStorage.js.map +0 -1
  420. package/lib/summarizer.d.ts.map +0 -1
  421. package/lib/summarizer.js.map +0 -1
  422. package/lib/summarizerClientElection.d.ts.map +0 -1
  423. package/lib/summarizerClientElection.js.map +0 -1
  424. package/lib/summarizerHandle.d.ts +0 -12
  425. package/lib/summarizerHandle.d.ts.map +0 -1
  426. package/lib/summarizerHandle.js +0 -18
  427. package/lib/summarizerHandle.js.map +0 -1
  428. package/lib/summarizerHeuristics.d.ts.map +0 -1
  429. package/lib/summarizerHeuristics.js.map +0 -1
  430. package/lib/summarizerTypes.d.ts.map +0 -1
  431. package/lib/summarizerTypes.js +0 -9
  432. package/lib/summarizerTypes.js.map +0 -1
  433. package/lib/summaryCollection.d.ts.map +0 -1
  434. package/lib/summaryCollection.js.map +0 -1
  435. package/lib/summaryFormat.d.ts.map +0 -1
  436. package/lib/summaryFormat.js.map +0 -1
  437. package/lib/summaryGenerator.d.ts.map +0 -1
  438. package/lib/summaryGenerator.js.map +0 -1
  439. package/lib/summaryManager.d.ts.map +0 -1
  440. package/lib/summaryManager.js.map +0 -1
  441. package/src/garbageCollectionConstants.ts +0 -38
  442. package/src/garbageCollectionHelpers.ts +0 -37
  443. package/src/serializedSnapshotStorage.ts +0 -151
  444. package/src/summarizerHandle.ts +0 -23
  445. /package/dist/{gcSweepReadyUsageDetection.d.ts → gc/gcSweepReadyUsageDetection.d.ts} +0 -0
  446. /package/dist/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
  447. /package/dist/{orderedClientElection.js → summary/orderedClientElection.js} +0 -0
  448. /package/dist/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
  449. /package/dist/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
  450. /package/dist/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
  451. /package/dist/{summaryCollection.js → summary/summaryCollection.js} +0 -0
  452. /package/dist/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
  453. /package/dist/{summaryManager.js → summary/summaryManager.js} +0 -0
  454. /package/lib/{gcSweepReadyUsageDetection.d.ts → gc/gcSweepReadyUsageDetection.d.ts} +0 -0
  455. /package/lib/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
  456. /package/lib/{orderedClientElection.js → summary/orderedClientElection.js} +0 -0
  457. /package/lib/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
  458. /package/lib/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
  459. /package/lib/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
  460. /package/lib/{summaryCollection.js → summary/summaryCollection.js} +0 -0
  461. /package/lib/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
  462. /package/lib/{summaryManager.js → summary/summaryManager.js} +0 -0
  463. /package/src/{orderedClientElection.ts → summary/orderedClientElection.ts} +0 -0
  464. /package/src/{summarizerClientElection.ts → summary/summarizerClientElection.ts} +0 -0
  465. /package/src/{summaryCollection.ts → summary/summaryCollection.ts} +0 -0
@@ -14,105 +14,17 @@ var __rest = (this && this.__rest) || function (s, e) {
14
14
  return t;
15
15
  };
16
16
  import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
17
- import { ClientSessionExpiredError, DataProcessingError, UsageError, } from "@fluidframework/container-utils";
18
- import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot, runGarbageCollection, trimLeadingSlashes, } from "@fluidframework/garbage-collector";
19
- import { SummaryType } from "@fluidframework/protocol-definitions";
20
- import { gcTreeKey, gcBlobPrefix, gcTombstoneBlobKey, gcDeletedBlobKey, } from "@fluidframework/runtime-definitions";
21
- import { mergeStats, packagePathToTelemetryProperty, SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
17
+ import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
18
+ import { gcTreeKey, } from "@fluidframework/runtime-definitions";
19
+ import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
22
20
  import { ChildLogger, generateStack, loggerToMonitoringContext, PerformanceEvent, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
23
- import { RuntimeHeaders } from "./containerRuntime";
24
- import { getSummaryForDatastores } from "./dataStores";
25
- import { currentGCVersion, defaultInactiveTimeoutMs, defaultSessionExpiryDurationMs, disableSweepLogKey, disableTombstoneKey, gcVersionUpgradeToV2Key, gcTestModeKey, oneDayMs, runGCKey, runSessionExpiryKey, runSweepKey, stableGCVersion, trackGCStateKey, } from "./garbageCollectionConstants";
26
- import { sendGCUnexpectedUsageEvent } from "./garbageCollectionHelpers";
27
- import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
28
- import { getGCVersion, metadataBlobName, dataStoreAttributesBlobName, } from "./summaryFormat";
29
- /** The types of GC nodes in the GC reference graph. */
30
- export const GCNodeType = {
31
- // Nodes that are for data stores.
32
- DataStore: "DataStore",
33
- // Nodes that are within a data store. For example, DDS nodes.
34
- SubDataStore: "SubDataStore",
35
- // Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
36
- Blob: "Blob",
37
- // Nodes that are neither of the above. For example, root node.
38
- Other: "Other",
39
- };
40
- /** The state of node that is unreferenced. */
41
- export const UnreferencedState = {
42
- /** The node is active, i.e., it can become referenced again. */
43
- Active: "Active",
44
- /** The node is inactive, i.e., it should not become referenced. */
45
- Inactive: "Inactive",
46
- /** The node is ready to be deleted by the sweep phase. */
47
- SweepReady: "SweepReady",
48
- };
49
- /**
50
- * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
51
- * be deleted by the sweep phase.
52
- */
53
- export class UnreferencedStateTracker {
54
- constructor(unreferencedTimestampMs,
55
- /** The time after which node transitions to Inactive state. */
56
- inactiveTimeoutMs,
57
- /** The current reference timestamp used to track how long this node has been unreferenced for. */
58
- currentReferenceTimestampMs,
59
- /** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
60
- sweepTimeoutMs) {
61
- this.unreferencedTimestampMs = unreferencedTimestampMs;
62
- this.inactiveTimeoutMs = inactiveTimeoutMs;
63
- this.sweepTimeoutMs = sweepTimeoutMs;
64
- this._state = UnreferencedState.Active;
65
- if (this.sweepTimeoutMs !== undefined) {
66
- assert(this.inactiveTimeoutMs <= this.sweepTimeoutMs, 0x3b0 /* inactive timeout must not be greater than the sweep timeout */);
67
- }
68
- this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
69
- this._state = UnreferencedState.SweepReady;
70
- assert(!this.inactiveTimer.hasTimer, 0x3b1 /* inactiveTimer still running after sweepTimer fired! */);
71
- });
72
- this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
73
- this._state = UnreferencedState.Inactive;
74
- // After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
75
- if (this.sweepTimeoutMs !== undefined) {
76
- this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
77
- }
78
- });
79
- this.updateTracking(currentReferenceTimestampMs);
80
- }
81
- get state() {
82
- return this._state;
83
- }
84
- /* Updates the unreferenced state based on the provided timestamp. */
85
- updateTracking(currentReferenceTimestampMs) {
86
- const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
87
- // If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
88
- if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
89
- this._state = UnreferencedState.SweepReady;
90
- this.clearTimers();
91
- return;
92
- }
93
- // If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
94
- // Also, start a timer for the sweep timeout.
95
- if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
96
- this._state = UnreferencedState.Inactive;
97
- this.inactiveTimer.clear();
98
- if (this.sweepTimeoutMs !== undefined) {
99
- this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
100
- }
101
- return;
102
- }
103
- // The node is still active. Ensure the inactive timer is running with the proper remaining duration.
104
- this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
105
- }
106
- clearTimers() {
107
- this.inactiveTimer.clear();
108
- this.sweepTimer.clear();
109
- }
110
- /** Stop tracking this node. Reset the unreferenced timers and state, if any. */
111
- stopTracking() {
112
- this.clearTimers();
113
- this._state = UnreferencedState.Active;
114
- }
115
- }
21
+ import { RuntimeHeaders } from "../containerRuntime";
22
+ import { generateGCConfigs } from "./gcConfigs";
23
+ import { disableSweepLogKey, GCNodeType, UnreferencedState, } from "./gcDefinitions";
24
+ import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot, sendGCUnexpectedUsageEvent, } from "./gcHelpers";
25
+ import { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
26
+ import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
27
+ import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
116
28
  /**
117
29
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
118
30
  * its state across summaries.
@@ -124,20 +36,19 @@ export class UnreferencedStateTracker {
124
36
  * Graph - all nodes with their respective routes
125
37
  *
126
38
  * ```
127
- * GC Graph
39
+ * GC Graph
128
40
  *
129
- * Node
130
- * NodeId = "datastore1"
131
- * / \\
132
- * OutboundRoute OutboundRoute
133
- * / \\
134
- * Node Node
135
- * NodeId = "dds1" NodeId = "dds2"
41
+ * Node
42
+ * NodeId = "datastore1"
43
+ * / \\
44
+ * OutboundRoute OutboundRoute
45
+ * / \\
46
+ * Node Node
47
+ * NodeId = "dds1" NodeId = "dds2"
136
48
  * ```
137
49
  */
138
50
  export class GarbageCollector {
139
51
  constructor(createParams) {
140
- var _a, _b, _c, _d, _e, _f, _g, _h;
141
52
  // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
142
53
  // outbound routes from that node.
143
54
  this.newReferencesSinceLastRun = new Map();
@@ -156,194 +67,61 @@ export class GarbageCollector {
156
67
  this.completedRuns = 0;
157
68
  this.runtime = createParams.runtime;
158
69
  this.isSummarizerClient = createParams.isSummarizerClient;
159
- this.gcOptions = createParams.gcOptions;
160
70
  this.createContainerMetadata = createParams.createContainerMetadata;
161
71
  this.getNodePackagePath = createParams.getNodePackagePath;
162
72
  this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
163
73
  this.activeConnection = createParams.activeConnection;
164
74
  const baseSnapshot = createParams.baseSnapshot;
165
- const metadata = createParams.metadata;
166
75
  const readAndParseBlob = createParams.readAndParseBlob;
167
76
  this.mc = loggerToMonitoringContext(ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
168
77
  all: { completedGCRuns: () => this.completedRuns },
169
78
  }));
170
- // If version upgrade is not enabled, fall back to the stable GC version.
171
- this.currentGCVersion =
172
- this.mc.config.getBoolean(gcVersionUpgradeToV2Key) === true
173
- ? currentGCVersion
174
- : stableGCVersion;
175
- this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
176
- let prevSummaryGCVersion;
177
- /**
178
- * Sweep timeout is the time after which unreferenced content can be swept.
179
- * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
180
- *
181
- * The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
182
- * The buffer is added to account for any clock skew or other edge cases.
183
- * We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
184
- */
185
- function computeSweepTimeout(sessionExpiryTimeoutMs) {
186
- const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
187
- const bufferMs = oneDayMs;
188
- return (sessionExpiryTimeoutMs &&
189
- sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
190
- }
191
- /**
192
- * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
193
- * 1. Whether running GC mark phase is allowed or not.
194
- * 2. Whether running GC sweep phase is allowed or not.
195
- * 3. Whether GC session expiry is enabled or not.
196
- * For existing containers, we get this information from the metadata blob of its summary.
197
- */
198
- if (createParams.existing) {
199
- prevSummaryGCVersion = getGCVersion(metadata);
200
- // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
201
- // other existing documents, GC is enabled.
202
- this.gcEnabled = prevSummaryGCVersion > 0;
203
- this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
204
- this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
205
- this.sweepTimeoutMs =
206
- (_b = metadata === null || metadata === void 0 ? void 0 : metadata.sweepTimeoutMs) !== null && _b !== void 0 ? _b : computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
207
- }
208
- else {
209
- // Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
210
- // scenario but explicitly failing makes it clearer and promotes correct usage.
211
- if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
212
- throw new UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
213
- }
214
- // This Test Override only applies for new containers
215
- const testOverrideSweepTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
216
- // For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
217
- // flag in GC options to false.
218
- this.gcEnabled = this.gcOptions.gcAllowed !== false;
219
- // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
220
- this.sweepEnabled = this.gcOptions.sweepAllowed === true;
221
- // Set the Session Expiry only if the flag is enabled and GC is enabled.
222
- if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
223
- this.sessionExpiryTimeoutMs =
224
- (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : defaultSessionExpiryDurationMs;
225
- }
226
- this.sweepTimeoutMs =
227
- testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
228
- }
79
+ this.configs = generateGCConfigs(this.mc, createParams);
229
80
  // If session expiry is enabled, we need to close the container when the session expiry timeout expires.
230
- if (this.sessionExpiryTimeoutMs !== undefined) {
81
+ if (this.configs.sessionExpiryTimeoutMs !== undefined) {
231
82
  // If Test Override config is set, override Session Expiry timeout.
232
83
  const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
233
- const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
84
+ const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.configs.sessionExpiryTimeoutMs;
234
85
  this.sessionExpiryTimer = new Timer(timeoutMs, () => {
235
86
  this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
236
87
  });
237
88
  this.sessionExpiryTimer.start();
238
89
  }
239
- // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
240
- // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
241
- this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
242
- /**
243
- * Whether GC should run or not. The following conditions have to be met to run sweep:
244
- *
245
- * 1. GC should be enabled for this container.
246
- *
247
- * 2. GC should not be disabled via disableGC GC option.
248
- *
249
- * These conditions can be overridden via runGCKey feature flag.
250
- */
251
- this.shouldRunGC =
252
- (_d = this.mc.config.getBoolean(runGCKey)) !== null && _d !== void 0 ? _d :
253
- // GC must be enabled for the document.
254
- (this.gcEnabled &&
255
- // GC must not be disabled via GC options.
256
- !this.gcOptions.disableGC);
257
- /**
258
- * Whether sweep should run or not. The following conditions have to be met to run sweep:
259
- *
260
- * 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
261
- * 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
262
- * 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
263
- * the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
264
- * 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
265
- * feature flag.
266
- */
267
- this.shouldRunSweep =
268
- this.shouldRunGC &&
269
- this.sweepTimeoutMs !== undefined &&
270
- ((_e = this.mc.config.getBoolean(runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
271
- this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
272
- // Override inactive timeout if test config or gc options to override it is set.
273
- this.inactiveTimeoutMs =
274
- (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : defaultInactiveTimeoutMs;
275
- // Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
276
- if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
277
- throw new UsageError("inactive timeout should not be greater than the sweep timeout");
278
- }
279
- // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
280
- this.testMode =
281
- (_h = this.mc.config.getBoolean(gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
282
- // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
283
- // via feature flags.
284
- this.tombstoneMode =
285
- !this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
286
- // If GC ran in the container that generated the base snapshot, it will have a GC tree.
287
- this.wasGCRunInLatestSummary = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined;
90
+ this.summaryStateTracker = new GCSummaryStateTracker(this.configs, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */);
288
91
  // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
289
92
  // it involves fetching blobs from storage which is expensive.
290
93
  this.baseSnapshotDataP = new LazyPromise(async () => {
291
- var _a;
292
94
  if (baseSnapshot === undefined) {
293
95
  return undefined;
294
96
  }
295
97
  try {
296
98
  // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
297
99
  const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
298
- if (gcSnapshotTree !== undefined) {
299
- return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
100
+ if (gcSnapshotTree === undefined) {
101
+ // back-compat - Older documents get their gc data reset for simplicity as there are few of them
102
+ // incremental gc summary will not work with older gc data as well
103
+ return undefined;
300
104
  }
301
- // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
302
- // consolidate into IGarbageCollectionState format.
303
- // Add a node for the root node that is not present in older snapshot format.
304
- const gcState = {
305
- gcNodes: { "/": { outboundRoutes: [] } },
306
- };
307
- const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
308
- assert(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
309
- for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
310
- const blobId = dsSnapshotTree.blobs[gcTreeKey];
311
- if (blobId === undefined) {
312
- continue;
313
- }
314
- const gcSummaryDetails = await readAndParseBlob(blobId);
315
- // If there are no nodes for this data store, skip it.
316
- if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
317
- continue;
318
- }
319
- const dsRootId = `/${dsId}`;
320
- // Since we used to write GC data at data store level, we won't have an entry for the root ("/").
321
- // Construct that entry by adding root data store ids to its outbound routes.
322
- const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[dataStoreAttributesBlobName]);
323
- if (initialSnapshotDetails.isRootDataStore) {
324
- gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
325
- }
326
- for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
327
- // Prefix the data store id to the GC node ids to make them relative to the root from being
328
- // relative to the data store. Similar to how its done in DataStore::getGCData.
329
- const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
330
- gcState.gcNodes[rootId] = {
331
- outboundRoutes: Array.from(outboundRoutes),
332
- };
333
- }
334
- assert(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
335
- gcState.gcNodes[dsRootId].unreferencedTimestampMs =
336
- gcSummaryDetails.unrefTimestamp;
105
+ const snapshotData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
106
+ // If the GC version in base snapshot does not match the GC version currently in effect, the GC data
107
+ // in the snapshot cannot be interpreted correctly. Set everything to undefined except for
108
+ // deletedNodes because irrespective of GC versions, these nodes have been deleted and cannot be
109
+ // brought back. The deletedNodes info is needed to identify when these nodes are used.
110
+ if (this.configs.gcVersionInBaseSnapshot !==
111
+ this.summaryStateTracker.currentGCVersion) {
112
+ return {
113
+ gcState: undefined,
114
+ tombstones: undefined,
115
+ deletedNodes: snapshotData.deletedNodes,
116
+ };
337
117
  }
338
- // If there is only one node (root node just added above), either GC is disabled or we are loading from
339
- // the first summary generated by detached container. In both cases, GC was not run - return undefined.
340
- return Object.keys(gcState.gcNodes).length === 1
341
- ? undefined
342
- : { gcState, tombstones: undefined, deletedNodes: undefined };
118
+ return snapshotData;
343
119
  }
344
120
  catch (error) {
345
121
  const dpe = DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
346
- dpe.addTelemetryProperties({ gcConfigs: JSON.stringify(this.configs) });
122
+ dpe.addTelemetryProperties({
123
+ gcConfigs: JSON.stringify(this.configs),
124
+ });
347
125
  throw dpe;
348
126
  }
349
127
  });
@@ -381,12 +159,13 @@ export class GarbageCollector {
381
159
  return;
382
160
  }
383
161
  this.updateStateFromSnapshotData(baseSnapshotData, currentReferenceTimestampMs);
162
+ this.summaryStateTracker.initializeBaseState(baseSnapshotData);
384
163
  });
385
164
  // Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
386
165
  // used to initialize the GC state of all the nodes in the container.
387
166
  this.baseGCDetailsP = new LazyPromise(async () => {
388
167
  const baseSnapshotData = await this.baseSnapshotDataP;
389
- if (baseSnapshotData === undefined) {
168
+ if ((baseSnapshotData === null || baseSnapshotData === void 0 ? void 0 : baseSnapshotData.gcState) === undefined) {
390
169
  return {};
391
170
  }
392
171
  const gcNodes = {};
@@ -405,51 +184,18 @@ export class GarbageCollector {
405
184
  this.mc.logger.sendTelemetryEvent({
406
185
  eventName: "GarbageCollectorLoaded",
407
186
  gcConfigs: JSON.stringify(this.configs),
187
+ gcOptions: JSON.stringify(createParams.gcOptions),
408
188
  });
409
189
  }
410
190
  }
411
191
  static create(createParams) {
412
192
  return new GarbageCollector(createParams);
413
193
  }
414
- /**
415
- * Tells whether the GC state needs to be reset in the next summary. We need to do this if:
416
- *
417
- * 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
418
- *
419
- * 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
420
- *
421
- * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
422
- *
423
- * 4. The GC version in the latest summary is different from the current GC version. This can happen if:
424
- *
425
- * 4.1. The summary this client loaded with has data from a different GC version.
426
- *
427
- * 4.2. This client's latest summary was updated from a snapshot that has a different GC version.
428
- */
429
- get summaryStateNeedsReset() {
430
- return (this.gcStateNeedsReset ||
431
- (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion));
194
+ get shouldRunGC() {
195
+ return this.configs.shouldRunGC;
432
196
  }
433
- /**
434
- * Tells whether the GC state needs to be reset. This can happen under 3 conditions:
435
- *
436
- * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
437
- * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
438
- *
439
- * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
440
- * a document and the first time GC is enabled after is was disabled before.
441
- *
442
- * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
443
- *
444
- * Note that the state will be reset only once for the first summary generated after this returns true. After that,
445
- * this will return false.
446
- */
447
- get gcStateNeedsReset() {
448
- return this.wasGCRunInLatestSummary !== this.shouldRunGC;
449
- }
450
- /** Returns a list of all the configurations for garbage collection. */
451
- get configs() {
452
- return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
197
+ get summaryStateNeedsReset() {
198
+ return this.summaryStateTracker.doesSummaryStateNeedReset;
453
199
  }
454
200
  /**
455
201
  * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
@@ -473,7 +219,8 @@ export class GarbageCollector {
473
219
  }
474
220
  // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
475
221
  // tombstone routes.
476
- if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
222
+ if (this.configs.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
223
+ // Create a copy since we are writing from a source we don't control
477
224
  this.tombstones = Array.from(baseSnapshotData.tombstones);
478
225
  this.runtime.updateTombstonedRoutes(this.tombstones);
479
226
  }
@@ -507,9 +254,9 @@ export class GarbageCollector {
507
254
  // tombstones.
508
255
  // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
509
256
  // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
510
- if (this.shouldRunSweep) {
511
- const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones)
512
- ? new Set(snapshotData.tombstones)
257
+ if (this.configs.shouldRunSweep) {
258
+ const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.deletedNodes)
259
+ ? new Set(snapshotData.deletedNodes)
513
260
  : undefined;
514
261
  // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
515
262
  // delete them.
@@ -525,16 +272,15 @@ export class GarbageCollector {
525
272
  }
526
273
  }
527
274
  }
528
- else if (this.tombstoneMode) {
275
+ else if (this.configs.tombstoneMode) {
529
276
  // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
530
277
  // notify the runtime to update its state as well.
531
278
  this.tombstones = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? Array.from(snapshotData.tombstones) : [];
532
279
  this.runtime.updateTombstonedRoutes(this.tombstones);
533
280
  }
534
281
  // If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
535
- if (snapshotData === undefined) {
282
+ if ((snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.gcState) === undefined) {
536
283
  this.gcDataFromLastRun = undefined;
537
- this.latestSummaryData = undefined;
538
284
  return;
539
285
  }
540
286
  // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
@@ -542,19 +288,11 @@ export class GarbageCollector {
542
288
  const gcNodes = {};
543
289
  for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
544
290
  if (nodeData.unreferencedTimestampMs !== undefined) {
545
- this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
291
+ this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
546
292
  }
547
293
  gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
548
294
  }
549
295
  this.gcDataFromLastRun = { gcNodes };
550
- // If tracking state across summaries, update latest summary data from the snapshot's GC data.
551
- if (this.trackGCState) {
552
- this.latestSummaryData = {
553
- serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
554
- serializedTombstones: JSON.stringify(snapshotData.tombstones),
555
- serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
556
- };
557
- }
558
296
  }
559
297
  /**
560
298
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
@@ -576,7 +314,7 @@ export class GarbageCollector {
576
314
  * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
577
315
  * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
578
316
  */
579
- if (this.activeConnection() && this.shouldRunGC) {
317
+ if (this.activeConnection() && this.configs.shouldRunGC) {
580
318
  this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
581
319
  }
582
320
  }
@@ -584,9 +322,9 @@ export class GarbageCollector {
584
322
  * Runs garbage collection and updates the reference / used state of the nodes in the container.
585
323
  * @returns stats of the GC run or undefined if GC did not run.
586
324
  */
587
- async collectGarbage(options) {
325
+ async collectGarbage(options, telemetryContext) {
588
326
  var _a;
589
- const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
327
+ const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.configs.runFullGC === true || this.summaryStateTracker.doesSummaryStateNeedReset);
590
328
  const logger = options.logger
591
329
  ? ChildLogger.create(options.logger, undefined, {
592
330
  all: { completedGCRuns: () => this.completedRuns },
@@ -609,6 +347,11 @@ export class GarbageCollector {
609
347
  });
610
348
  return undefined;
611
349
  }
350
+ // Add the options that are used to run GC to the telemetry context.
351
+ telemetryContext === null || telemetryContext === void 0 ? void 0 : telemetryContext.setMultiple("fluid_GC", "Options", {
352
+ fullGC,
353
+ runSweep: options.runSweep,
354
+ });
612
355
  return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
613
356
  await this.runPreGCSteps();
614
357
  // Get the runtime's GC data and run GC on the reference graph in it.
@@ -630,26 +373,29 @@ export class GarbageCollector {
630
373
  // Generate statistics from the current run. This is done before updating the current state because it
631
374
  // generates some of its data based on previous state of the system.
632
375
  const gcStats = this.generateStats(gcResult);
633
- // Update the state since the last GC run. There can be nodes that were referenced between the last and
634
- // the current run. We need to identify than and update their unreferenced state if needed.
635
- this.updateStateSinceLastRun(gcData, logger);
636
- // Update the current state and update the runtime of all routes or ids that used as per the GC run.
637
- this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
376
+ // Update the current mark state and update the runtime of all used routes or ids that used as per the GC run.
377
+ const sweepReadyNodes = this.updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger);
638
378
  this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
639
379
  // Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
640
380
  // delete these objects here instead.
641
381
  this.logSweepEvents(logger, currentReferenceTimestampMs);
642
- // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
643
- // involving access to deleted data.
644
- if (this.testMode) {
382
+ let updatedGCData = gcData;
383
+ if (this.configs.shouldRunSweep) {
384
+ updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
385
+ }
386
+ else if (this.configs.testMode) {
387
+ // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
388
+ // involving access to deleted data.
645
389
  this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
646
390
  }
647
- else if (this.tombstoneMode) {
391
+ else if (this.configs.tombstoneMode) {
392
+ this.tombstones = sweepReadyNodes;
648
393
  // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
649
394
  // involving access to "deleted" data without actually deleting the data from summaries.
650
395
  // Note: we will not tombstone in test mode.
651
396
  this.runtime.updateTombstonedRoutes(this.tombstones);
652
397
  }
398
+ this.gcDataFromLastRun = cloneGCData(updatedGCData);
653
399
  // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
654
400
  // updates its state so that we don't send false positives based on intermediate state. For example, we may get
655
401
  // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
@@ -663,7 +409,7 @@ export class GarbageCollector {
663
409
  */
664
410
  summarize(fullTree, trackState, telemetryContext) {
665
411
  var _a;
666
- if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
412
+ if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
667
413
  return;
668
414
  }
669
415
  const gcState = { gcNodes: {} };
@@ -673,97 +419,7 @@ export class GarbageCollector {
673
419
  unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
674
420
  };
675
421
  }
676
- const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
677
- // Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
678
- // to identify deleted nodes' usage.
679
- const serializedDeletedNodes = this.deletedNodes.size > 0
680
- ? JSON.stringify(Array.from(this.deletedNodes).sort())
681
- : undefined;
682
- // If running in tombstone mode, serialize and write tombstones, if any.
683
- const serializedTombstones = this.tombstoneMode
684
- ? this.tombstones.length > 0
685
- ? JSON.stringify(this.tombstones.sort())
686
- : undefined
687
- : undefined;
688
- /**
689
- * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
690
- * write summary handle instead of summary tree for GC.
691
- * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
692
- * for each of these that did not change, write a summary handle.
693
- */
694
- if (this.trackGCState) {
695
- this.pendingSummaryData = {
696
- serializedGCState,
697
- serializedTombstones,
698
- serializedDeletedNodes,
699
- };
700
- if (trackState && !fullTree && this.latestSummaryData !== undefined) {
701
- // If nothing changed since last summary, send a summary handle for the entire GC data.
702
- if (this.latestSummaryData.serializedGCState === serializedGCState &&
703
- this.latestSummaryData.serializedTombstones === serializedTombstones) {
704
- const stats = mergeStats();
705
- stats.handleNodeCount++;
706
- return {
707
- summary: {
708
- type: SummaryType.Handle,
709
- handle: `/${gcTreeKey}`,
710
- handleType: SummaryType.Tree,
711
- },
712
- stats,
713
- };
714
- }
715
- // If some state changed, build a GC summary tree.
716
- return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
717
- }
718
- }
719
- // If not tracking GC state, build a GC summary tree without any summary handles.
720
- return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
721
- }
722
- /**
723
- * Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
724
- * If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
725
- * If trackState is true, only states that changed are written. Rest are written as handles.
726
- * @param serializedGCState - The GC state serialized as string.
727
- * @param serializedTombstones - The tombstone state serialized as string.
728
- * @param serializedDeletedNodes - Deleted nodes serialized as string.
729
- * @param trackState - Whether we are tracking GC state across summaries.
730
- * @returns the GC summary tree.
731
- */
732
- buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, trackState) {
733
- var _a, _b, _c;
734
- const gcStateBlobKey = `${gcBlobPrefix}_root`;
735
- const builder = new SummaryTreeBuilder();
736
- // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
737
- if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
738
- builder.addHandle(gcStateBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcStateBlobKey}`);
739
- }
740
- else {
741
- builder.addBlob(gcStateBlobKey, serializedGCState);
742
- }
743
- // If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
744
- // summary blob.
745
- if (serializedTombstones !== undefined) {
746
- if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones &&
747
- trackState) {
748
- builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
749
- }
750
- else {
751
- builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
752
- }
753
- }
754
- // If there are no deleted nodes, return the summary tree.
755
- if (serializedDeletedNodes === undefined) {
756
- return builder.getSummaryTree();
757
- }
758
- // If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
759
- if (((_c = this.latestSummaryData) === null || _c === void 0 ? void 0 : _c.serializedDeletedNodes) === serializedDeletedNodes &&
760
- trackState) {
761
- builder.addHandle(gcDeletedBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcDeletedBlobKey}`);
762
- }
763
- else {
764
- builder.addBlob(gcDeletedBlobKey, serializedDeletedNodes);
765
- }
766
- return builder.getSummaryTree();
422
+ return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
767
423
  }
768
424
  getMetadata() {
769
425
  return {
@@ -771,10 +427,11 @@ export class GarbageCollector {
771
427
  * If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
772
428
  * into the metadata blob. If GC is disabled, the gcFeature is 0.
773
429
  */
774
- gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
775
- sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
776
- sweepEnabled: this.sweepEnabled,
777
- sweepTimeoutMs: this.sweepTimeoutMs,
430
+ gcFeature: this.configs.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
431
+ gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
432
+ sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
433
+ sweepEnabled: false,
434
+ sweepTimeoutMs: this.configs.sweepTimeoutMs,
778
435
  };
779
436
  }
780
437
  /**
@@ -788,49 +445,23 @@ export class GarbageCollector {
788
445
  * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
789
446
  * is downloaded and should be used to update the state.
790
447
  */
791
- async refreshLatestSummary(result, proposalHandle, summaryRefSeq, readAndParseBlob) {
792
- // If the latest summary was updated and the summary was tracked, this client is the one that generated this
793
- // summary. So, update wasGCRunInLatestSummary.
794
- // Note that this has to be updated if GC did not run too. Otherwise, `gcStateNeedsReset` will always return
795
- // true in scenarios where GC is disabled but enabled in the snapshot we loaded from.
796
- if (result.latestSummaryUpdated && result.wasSummaryTracked) {
797
- this.wasGCRunInLatestSummary = this.shouldRunGC;
798
- }
799
- if (!result.latestSummaryUpdated || !this.shouldRunGC) {
800
- return;
801
- }
802
- // If the summary was tracked by this client, it was the one that generated the summary in the first place.
803
- // Update latest state from pending.
804
- if (result.wasSummaryTracked) {
805
- this.latestSummaryGCVersion = this.currentGCVersion;
806
- if (this.trackGCState) {
807
- this.latestSummaryData = this.pendingSummaryData;
808
- this.pendingSummaryData = undefined;
448
+ async refreshLatestSummary(proposalHandle, result, readAndParseBlob) {
449
+ const latestSnapshotData = await this.summaryStateTracker.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
450
+ // If the latest summary was updated but it was not tracked by this client, our state needs to be updated from
451
+ // this snapshot data.
452
+ if (this.shouldRunGC && result.latestSummaryUpdated && !result.wasSummaryTracked) {
453
+ // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
454
+ // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
455
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
456
+ if (currentReferenceTimestampMs === undefined) {
457
+ throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
458
+ proposalHandle,
459
+ summaryRefSeq: result.summaryRefSeq,
460
+ gcConfigs: JSON.stringify(this.configs),
461
+ });
809
462
  }
810
- return;
811
- }
812
- // If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
813
- const snapshot = result.snapshot;
814
- const metadataBlobId = snapshot.blobs[metadataBlobName];
815
- if (metadataBlobId) {
816
- const metadata = await readAndParseBlob(metadataBlobId);
817
- this.latestSummaryGCVersion = getGCVersion(metadata);
818
- }
819
- // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
820
- // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
821
- const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
822
- if (currentReferenceTimestampMs === undefined) {
823
- throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, { proposalHandle, summaryRefSeq, details: JSON.stringify(this.configs) });
463
+ this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
824
464
  }
825
- const gcSnapshotTree = snapshot.trees[gcTreeKey];
826
- // If GC ran in the container that generated this snapshot, it will have a GC tree.
827
- this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
828
- let latestGCData;
829
- if (gcSnapshotTree !== undefined) {
830
- latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
831
- }
832
- this.updateStateFromSnapshotData(latestGCData, currentReferenceTimestampMs);
833
- this.pendingSummaryData = undefined;
834
465
  }
835
466
  /**
836
467
  * Called when a node with the given id is updated. If the node is inactive, log an error.
@@ -841,7 +472,7 @@ export class GarbageCollector {
841
472
  * @param requestHeaders - If the node was loaded via request path, the headers in the request.
842
473
  */
843
474
  nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
844
- if (!this.shouldRunGC) {
475
+ if (!this.configs.shouldRunGC) {
845
476
  return;
846
477
  }
847
478
  const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
@@ -858,7 +489,7 @@ export class GarbageCollector {
858
489
  */
859
490
  addedOutboundReference(fromNodePath, toNodePath) {
860
491
  var _a;
861
- if (!this.shouldRunGC) {
492
+ if (!this.configs.shouldRunGC) {
862
493
  return;
863
494
  }
864
495
  const outboundRoutes = (_a = this.newReferencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
@@ -880,9 +511,9 @@ export class GarbageCollector {
880
511
  sendGCUnexpectedUsageEvent(this.mc, {
881
512
  eventName,
882
513
  category: "generic",
883
- isSummarizerClient: this.isSummarizerClient,
884
- url: trimLeadingSlashes(toNodePath),
514
+ url: toNodePath,
885
515
  nodeType,
516
+ gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
886
517
  }, undefined /* packagePath */);
887
518
  }
888
519
  }
@@ -906,13 +537,16 @@ export class GarbageCollector {
906
537
  * @param gcData - The data representing the reference graph on which GC is run.
907
538
  * @param gcResult - The result of the GC run on the gcData.
908
539
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
540
+ * @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
909
541
  */
910
- updateCurrentState(gcData, gcResult, currentReferenceTimestampMs) {
911
- this.gcDataFromLastRun = cloneGCData(gcData);
912
- this.tombstones = [];
542
+ updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger) {
543
+ var _a;
544
+ // Get references from the current GC run + references between previous and current run and then update each
545
+ // node's state
546
+ const allNodesReferencedBetweenGCs = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
913
547
  this.newReferencesSinceLastRun.clear();
914
548
  // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
915
- for (const nodeId of gcResult.referencedNodeIds) {
549
+ for (const nodeId of allNodesReferencedBetweenGCs) {
916
550
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
917
551
  if (nodeStateTracker !== undefined) {
918
552
  // Stop tracking so as to clear out any running timers.
@@ -925,22 +559,62 @@ export class GarbageCollector {
925
559
  * If a node became unreferenced in this run, start tracking it.
926
560
  * If a node was already unreferenced, update its tracking information. Since the current reference time is
927
561
  * from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
562
+ *
563
+ * If a node is sweep ready, store and then return it.
928
564
  */
565
+ const sweepReadyNodes = [];
929
566
  for (const nodeId of gcResult.deletedNodeIds) {
930
567
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
931
568
  if (nodeStateTracker === undefined) {
932
- this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
569
+ this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
933
570
  }
934
571
  else {
935
572
  nodeStateTracker.updateTracking(currentReferenceTimestampMs);
936
- if (this.tombstoneMode && nodeStateTracker.state === UnreferencedState.SweepReady) {
937
- const nodeType = this.runtime.getNodeType(nodeId);
938
- if (nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) {
939
- this.tombstones.push(nodeId);
940
- }
573
+ if (nodeStateTracker.state === UnreferencedState.SweepReady) {
574
+ sweepReadyNodes.push(nodeId);
941
575
  }
942
576
  }
943
577
  }
578
+ return sweepReadyNodes;
579
+ }
580
+ /**
581
+ * Deletes nodes from both the runtime and garbage collection
582
+ * @param sweepReadyNodes - nodes that are ready to be deleted
583
+ */
584
+ runSweepPhase(sweepReadyNodes, gcData) {
585
+ // TODO: GC:Validation - validate that removed routes are not double deleted
586
+ // TODO: GC:Validation - validate that the child routes of removed routes are deleted as well
587
+ const sweptRoutes = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
588
+ const updatedGCData = this.deleteSweptRoutes(sweptRoutes, gcData);
589
+ for (const nodeId of sweptRoutes) {
590
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
591
+ // TODO: GC:Validation - assert that the nodeStateTracker is defined
592
+ if (nodeStateTracker !== undefined) {
593
+ // Stop tracking so as to clear out any running timers.
594
+ nodeStateTracker.stopTracking();
595
+ // Delete the node as we don't need to track it any more.
596
+ this.unreferencedNodesState.delete(nodeId);
597
+ }
598
+ // TODO: GC:Validation - assert that the deleted node is not a duplicate
599
+ this.deletedNodes.add(nodeId);
600
+ }
601
+ return updatedGCData;
602
+ }
603
+ /**
604
+ * @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
605
+ */
606
+ deleteSweptRoutes(sweptRoutes, gcData) {
607
+ const sweptRoutesSet = new Set(sweptRoutes);
608
+ const gcNodes = {};
609
+ for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
610
+ if (!sweptRoutesSet.has(id)) {
611
+ gcNodes[id] = Array.from(outboundRoutes);
612
+ }
613
+ }
614
+ // TODO: GC:Validation - assert that the nodeId is in gcData
615
+ return {
616
+ gcNodes,
617
+ };
944
618
  }
945
619
  /**
946
620
  * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
@@ -952,16 +626,19 @@ export class GarbageCollector {
952
626
  * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
953
627
  * unreferenced, they could have been accessed and in-memory reference to them added.
954
628
  *
955
- * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
629
+ * This function identifies nodes that were referenced since the last run.
956
630
  * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
631
+ *
632
+ * @returns - a list of all nodes referenced from the last local summary until now.
957
633
  */
958
- updateStateSinceLastRun(currentGCData, logger) {
634
+ findAllNodesReferencedBetweenGCs(currentGCData, previousGCData, logger) {
959
635
  // If we haven't run GC before there is nothing to do.
960
- if (this.gcDataFromLastRun === undefined) {
961
- return;
636
+ // No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
637
+ if (previousGCData === undefined) {
638
+ return undefined;
962
639
  }
963
640
  // Find any references that haven't been identified correctly.
964
- const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, this.gcDataFromLastRun, this.newReferencesSinceLastRun);
641
+ const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun);
965
642
  if (missingExplicitReferences.length > 0) {
966
643
  missingExplicitReferences.forEach((missingExplicitReference) => {
967
644
  logger.sendErrorEvent({
@@ -972,9 +649,9 @@ export class GarbageCollector {
972
649
  });
973
650
  }
974
651
  // No references were added since the last run so we don't have to update reference states of any unreferenced
975
- // nodes
652
+ // nodes. There is no in between state at this point.
976
653
  if (this.newReferencesSinceLastRun.size === 0) {
977
- return;
654
+ return undefined;
978
655
  }
979
656
  /**
980
657
  * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
@@ -992,7 +669,7 @@ export class GarbageCollector {
992
669
  * - We don't require DDSes handles to be stored in a referenced DDS.
993
670
  * - A new data store may have "root" DDSes already created and we don't detect them today.
994
671
  */
995
- const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
672
+ const gcDataSuperSet = concatGarbageCollectionData(previousGCData, currentGCData);
996
673
  const newOutboundRoutesSinceLastRun = [];
997
674
  this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
998
675
  if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
@@ -1014,15 +691,7 @@ export class GarbageCollector {
1014
691
  "/",
1015
692
  ...newOutboundRoutesSinceLastRun,
1016
693
  ]);
1017
- for (const nodeId of gcResult.referencedNodeIds) {
1018
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
1019
- if (nodeStateTracker !== undefined) {
1020
- // Stop tracking so as to clear out any running timers.
1021
- nodeStateTracker.stopTracking();
1022
- // Delete the unreferenced state as we don't need to track it any more.
1023
- this.unreferencedNodesState.delete(nodeId);
1024
- }
1025
- }
694
+ return gcResult.referencedNodeIds;
1026
695
  }
1027
696
  /**
1028
697
  * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
@@ -1132,7 +801,7 @@ export class GarbageCollector {
1132
801
  */
1133
802
  logSweepEvents(logger, currentReferenceTimestampMs) {
1134
803
  if (this.mc.config.getBoolean(disableSweepLogKey) === true ||
1135
- this.sweepTimeoutMs === undefined) {
804
+ this.configs.sweepTimeoutMs === undefined) {
1136
805
  return;
1137
806
  }
1138
807
  this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
@@ -1154,7 +823,7 @@ export class GarbageCollector {
1154
823
  id: nodeId,
1155
824
  type: nodeType,
1156
825
  age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
1157
- timeout: this.sweepTimeoutMs,
826
+ timeout: this.configs.sweepTimeoutMs,
1158
827
  completedGCRuns: this.completedRuns,
1159
828
  lastSummaryTime: this.getLastSummaryTimestampMs(),
1160
829
  });
@@ -1184,8 +853,8 @@ export class GarbageCollector {
1184
853
  }
1185
854
  this.loggedUnreferencedEvents.add(uniqueEventId);
1186
855
  const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === UnreferencedState.Inactive
1187
- ? this.inactiveTimeoutMs
1188
- : this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.externalRequest], viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
856
+ ? this.configs.inactiveTimeoutMs
857
+ : this.configs.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
1189
858
  // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
1190
859
  // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
1191
860
  // but it's a good signal nonetheless and we can consume it with a grain of salt.
@@ -1210,12 +879,6 @@ export class GarbageCollector {
1210
879
  this.mc.logger.sendErrorEvent(event);
1211
880
  }
1212
881
  }
1213
- // If SweepReady Usage Detection is enabled, the handler may close the interactive container.
1214
- // Once Sweep is fully implemented, this will be removed since the objects will be gone
1215
- // and errors will arise elsewhere in the runtime
1216
- if (state === UnreferencedState.SweepReady) {
1217
- this.sweepReadyUsageHandler.usageDetectedInInteractiveClient(Object.assign(Object.assign({}, propsToLog), { usageType }));
1218
- }
1219
882
  }
1220
883
  }
1221
884
  async logUnreferencedEvents(logger) {
@@ -1256,30 +919,4 @@ export class GarbageCollector {
1256
919
  this.pendingEventsQueue = [];
1257
920
  }
1258
921
  }
1259
- function generateSortedGCState(gcState) {
1260
- const sortableArray = Object.entries(gcState.gcNodes);
1261
- sortableArray.sort(([a], [b]) => a.localeCompare(b));
1262
- const sortedGCState = { gcNodes: {} };
1263
- for (const [nodeId, nodeData] of sortableArray) {
1264
- nodeData.outboundRoutes.sort();
1265
- sortedGCState.gcNodes[nodeId] = nodeData;
1266
- }
1267
- return sortedGCState;
1268
- }
1269
- /** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
1270
- class TimerWithNoDefaultTimeout extends Timer {
1271
- constructor(callback) {
1272
- // The default timeout/handlers will never be used since start/restart pass overrides below
1273
- super(0, () => {
1274
- throw new Error("DefaultHandler should not be used");
1275
- });
1276
- this.callback = callback;
1277
- }
1278
- start(timeoutMs) {
1279
- super.start(timeoutMs, this.callback);
1280
- }
1281
- restart(timeoutMs) {
1282
- super.restart(timeoutMs, this.callback);
1283
- }
1284
- }
1285
922
  //# sourceMappingURL=garbageCollection.js.map