@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.3.1.0.125672

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (365) hide show
  1. package/.eslintrc.js +21 -10
  2. package/.mocharc.js +2 -2
  3. package/api-extractor.json +2 -2
  4. package/dist/batchTracker.d.ts +1 -2
  5. package/dist/batchTracker.d.ts.map +1 -1
  6. package/dist/batchTracker.js +2 -1
  7. package/dist/batchTracker.js.map +1 -1
  8. package/dist/blobManager.d.ts +62 -28
  9. package/dist/blobManager.d.ts.map +1 -1
  10. package/dist/blobManager.js +256 -102
  11. package/dist/blobManager.js.map +1 -1
  12. package/dist/connectionTelemetry.d.ts.map +1 -1
  13. package/dist/connectionTelemetry.js +11 -9
  14. package/dist/connectionTelemetry.js.map +1 -1
  15. package/dist/containerHandleContext.d.ts.map +1 -1
  16. package/dist/containerHandleContext.js +3 -1
  17. package/dist/containerHandleContext.js.map +1 -1
  18. package/dist/containerRuntime.d.ts +110 -78
  19. package/dist/containerRuntime.d.ts.map +1 -1
  20. package/dist/containerRuntime.js +336 -331
  21. package/dist/containerRuntime.js.map +1 -1
  22. package/dist/dataStore.d.ts.map +1 -1
  23. package/dist/dataStore.js +11 -9
  24. package/dist/dataStore.js.map +1 -1
  25. package/dist/dataStoreContext.d.ts +2 -1
  26. package/dist/dataStoreContext.d.ts.map +1 -1
  27. package/dist/dataStoreContext.js +40 -23
  28. package/dist/dataStoreContext.js.map +1 -1
  29. package/dist/dataStoreContexts.d.ts.map +1 -1
  30. package/dist/dataStoreContexts.js +7 -3
  31. package/dist/dataStoreContexts.js.map +1 -1
  32. package/dist/dataStoreRegistry.d.ts.map +1 -1
  33. package/dist/dataStoreRegistry.js +3 -1
  34. package/dist/dataStoreRegistry.js.map +1 -1
  35. package/dist/dataStores.d.ts +12 -9
  36. package/dist/dataStores.d.ts.map +1 -1
  37. package/dist/dataStores.js +69 -46
  38. package/dist/dataStores.js.map +1 -1
  39. package/dist/deltaScheduler.d.ts.map +1 -1
  40. package/dist/deltaScheduler.js +8 -3
  41. package/dist/deltaScheduler.js.map +1 -1
  42. package/dist/garbageCollection.d.ts +57 -42
  43. package/dist/garbageCollection.d.ts.map +1 -1
  44. package/dist/garbageCollection.js +371 -239
  45. package/dist/garbageCollection.js.map +1 -1
  46. package/dist/garbageCollectionConstants.d.ts +23 -0
  47. package/dist/garbageCollectionConstants.d.ts.map +1 -0
  48. package/dist/garbageCollectionConstants.js +36 -0
  49. package/dist/garbageCollectionConstants.js.map +1 -0
  50. package/dist/garbageCollectionHelpers.d.ts +15 -0
  51. package/dist/garbageCollectionHelpers.d.ts.map +1 -0
  52. package/dist/garbageCollectionHelpers.js +27 -0
  53. package/dist/garbageCollectionHelpers.js.map +1 -0
  54. package/dist/gcSweepReadyUsageDetection.d.ts +5 -5
  55. package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
  56. package/dist/gcSweepReadyUsageDetection.js +15 -11
  57. package/dist/gcSweepReadyUsageDetection.js.map +1 -1
  58. package/dist/index.d.ts +4 -3
  59. package/dist/index.d.ts.map +1 -1
  60. package/dist/index.js +5 -6
  61. package/dist/index.js.map +1 -1
  62. package/dist/opLifecycle/batchManager.d.ts +42 -0
  63. package/dist/opLifecycle/batchManager.d.ts.map +1 -0
  64. package/dist/opLifecycle/batchManager.js +124 -0
  65. package/dist/opLifecycle/batchManager.js.map +1 -0
  66. package/dist/opLifecycle/definitions.d.ts +64 -0
  67. package/dist/opLifecycle/definitions.d.ts.map +1 -0
  68. package/dist/opLifecycle/definitions.js +7 -0
  69. package/dist/opLifecycle/definitions.js.map +1 -0
  70. package/dist/opLifecycle/index.d.ts +12 -0
  71. package/dist/opLifecycle/index.d.ts.map +1 -0
  72. package/dist/opLifecycle/index.js +22 -0
  73. package/dist/opLifecycle/index.js.map +1 -0
  74. package/dist/{opCompressor.d.ts → opLifecycle/opCompressor.d.ts} +3 -3
  75. package/dist/opLifecycle/opCompressor.d.ts.map +1 -0
  76. package/dist/opLifecycle/opCompressor.js +67 -0
  77. package/dist/opLifecycle/opCompressor.js.map +1 -0
  78. package/dist/{opDecompressor.d.ts → opLifecycle/opDecompressor.d.ts} +2 -1
  79. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -0
  80. package/dist/{opDecompressor.js → opLifecycle/opDecompressor.js} +37 -21
  81. package/dist/opLifecycle/opDecompressor.js.map +1 -0
  82. package/dist/opLifecycle/opSplitter.d.ts +49 -0
  83. package/dist/opLifecycle/opSplitter.d.ts.map +1 -0
  84. package/dist/opLifecycle/opSplitter.js +173 -0
  85. package/dist/opLifecycle/opSplitter.js.map +1 -0
  86. package/dist/opLifecycle/outbox.d.ts +52 -0
  87. package/dist/opLifecycle/outbox.d.ts.map +1 -0
  88. package/dist/opLifecycle/outbox.js +164 -0
  89. package/dist/opLifecycle/outbox.js.map +1 -0
  90. package/dist/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  91. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  92. package/dist/opLifecycle/remoteMessageProcessor.js +96 -0
  93. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -0
  94. package/dist/opProperties.d.ts.map +1 -1
  95. package/dist/opProperties.js +1 -3
  96. package/dist/opProperties.js.map +1 -1
  97. package/dist/orderedClientElection.d.ts.map +1 -1
  98. package/dist/orderedClientElection.js +10 -4
  99. package/dist/orderedClientElection.js.map +1 -1
  100. package/dist/packageVersion.d.ts +1 -1
  101. package/dist/packageVersion.js +1 -1
  102. package/dist/packageVersion.js.map +1 -1
  103. package/dist/pendingStateManager.d.ts +4 -13
  104. package/dist/pendingStateManager.d.ts.map +1 -1
  105. package/dist/pendingStateManager.js +134 -161
  106. package/dist/pendingStateManager.js.map +1 -1
  107. package/dist/runWhileConnectedCoordinator.d.ts.map +1 -1
  108. package/dist/runWhileConnectedCoordinator.js.map +1 -1
  109. package/dist/runningSummarizer.d.ts.map +1 -1
  110. package/dist/runningSummarizer.js +34 -22
  111. package/dist/runningSummarizer.js.map +1 -1
  112. package/dist/scheduleManager.d.ts +0 -1
  113. package/dist/scheduleManager.d.ts.map +1 -1
  114. package/dist/scheduleManager.js +11 -21
  115. package/dist/scheduleManager.js.map +1 -1
  116. package/dist/serializedSnapshotStorage.d.ts.map +1 -1
  117. package/dist/serializedSnapshotStorage.js +3 -1
  118. package/dist/serializedSnapshotStorage.js.map +1 -1
  119. package/dist/summarizer.d.ts +2 -3
  120. package/dist/summarizer.d.ts.map +1 -1
  121. package/dist/summarizer.js +39 -18
  122. package/dist/summarizer.js.map +1 -1
  123. package/dist/summarizerClientElection.d.ts +1 -2
  124. package/dist/summarizerClientElection.d.ts.map +1 -1
  125. package/dist/summarizerClientElection.js +3 -30
  126. package/dist/summarizerClientElection.js.map +1 -1
  127. package/dist/summarizerHandle.d.ts.map +1 -1
  128. package/dist/summarizerHandle.js.map +1 -1
  129. package/dist/summarizerHeuristics.d.ts.map +1 -1
  130. package/dist/summarizerHeuristics.js +6 -9
  131. package/dist/summarizerHeuristics.js.map +1 -1
  132. package/dist/summarizerTypes.d.ts +22 -25
  133. package/dist/summarizerTypes.d.ts.map +1 -1
  134. package/dist/summarizerTypes.js.map +1 -1
  135. package/dist/summaryCollection.d.ts.map +1 -1
  136. package/dist/summaryCollection.js +18 -8
  137. package/dist/summaryCollection.js.map +1 -1
  138. package/dist/summaryFormat.d.ts.map +1 -1
  139. package/dist/summaryFormat.js +18 -11
  140. package/dist/summaryFormat.js.map +1 -1
  141. package/dist/summaryGenerator.d.ts.map +1 -1
  142. package/dist/summaryGenerator.js +32 -14
  143. package/dist/summaryGenerator.js.map +1 -1
  144. package/dist/summaryManager.d.ts.map +1 -1
  145. package/dist/summaryManager.js +21 -9
  146. package/dist/summaryManager.js.map +1 -1
  147. package/dist/throttler.d.ts +2 -2
  148. package/dist/throttler.d.ts.map +1 -1
  149. package/dist/throttler.js +4 -4
  150. package/dist/throttler.js.map +1 -1
  151. package/garbageCollection.md +15 -2
  152. package/lib/batchTracker.d.ts +1 -2
  153. package/lib/batchTracker.d.ts.map +1 -1
  154. package/lib/batchTracker.js +2 -1
  155. package/lib/batchTracker.js.map +1 -1
  156. package/lib/blobManager.d.ts +62 -28
  157. package/lib/blobManager.d.ts.map +1 -1
  158. package/lib/blobManager.js +259 -105
  159. package/lib/blobManager.js.map +1 -1
  160. package/lib/connectionTelemetry.d.ts.map +1 -1
  161. package/lib/connectionTelemetry.js +11 -9
  162. package/lib/connectionTelemetry.js.map +1 -1
  163. package/lib/containerHandleContext.d.ts.map +1 -1
  164. package/lib/containerHandleContext.js +3 -1
  165. package/lib/containerHandleContext.js.map +1 -1
  166. package/lib/containerRuntime.d.ts +110 -78
  167. package/lib/containerRuntime.d.ts.map +1 -1
  168. package/lib/containerRuntime.js +340 -334
  169. package/lib/containerRuntime.js.map +1 -1
  170. package/lib/dataStore.d.ts.map +1 -1
  171. package/lib/dataStore.js +11 -9
  172. package/lib/dataStore.js.map +1 -1
  173. package/lib/dataStoreContext.d.ts +2 -1
  174. package/lib/dataStoreContext.d.ts.map +1 -1
  175. package/lib/dataStoreContext.js +41 -24
  176. package/lib/dataStoreContext.js.map +1 -1
  177. package/lib/dataStoreContexts.d.ts.map +1 -1
  178. package/lib/dataStoreContexts.js +7 -3
  179. package/lib/dataStoreContexts.js.map +1 -1
  180. package/lib/dataStoreRegistry.d.ts.map +1 -1
  181. package/lib/dataStoreRegistry.js +3 -1
  182. package/lib/dataStoreRegistry.js.map +1 -1
  183. package/lib/dataStores.d.ts +12 -9
  184. package/lib/dataStores.d.ts.map +1 -1
  185. package/lib/dataStores.js +75 -52
  186. package/lib/dataStores.js.map +1 -1
  187. package/lib/deltaScheduler.d.ts.map +1 -1
  188. package/lib/deltaScheduler.js +9 -4
  189. package/lib/deltaScheduler.js.map +1 -1
  190. package/lib/garbageCollection.d.ts +57 -42
  191. package/lib/garbageCollection.d.ts.map +1 -1
  192. package/lib/garbageCollection.js +364 -232
  193. package/lib/garbageCollection.js.map +1 -1
  194. package/lib/garbageCollectionConstants.d.ts +23 -0
  195. package/lib/garbageCollectionConstants.d.ts.map +1 -0
  196. package/lib/garbageCollectionConstants.js +33 -0
  197. package/lib/garbageCollectionConstants.js.map +1 -0
  198. package/lib/garbageCollectionHelpers.d.ts +15 -0
  199. package/lib/garbageCollectionHelpers.d.ts.map +1 -0
  200. package/lib/garbageCollectionHelpers.js +23 -0
  201. package/lib/garbageCollectionHelpers.js.map +1 -0
  202. package/lib/gcSweepReadyUsageDetection.d.ts +5 -5
  203. package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
  204. package/lib/gcSweepReadyUsageDetection.js +15 -11
  205. package/lib/gcSweepReadyUsageDetection.js.map +1 -1
  206. package/lib/index.d.ts +4 -3
  207. package/lib/index.d.ts.map +1 -1
  208. package/lib/index.js +3 -3
  209. package/lib/index.js.map +1 -1
  210. package/lib/opLifecycle/batchManager.d.ts +42 -0
  211. package/lib/opLifecycle/batchManager.d.ts.map +1 -0
  212. package/lib/opLifecycle/batchManager.js +120 -0
  213. package/lib/opLifecycle/batchManager.js.map +1 -0
  214. package/lib/opLifecycle/definitions.d.ts +64 -0
  215. package/lib/opLifecycle/definitions.d.ts.map +1 -0
  216. package/lib/opLifecycle/definitions.js +6 -0
  217. package/lib/opLifecycle/definitions.js.map +1 -0
  218. package/lib/opLifecycle/index.d.ts +12 -0
  219. package/lib/opLifecycle/index.d.ts.map +1 -0
  220. package/lib/opLifecycle/index.js +11 -0
  221. package/lib/opLifecycle/index.js.map +1 -0
  222. package/lib/{opCompressor.d.ts → opLifecycle/opCompressor.d.ts} +3 -3
  223. package/lib/opLifecycle/opCompressor.d.ts.map +1 -0
  224. package/lib/opLifecycle/opCompressor.js +63 -0
  225. package/lib/opLifecycle/opCompressor.js.map +1 -0
  226. package/lib/{opDecompressor.d.ts → opLifecycle/opDecompressor.d.ts} +2 -1
  227. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -0
  228. package/lib/{opDecompressor.js → opLifecycle/opDecompressor.js} +37 -21
  229. package/lib/opLifecycle/opDecompressor.js.map +1 -0
  230. package/lib/opLifecycle/opSplitter.d.ts +49 -0
  231. package/lib/opLifecycle/opSplitter.d.ts.map +1 -0
  232. package/lib/opLifecycle/opSplitter.js +168 -0
  233. package/lib/opLifecycle/opSplitter.js.map +1 -0
  234. package/lib/opLifecycle/outbox.d.ts +52 -0
  235. package/lib/opLifecycle/outbox.d.ts.map +1 -0
  236. package/lib/opLifecycle/outbox.js +160 -0
  237. package/lib/opLifecycle/outbox.js.map +1 -0
  238. package/lib/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  239. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  240. package/lib/opLifecycle/remoteMessageProcessor.js +91 -0
  241. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -0
  242. package/lib/opProperties.d.ts.map +1 -1
  243. package/lib/opProperties.js +1 -3
  244. package/lib/opProperties.js.map +1 -1
  245. package/lib/orderedClientElection.d.ts.map +1 -1
  246. package/lib/orderedClientElection.js +10 -4
  247. package/lib/orderedClientElection.js.map +1 -1
  248. package/lib/packageVersion.d.ts +1 -1
  249. package/lib/packageVersion.js +1 -1
  250. package/lib/packageVersion.js.map +1 -1
  251. package/lib/pendingStateManager.d.ts +4 -13
  252. package/lib/pendingStateManager.d.ts.map +1 -1
  253. package/lib/pendingStateManager.js +134 -161
  254. package/lib/pendingStateManager.js.map +1 -1
  255. package/lib/runWhileConnectedCoordinator.d.ts.map +1 -1
  256. package/lib/runWhileConnectedCoordinator.js.map +1 -1
  257. package/lib/runningSummarizer.d.ts.map +1 -1
  258. package/lib/runningSummarizer.js +35 -23
  259. package/lib/runningSummarizer.js.map +1 -1
  260. package/lib/scheduleManager.d.ts +0 -1
  261. package/lib/scheduleManager.d.ts.map +1 -1
  262. package/lib/scheduleManager.js +11 -21
  263. package/lib/scheduleManager.js.map +1 -1
  264. package/lib/serializedSnapshotStorage.d.ts.map +1 -1
  265. package/lib/serializedSnapshotStorage.js +3 -1
  266. package/lib/serializedSnapshotStorage.js.map +1 -1
  267. package/lib/summarizer.d.ts +2 -3
  268. package/lib/summarizer.d.ts.map +1 -1
  269. package/lib/summarizer.js +39 -18
  270. package/lib/summarizer.js.map +1 -1
  271. package/lib/summarizerClientElection.d.ts +1 -2
  272. package/lib/summarizerClientElection.d.ts.map +1 -1
  273. package/lib/summarizerClientElection.js +3 -30
  274. package/lib/summarizerClientElection.js.map +1 -1
  275. package/lib/summarizerHandle.d.ts.map +1 -1
  276. package/lib/summarizerHandle.js.map +1 -1
  277. package/lib/summarizerHeuristics.d.ts.map +1 -1
  278. package/lib/summarizerHeuristics.js +6 -9
  279. package/lib/summarizerHeuristics.js.map +1 -1
  280. package/lib/summarizerTypes.d.ts +22 -25
  281. package/lib/summarizerTypes.d.ts.map +1 -1
  282. package/lib/summarizerTypes.js.map +1 -1
  283. package/lib/summaryCollection.d.ts.map +1 -1
  284. package/lib/summaryCollection.js +18 -8
  285. package/lib/summaryCollection.js.map +1 -1
  286. package/lib/summaryFormat.d.ts.map +1 -1
  287. package/lib/summaryFormat.js +20 -13
  288. package/lib/summaryFormat.js.map +1 -1
  289. package/lib/summaryGenerator.d.ts.map +1 -1
  290. package/lib/summaryGenerator.js +32 -14
  291. package/lib/summaryGenerator.js.map +1 -1
  292. package/lib/summaryManager.d.ts.map +1 -1
  293. package/lib/summaryManager.js +21 -9
  294. package/lib/summaryManager.js.map +1 -1
  295. package/lib/throttler.d.ts +2 -2
  296. package/lib/throttler.d.ts.map +1 -1
  297. package/lib/throttler.js +4 -4
  298. package/lib/throttler.js.map +1 -1
  299. package/package.json +28 -38
  300. package/prettier.config.cjs +1 -1
  301. package/src/batchTracker.ts +55 -50
  302. package/src/blobManager.ts +802 -541
  303. package/src/connectionTelemetry.ts +280 -249
  304. package/src/containerHandleContext.ts +27 -29
  305. package/src/containerRuntime.ts +3125 -2982
  306. package/src/dataStore.ts +172 -159
  307. package/src/dataStoreContext.ts +1049 -992
  308. package/src/dataStoreContexts.ts +178 -161
  309. package/src/dataStoreRegistry.ts +25 -20
  310. package/src/dataStores.ts +785 -711
  311. package/src/deltaScheduler.ts +158 -150
  312. package/src/garbageCollection.ts +1797 -1558
  313. package/src/garbageCollectionConstants.ts +38 -0
  314. package/src/garbageCollectionHelpers.ts +37 -0
  315. package/src/gcSweepReadyUsageDetection.ts +90 -84
  316. package/src/index.ts +68 -69
  317. package/src/opLifecycle/batchManager.ts +167 -0
  318. package/src/opLifecycle/definitions.ts +70 -0
  319. package/src/opLifecycle/index.ts +18 -0
  320. package/src/opLifecycle/opCompressor.ts +82 -0
  321. package/src/opLifecycle/opDecompressor.ts +124 -0
  322. package/src/opLifecycle/opSplitter.ts +238 -0
  323. package/src/opLifecycle/outbox.ts +228 -0
  324. package/src/opLifecycle/remoteMessageProcessor.ts +106 -0
  325. package/src/opProperties.ts +11 -9
  326. package/src/orderedClientElection.ts +489 -457
  327. package/src/packageVersion.ts +1 -1
  328. package/src/pendingStateManager.ts +379 -381
  329. package/src/runWhileConnectedCoordinator.ts +78 -71
  330. package/src/runningSummarizer.ts +619 -582
  331. package/src/scheduleManager.ts +299 -280
  332. package/src/serializedSnapshotStorage.ts +116 -111
  333. package/src/summarizer.ts +417 -381
  334. package/src/summarizerClientElection.ts +107 -129
  335. package/src/summarizerHandle.ts +11 -9
  336. package/src/summarizerHeuristics.ts +183 -186
  337. package/src/summarizerTypes.ts +344 -333
  338. package/src/summaryCollection.ts +378 -349
  339. package/src/summaryFormat.ts +146 -127
  340. package/src/summaryGenerator.ts +464 -406
  341. package/src/summaryManager.ts +377 -348
  342. package/src/throttler.ts +131 -122
  343. package/tsconfig.esnext.json +6 -6
  344. package/tsconfig.json +9 -13
  345. package/dist/batchManager.d.ts +0 -42
  346. package/dist/batchManager.d.ts.map +0 -1
  347. package/dist/batchManager.js +0 -83
  348. package/dist/batchManager.js.map +0 -1
  349. package/dist/opCompressor.d.ts.map +0 -1
  350. package/dist/opCompressor.js +0 -50
  351. package/dist/opCompressor.js.map +0 -1
  352. package/dist/opDecompressor.d.ts.map +0 -1
  353. package/dist/opDecompressor.js.map +0 -1
  354. package/lib/batchManager.d.ts +0 -42
  355. package/lib/batchManager.d.ts.map +0 -1
  356. package/lib/batchManager.js +0 -79
  357. package/lib/batchManager.js.map +0 -1
  358. package/lib/opCompressor.d.ts.map +0 -1
  359. package/lib/opCompressor.js +0 -46
  360. package/lib/opCompressor.js.map +0 -1
  361. package/lib/opDecompressor.d.ts.map +0 -1
  362. package/lib/opDecompressor.js.map +0 -1
  363. package/src/batchManager.ts +0 -108
  364. package/src/opCompressor.ts +0 -59
  365. package/src/opDecompressor.ts +0 -82
@@ -15,7 +15,7 @@ var __rest = (this && this.__rest) || function (s, e) {
15
15
  return t;
16
16
  };
17
17
  Object.defineProperty(exports, "__esModule", { value: true });
18
- exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = exports.defaultSessionExpiryDurationMs = exports.defaultInactiveTimeoutMs = exports.oneDayMs = exports.throwOnTombstoneUsageKey = exports.disableTombstoneKey = exports.disableSweepLogKey = exports.trackGCStateKey = exports.runSessionExpiryKey = exports.gcTestModeKey = exports.runSweepKey = exports.runGCKey = exports.gcTombstoneBlobKey = exports.gcBlobPrefix = exports.gcTreeKey = void 0;
18
+ exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = void 0;
19
19
  const common_utils_1 = require("@fluidframework/common-utils");
20
20
  const container_utils_1 = require("@fluidframework/container-utils");
21
21
  const garbage_collector_1 = require("@fluidframework/garbage-collector");
@@ -25,36 +25,10 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
25
25
  const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
26
26
  const containerRuntime_1 = require("./containerRuntime");
27
27
  const dataStores_1 = require("./dataStores");
28
+ const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
29
+ const garbageCollectionHelpers_1 = require("./garbageCollectionHelpers");
28
30
  const gcSweepReadyUsageDetection_1 = require("./gcSweepReadyUsageDetection");
29
31
  const summaryFormat_1 = require("./summaryFormat");
30
- /** This is the current version of garbage collection. */
31
- const GCVersion = 1;
32
- // The key for the GC tree in summary.
33
- exports.gcTreeKey = "gc";
34
- // They prefix for GC blobs in the GC tree in summary.
35
- exports.gcBlobPrefix = "__gc";
36
- // The key for tombstone blob in the GC tree in summary.
37
- exports.gcTombstoneBlobKey = "__tombstones";
38
- // Feature gate key to turn GC on / off.
39
- exports.runGCKey = "Fluid.GarbageCollection.RunGC";
40
- // Feature gate key to turn GC sweep on / off.
41
- exports.runSweepKey = "Fluid.GarbageCollection.RunSweep";
42
- // Feature gate key to turn GC test mode on / off.
43
- exports.gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
44
- // Feature gate key to expire a session after a set period of time.
45
- exports.runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
46
- // Feature gate key to write the gc blob as a handle if the data is the same.
47
- exports.trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
48
- // Feature gate key to turn GC sweep log off.
49
- exports.disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
50
- // Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
51
- exports.disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
52
- // Feature gate to enable throwing an error when tombstone object is used.
53
- exports.throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
54
- // One day in milliseconds.
55
- exports.oneDayMs = 1 * 24 * 60 * 60 * 1000;
56
- exports.defaultInactiveTimeoutMs = 7 * exports.oneDayMs; // 7 days
57
- exports.defaultSessionExpiryDurationMs = 30 * exports.oneDayMs; // 30 days
58
32
  /** The types of GC nodes in the GC reference graph. */
59
33
  exports.GCNodeType = {
60
34
  // Nodes that are for data stores.
@@ -168,25 +142,13 @@ exports.UnreferencedStateTracker = UnreferencedStateTracker;
168
142
  class GarbageCollector {
169
143
  constructor(createParams) {
170
144
  var _a, _b, _c, _d, _e, _f, _g, _h;
171
- /**
172
- * Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
173
- *
174
- * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
175
- * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
176
- *
177
- * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
178
- * a document and the first time GC is enabled after is was disabled before.
179
- *
180
- * Note that the state needs reset only for the very first time summary is generated by this client. After that, the
181
- * state will be up-to-date and this flag will be reset.
182
- */
183
- this.initialStateNeedsReset = false;
184
- // The current GC version that this container is running.
185
- this.currentGCVersion = GCVersion;
186
145
  // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
187
146
  // outbound routes from that node.
188
147
  this.newReferencesSinceLastRun = new Map();
148
+ // A list of nodes that have been tombstoned.
189
149
  this.tombstones = [];
150
+ // A list of nodes that have been deleted during sweep phase.
151
+ this.deletedNodes = new Set();
190
152
  // Map of node ids to their unreferenced state tracker.
191
153
  this.unreferencedNodesState = new Map();
192
154
  // Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
@@ -206,7 +168,14 @@ class GarbageCollector {
206
168
  const baseSnapshot = createParams.baseSnapshot;
207
169
  const metadata = createParams.metadata;
208
170
  const readAndParseBlob = createParams.readAndParseBlob;
209
- this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } }));
171
+ this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
172
+ all: { completedGCRuns: () => this.completedRuns },
173
+ }));
174
+ // If version upgrade is not enabled, fall back to the stable GC version.
175
+ this.currentGCVersion =
176
+ this.mc.config.getBoolean(garbageCollectionConstants_1.gcVersionUpgradeToV2Key) === true
177
+ ? garbageCollectionConstants_1.currentGCVersion
178
+ : garbageCollectionConstants_1.stableGCVersion;
210
179
  this.sweepReadyUsageHandler = new gcSweepReadyUsageDetection_1.SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
211
180
  let prevSummaryGCVersion;
212
181
  /**
@@ -218,10 +187,10 @@ class GarbageCollector {
218
187
  * We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
219
188
  */
220
189
  function computeSweepTimeout(sessionExpiryTimeoutMs) {
221
- const maxSnapshotCacheExpiryMs = 5 * exports.oneDayMs;
222
- const bufferMs = exports.oneDayMs;
223
- return sessionExpiryTimeoutMs &&
224
- (sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
190
+ const maxSnapshotCacheExpiryMs = 5 * garbageCollectionConstants_1.oneDayMs;
191
+ const bufferMs = garbageCollectionConstants_1.oneDayMs;
192
+ return (sessionExpiryTimeoutMs &&
193
+ sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
225
194
  }
226
195
  /**
227
196
  * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
@@ -252,11 +221,11 @@ class GarbageCollector {
252
221
  // flag in GC options to false.
253
222
  this.gcEnabled = this.gcOptions.gcAllowed !== false;
254
223
  // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
255
- // ...unless we're using the TestOverride
256
- this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
224
+ this.sweepEnabled = this.gcOptions.sweepAllowed === true;
257
225
  // Set the Session Expiry only if the flag is enabled and GC is enabled.
258
- if (this.mc.config.getBoolean(exports.runSessionExpiryKey) && this.gcEnabled) {
259
- this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : exports.defaultSessionExpiryDurationMs;
226
+ if (this.mc.config.getBoolean(garbageCollectionConstants_1.runSessionExpiryKey) && this.gcEnabled) {
227
+ this.sessionExpiryTimeoutMs =
228
+ (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : garbageCollectionConstants_1.defaultSessionExpiryDurationMs;
260
229
  }
261
230
  this.sweepTimeoutMs =
262
231
  testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
@@ -266,7 +235,9 @@ class GarbageCollector {
266
235
  // If Test Override config is set, override Session Expiry timeout.
267
236
  const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
268
237
  const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
269
- this.sessionExpiryTimer = new common_utils_1.Timer(timeoutMs, () => { this.runtime.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs)); });
238
+ this.sessionExpiryTimer = new common_utils_1.Timer(timeoutMs, () => {
239
+ this.runtime.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
240
+ });
270
241
  this.sessionExpiryTimer.start();
271
242
  }
272
243
  // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
@@ -281,11 +252,12 @@ class GarbageCollector {
281
252
  *
282
253
  * These conditions can be overridden via runGCKey feature flag.
283
254
  */
284
- this.shouldRunGC = (_d = this.mc.config.getBoolean(exports.runGCKey)) !== null && _d !== void 0 ? _d : (
285
- // GC must be enabled for the document.
286
- this.gcEnabled
287
- // GC must not be disabled via GC options.
288
- && !this.gcOptions.disableGC);
255
+ this.shouldRunGC =
256
+ (_d = this.mc.config.getBoolean(garbageCollectionConstants_1.runGCKey)) !== null && _d !== void 0 ? _d :
257
+ // GC must be enabled for the document.
258
+ (this.gcEnabled &&
259
+ // GC must not be disabled via GC options.
260
+ !this.gcOptions.disableGC);
289
261
  /**
290
262
  * Whether sweep should run or not. The following conditions have to be met to run sweep:
291
263
  *
@@ -297,24 +269,26 @@ class GarbageCollector {
297
269
  * feature flag.
298
270
  */
299
271
  this.shouldRunSweep =
300
- this.shouldRunGC
301
- && this.sweepTimeoutMs !== undefined
302
- && ((_e = this.mc.config.getBoolean(exports.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
303
- this.trackGCState = this.mc.config.getBoolean(exports.trackGCStateKey) === true;
272
+ this.shouldRunGC &&
273
+ this.sweepTimeoutMs !== undefined &&
274
+ ((_e = this.mc.config.getBoolean(garbageCollectionConstants_1.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
275
+ this.trackGCState = this.mc.config.getBoolean(garbageCollectionConstants_1.trackGCStateKey) === true;
304
276
  // Override inactive timeout if test config or gc options to override it is set.
305
- this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : exports.defaultInactiveTimeoutMs;
277
+ this.inactiveTimeoutMs =
278
+ (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : garbageCollectionConstants_1.defaultInactiveTimeoutMs;
306
279
  // Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
307
280
  if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
308
281
  throw new container_utils_1.UsageError("inactive timeout should not be greater than the sweep timeout");
309
282
  }
310
283
  // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
311
- this.testMode = (_h = this.mc.config.getBoolean(exports.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
312
- // Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
313
- this.tombstoneMode = this.mc.config.getBoolean(exports.disableTombstoneKey) !== true;
314
- // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
315
- // contain GC tree and GC is enabled.
316
- const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[exports.gcTreeKey]) !== undefined;
317
- this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
284
+ this.testMode =
285
+ (_h = this.mc.config.getBoolean(garbageCollectionConstants_1.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
286
+ // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
287
+ // via feature flags.
288
+ this.tombstoneMode =
289
+ !this.shouldRunSweep && this.mc.config.getBoolean(garbageCollectionConstants_1.disableTombstoneKey) !== true;
290
+ // If GC ran in the container that generated the base snapshot, it will have a GC tree.
291
+ this.wasGCRunInLatestSummary = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[runtime_definitions_1.gcTreeKey]) !== undefined;
318
292
  // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
319
293
  // it involves fetching blobs from storage which is expensive.
320
294
  this.baseSnapshotDataP = new common_utils_1.LazyPromise(async () => {
@@ -324,18 +298,20 @@ class GarbageCollector {
324
298
  }
325
299
  try {
326
300
  // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
327
- const gcSnapshotTree = baseSnapshot.trees[exports.gcTreeKey];
301
+ const gcSnapshotTree = baseSnapshot.trees[runtime_definitions_1.gcTreeKey];
328
302
  if (gcSnapshotTree !== undefined) {
329
- return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
303
+ return (0, garbage_collector_1.getGCDataFromSnapshot)(gcSnapshotTree, readAndParseBlob);
330
304
  }
331
305
  // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
332
306
  // consolidate into IGarbageCollectionState format.
333
307
  // Add a node for the root node that is not present in older snapshot format.
334
- const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
308
+ const gcState = {
309
+ gcNodes: { "/": { outboundRoutes: [] } },
310
+ };
335
311
  const dataStoreSnapshotTree = (0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata);
336
312
  (0, common_utils_1.assert)(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
337
313
  for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
338
- const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcBlobKey];
314
+ const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcTreeKey];
339
315
  if (blobId === undefined) {
340
316
  continue;
341
317
  }
@@ -355,14 +331,19 @@ class GarbageCollector {
355
331
  // Prefix the data store id to the GC node ids to make them relative to the root from being
356
332
  // relative to the data store. Similar to how its done in DataStore::getGCData.
357
333
  const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
358
- gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
334
+ gcState.gcNodes[rootId] = {
335
+ outboundRoutes: Array.from(outboundRoutes),
336
+ };
359
337
  }
360
338
  (0, common_utils_1.assert)(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
361
- gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
339
+ gcState.gcNodes[dsRootId].unreferencedTimestampMs =
340
+ gcSummaryDetails.unrefTimestamp;
362
341
  }
363
342
  // If there is only one node (root node just added above), either GC is disabled or we are loading from
364
343
  // the first summary generated by detached container. In both cases, GC was not run - return undefined.
365
- return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
344
+ return Object.keys(gcState.gcNodes).length === 1
345
+ ? undefined
346
+ : { gcState, tombstones: undefined, deletedNodes: undefined };
366
347
  }
367
348
  catch (error) {
368
349
  const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
@@ -376,7 +357,6 @@ class GarbageCollector {
376
357
  * GC state and updates their inactive or sweep ready state.
377
358
  */
378
359
  this.initializeGCStateFromBaseSnapshotP = new common_utils_1.LazyPromise(async () => {
379
- const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
380
360
  /**
381
361
  * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
382
362
  * how long objects have been unreferenced and if they can be deleted.
@@ -385,6 +365,7 @@ class GarbageCollector {
385
365
  * for this container and it is in read mode. In this scenario, there is no point in running GC anyway
386
366
  * because references in the container do not change without any ops, i.e., there is nothing to collect.
387
367
  */
368
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
388
369
  if (currentReferenceTimestampMs === undefined) {
389
370
  // Log an event so we can evaluate how often we run into this scenario.
390
371
  this.mc.logger.sendErrorEvent({
@@ -393,38 +374,24 @@ class GarbageCollector {
393
374
  });
394
375
  return;
395
376
  }
396
- const baseSnapshotData = await this.baseSnapshotDataP;
397
377
  /**
398
378
  * The base snapshot data will not be present if the container is loaded from:
399
379
  * 1. The first summary created by the detached container.
400
380
  * 2. A summary that was generated with GC disabled.
401
381
  * 3. A summary that was generated before GC even existed.
402
382
  */
383
+ const baseSnapshotData = await this.baseSnapshotDataP;
403
384
  if (baseSnapshotData === undefined) {
404
385
  return;
405
386
  }
406
- const gcNodes = {};
407
- for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
408
- if (nodeData.unreferencedTimestampMs !== undefined) {
409
- this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
410
- }
411
- gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
412
- }
413
- this.previousGCDataFromLastRun = { gcNodes };
414
- // If tracking state across summaries, update latest summary data from the base snapshot's GC data.
415
- if (this.trackGCState) {
416
- this.latestSummaryData = {
417
- serializedGCState: JSON.stringify(generateSortedGCState(baseSnapshotData.gcState)),
418
- serializedTombstones: JSON.stringify(baseSnapshotData.tombstones),
419
- };
420
- }
387
+ this.updateStateFromSnapshotData(baseSnapshotData, currentReferenceTimestampMs);
421
388
  });
422
- // Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
423
- // which the caller uses to initialize each node's GC state.
389
+ // Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
390
+ // used to initialize the GC state of all the nodes in the container.
424
391
  this.baseGCDetailsP = new common_utils_1.LazyPromise(async () => {
425
392
  const baseSnapshotData = await this.baseSnapshotDataP;
426
393
  if (baseSnapshotData === undefined) {
427
- return new Map();
394
+ return {};
428
395
  }
429
396
  const gcNodes = {};
430
397
  for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
@@ -434,18 +401,7 @@ class GarbageCollector {
434
401
  // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
435
402
  // each node in the summary.
436
403
  const usedRoutes = (0, garbage_collector_1.runGarbageCollection)(gcNodes, ["/"]).referencedNodeIds;
437
- const baseGCDetailsMap = (0, garbage_collector_1.unpackChildNodesGCDetails)({ gcData: { gcNodes }, usedRoutes });
438
- // Currently, the nodes may write the GC data. So, we need to update its base GC details with the
439
- // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
440
- for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
441
- if (nodeData.unreferencedTimestampMs !== undefined) {
442
- const dataStoreGCDetails = baseGCDetailsMap.get(nodeId.slice(1));
443
- if (dataStoreGCDetails !== undefined) {
444
- dataStoreGCDetails.unrefTimestamp = nodeData.unreferencedTimestampMs;
445
- }
446
- }
447
- }
448
- return baseGCDetailsMap;
404
+ return { gcData: { gcNodes }, usedRoutes };
449
405
  });
450
406
  // Log all the GC options and the state determined by the garbage collector. This is interesting only for the
451
407
  // summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
@@ -466,39 +422,143 @@ class GarbageCollector {
466
422
  *
467
423
  * 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
468
424
  *
469
- * 3. The GC version in the latest summary is different from the current GC version. This can happen if:
425
+ * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
426
+ *
427
+ * 4. The GC version in the latest summary is different from the current GC version. This can happen if:
470
428
  *
471
- * 3.1. The summary this client loaded with has data from a different GC version.
429
+ * 4.1. The summary this client loaded with has data from a different GC version.
472
430
  *
473
- * 3.2. This client's latest summary was updated from a snapshot that has a different GC version.
431
+ * 4.2. This client's latest summary was updated from a snapshot that has a different GC version.
474
432
  */
475
433
  get summaryStateNeedsReset() {
476
- return this.initialStateNeedsReset ||
477
- (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion);
434
+ return (this.gcStateNeedsReset ||
435
+ (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion));
436
+ }
437
+ /**
438
+ * Tells whether the GC state needs to be reset. This can happen under 3 conditions:
439
+ *
440
+ * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
441
+ * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
442
+ *
443
+ * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
444
+ * a document and the first time GC is enabled after is was disabled before.
445
+ *
446
+ * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
447
+ *
448
+ * Note that the state will be reset only once for the first summary generated after this returns true. After that,
449
+ * this will return false.
450
+ */
451
+ get gcStateNeedsReset() {
452
+ return this.wasGCRunInLatestSummary !== this.shouldRunGC;
478
453
  }
479
454
  /** Returns a list of all the configurations for garbage collection. */
480
455
  get configs() {
481
456
  return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
482
457
  }
483
458
  /**
484
- * Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
485
- * before they are loaded or used. This is important to get accurate information of whether tombstoned object are
486
- * in use or not.
459
+ * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
460
+ * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
487
461
  */
488
462
  async initializeBaseState() {
489
463
  const baseSnapshotData = await this.baseSnapshotDataP;
490
464
  /**
491
- * The base snapshot data or tombstone state will not be present if the container is loaded from:
465
+ * The base snapshot data will not be present if the container is loaded from:
492
466
  * 1. The first summary created by the detached container.
493
467
  * 2. A summary that was generated with GC disabled.
494
468
  * 3. A summary that was generated before GC even existed.
495
- * 4. A summary that was generated with tombstone feature disabled.
496
469
  */
497
- if (!this.tombstoneMode || (baseSnapshotData === null || baseSnapshotData === void 0 ? void 0 : baseSnapshotData.tombstones) === undefined) {
470
+ if (baseSnapshotData === undefined) {
498
471
  return;
499
472
  }
500
- this.tombstones = baseSnapshotData.tombstones;
501
- this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
473
+ // Initialize the deleted nodes from the snapshot. This is done irrespective of whether sweep is enabled or not
474
+ // to identify deleted nodes' usage.
475
+ if (baseSnapshotData.deletedNodes !== undefined) {
476
+ this.deletedNodes = new Set(baseSnapshotData.deletedNodes);
477
+ }
478
+ // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
479
+ // tombstone routes.
480
+ if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
481
+ this.tombstones = Array.from(baseSnapshotData.tombstones);
482
+ this.runtime.updateTombstonedRoutes(this.tombstones);
483
+ }
484
+ }
485
+ /**
486
+ * Update state from the given snapshot data. This is done during load and during refreshing state from a snapshot.
487
+ * All current tracking is reset and updated from the data in the snapshot.
488
+ * @param snapshotData - The snapshot data to update state from. If this is undefined, all GC state and tracking
489
+ * is reset.
490
+ * @param currentReferenceTimestampMs - The current reference timestamp for marking unreferenced nodes' unreferenced
491
+ * timestamp.
492
+ */
493
+ updateStateFromSnapshotData(snapshotData, currentReferenceTimestampMs) {
494
+ /**
495
+ * Note: "newReferencesSinceLastRun" is not reset here. This is done because there may be references since the
496
+ * snapshot that we are updating state from. For example, this client may have processed ops till seq#1000 and
497
+ * its refreshing state from a summary that happened at seq#900. In this case, there may be references between
498
+ * seq#901 and seq#1000 that we don't want to reset.
499
+ * Unfortunately, there is no way to track the seq# of ops that add references, so we choose to not reset any
500
+ * references here. This should be fine because, in the worst case, we may end up updating the unreferenced
501
+ * timestamp of a node which will delay its deletion. Although not ideal, this will only happen in rare
502
+ * scenarios, so it should be okay.
503
+ */
504
+ // Clear all existing unreferenced state tracking.
505
+ for (const [, nodeStateTracker] of this.unreferencedNodesState) {
506
+ nodeStateTracker.stopTracking();
507
+ }
508
+ this.unreferencedNodesState.clear();
509
+ // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
510
+ // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
511
+ // tombstones.
512
+ // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
513
+ // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
514
+ if (this.shouldRunSweep) {
515
+ const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones)
516
+ ? new Set(snapshotData.tombstones)
517
+ : undefined;
518
+ // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
519
+ // delete them.
520
+ if (snapshotDeletedNodes !== undefined) {
521
+ const newDeletedNodes = [];
522
+ for (const nodeId of snapshotDeletedNodes) {
523
+ if (!this.deletedNodes.has(nodeId)) {
524
+ newDeletedNodes.push(nodeId);
525
+ }
526
+ }
527
+ if (newDeletedNodes.length > 0) {
528
+ // Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
529
+ }
530
+ }
531
+ }
532
+ else if (this.tombstoneMode) {
533
+ // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
534
+ // notify the runtime to update its state as well.
535
+ this.tombstones = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? Array.from(snapshotData.tombstones) : [];
536
+ this.runtime.updateTombstonedRoutes(this.tombstones);
537
+ }
538
+ // If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
539
+ if (snapshotData === undefined) {
540
+ this.gcDataFromLastRun = undefined;
541
+ this.latestSummaryData = undefined;
542
+ return;
543
+ }
544
+ // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
545
+ // to the GC data from the snapshot data.
546
+ const gcNodes = {};
547
+ for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
548
+ if (nodeData.unreferencedTimestampMs !== undefined) {
549
+ this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
550
+ }
551
+ gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
552
+ }
553
+ this.gcDataFromLastRun = { gcNodes };
554
+ // If tracking state across summaries, update latest summary data from the snapshot's GC data.
555
+ if (this.trackGCState) {
556
+ this.latestSummaryData = {
557
+ serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
558
+ serializedTombstones: JSON.stringify(snapshotData.tombstones),
559
+ serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
560
+ };
561
+ }
502
562
  }
503
563
  /**
504
564
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
@@ -532,7 +592,9 @@ class GarbageCollector {
532
592
  var _a;
533
593
  const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
534
594
  const logger = options.logger
535
- ? telemetry_utils_1.ChildLogger.create(options.logger, undefined, { all: { completedGCRuns: () => this.completedRuns } })
595
+ ? telemetry_utils_1.ChildLogger.create(options.logger, undefined, {
596
+ all: { completedGCRuns: () => this.completedRuns },
597
+ })
536
598
  : this.mc.logger;
537
599
  /**
538
600
  * If there is no current reference timestamp, skip running GC. We need the current timestamp to track
@@ -584,13 +646,13 @@ class GarbageCollector {
584
646
  // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
585
647
  // involving access to deleted data.
586
648
  if (this.testMode) {
587
- this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds, false /* tombstone */);
649
+ this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
588
650
  }
589
651
  else if (this.tombstoneMode) {
590
- // If we are running in GC tombstone mode, tombstone objects for unused routes. This enables testing
591
- // scenarios involving access to "deleted" data without actually deleting the data from summaries.
592
- // Note: we will not tombstone in test mode
593
- this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
652
+ // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
653
+ // involving access to "deleted" data without actually deleting the data from summaries.
654
+ // Note: we will not tombstone in test mode.
655
+ this.runtime.updateTombstonedRoutes(this.tombstones);
594
656
  }
595
657
  // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
596
658
  // updates its state so that we don't send false positives based on intermediate state. For example, we may get
@@ -605,78 +667,105 @@ class GarbageCollector {
605
667
  */
606
668
  summarize(fullTree, trackState, telemetryContext) {
607
669
  var _a;
608
- if (!this.shouldRunGC || this.previousGCDataFromLastRun === undefined) {
670
+ if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
609
671
  return;
610
672
  }
611
673
  const gcState = { gcNodes: {} };
612
- for (const [nodeId, outboundRoutes] of Object.entries(this.previousGCDataFromLastRun.gcNodes)) {
674
+ for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
613
675
  gcState.gcNodes[nodeId] = {
614
676
  outboundRoutes,
615
677
  unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
616
678
  };
617
679
  }
618
680
  const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
681
+ // Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
682
+ // to identify deleted nodes' usage.
683
+ const serializedDeletedNodes = this.deletedNodes.size > 0
684
+ ? JSON.stringify(Array.from(this.deletedNodes).sort())
685
+ : undefined;
686
+ // If running in tombstone mode, serialize and write tombstones, if any.
619
687
  const serializedTombstones = this.tombstoneMode
620
- ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
688
+ ? this.tombstones.length > 0
689
+ ? JSON.stringify(this.tombstones.sort())
690
+ : undefined
621
691
  : undefined;
622
692
  /**
623
- * Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
624
- * summary, send summary handles for them. Otherwise, send the data in summary blobs.
693
+ * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
694
+ * write summary handle instead of summary tree for GC.
695
+ * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
696
+ * for each of these that did not change, write a summary handle.
625
697
  */
626
698
  if (this.trackGCState) {
627
- this.pendingSummaryData = { serializedGCState, serializedTombstones };
699
+ this.pendingSummaryData = {
700
+ serializedGCState,
701
+ serializedTombstones,
702
+ serializedDeletedNodes,
703
+ };
628
704
  if (trackState && !fullTree && this.latestSummaryData !== undefined) {
629
- // If neither GC state or tombstone state changed, send a summary handle for the entire GC data.
630
- if (this.latestSummaryData.serializedGCState === serializedGCState
631
- && this.latestSummaryData.serializedTombstones === serializedTombstones) {
705
+ // If nothing changed since last summary, send a summary handle for the entire GC data.
706
+ if (this.latestSummaryData.serializedGCState === serializedGCState &&
707
+ this.latestSummaryData.serializedTombstones === serializedTombstones) {
632
708
  const stats = (0, runtime_utils_1.mergeStats)();
633
709
  stats.handleNodeCount++;
634
710
  return {
635
711
  summary: {
636
712
  type: protocol_definitions_1.SummaryType.Handle,
637
- handle: `/${exports.gcTreeKey}`,
713
+ handle: `/${runtime_definitions_1.gcTreeKey}`,
638
714
  handleType: protocol_definitions_1.SummaryType.Tree,
639
715
  },
640
716
  stats,
641
717
  };
642
718
  }
643
- // If either or both of GC state or tombstone state changed, build a GC summary tree.
644
- return this.buildGCSummaryTree(serializedGCState, serializedTombstones, true /* trackState */);
719
+ // If some state changed, build a GC summary tree.
720
+ return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
645
721
  }
646
722
  }
647
723
  // If not tracking GC state, build a GC summary tree without any summary handles.
648
- return this.buildGCSummaryTree(serializedGCState, serializedTombstones, false /* trackState */);
724
+ return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
649
725
  }
650
726
  /**
651
- * Builds the GC summary tree which contains GC state and tombstone state.
652
- * If trackState is false, both GC state and tombstone state are written as summary blobs.
653
- * If trackState is true, summary blob is written for GC state or tombstone state if they changed.
727
+ * Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
728
+ * If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
729
+ * If trackState is true, only states that changed are written. Rest are written as handles.
654
730
  * @param serializedGCState - The GC state serialized as string.
655
- * @param serializedTombstones - THe tombstone state serialized as string.
731
+ * @param serializedTombstones - The tombstone state serialized as string.
732
+ * @param serializedDeletedNodes - Deleted nodes serialized as string.
656
733
  * @param trackState - Whether we are tracking GC state across summaries.
657
734
  * @returns the GC summary tree.
658
735
  */
659
- buildGCSummaryTree(serializedGCState, serializedTombstones, trackState) {
660
- var _a, _b;
661
- const gcStateBlobKey = `${exports.gcBlobPrefix}_root`;
736
+ buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, trackState) {
737
+ var _a, _b, _c;
738
+ const gcStateBlobKey = `${runtime_definitions_1.gcBlobPrefix}_root`;
662
739
  const builder = new runtime_utils_1.SummaryTreeBuilder();
663
740
  // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
664
741
  if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
665
- builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${exports.gcTreeKey}/${gcStateBlobKey}`);
742
+ builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${runtime_definitions_1.gcTreeKey}/${gcStateBlobKey}`);
666
743
  }
667
744
  else {
668
745
  builder.addBlob(gcStateBlobKey, serializedGCState);
669
746
  }
670
- // If there is no tombstone data, return only the GC state.
671
- if (serializedTombstones === undefined) {
747
+ // If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
748
+ // summary blob.
749
+ if (serializedTombstones !== undefined) {
750
+ if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones &&
751
+ trackState) {
752
+ builder.addHandle(runtime_definitions_1.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${runtime_definitions_1.gcTreeKey}/${runtime_definitions_1.gcTombstoneBlobKey}`);
753
+ }
754
+ else {
755
+ builder.addBlob(runtime_definitions_1.gcTombstoneBlobKey, serializedTombstones);
756
+ }
757
+ }
758
+ // If there are no deleted nodes, return the summary tree.
759
+ if (serializedDeletedNodes === undefined) {
672
760
  return builder.getSummaryTree();
673
761
  }
674
- // If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
675
- if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones && trackState) {
676
- builder.addHandle(exports.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${exports.gcTreeKey}/${exports.gcTombstoneBlobKey}`);
762
+ // If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
763
+ if (((_c = this.latestSummaryData) === null || _c === void 0 ? void 0 : _c.serializedDeletedNodes) === serializedDeletedNodes &&
764
+ trackState) {
765
+ builder.addHandle(runtime_definitions_1.gcDeletedBlobKey, protocol_definitions_1.SummaryType.Blob, `/${runtime_definitions_1.gcTreeKey}/${runtime_definitions_1.gcDeletedBlobKey}`);
677
766
  }
678
767
  else {
679
- builder.addBlob(exports.gcTombstoneBlobKey, serializedTombstones);
768
+ builder.addBlob(runtime_definitions_1.gcDeletedBlobKey, serializedDeletedNodes);
680
769
  }
681
770
  return builder.getSummaryTree();
682
771
  }
@@ -693,50 +782,58 @@ class GarbageCollector {
693
782
  };
694
783
  }
695
784
  /**
696
- * Returns a map of node ids to their base GC details generated from the base summary. This is used by the caller
697
- * to initialize the GC state of the nodes.
785
+ * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
786
+ * in the container.
698
787
  */
699
788
  async getBaseGCDetails() {
700
789
  return this.baseGCDetailsP;
701
790
  }
702
791
  /**
703
- * Called when the latest summary of the system has been refreshed. This will be used to update the state of the
704
- * latest summary tracked.
792
+ * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
793
+ * is downloaded and should be used to update the state.
705
794
  */
706
- async latestSummaryStateRefreshed(result, readAndParseBlob) {
707
- if (!this.shouldRunGC || !result.latestSummaryUpdated) {
795
+ async refreshLatestSummary(result, proposalHandle, summaryRefSeq, readAndParseBlob) {
796
+ // If the latest summary was updated and the summary was tracked, this client is the one that generated this
797
+ // summary. So, update wasGCRunInLatestSummary.
798
+ // Note that this has to be updated if GC did not run too. Otherwise, `gcStateNeedsReset` will always return
799
+ // true in scenarios where GC is disabled but enabled in the snapshot we loaded from.
800
+ if (result.latestSummaryUpdated && result.wasSummaryTracked) {
801
+ this.wasGCRunInLatestSummary = this.shouldRunGC;
802
+ }
803
+ if (!result.latestSummaryUpdated || !this.shouldRunGC) {
708
804
  return;
709
805
  }
710
806
  // If the summary was tracked by this client, it was the one that generated the summary in the first place.
711
- // Basically, it was written in the current GC version.
807
+ // Update latest state from pending.
712
808
  if (result.wasSummaryTracked) {
713
809
  this.latestSummaryGCVersion = this.currentGCVersion;
714
- this.initialStateNeedsReset = false;
715
810
  if (this.trackGCState) {
716
811
  this.latestSummaryData = this.pendingSummaryData;
717
812
  this.pendingSummaryData = undefined;
718
813
  }
719
814
  return;
720
815
  }
721
- // If the summary was not tracked by this client, update latest GC version and blob from the snapshot in the
722
- // result as that is now the latest summary.
816
+ // If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
723
817
  const snapshot = result.snapshot;
724
818
  const metadataBlobId = snapshot.blobs[summaryFormat_1.metadataBlobName];
725
819
  if (metadataBlobId) {
726
820
  const metadata = await readAndParseBlob(metadataBlobId);
727
821
  this.latestSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
728
822
  }
729
- const gcSnapshotTree = snapshot.trees[exports.gcTreeKey];
730
- if (gcSnapshotTree !== undefined && this.trackGCState) {
731
- const latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
732
- this.latestSummaryData = {
733
- serializedGCState: JSON.stringify(generateSortedGCState(latestGCData.gcState)),
734
- serializedTombstones: JSON.stringify(latestGCData.tombstones),
735
- };
823
+ // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
824
+ // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
825
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
826
+ if (currentReferenceTimestampMs === undefined) {
827
+ throw container_utils_1.DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, { proposalHandle, summaryRefSeq, details: JSON.stringify(this.configs) });
736
828
  }
737
- else {
738
- this.latestSummaryData = undefined;
829
+ const gcSnapshotTree = snapshot.trees[runtime_definitions_1.gcTreeKey];
830
+ // If GC ran in the container that generated this snapshot, it will have a GC tree.
831
+ this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
832
+ let latestGCData;
833
+ if (gcSnapshotTree !== undefined) {
834
+ latestGCData = await (0, garbage_collector_1.getGCDataFromSnapshot)(gcSnapshotTree, readAndParseBlob);
739
835
  }
836
+ this.updateStateFromSnapshotData(latestGCData, currentReferenceTimestampMs);
740
837
  this.pendingSummaryData = undefined;
741
838
  }
742
839
  /**
@@ -775,6 +872,30 @@ class GarbageCollector {
775
872
  if (nodeStateTracker && nodeStateTracker.state !== exports.UnreferencedState.Active) {
776
873
  this.inactiveNodeUsed("Revived", toNodePath, nodeStateTracker, fromNodePath);
777
874
  }
875
+ if (this.tombstones.includes(toNodePath)) {
876
+ const nodeType = this.runtime.getNodeType(toNodePath);
877
+ let eventName = "GC_Tombstone_SubDatastore_Revived";
878
+ if (nodeType === exports.GCNodeType.DataStore) {
879
+ eventName = "GC_Tombstone_Datastore_Revived";
880
+ }
881
+ else if (nodeType === exports.GCNodeType.Blob) {
882
+ eventName = "GC_Tombstone_Blob_Revived";
883
+ }
884
+ (0, garbageCollectionHelpers_1.sendGCUnexpectedUsageEvent)(this.mc, {
885
+ eventName,
886
+ category: "generic",
887
+ isSummarizerClient: this.isSummarizerClient,
888
+ url: (0, garbage_collector_1.trimLeadingSlashes)(toNodePath),
889
+ nodeType,
890
+ }, undefined /* packagePath */);
891
+ }
892
+ }
893
+ /**
894
+ * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
895
+ * cases where objects are used after they are deleted and throw / log errors accordingly.
896
+ */
897
+ isNodeDeleted(nodePath) {
898
+ return this.deletedNodes.has(nodePath);
778
899
  }
779
900
  dispose() {
780
901
  var _a;
@@ -791,7 +912,7 @@ class GarbageCollector {
791
912
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
792
913
  */
793
914
  updateCurrentState(gcData, gcResult, currentReferenceTimestampMs) {
794
- this.previousGCDataFromLastRun = (0, garbage_collector_1.cloneGCData)(gcData);
915
+ this.gcDataFromLastRun = (0, garbage_collector_1.cloneGCData)(gcData);
795
916
  this.tombstones = [];
796
917
  this.newReferencesSinceLastRun.clear();
797
918
  // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
@@ -827,27 +948,31 @@ class GarbageCollector {
827
948
  }
828
949
  /**
829
950
  * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
830
- * time. It's possible that nodes transition from `unreferenced -> referenced -> unreferenced` between two runs. The
831
- * unreferenced timestamp of such nodes needs to be reset as they may have been accessed when they were referenced.
951
+ * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
952
+ * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
953
+ * these objects while there can be in-memory referenced to it:
954
+ * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
955
+ * added, the object may have been accessed and in-memory reference to it added.
956
+ * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
957
+ * unreferenced, they could have been accessed and in-memory reference to them added.
832
958
  *
833
959
  * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
834
960
  * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
835
961
  */
836
962
  updateStateSinceLastRun(currentGCData, logger) {
837
963
  // If we haven't run GC before there is nothing to do.
838
- if (this.previousGCDataFromLastRun === undefined) {
964
+ if (this.gcDataFromLastRun === undefined) {
839
965
  return;
840
966
  }
841
967
  // Find any references that haven't been identified correctly.
842
- const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, this.previousGCDataFromLastRun, this.newReferencesSinceLastRun);
968
+ const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, this.gcDataFromLastRun, this.newReferencesSinceLastRun);
843
969
  if (missingExplicitReferences.length > 0) {
844
970
  missingExplicitReferences.forEach((missingExplicitReference) => {
845
- const event = {
971
+ logger.sendErrorEvent({
846
972
  eventName: "gcUnknownOutboundReferences",
847
973
  gcNodeId: missingExplicitReference[0],
848
974
  gcRoutes: JSON.stringify(missingExplicitReference[1]),
849
- };
850
- logger.sendPerformanceEvent(event);
975
+ });
851
976
  });
852
977
  }
853
978
  // No references were added since the last run so we don't have to update reference states of any unreferenced
@@ -861,21 +986,18 @@ class GarbageCollector {
861
986
  * run, and then add the references since last run.
862
987
  *
863
988
  * Note on why we need to combine the data from previous run, current run and all references in between -
864
- *
865
989
  * 1. We need data from last run because some of its references may have been deleted since then. If those
866
- * references added new outbound references before getting deleted, we need to detect them.
990
+ * references added new outbound references before they were deleted, we need to detect them.
867
991
  *
868
992
  * 2. We need new outbound references since last run because some of them may have been deleted later. If those
869
- * references added new outbound references before getting deleted, we need to detect them.
993
+ * references added new outbound references before they were deleted, we need to detect them.
870
994
  *
871
995
  * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
872
- *
873
- * - We don't require DDSes handles to be stored in a referenced DDS. For this, we need GC at DDS level
874
- * which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
875
- *
996
+ * - We don't require DDSes handles to be stored in a referenced DDS.
876
997
  * - A new data store may have "root" DDSes already created and we don't detect them today.
877
998
  */
878
- const gcDataSuperSet = (0, garbage_collector_1.concatGarbageCollectionData)(this.previousGCDataFromLastRun, currentGCData);
999
+ const gcDataSuperSet = (0, garbage_collector_1.concatGarbageCollectionData)(this.gcDataFromLastRun, currentGCData);
1000
+ const newOutboundRoutesSinceLastRun = [];
879
1001
  this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
880
1002
  if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
881
1003
  gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
@@ -883,19 +1005,25 @@ class GarbageCollector {
883
1005
  else {
884
1006
  gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
885
1007
  }
1008
+ newOutboundRoutesSinceLastRun.push(...outboundRoutes);
886
1009
  });
887
1010
  /**
888
- * Run GC on the above reference graph to find all nodes that are referenced. For each one, if they are
1011
+ * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
1012
+ * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
889
1013
  * unreferenced, stop tracking them and remove from unreferenced list.
890
- * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
1014
+ * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
1015
+ * unreferenced and add unreferenced state.
891
1016
  */
892
- const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcDataSuperSet.gcNodes, ["/"]);
1017
+ const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcDataSuperSet.gcNodes, [
1018
+ "/",
1019
+ ...newOutboundRoutesSinceLastRun,
1020
+ ]);
893
1021
  for (const nodeId of gcResult.referencedNodeIds) {
894
1022
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
895
1023
  if (nodeStateTracker !== undefined) {
896
1024
  // Stop tracking so as to clear out any running timers.
897
1025
  nodeStateTracker.stopTracking();
898
- // Delete the node as we don't need to track it any more.
1026
+ // Delete the unreferenced state as we don't need to track it any more.
899
1027
  this.unreferencedNodesState.delete(nodeId);
900
1028
  }
901
1029
  }
@@ -914,7 +1042,7 @@ class GarbageCollector {
914
1042
  * @returns - a list of missing explicit references
915
1043
  */
916
1044
  findMissingExplicitReferences(currentGCData, previousGCData, explicitReferences) {
917
- (0, common_utils_1.assert)(previousGCData !== undefined, 0x2b7);
1045
+ (0, common_utils_1.assert)(previousGCData !== undefined, 0x2b7 /* "Can't validate correctness without GC data from last run" */);
918
1046
  const currentGraph = Object.entries(currentGCData.gcNodes);
919
1047
  const missingExplicitReferences = [];
920
1048
  currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
@@ -922,14 +1050,20 @@ class GarbageCollector {
922
1050
  const previousRoutes = (_a = previousGCData.gcNodes[nodeId]) !== null && _a !== void 0 ? _a : [];
923
1051
  const explicitRoutes = (_b = explicitReferences.get(nodeId)) !== null && _b !== void 0 ? _b : [];
924
1052
  const missingExplicitRoutes = [];
1053
+ /**
1054
+ * 1. For routes in the current GC data, routes that were not present in previous GC data and did not have
1055
+ * explicit references should be added to missing explicit routes list.
1056
+ * 2. Only include data store and blob routes since GC only works for these two.
1057
+ * Note: Due to a bug with de-duped blobs, only adding data store routes for now.
1058
+ * 3. Ignore DDS routes to their parent datastores since those were added implicitly. So, there won't be
1059
+ * explicit routes to them.
1060
+ */
925
1061
  currentOutboundRoutes.forEach((route) => {
926
- const isBlobOrDataStoreRoute = this.runtime.getNodeType(route) === exports.GCNodeType.Blob ||
927
- this.runtime.getNodeType(route) === exports.GCNodeType.DataStore;
928
- // Ignore implicitly added DDS routes to their parent datastores
929
- const notRouteFromDDSToParentDataStore = !nodeId.startsWith(route);
930
- if (isBlobOrDataStoreRoute &&
931
- notRouteFromDDSToParentDataStore &&
932
- (!previousRoutes.includes(route) && !explicitRoutes.includes(route))) {
1062
+ const nodeType = this.runtime.getNodeType(route);
1063
+ if ((nodeType === exports.GCNodeType.DataStore || nodeType === exports.GCNodeType.Blob) &&
1064
+ !nodeId.startsWith(route) &&
1065
+ !previousRoutes.includes(route) &&
1066
+ !explicitRoutes.includes(route)) {
933
1067
  missingExplicitRoutes.push(route);
934
1068
  }
935
1069
  });
@@ -961,7 +1095,7 @@ class GarbageCollector {
961
1095
  gcStats.nodeCount++;
962
1096
  // If there is no previous GC data, every node's state is generated and is considered as updated.
963
1097
  // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
964
- const stateUpdated = this.previousGCDataFromLastRun === undefined ||
1098
+ const stateUpdated = this.gcDataFromLastRun === undefined ||
965
1099
  this.unreferencedNodesState.has(nodeId) === referenced;
966
1100
  if (stateUpdated) {
967
1101
  gcStats.updatedNodeCount++;
@@ -1001,7 +1135,8 @@ class GarbageCollector {
1001
1135
  * this will give us a view into how much deleted content a container has.
1002
1136
  */
1003
1137
  logSweepEvents(logger, currentReferenceTimestampMs) {
1004
- if (this.mc.config.getBoolean(exports.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
1138
+ if (this.mc.config.getBoolean(garbageCollectionConstants_1.disableSweepLogKey) === true ||
1139
+ this.sweepTimeoutMs === undefined) {
1005
1140
  return;
1006
1141
  }
1007
1142
  this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
@@ -1036,7 +1171,8 @@ class GarbageCollector {
1036
1171
  // If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
1037
1172
  // logging as nothing interesting would have happened worth logging.
1038
1173
  // If the node is active, skip logging.
1039
- if (currentReferenceTimestampMs === undefined || nodeStateTracker.state === exports.UnreferencedState.Active) {
1174
+ if (currentReferenceTimestampMs === undefined ||
1175
+ nodeStateTracker.state === exports.UnreferencedState.Active) {
1040
1176
  return;
1041
1177
  }
1042
1178
  // We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
@@ -1068,9 +1204,17 @@ class GarbageCollector {
1068
1204
  // Events generated:
1069
1205
  // InactiveObject_Loaded, SweepReadyObject_Loaded
1070
1206
  if (usageType === "Loaded") {
1071
- this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(packagePath), stack: (0, telemetry_utils_1.generateStack)() }));
1207
+ const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(packagePath), stack: (0, telemetry_utils_1.generateStack)() });
1208
+ // Do not log the inactive object x events as error events as they are not the best signal for
1209
+ // detecting something wrong with GC either from the partner or from the runtime itself.
1210
+ if (state === exports.UnreferencedState.Inactive) {
1211
+ this.mc.logger.sendTelemetryEvent(event);
1212
+ }
1213
+ else {
1214
+ this.mc.logger.sendErrorEvent(event);
1215
+ }
1072
1216
  }
1073
- // If SweepReady Usage Detection is enabed, the handler may close the interactive container.
1217
+ // If SweepReady Usage Detection is enabled, the handler may close the interactive container.
1074
1218
  // Once Sweep is fully implemented, this will be removed since the objects will be gone
1075
1219
  // and errors will arise elsewhere in the runtime
1076
1220
  if (state === exports.UnreferencedState.SweepReady) {
@@ -1093,44 +1237,30 @@ class GarbageCollector {
1093
1237
  * revived and a Revived event will be logged for it.
1094
1238
  */
1095
1239
  const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
1096
- const active = nodeStateTracker === undefined || nodeStateTracker.state === exports.UnreferencedState.Active;
1240
+ const active = nodeStateTracker === undefined ||
1241
+ nodeStateTracker.state === exports.UnreferencedState.Active;
1097
1242
  if ((usageType === "Revived") === active) {
1098
1243
  const pkg = await this.getNodePackagePath(eventProps.id);
1099
- const fromPkg = eventProps.fromId ? await this.getNodePackagePath(eventProps.fromId) : undefined;
1100
- logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: pkg ? { value: pkg.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined, fromPkg: fromPkg ? { value: fromPkg.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined }));
1244
+ const fromPkg = eventProps.fromId
1245
+ ? await this.getNodePackagePath(eventProps.fromId)
1246
+ : undefined;
1247
+ const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: pkg
1248
+ ? { value: pkg.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact }
1249
+ : undefined, fromPkg: fromPkg
1250
+ ? { value: fromPkg.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact }
1251
+ : undefined });
1252
+ if (state === exports.UnreferencedState.Inactive) {
1253
+ logger.sendTelemetryEvent(event);
1254
+ }
1255
+ else {
1256
+ logger.sendErrorEvent(event);
1257
+ }
1101
1258
  }
1102
1259
  }
1103
1260
  this.pendingEventsQueue = [];
1104
1261
  }
1105
1262
  }
1106
1263
  exports.GarbageCollector = GarbageCollector;
1107
- /**
1108
- * Gets the base garbage collection state from the given snapshot tree. It contains GC state and tombstone state.
1109
- * The GC state may be written into multiple blobs. Merge the GC state from all such blobs into one.
1110
- */
1111
- async function getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob) {
1112
- let rootGCState = { gcNodes: {} };
1113
- let tombstones;
1114
- for (const key of Object.keys(gcSnapshotTree.blobs)) {
1115
- if (key === exports.gcTombstoneBlobKey) {
1116
- tombstones = await readAndParseBlob(gcSnapshotTree.blobs[key]);
1117
- continue;
1118
- }
1119
- // Skip blobs that do not start with the GC prefix.
1120
- if (!key.startsWith(exports.gcBlobPrefix)) {
1121
- continue;
1122
- }
1123
- const blobId = gcSnapshotTree.blobs[key];
1124
- if (blobId === undefined) {
1125
- continue;
1126
- }
1127
- const gcState = await readAndParseBlob(blobId);
1128
- (0, common_utils_1.assert)(gcState !== undefined, 0x2ad /* "GC blob missing from snapshot" */);
1129
- // Merge the GC state of this blob into the root GC state.
1130
- rootGCState = (0, garbage_collector_1.concatGarbageCollectionStates)(rootGCState, gcState);
1131
- }
1132
- return { gcState: rootGCState, tombstones };
1133
- }
1134
1264
  function generateSortedGCState(gcState) {
1135
1265
  const sortableArray = Object.entries(gcState.gcNodes);
1136
1266
  sortableArray.sort(([a], [b]) => a.localeCompare(b));
@@ -1145,7 +1275,9 @@ function generateSortedGCState(gcState) {
1145
1275
  class TimerWithNoDefaultTimeout extends common_utils_1.Timer {
1146
1276
  constructor(callback) {
1147
1277
  // The default timeout/handlers will never be used since start/restart pass overrides below
1148
- super(0, () => { throw new Error("DefaultHandler should not be used"); });
1278
+ super(0, () => {
1279
+ throw new Error("DefaultHandler should not be used");
1280
+ });
1149
1281
  this.callback = callback;
1150
1282
  }
1151
1283
  start(timeoutMs) {