@fluidframework/container-runtime 2.0.0-internal.3.0.1 → 2.0.0-internal.3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (327) hide show
  1. package/.eslintrc.js +19 -19
  2. package/.mocharc.js +2 -2
  3. package/api-extractor.json +2 -2
  4. package/dist/batchTracker.d.ts.map +1 -1
  5. package/dist/batchTracker.js +2 -1
  6. package/dist/batchTracker.js.map +1 -1
  7. package/dist/blobManager.d.ts +9 -2
  8. package/dist/blobManager.d.ts.map +1 -1
  9. package/dist/blobManager.js +80 -33
  10. package/dist/blobManager.js.map +1 -1
  11. package/dist/connectionTelemetry.d.ts.map +1 -1
  12. package/dist/connectionTelemetry.js +11 -9
  13. package/dist/connectionTelemetry.js.map +1 -1
  14. package/dist/containerHandleContext.d.ts.map +1 -1
  15. package/dist/containerHandleContext.js +3 -1
  16. package/dist/containerHandleContext.js.map +1 -1
  17. package/dist/containerRuntime.d.ts +10 -0
  18. package/dist/containerRuntime.d.ts.map +1 -1
  19. package/dist/containerRuntime.js +140 -72
  20. package/dist/containerRuntime.js.map +1 -1
  21. package/dist/dataStore.d.ts.map +1 -1
  22. package/dist/dataStore.js +11 -9
  23. package/dist/dataStore.js.map +1 -1
  24. package/dist/dataStoreContext.d.ts +18 -1
  25. package/dist/dataStoreContext.d.ts.map +1 -1
  26. package/dist/dataStoreContext.js +66 -15
  27. package/dist/dataStoreContext.js.map +1 -1
  28. package/dist/dataStoreContexts.d.ts.map +1 -1
  29. package/dist/dataStoreContexts.js +7 -3
  30. package/dist/dataStoreContexts.js.map +1 -1
  31. package/dist/dataStoreRegistry.d.ts.map +1 -1
  32. package/dist/dataStoreRegistry.js +3 -1
  33. package/dist/dataStoreRegistry.js.map +1 -1
  34. package/dist/dataStores.d.ts +26 -1
  35. package/dist/dataStores.d.ts.map +1 -1
  36. package/dist/dataStores.js +103 -18
  37. package/dist/dataStores.js.map +1 -1
  38. package/dist/deltaScheduler.d.ts.map +1 -1
  39. package/dist/deltaScheduler.js +8 -3
  40. package/dist/deltaScheduler.js.map +1 -1
  41. package/dist/garbageCollection.d.ts +34 -14
  42. package/dist/garbageCollection.d.ts.map +1 -1
  43. package/dist/garbageCollection.js +188 -93
  44. package/dist/garbageCollection.js.map +1 -1
  45. package/dist/garbageCollectionConstants.d.ts +3 -0
  46. package/dist/garbageCollectionConstants.d.ts.map +1 -1
  47. package/dist/garbageCollectionConstants.js +6 -1
  48. package/dist/garbageCollectionConstants.js.map +1 -1
  49. package/dist/garbageCollectionHelpers.d.ts +26 -0
  50. package/dist/garbageCollectionHelpers.d.ts.map +1 -0
  51. package/dist/garbageCollectionHelpers.js +45 -0
  52. package/dist/garbageCollectionHelpers.js.map +1 -0
  53. package/dist/gcSweepReadyUsageDetection.d.ts +5 -5
  54. package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
  55. package/dist/gcSweepReadyUsageDetection.js +14 -10
  56. package/dist/gcSweepReadyUsageDetection.js.map +1 -1
  57. package/dist/index.d.ts +2 -2
  58. package/dist/index.d.ts.map +1 -1
  59. package/dist/index.js.map +1 -1
  60. package/dist/opLifecycle/batchManager.d.ts +5 -5
  61. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  62. package/dist/opLifecycle/batchManager.js +19 -12
  63. package/dist/opLifecycle/batchManager.js.map +1 -1
  64. package/dist/opLifecycle/definitions.d.ts.map +1 -1
  65. package/dist/opLifecycle/definitions.js.map +1 -1
  66. package/dist/opLifecycle/index.d.ts.map +1 -1
  67. package/dist/opLifecycle/index.js.map +1 -1
  68. package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
  69. package/dist/opLifecycle/opCompressor.js.map +1 -1
  70. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
  71. package/dist/opLifecycle/opDecompressor.js +5 -2
  72. package/dist/opLifecycle/opDecompressor.js.map +1 -1
  73. package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
  74. package/dist/opLifecycle/opSplitter.js +4 -1
  75. package/dist/opLifecycle/opSplitter.js.map +1 -1
  76. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  77. package/dist/opLifecycle/outbox.js +19 -17
  78. package/dist/opLifecycle/outbox.js.map +1 -1
  79. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  80. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
  81. package/dist/opProperties.d.ts.map +1 -1
  82. package/dist/opProperties.js +1 -3
  83. package/dist/opProperties.js.map +1 -1
  84. package/dist/orderedClientElection.d.ts.map +1 -1
  85. package/dist/orderedClientElection.js +10 -4
  86. package/dist/orderedClientElection.js.map +1 -1
  87. package/dist/packageVersion.d.ts +1 -1
  88. package/dist/packageVersion.js +1 -1
  89. package/dist/packageVersion.js.map +1 -1
  90. package/dist/pendingStateManager.d.ts +7 -0
  91. package/dist/pendingStateManager.d.ts.map +1 -1
  92. package/dist/pendingStateManager.js +7 -4
  93. package/dist/pendingStateManager.js.map +1 -1
  94. package/dist/runWhileConnectedCoordinator.d.ts.map +1 -1
  95. package/dist/runWhileConnectedCoordinator.js.map +1 -1
  96. package/dist/runningSummarizer.d.ts.map +1 -1
  97. package/dist/runningSummarizer.js +34 -21
  98. package/dist/runningSummarizer.js.map +1 -1
  99. package/dist/scheduleManager.d.ts.map +1 -1
  100. package/dist/scheduleManager.js +3 -2
  101. package/dist/scheduleManager.js.map +1 -1
  102. package/dist/serializedSnapshotStorage.d.ts +2 -2
  103. package/dist/serializedSnapshotStorage.d.ts.map +1 -1
  104. package/dist/serializedSnapshotStorage.js +5 -3
  105. package/dist/serializedSnapshotStorage.js.map +1 -1
  106. package/dist/summarizer.d.ts +2 -2
  107. package/dist/summarizer.d.ts.map +1 -1
  108. package/dist/summarizer.js +37 -17
  109. package/dist/summarizer.js.map +1 -1
  110. package/dist/summarizerClientElection.d.ts.map +1 -1
  111. package/dist/summarizerClientElection.js.map +1 -1
  112. package/dist/summarizerHandle.d.ts.map +1 -1
  113. package/dist/summarizerHandle.js.map +1 -1
  114. package/dist/summarizerHeuristics.d.ts.map +1 -1
  115. package/dist/summarizerHeuristics.js +6 -9
  116. package/dist/summarizerHeuristics.js.map +1 -1
  117. package/dist/summarizerTypes.d.ts +21 -21
  118. package/dist/summarizerTypes.d.ts.map +1 -1
  119. package/dist/summarizerTypes.js.map +1 -1
  120. package/dist/summaryCollection.d.ts.map +1 -1
  121. package/dist/summaryCollection.js +18 -8
  122. package/dist/summaryCollection.js.map +1 -1
  123. package/dist/summaryFormat.d.ts +22 -0
  124. package/dist/summaryFormat.d.ts.map +1 -1
  125. package/dist/summaryFormat.js +18 -10
  126. package/dist/summaryFormat.js.map +1 -1
  127. package/dist/summaryGenerator.d.ts.map +1 -1
  128. package/dist/summaryGenerator.js +34 -15
  129. package/dist/summaryGenerator.js.map +1 -1
  130. package/dist/summaryManager.d.ts.map +1 -1
  131. package/dist/summaryManager.js +21 -9
  132. package/dist/summaryManager.js.map +1 -1
  133. package/dist/throttler.d.ts +2 -2
  134. package/dist/throttler.d.ts.map +1 -1
  135. package/dist/throttler.js +4 -4
  136. package/dist/throttler.js.map +1 -1
  137. package/garbageCollection.md +15 -2
  138. package/lib/batchTracker.d.ts.map +1 -1
  139. package/lib/batchTracker.js +2 -1
  140. package/lib/batchTracker.js.map +1 -1
  141. package/lib/blobManager.d.ts +9 -2
  142. package/lib/blobManager.d.ts.map +1 -1
  143. package/lib/blobManager.js +82 -35
  144. package/lib/blobManager.js.map +1 -1
  145. package/lib/connectionTelemetry.d.ts.map +1 -1
  146. package/lib/connectionTelemetry.js +11 -9
  147. package/lib/connectionTelemetry.js.map +1 -1
  148. package/lib/containerHandleContext.d.ts.map +1 -1
  149. package/lib/containerHandleContext.js +3 -1
  150. package/lib/containerHandleContext.js.map +1 -1
  151. package/lib/containerRuntime.d.ts +10 -0
  152. package/lib/containerRuntime.d.ts.map +1 -1
  153. package/lib/containerRuntime.js +146 -78
  154. package/lib/containerRuntime.js.map +1 -1
  155. package/lib/dataStore.d.ts.map +1 -1
  156. package/lib/dataStore.js +11 -9
  157. package/lib/dataStore.js.map +1 -1
  158. package/lib/dataStoreContext.d.ts +18 -1
  159. package/lib/dataStoreContext.d.ts.map +1 -1
  160. package/lib/dataStoreContext.js +68 -17
  161. package/lib/dataStoreContext.js.map +1 -1
  162. package/lib/dataStoreContexts.d.ts.map +1 -1
  163. package/lib/dataStoreContexts.js +7 -3
  164. package/lib/dataStoreContexts.js.map +1 -1
  165. package/lib/dataStoreRegistry.d.ts.map +1 -1
  166. package/lib/dataStoreRegistry.js +3 -1
  167. package/lib/dataStoreRegistry.js.map +1 -1
  168. package/lib/dataStores.d.ts +26 -1
  169. package/lib/dataStores.d.ts.map +1 -1
  170. package/lib/dataStores.js +109 -24
  171. package/lib/dataStores.js.map +1 -1
  172. package/lib/deltaScheduler.d.ts.map +1 -1
  173. package/lib/deltaScheduler.js +9 -4
  174. package/lib/deltaScheduler.js.map +1 -1
  175. package/lib/garbageCollection.d.ts +34 -14
  176. package/lib/garbageCollection.d.ts.map +1 -1
  177. package/lib/garbageCollection.js +190 -95
  178. package/lib/garbageCollection.js.map +1 -1
  179. package/lib/garbageCollectionConstants.d.ts +3 -0
  180. package/lib/garbageCollectionConstants.d.ts.map +1 -1
  181. package/lib/garbageCollectionConstants.js +5 -0
  182. package/lib/garbageCollectionConstants.js.map +1 -1
  183. package/lib/garbageCollectionHelpers.d.ts +26 -0
  184. package/lib/garbageCollectionHelpers.d.ts.map +1 -0
  185. package/lib/garbageCollectionHelpers.js +40 -0
  186. package/lib/garbageCollectionHelpers.js.map +1 -0
  187. package/lib/gcSweepReadyUsageDetection.d.ts +5 -5
  188. package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
  189. package/lib/gcSweepReadyUsageDetection.js +14 -10
  190. package/lib/gcSweepReadyUsageDetection.js.map +1 -1
  191. package/lib/index.d.ts +2 -2
  192. package/lib/index.d.ts.map +1 -1
  193. package/lib/index.js +1 -1
  194. package/lib/index.js.map +1 -1
  195. package/lib/opLifecycle/batchManager.d.ts +5 -5
  196. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  197. package/lib/opLifecycle/batchManager.js +19 -12
  198. package/lib/opLifecycle/batchManager.js.map +1 -1
  199. package/lib/opLifecycle/definitions.d.ts.map +1 -1
  200. package/lib/opLifecycle/definitions.js.map +1 -1
  201. package/lib/opLifecycle/index.d.ts.map +1 -1
  202. package/lib/opLifecycle/index.js.map +1 -1
  203. package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
  204. package/lib/opLifecycle/opCompressor.js.map +1 -1
  205. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
  206. package/lib/opLifecycle/opDecompressor.js +5 -2
  207. package/lib/opLifecycle/opDecompressor.js.map +1 -1
  208. package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
  209. package/lib/opLifecycle/opSplitter.js +5 -2
  210. package/lib/opLifecycle/opSplitter.js.map +1 -1
  211. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  212. package/lib/opLifecycle/outbox.js +19 -17
  213. package/lib/opLifecycle/outbox.js.map +1 -1
  214. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  215. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
  216. package/lib/opProperties.d.ts.map +1 -1
  217. package/lib/opProperties.js +1 -3
  218. package/lib/opProperties.js.map +1 -1
  219. package/lib/orderedClientElection.d.ts.map +1 -1
  220. package/lib/orderedClientElection.js +10 -4
  221. package/lib/orderedClientElection.js.map +1 -1
  222. package/lib/packageVersion.d.ts +1 -1
  223. package/lib/packageVersion.js +1 -1
  224. package/lib/packageVersion.js.map +1 -1
  225. package/lib/pendingStateManager.d.ts +7 -0
  226. package/lib/pendingStateManager.d.ts.map +1 -1
  227. package/lib/pendingStateManager.js +7 -4
  228. package/lib/pendingStateManager.js.map +1 -1
  229. package/lib/runWhileConnectedCoordinator.d.ts.map +1 -1
  230. package/lib/runWhileConnectedCoordinator.js.map +1 -1
  231. package/lib/runningSummarizer.d.ts.map +1 -1
  232. package/lib/runningSummarizer.js +35 -22
  233. package/lib/runningSummarizer.js.map +1 -1
  234. package/lib/scheduleManager.d.ts.map +1 -1
  235. package/lib/scheduleManager.js +3 -2
  236. package/lib/scheduleManager.js.map +1 -1
  237. package/lib/serializedSnapshotStorage.d.ts +2 -2
  238. package/lib/serializedSnapshotStorage.d.ts.map +1 -1
  239. package/lib/serializedSnapshotStorage.js +5 -3
  240. package/lib/serializedSnapshotStorage.js.map +1 -1
  241. package/lib/summarizer.d.ts +2 -2
  242. package/lib/summarizer.d.ts.map +1 -1
  243. package/lib/summarizer.js +37 -17
  244. package/lib/summarizer.js.map +1 -1
  245. package/lib/summarizerClientElection.d.ts.map +1 -1
  246. package/lib/summarizerClientElection.js.map +1 -1
  247. package/lib/summarizerHandle.d.ts.map +1 -1
  248. package/lib/summarizerHandle.js.map +1 -1
  249. package/lib/summarizerHeuristics.d.ts.map +1 -1
  250. package/lib/summarizerHeuristics.js +6 -9
  251. package/lib/summarizerHeuristics.js.map +1 -1
  252. package/lib/summarizerTypes.d.ts +21 -21
  253. package/lib/summarizerTypes.d.ts.map +1 -1
  254. package/lib/summarizerTypes.js.map +1 -1
  255. package/lib/summaryCollection.d.ts.map +1 -1
  256. package/lib/summaryCollection.js +18 -8
  257. package/lib/summaryCollection.js.map +1 -1
  258. package/lib/summaryFormat.d.ts +22 -0
  259. package/lib/summaryFormat.d.ts.map +1 -1
  260. package/lib/summaryFormat.js +20 -12
  261. package/lib/summaryFormat.js.map +1 -1
  262. package/lib/summaryGenerator.d.ts.map +1 -1
  263. package/lib/summaryGenerator.js +34 -15
  264. package/lib/summaryGenerator.js.map +1 -1
  265. package/lib/summaryManager.d.ts.map +1 -1
  266. package/lib/summaryManager.js +21 -9
  267. package/lib/summaryManager.js.map +1 -1
  268. package/lib/throttler.d.ts +2 -2
  269. package/lib/throttler.d.ts.map +1 -1
  270. package/lib/throttler.js +4 -4
  271. package/lib/throttler.js.map +1 -1
  272. package/package.json +121 -149
  273. package/prettier.config.cjs +1 -1
  274. package/src/batchTracker.ts +54 -49
  275. package/src/blobManager.ts +793 -672
  276. package/src/connectionTelemetry.ts +280 -249
  277. package/src/containerHandleContext.ts +27 -29
  278. package/src/containerRuntime.ts +3168 -2940
  279. package/src/dataStore.ts +172 -159
  280. package/src/dataStoreContext.ts +1098 -996
  281. package/src/dataStoreContexts.ts +178 -161
  282. package/src/dataStoreRegistry.ts +25 -20
  283. package/src/dataStores.ts +884 -728
  284. package/src/deltaScheduler.ts +158 -150
  285. package/src/garbageCollection.ts +1883 -1692
  286. package/src/garbageCollectionConstants.ts +6 -0
  287. package/src/garbageCollectionHelpers.ts +61 -0
  288. package/src/gcSweepReadyUsageDetection.ts +89 -83
  289. package/src/index.ts +67 -66
  290. package/src/opLifecycle/README.md +152 -0
  291. package/src/opLifecycle/batchManager.ts +145 -141
  292. package/src/opLifecycle/definitions.ts +29 -29
  293. package/src/opLifecycle/index.ts +5 -5
  294. package/src/opLifecycle/opCompressor.ts +54 -53
  295. package/src/opLifecycle/opDecompressor.ts +100 -81
  296. package/src/opLifecycle/opSplitter.ts +214 -188
  297. package/src/opLifecycle/outbox.ts +204 -194
  298. package/src/opLifecycle/remoteMessageProcessor.ts +62 -62
  299. package/src/opProperties.ts +11 -9
  300. package/src/orderedClientElection.ts +489 -457
  301. package/src/packageVersion.ts +1 -1
  302. package/src/pendingStateManager.ts +384 -338
  303. package/src/runWhileConnectedCoordinator.ts +78 -71
  304. package/src/runningSummarizer.ts +619 -581
  305. package/src/scheduleManager.ts +299 -269
  306. package/src/serializedSnapshotStorage.ts +126 -112
  307. package/src/summarizer.ts +417 -381
  308. package/src/summarizerClientElection.ts +107 -100
  309. package/src/summarizerHandle.ts +11 -9
  310. package/src/summarizerHeuristics.ts +183 -186
  311. package/src/summarizerTypes.ts +344 -330
  312. package/src/summaryCollection.ts +378 -349
  313. package/src/summaryFormat.ts +170 -126
  314. package/src/summaryGenerator.ts +465 -406
  315. package/src/summaryManager.ts +377 -348
  316. package/src/throttler.ts +131 -122
  317. package/tsconfig.esnext.json +6 -6
  318. package/tsconfig.json +9 -13
  319. package/dist/garbageCollectionTombstoneUtils.d.ts +0 -14
  320. package/dist/garbageCollectionTombstoneUtils.d.ts.map +0 -1
  321. package/dist/garbageCollectionTombstoneUtils.js +0 -23
  322. package/dist/garbageCollectionTombstoneUtils.js.map +0 -1
  323. package/lib/garbageCollectionTombstoneUtils.d.ts +0 -14
  324. package/lib/garbageCollectionTombstoneUtils.d.ts.map +0 -1
  325. package/lib/garbageCollectionTombstoneUtils.js +0 -19
  326. package/lib/garbageCollectionTombstoneUtils.js.map +0 -1
  327. package/src/garbageCollectionTombstoneUtils.ts +0 -28
@@ -6,230 +6,244 @@
6
6
  import { ITelemetryLogger } from "@fluidframework/common-definitions";
7
7
  import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
8
8
  import { ICriticalContainerError } from "@fluidframework/container-definitions";
9
- import { ClientSessionExpiredError, DataProcessingError, UsageError } from "@fluidframework/container-utils";
9
+ import {
10
+ ClientSessionExpiredError,
11
+ DataProcessingError,
12
+ UsageError,
13
+ } from "@fluidframework/container-utils";
10
14
  import { IRequestHeader } from "@fluidframework/core-interfaces";
11
15
  import {
12
- cloneGCData,
13
- concatGarbageCollectionData,
14
- getGCDataFromSnapshot,
15
- IGCResult,
16
- runGarbageCollection,
17
- trimLeadingSlashes,
16
+ cloneGCData,
17
+ concatGarbageCollectionData,
18
+ getGCDataFromSnapshot,
19
+ IGCResult,
20
+ runGarbageCollection,
21
+ trimLeadingSlashes,
18
22
  } from "@fluidframework/garbage-collector";
19
23
  import { ISnapshotTree, SummaryType } from "@fluidframework/protocol-definitions";
20
24
  import {
21
- gcTreeKey,
22
- gcBlobPrefix,
23
- gcTombstoneBlobKey,
24
- IGarbageCollectionData,
25
- IGarbageCollectionDetailsBase,
26
- IGarbageCollectionSnapshotData,
27
- IGarbageCollectionState,
28
- ISummarizeResult,
29
- ITelemetryContext,
30
- IGarbageCollectionNodeData,
31
- IGarbageCollectionSummaryDetailsLegacy,
32
- ISummaryTreeWithStats,
33
- gcDeletedBlobKey,
25
+ gcTreeKey,
26
+ gcBlobPrefix,
27
+ gcTombstoneBlobKey,
28
+ IGarbageCollectionData,
29
+ IGarbageCollectionDetailsBase,
30
+ IGarbageCollectionSnapshotData,
31
+ IGarbageCollectionState,
32
+ ISummarizeResult,
33
+ ITelemetryContext,
34
+ IGarbageCollectionNodeData,
35
+ IGarbageCollectionSummaryDetailsLegacy,
36
+ ISummaryTreeWithStats,
37
+ gcDeletedBlobKey,
34
38
  } from "@fluidframework/runtime-definitions";
35
39
  import {
36
- mergeStats,
37
- packagePathToTelemetryProperty,
38
- ReadAndParseBlob,
39
- RefreshSummaryResult,
40
- SummaryTreeBuilder,
40
+ mergeStats,
41
+ packagePathToTelemetryProperty,
42
+ ReadAndParseBlob,
43
+ RefreshSummaryResult,
44
+ SummaryTreeBuilder,
41
45
  } from "@fluidframework/runtime-utils";
42
46
  import {
43
- ChildLogger,
44
- generateStack,
45
- loggerToMonitoringContext,
46
- MonitoringContext,
47
- PerformanceEvent,
48
- TelemetryDataTag,
47
+ ChildLogger,
48
+ generateStack,
49
+ loggerToMonitoringContext,
50
+ MonitoringContext,
51
+ PerformanceEvent,
52
+ TelemetryDataTag,
49
53
  } from "@fluidframework/telemetry-utils";
50
54
 
51
55
  import { IGCRuntimeOptions, RuntimeHeaders } from "./containerRuntime";
52
56
  import { getSummaryForDatastores } from "./dataStores";
53
57
  import {
54
- currentGCVersion,
55
- defaultInactiveTimeoutMs,
56
- defaultSessionExpiryDurationMs,
57
- disableSweepLogKey,
58
- disableTombstoneKey,
59
- gcVersionUpgradeToV2Key,
60
- gcTestModeKey,
61
- oneDayMs,
62
- runGCKey,
63
- runSessionExpiryKey,
64
- runSweepKey,
65
- stableGCVersion,
66
- trackGCStateKey
58
+ currentGCVersion,
59
+ defaultInactiveTimeoutMs,
60
+ defaultSessionExpiryDurationMs,
61
+ disableSweepLogKey,
62
+ disableTombstoneKey,
63
+ gcVersionUpgradeToV2Key,
64
+ gcTestModeKey,
65
+ oneDayMs,
66
+ runGCKey,
67
+ runSessionExpiryKey,
68
+ runSweepKey,
69
+ stableGCVersion,
70
+ trackGCStateKey,
71
+ gcTombstoneGenerationOptionName,
67
72
  } from "./garbageCollectionConstants";
68
- import { sendGCTombstoneEvent } from "./garbageCollectionTombstoneUtils";
73
+ import { sendGCUnexpectedUsageEvent } from "./garbageCollectionHelpers";
69
74
  import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
70
75
  import {
71
- getGCVersion,
72
- GCVersion,
73
- IContainerRuntimeMetadata,
74
- metadataBlobName,
75
- ReadFluidDataStoreAttributes,
76
- dataStoreAttributesBlobName,
77
- IGCMetadata,
78
- ICreateContainerMetadata,
76
+ getGCVersion,
77
+ GCVersion,
78
+ IContainerRuntimeMetadata,
79
+ metadataBlobName,
80
+ ReadFluidDataStoreAttributes,
81
+ dataStoreAttributesBlobName,
82
+ IGCMetadata,
83
+ ICreateContainerMetadata,
84
+ GCFeatureMatrix,
79
85
  } from "./summaryFormat";
80
86
 
81
87
  /** The statistics of the system state after a garbage collection run. */
82
88
  export interface IGCStats {
83
- /** The number of nodes in the container. */
84
- nodeCount: number;
85
- /** The number of data stores in the container. */
86
- dataStoreCount: number;
87
- /** The number of attachment blobs in the container. */
88
- attachmentBlobCount: number;
89
- /** The number of unreferenced nodes in the container. */
90
- unrefNodeCount: number;
91
- /** The number of unreferenced data stores in the container. */
92
- unrefDataStoreCount: number;
93
- /** The number of unreferenced attachment blobs in the container. */
94
- unrefAttachmentBlobCount: number;
95
- /** The number of nodes whose reference state updated since last GC run. */
96
- updatedNodeCount: number;
97
- /** The number of data stores whose reference state updated since last GC run. */
98
- updatedDataStoreCount: number;
99
- /** The number of attachment blobs whose reference state updated since last GC run. */
100
- updatedAttachmentBlobCount: number;
89
+ /** The number of nodes in the container. */
90
+ nodeCount: number;
91
+ /** The number of data stores in the container. */
92
+ dataStoreCount: number;
93
+ /** The number of attachment blobs in the container. */
94
+ attachmentBlobCount: number;
95
+ /** The number of unreferenced nodes in the container. */
96
+ unrefNodeCount: number;
97
+ /** The number of unreferenced data stores in the container. */
98
+ unrefDataStoreCount: number;
99
+ /** The number of unreferenced attachment blobs in the container. */
100
+ unrefAttachmentBlobCount: number;
101
+ /** The number of nodes whose reference state updated since last GC run. */
102
+ updatedNodeCount: number;
103
+ /** The number of data stores whose reference state updated since last GC run. */
104
+ updatedDataStoreCount: number;
105
+ /** The number of attachment blobs whose reference state updated since last GC run. */
106
+ updatedAttachmentBlobCount: number;
101
107
  }
102
108
 
103
109
  /** The types of GC nodes in the GC reference graph. */
104
110
  export const GCNodeType = {
105
- // Nodes that are for data stores.
106
- DataStore: "DataStore",
107
- // Nodes that are within a data store. For example, DDS nodes.
108
- SubDataStore: "SubDataStore",
109
- // Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
110
- Blob: "Blob",
111
- // Nodes that are neither of the above. For example, root node.
112
- Other: "Other",
111
+ // Nodes that are for data stores.
112
+ DataStore: "DataStore",
113
+ // Nodes that are within a data store. For example, DDS nodes.
114
+ SubDataStore: "SubDataStore",
115
+ // Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
116
+ Blob: "Blob",
117
+ // Nodes that are neither of the above. For example, root node.
118
+ Other: "Other",
113
119
  };
114
120
  export type GCNodeType = typeof GCNodeType[keyof typeof GCNodeType];
115
121
 
116
122
  /** Defines the APIs for the runtime object to be passed to the garbage collector. */
117
123
  export interface IGarbageCollectionRuntime {
118
- /** Before GC runs, called to notify the runtime to update any pending GC state. */
119
- updateStateBeforeGC(): Promise<void>;
120
- /** Returns the garbage collection data of the runtime. */
121
- getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
122
- /** After GC has run, called to notify the runtime of routes that are used in it. */
123
- updateUsedRoutes(usedRoutes: string[]): void;
124
- /** After GC has run, called to notify the runtime of routes that are unused in it. */
125
- updateUnusedRoutes(unusedRoutes: string[]): void;
126
- /** Called to notify the runtime of routes that are tombstones. */
127
- updateTombstonedRoutes(tombstoneRoutes: string[]): void;
128
- /** Returns a referenced timestamp to be used to track unreferenced nodes. */
129
- getCurrentReferenceTimestampMs(): number | undefined;
130
- /** Returns the type of the GC node. */
131
- getNodeType(nodePath: string): GCNodeType;
132
- /** Called when the runtime should close because of an error. */
133
- closeFn: (error?: ICriticalContainerError) => void;
124
+ /** Before GC runs, called to notify the runtime to update any pending GC state. */
125
+ updateStateBeforeGC(): Promise<void>;
126
+ /** Returns the garbage collection data of the runtime. */
127
+ getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
128
+ /** After GC has run, called to notify the runtime of routes that are used in it. */
129
+ updateUsedRoutes(usedRoutes: string[]): void;
130
+ /** After GC has run, called to notify the runtime of routes that are unused in it. */
131
+ updateUnusedRoutes(unusedRoutes: string[]): void;
132
+ /**
133
+ * After GC has run, called to notify the runtime of deletable routes. The runtime is responsible
134
+ * for telling the garbage collector the routes of the objects it has deleted
135
+ */
136
+ deleteUnusedNodes(unusedNodes: string[]): string[];
137
+ /** Called to notify the runtime of routes that are tombstones. */
138
+ updateTombstonedRoutes(tombstoneRoutes: string[]): void;
139
+ /** Returns a referenced timestamp to be used to track unreferenced nodes. */
140
+ getCurrentReferenceTimestampMs(): number | undefined;
141
+ /** Returns the type of the GC node. */
142
+ getNodeType(nodePath: string): GCNodeType;
143
+ /** Called when the runtime should close because of an error. */
144
+ closeFn: (error?: ICriticalContainerError) => void;
145
+ /** If false, loading or using a Tombstoned object should merely log, not fail */
146
+ gcTombstoneEnforcementAllowed: boolean;
134
147
  }
135
148
 
136
149
  /** Defines the contract for the garbage collector. */
137
150
  export interface IGarbageCollector {
138
- /** Tells whether GC should run or not. */
139
- readonly shouldRunGC: boolean;
140
- /** Tells whether the GC state in summary needs to be reset in the next summary. */
141
- readonly summaryStateNeedsReset: boolean;
142
- readonly trackGCState: boolean;
143
- /** Initialize the state from the base snapshot after its creation. */
144
- initializeBaseState(): Promise<void>;
145
- /** Run garbage collection and update the reference / used state of the system. */
146
- collectGarbage(
147
- options: { logger?: ITelemetryLogger; runSweep?: boolean; fullGC?: boolean; },
148
- ): Promise<IGCStats | undefined>;
149
- /** Summarizes the GC data and returns it as a summary tree. */
150
- summarize(
151
- fullTree: boolean,
152
- trackState: boolean,
153
- telemetryContext?: ITelemetryContext,
154
- ): ISummarizeResult | undefined;
155
- /** Returns the garbage collector specific metadata to be written into the summary. */
156
- getMetadata(): IGCMetadata;
157
- /** Returns the GC details generated from the base snapshot. */
158
- getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
159
- /** Called when the latest summary of the system has been refreshed. */
160
- refreshLatestSummary(
161
- result: RefreshSummaryResult,
162
- proposalHandle: string | undefined,
163
- summaryRefSeq: number,
164
- readAndParseBlob: ReadAndParseBlob,
165
- ): Promise<void>;
166
- /** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
167
- nodeUpdated(
168
- nodePath: string,
169
- reason: "Loaded" | "Changed",
170
- timestampMs?: number,
171
- packagePath?: readonly string[],
172
- requestHeaders?: IRequestHeader,
173
- ): void;
174
- /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
175
- addedOutboundReference(fromNodePath: string, toNodePath: string): void;
176
- /** Returns true if this node has been deleted by GC during sweep phase. */
177
- isNodeDeleted(nodePath: string): boolean;
178
- setConnectionState(connected: boolean, clientId?: string): void;
179
- dispose(): void;
151
+ /** Tells whether GC should run or not. */
152
+ readonly shouldRunGC: boolean;
153
+ /** Tells whether the GC state in summary needs to be reset in the next summary. */
154
+ readonly summaryStateNeedsReset: boolean;
155
+ readonly trackGCState: boolean;
156
+ /** Initialize the state from the base snapshot after its creation. */
157
+ initializeBaseState(): Promise<void>;
158
+ /** Run garbage collection and update the reference / used state of the system. */
159
+ collectGarbage(options: {
160
+ logger?: ITelemetryLogger;
161
+ runSweep?: boolean;
162
+ fullGC?: boolean;
163
+ }): Promise<IGCStats | undefined>;
164
+ /** Summarizes the GC data and returns it as a summary tree. */
165
+ summarize(
166
+ fullTree: boolean,
167
+ trackState: boolean,
168
+ telemetryContext?: ITelemetryContext,
169
+ ): ISummarizeResult | undefined;
170
+ /** Returns the garbage collector specific metadata to be written into the summary. */
171
+ getMetadata(): IGCMetadata;
172
+ /** Returns the GC details generated from the base snapshot. */
173
+ getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
174
+ /** Called when the latest summary of the system has been refreshed. */
175
+ refreshLatestSummary(
176
+ proposalHandle: string | undefined,
177
+ result: RefreshSummaryResult,
178
+ readAndParseBlob: ReadAndParseBlob,
179
+ ): Promise<void>;
180
+ /** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
181
+ nodeUpdated(
182
+ nodePath: string,
183
+ reason: "Loaded" | "Changed",
184
+ timestampMs?: number,
185
+ packagePath?: readonly string[],
186
+ requestHeaders?: IRequestHeader,
187
+ ): void;
188
+ /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
189
+ addedOutboundReference(fromNodePath: string, toNodePath: string): void;
190
+ /** Returns true if this node has been deleted by GC during sweep phase. */
191
+ isNodeDeleted(nodePath: string): boolean;
192
+ setConnectionState(connected: boolean, clientId?: string): void;
193
+ dispose(): void;
180
194
  }
181
195
 
182
196
  /** Parameters necessary for creating a GarbageCollector. */
183
197
  export interface IGarbageCollectorCreateParams {
184
- readonly runtime: IGarbageCollectionRuntime;
185
- readonly gcOptions: IGCRuntimeOptions;
186
- readonly baseLogger: ITelemetryLogger;
187
- readonly existing: boolean;
188
- readonly metadata: IContainerRuntimeMetadata | undefined;
189
- readonly createContainerMetadata: ICreateContainerMetadata;
190
- readonly baseSnapshot: ISnapshotTree | undefined;
191
- readonly isSummarizerClient: boolean;
192
- readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
193
- readonly getLastSummaryTimestampMs: () => number | undefined;
194
- readonly readAndParseBlob: ReadAndParseBlob;
195
- readonly activeConnection: () => boolean;
196
- readonly getContainerDiagnosticId: () => string;
198
+ readonly runtime: IGarbageCollectionRuntime;
199
+ readonly gcOptions: IGCRuntimeOptions;
200
+ readonly baseLogger: ITelemetryLogger;
201
+ readonly existing: boolean;
202
+ readonly metadata: IContainerRuntimeMetadata | undefined;
203
+ readonly createContainerMetadata: ICreateContainerMetadata;
204
+ readonly baseSnapshot: ISnapshotTree | undefined;
205
+ readonly isSummarizerClient: boolean;
206
+ readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
207
+ readonly getLastSummaryTimestampMs: () => number | undefined;
208
+ readonly readAndParseBlob: ReadAndParseBlob;
209
+ readonly activeConnection: () => boolean;
210
+ readonly getContainerDiagnosticId: () => string;
197
211
  }
198
212
 
199
213
  /** The state of node that is unreferenced. */
200
214
  export const UnreferencedState = {
201
- /** The node is active, i.e., it can become referenced again. */
202
- Active: "Active",
203
- /** The node is inactive, i.e., it should not become referenced. */
204
- Inactive: "Inactive",
205
- /** The node is ready to be deleted by the sweep phase. */
206
- SweepReady: "SweepReady",
215
+ /** The node is active, i.e., it can become referenced again. */
216
+ Active: "Active",
217
+ /** The node is inactive, i.e., it should not become referenced. */
218
+ Inactive: "Inactive",
219
+ /** The node is ready to be deleted by the sweep phase. */
220
+ SweepReady: "SweepReady",
207
221
  } as const;
208
222
  export type UnreferencedState = typeof UnreferencedState[keyof typeof UnreferencedState];
209
223
 
210
224
  /** The event that is logged when unreferenced node is used after a certain time. */
211
225
  interface IUnreferencedEventProps {
212
- usageType: "Changed" | "Loaded" | "Revived";
213
- state: UnreferencedState;
214
- id: string;
215
- type: GCNodeType;
216
- unrefTime: number;
217
- age: number;
218
- completedGCRuns: number;
219
- fromId?: string;
220
- timeout?: number;
221
- lastSummaryTime?: number;
222
- externalRequest?: boolean;
223
- viaHandle?: boolean;
226
+ usageType: "Changed" | "Loaded" | "Revived";
227
+ state: UnreferencedState;
228
+ id: string;
229
+ type: GCNodeType;
230
+ unrefTime: number;
231
+ age: number;
232
+ completedGCRuns: number;
233
+ fromId?: string;
234
+ timeout?: number;
235
+ lastSummaryTime?: number;
236
+ externalRequest?: boolean;
237
+ viaHandle?: boolean;
224
238
  }
225
239
 
226
240
  /**
227
241
  * The GC data that is tracked for a summary that is submitted.
228
242
  */
229
243
  interface IGCSummaryTrackingData {
230
- serializedGCState: string | undefined;
231
- serializedTombstones: string | undefined;
232
- serializedDeletedNodes: string | undefined;
244
+ serializedGCState: string | undefined;
245
+ serializedTombstones: string | undefined;
246
+ serializedDeletedNodes: string | undefined;
233
247
  }
234
248
 
235
249
  /**
@@ -237,85 +251,88 @@ interface IGCSummaryTrackingData {
237
251
  * be deleted by the sweep phase.
238
252
  */
239
253
  export class UnreferencedStateTracker {
240
- private _state: UnreferencedState = UnreferencedState.Active;
241
- public get state(): UnreferencedState {
242
- return this._state;
243
- }
244
-
245
- /** Timer to indicate when an unreferenced object is considered Inactive */
246
- private readonly inactiveTimer: TimerWithNoDefaultTimeout;
247
- /** Timer to indicate when an unreferenced object is Sweep-Ready */
248
- private readonly sweepTimer: TimerWithNoDefaultTimeout;
249
-
250
- constructor(
251
- public readonly unreferencedTimestampMs: number,
252
- /** The time after which node transitions to Inactive state. */
253
- private readonly inactiveTimeoutMs: number,
254
- /** The current reference timestamp used to track how long this node has been unreferenced for. */
255
- currentReferenceTimestampMs: number,
256
- /** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
257
- private readonly sweepTimeoutMs: number | undefined,
258
- ) {
259
- if (this.sweepTimeoutMs !== undefined) {
260
- assert(this.inactiveTimeoutMs <= this.sweepTimeoutMs,
261
- 0x3b0 /* inactive timeout must not be greater than the sweep timeout */);
262
- }
263
-
264
- this.sweepTimer = new TimerWithNoDefaultTimeout(
265
- () => {
266
- this._state = UnreferencedState.SweepReady;
267
- assert(!this.inactiveTimer.hasTimer, 0x3b1 /* inactiveTimer still running after sweepTimer fired! */);
268
- },
269
- );
270
-
271
- this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
272
- this._state = UnreferencedState.Inactive;
273
-
274
- // After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
275
- if (this.sweepTimeoutMs !== undefined) {
276
- this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
277
- }
278
- });
279
- this.updateTracking(currentReferenceTimestampMs);
280
- }
281
-
282
- /* Updates the unreferenced state based on the provided timestamp. */
283
- public updateTracking(currentReferenceTimestampMs: number) {
284
- const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
285
-
286
- // If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
287
- if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
288
- this._state = UnreferencedState.SweepReady;
289
- this.clearTimers();
290
- return;
291
- }
292
-
293
- // If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
294
- // Also, start a timer for the sweep timeout.
295
- if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
296
- this._state = UnreferencedState.Inactive;
297
- this.inactiveTimer.clear();
298
-
299
- if (this.sweepTimeoutMs !== undefined) {
300
- this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
301
- }
302
- return;
303
- }
304
-
305
- // The node is still active. Ensure the inactive timer is running with the proper remaining duration.
306
- this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
307
- }
308
-
309
- private clearTimers() {
310
- this.inactiveTimer.clear();
311
- this.sweepTimer.clear();
312
- }
313
-
314
- /** Stop tracking this node. Reset the unreferenced timers and state, if any. */
315
- public stopTracking() {
316
- this.clearTimers();
317
- this._state = UnreferencedState.Active;
318
- }
254
+ private _state: UnreferencedState = UnreferencedState.Active;
255
+ public get state(): UnreferencedState {
256
+ return this._state;
257
+ }
258
+
259
+ /** Timer to indicate when an unreferenced object is considered Inactive */
260
+ private readonly inactiveTimer: TimerWithNoDefaultTimeout;
261
+ /** Timer to indicate when an unreferenced object is Sweep-Ready */
262
+ private readonly sweepTimer: TimerWithNoDefaultTimeout;
263
+
264
+ constructor(
265
+ public readonly unreferencedTimestampMs: number,
266
+ /** The time after which node transitions to Inactive state. */
267
+ private readonly inactiveTimeoutMs: number,
268
+ /** The current reference timestamp used to track how long this node has been unreferenced for. */
269
+ currentReferenceTimestampMs: number,
270
+ /** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
271
+ private readonly sweepTimeoutMs: number | undefined,
272
+ ) {
273
+ if (this.sweepTimeoutMs !== undefined) {
274
+ assert(
275
+ this.inactiveTimeoutMs <= this.sweepTimeoutMs,
276
+ 0x3b0 /* inactive timeout must not be greater than the sweep timeout */,
277
+ );
278
+ }
279
+
280
+ this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
281
+ this._state = UnreferencedState.SweepReady;
282
+ assert(
283
+ !this.inactiveTimer.hasTimer,
284
+ 0x3b1 /* inactiveTimer still running after sweepTimer fired! */,
285
+ );
286
+ });
287
+
288
+ this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
289
+ this._state = UnreferencedState.Inactive;
290
+
291
+ // After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
292
+ if (this.sweepTimeoutMs !== undefined) {
293
+ this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
294
+ }
295
+ });
296
+ this.updateTracking(currentReferenceTimestampMs);
297
+ }
298
+
299
+ /* Updates the unreferenced state based on the provided timestamp. */
300
+ public updateTracking(currentReferenceTimestampMs: number) {
301
+ const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
302
+
303
+ // If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
304
+ if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
305
+ this._state = UnreferencedState.SweepReady;
306
+ this.clearTimers();
307
+ return;
308
+ }
309
+
310
+ // If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
311
+ // Also, start a timer for the sweep timeout.
312
+ if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
313
+ this._state = UnreferencedState.Inactive;
314
+ this.inactiveTimer.clear();
315
+
316
+ if (this.sweepTimeoutMs !== undefined) {
317
+ this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
318
+ }
319
+ return;
320
+ }
321
+
322
+ // The node is still active. Ensure the inactive timer is running with the proper remaining duration.
323
+ this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
324
+ }
325
+
326
+ private clearTimers() {
327
+ this.inactiveTimer.clear();
328
+ this.sweepTimer.clear();
329
+ }
330
+
331
+ /** Stop tracking this node. Reset the unreferenced timers and state, if any. */
332
+ public stopTracking() {
333
+ this.clearTimers();
334
+ this._state = UnreferencedState.Active;
335
+ }
319
336
  }
320
337
 
321
338
  /**
@@ -329,1457 +346,1631 @@ export class UnreferencedStateTracker {
329
346
  * Graph - all nodes with their respective routes
330
347
  *
331
348
  * ```
332
- * GC Graph
349
+ * GC Graph
333
350
  *
334
- * Node
335
- * NodeId = "datastore1"
336
- * / \\
337
- * OutboundRoute OutboundRoute
338
- * / \\
339
- * Node Node
340
- * NodeId = "dds1" NodeId = "dds2"
351
+ * Node
352
+ * NodeId = "datastore1"
353
+ * / \\
354
+ * OutboundRoute OutboundRoute
355
+ * / \\
356
+ * Node Node
357
+ * NodeId = "dds1" NodeId = "dds2"
341
358
  * ```
342
359
  */
343
360
  export class GarbageCollector implements IGarbageCollector {
344
- public static create(createParams: IGarbageCollectorCreateParams): IGarbageCollector {
345
- return new GarbageCollector(createParams);
346
- }
347
-
348
- /**
349
- * Tells whether the GC state needs to be reset in the next summary. We need to do this if:
350
- *
351
- * 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
352
- *
353
- * 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
354
- *
355
- * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
356
- *
357
- * 4. The GC version in the latest summary is different from the current GC version. This can happen if:
358
- *
359
- * 4.1. The summary this client loaded with has data from a different GC version.
360
- *
361
- * 4.2. This client's latest summary was updated from a snapshot that has a different GC version.
362
- */
363
- public get summaryStateNeedsReset(): boolean {
364
- return this.gcStateNeedsReset ||
365
- (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion);
366
- }
367
-
368
- /**
369
- * Tracks if GC is enabled for this document. This is specified during document creation and doesn't change
370
- * throughout its lifetime.
371
- */
372
- private readonly gcEnabled: boolean;
373
- /**
374
- * Tracks if sweep phase is enabled for this document. This is specified during document creation and doesn't change
375
- * throughout its lifetime.
376
- */
377
- private readonly sweepEnabled: boolean;
378
-
379
- /**
380
- * Tracks if GC should run or not. Even if GC is enabled for a document (see gcEnabled), it can be explicitly
381
- * disabled via runtime options or feature flags.
382
- */
383
- public readonly shouldRunGC: boolean;
384
- /**
385
- * Tracks if sweep phase should run or not. Even if the sweep phase is enabled for a document (see sweepEnabled), it
386
- * can be explicitly disabled via feature flags. It also won't run if session expiry is not enabled.
387
- */
388
- private readonly shouldRunSweep: boolean;
389
-
390
- public readonly trackGCState: boolean;
391
-
392
- private readonly testMode: boolean;
393
- private readonly tombstoneMode: boolean;
394
- private readonly mc: MonitoringContext;
395
-
396
- /**
397
- * Tells whether the GC state needs to be reset. This can happen under 3 conditions:
398
- *
399
- * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
400
- * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
401
- *
402
- * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
403
- * a document and the first time GC is enabled after is was disabled before.
404
- *
405
- * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
406
- *
407
- * Note that the state will be reset only once for the first summary generated after this returns true. After that,
408
- * this will return false.
409
- */
410
- private get gcStateNeedsReset(): boolean {
411
- return this.wasGCRunInLatestSummary !== this.shouldRunGC;
412
- }
413
- // Tracks whether there was GC was run in latest summary being tracked.
414
- private wasGCRunInLatestSummary: boolean;
415
-
416
- // The current GC version that this container is running.
417
- private readonly currentGCVersion: GCVersion;
418
- // This is the version of GC data in the latest summary being tracked.
419
- private latestSummaryGCVersion: GCVersion;
420
-
421
- // Keeps track of the GC state from the last run.
422
- private gcDataFromLastRun: IGarbageCollectionData | undefined;
423
- // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
424
- // outbound routes from that node.
425
- private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
426
- // A list of nodes that have been tombstoned.
427
- private tombstones: string[] = [];
428
- // A list of nodes that have been deleted during sweep phase.
429
- private deletedNodes: Set<string> = new Set();
430
-
431
- /**
432
- * Keeps track of the GC data from the latest summary successfully submitted to and acked from the server.
433
- */
434
- private latestSummaryData: IGCSummaryTrackingData | undefined;
435
- /**
436
- * Keeps track of the GC data from the last summary submitted to the server but not yet acked.
437
- */
438
- private pendingSummaryData: IGCSummaryTrackingData | undefined;
439
-
440
- // Promise when resolved returns the GC data data in the base snapshot.
441
- private readonly baseSnapshotDataP: Promise<IGarbageCollectionSnapshotData | undefined>;
442
- // Promise when resolved initializes the GC state from the data in the base snapshot.
443
- private readonly initializeGCStateFromBaseSnapshotP: Promise<void>;
444
- // The GC details generated from the base snapshot.
445
- private readonly baseGCDetailsP: Promise<IGarbageCollectionDetailsBase>;
446
- // Map of node ids to their unreferenced state tracker.
447
- private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();
448
- // The Timer responsible for closing the container when the session has expired
449
- private sessionExpiryTimer: Timer | undefined;
450
-
451
- // Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
452
- // per event per node.
453
- private readonly loggedUnreferencedEvents: Set<string> = new Set();
454
- // Queue for unreferenced events that should be logged the next time GC runs.
455
- private pendingEventsQueue: IUnreferencedEventProps[] = [];
456
-
457
- // The number of times GC has successfully completed on this instance of GarbageCollector.
458
- private completedRuns = 0;
459
-
460
- private readonly runtime: IGarbageCollectionRuntime;
461
- private readonly createContainerMetadata: ICreateContainerMetadata;
462
- private readonly gcOptions: IGCRuntimeOptions;
463
- private readonly isSummarizerClient: boolean;
464
-
465
- /** The time in ms to expire a session for a client for gc. */
466
- private readonly sessionExpiryTimeoutMs: number | undefined;
467
- /** The time after which an unreferenced node is inactive. */
468
- private readonly inactiveTimeoutMs: number;
469
- /** The time after which an unreferenced node is ready to be swept. */
470
- private readonly sweepTimeoutMs: number | undefined;
471
-
472
- /** For a given node path, returns the node's package path. */
473
- private readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
474
- /** Returns the timestamp of the last summary generated for this container. */
475
- private readonly getLastSummaryTimestampMs: () => number | undefined;
476
- /** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
477
- private readonly activeConnection: () => boolean;
478
-
479
- /** Returns a list of all the configurations for garbage collection. */
480
- private get configs() {
481
- return {
482
- gcEnabled: this.gcEnabled,
483
- sweepEnabled: this.sweepEnabled,
484
- runGC: this.shouldRunGC,
485
- runSweep: this.shouldRunSweep,
486
- testMode: this.testMode,
487
- tombstoneMode: this.tombstoneMode,
488
- sessionExpiry: this.sessionExpiryTimeoutMs,
489
- sweepTimeout: this.sweepTimeoutMs,
490
- inactiveTimeout: this.inactiveTimeoutMs,
491
- trackGCState: this.trackGCState,
492
- ...this.gcOptions,
493
- };
494
- }
495
-
496
- /** Handler to respond to when a SweepReady object is used */
497
- private readonly sweepReadyUsageHandler: SweepReadyUsageDetectionHandler;
498
-
499
- protected constructor(createParams: IGarbageCollectorCreateParams) {
500
- this.runtime = createParams.runtime;
501
- this.isSummarizerClient = createParams.isSummarizerClient;
502
- this.gcOptions = createParams.gcOptions;
503
- this.createContainerMetadata = createParams.createContainerMetadata;
504
- this.getNodePackagePath = createParams.getNodePackagePath;
505
- this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
506
- this.activeConnection = createParams.activeConnection;
507
-
508
- const baseSnapshot = createParams.baseSnapshot;
509
- const metadata = createParams.metadata;
510
- const readAndParseBlob = createParams.readAndParseBlob;
511
-
512
- this.mc = loggerToMonitoringContext(ChildLogger.create(
513
- createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } },
514
- ));
515
-
516
- // If version upgrade is not enabled, fall back to the stable GC version.
517
- this.currentGCVersion =
518
- this.mc.config.getBoolean(gcVersionUpgradeToV2Key) === true ? currentGCVersion : stableGCVersion;
519
-
520
- this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(
521
- createParams.getContainerDiagnosticId(),
522
- this.mc,
523
- this.runtime.closeFn,
524
- );
525
-
526
- let prevSummaryGCVersion: number | undefined;
527
-
528
- /**
529
- * Sweep timeout is the time after which unreferenced content can be swept.
530
- * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
531
- *
532
- * The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
533
- * The buffer is added to account for any clock skew or other edge cases.
534
- * We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
535
- */
536
- function computeSweepTimeout(sessionExpiryTimeoutMs: number | undefined) {
537
- const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
538
- const bufferMs = oneDayMs;
539
- return sessionExpiryTimeoutMs &&
540
- (sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
541
- }
542
-
543
- /**
544
- * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
545
- * 1. Whether running GC mark phase is allowed or not.
546
- * 2. Whether running GC sweep phase is allowed or not.
547
- * 3. Whether GC session expiry is enabled or not.
548
- * For existing containers, we get this information from the metadata blob of its summary.
549
- */
550
- if (createParams.existing) {
551
- prevSummaryGCVersion = getGCVersion(metadata);
552
- // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
553
- // other existing documents, GC is enabled.
554
- this.gcEnabled = prevSummaryGCVersion > 0;
555
- this.sweepEnabled = metadata?.sweepEnabled ?? false;
556
- this.sessionExpiryTimeoutMs = metadata?.sessionExpiryTimeoutMs;
557
- this.sweepTimeoutMs =
558
- metadata?.sweepTimeoutMs
559
- ?? computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
560
- } else {
561
- // Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
562
- // scenario but explicitly failing makes it clearer and promotes correct usage.
563
- if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
564
- throw new UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
565
- }
566
-
567
- // This Test Override only applies for new containers
568
- const testOverrideSweepTimeoutMs =
569
- this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
570
-
571
- // For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
572
- // flag in GC options to false.
573
- this.gcEnabled = this.gcOptions.gcAllowed !== false;
574
- // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
575
- this.sweepEnabled = this.gcOptions.sweepAllowed === true;
576
-
577
- // Set the Session Expiry only if the flag is enabled and GC is enabled.
578
- if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
579
- this.sessionExpiryTimeoutMs = this.gcOptions.sessionExpiryTimeoutMs ?? defaultSessionExpiryDurationMs;
580
- }
581
- this.sweepTimeoutMs =
582
- testOverrideSweepTimeoutMs
583
- ?? computeSweepTimeout(this.sessionExpiryTimeoutMs);
584
- }
585
-
586
- // If session expiry is enabled, we need to close the container when the session expiry timeout expires.
587
- if (this.sessionExpiryTimeoutMs !== undefined) {
588
- // If Test Override config is set, override Session Expiry timeout.
589
- const overrideSessionExpiryTimeoutMs =
590
- this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
591
- const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.sessionExpiryTimeoutMs;
592
-
593
- this.sessionExpiryTimer = new Timer(
594
- timeoutMs,
595
- () => { this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs)); },
596
- );
597
- this.sessionExpiryTimer.start();
598
- }
599
-
600
- // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
601
- // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
602
- this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;
603
-
604
- /**
605
- * Whether GC should run or not. The following conditions have to be met to run sweep:
606
- *
607
- * 1. GC should be enabled for this container.
608
- *
609
- * 2. GC should not be disabled via disableGC GC option.
610
- *
611
- * These conditions can be overridden via runGCKey feature flag.
612
- */
613
- this.shouldRunGC = this.mc.config.getBoolean(runGCKey) ?? (
614
- // GC must be enabled for the document.
615
- this.gcEnabled
616
- // GC must not be disabled via GC options.
617
- && !this.gcOptions.disableGC
618
- );
619
-
620
- /**
621
- * Whether sweep should run or not. The following conditions have to be met to run sweep:
622
- *
623
- * 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
624
- * 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
625
- * 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
626
- * the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
627
- * 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
628
- * feature flag.
629
- */
630
- this.shouldRunSweep =
631
- this.shouldRunGC
632
- && this.sweepTimeoutMs !== undefined
633
- && (this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
634
-
635
- this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
636
-
637
- // Override inactive timeout if test config or gc options to override it is set.
638
- this.inactiveTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs") ??
639
- this.gcOptions.inactiveTimeoutMs ??
640
- defaultInactiveTimeoutMs;
641
-
642
- // Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
643
- if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
644
- throw new UsageError("inactive timeout should not be greater than the sweep timeout");
645
- }
646
-
647
- // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
648
- this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
649
- // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
650
- // via feature flags.
651
- this.tombstoneMode = !this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
652
-
653
- // If GC ran in the container that generated the base snapshot, it will have a GC tree.
654
- this.wasGCRunInLatestSummary = baseSnapshot?.trees[gcTreeKey] !== undefined;
655
-
656
- // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
657
- // it involves fetching blobs from storage which is expensive.
658
- this.baseSnapshotDataP = new LazyPromise<IGarbageCollectionSnapshotData | undefined>(async () => {
659
- if (baseSnapshot === undefined) {
660
- return undefined;
661
- }
662
-
663
- try {
664
- // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
665
- const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
666
- if (gcSnapshotTree !== undefined) {
667
- return getGCDataFromSnapshot(
668
- gcSnapshotTree,
669
- readAndParseBlob,
670
- );
671
- }
672
-
673
- // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
674
- // consolidate into IGarbageCollectionState format.
675
- // Add a node for the root node that is not present in older snapshot format.
676
- const gcState: IGarbageCollectionState = { gcNodes: { "/": { outboundRoutes: [] } } };
677
- const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
678
- assert(dataStoreSnapshotTree !== undefined,
679
- 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
680
- for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
681
- const blobId = dsSnapshotTree.blobs[gcTreeKey];
682
- if (blobId === undefined) {
683
- continue;
684
- }
685
-
686
- const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionSummaryDetailsLegacy>(blobId);
687
- // If there are no nodes for this data store, skip it.
688
- if (gcSummaryDetails.gcData?.gcNodes === undefined) {
689
- continue;
690
- }
691
-
692
- const dsRootId = `/${dsId}`;
693
- // Since we used to write GC data at data store level, we won't have an entry for the root ("/").
694
- // Construct that entry by adding root data store ids to its outbound routes.
695
- const initialSnapshotDetails = await readAndParseBlob<ReadFluidDataStoreAttributes>(
696
- dsSnapshotTree.blobs[dataStoreAttributesBlobName],
697
- );
698
- if (initialSnapshotDetails.isRootDataStore) {
699
- gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
700
- }
701
-
702
- for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
703
- // Prefix the data store id to the GC node ids to make them relative to the root from being
704
- // relative to the data store. Similar to how its done in DataStore::getGCData.
705
- const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
706
- gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
707
- }
708
- assert(gcState.gcNodes[dsRootId] !== undefined,
709
- 0x2a9 /* GC nodes for data store not in GC blob */);
710
- gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
711
- }
712
- // If there is only one node (root node just added above), either GC is disabled or we are loading from
713
- // the first summary generated by detached container. In both cases, GC was not run - return undefined.
714
- return Object.keys(gcState.gcNodes).length === 1
715
- ? undefined
716
- : { gcState, tombstones: undefined, deletedNodes: undefined };
717
- } catch (error) {
718
- const dpe = DataProcessingError.wrapIfUnrecognized(
719
- error,
720
- "FailedToInitializeGC",
721
- );
722
- dpe.addTelemetryProperties({ gcConfigs: JSON.stringify(this.configs) });
723
- throw dpe;
724
- }
725
- });
726
-
727
- /**
728
- * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
729
- * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
730
- * GC state and updates their inactive or sweep ready state.
731
- */
732
- this.initializeGCStateFromBaseSnapshotP = new LazyPromise<void>(async () => {
733
- /**
734
- * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
735
- * how long objects have been unreferenced and if they can be deleted.
736
- *
737
- * Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
738
- * for this container and it is in read mode. In this scenario, there is no point in running GC anyway
739
- * because references in the container do not change without any ops, i.e., there is nothing to collect.
740
- */
741
- const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
742
- if (currentReferenceTimestampMs === undefined) {
743
- // Log an event so we can evaluate how often we run into this scenario.
744
- this.mc.logger.sendErrorEvent({
745
- eventName: "GarbageCollectorInitializedWithoutTimestamp",
746
- gcConfigs: JSON.stringify(this.configs),
747
- });
748
- return;
749
- }
750
- /**
751
- * The base snapshot data will not be present if the container is loaded from:
752
- * 1. The first summary created by the detached container.
753
- * 2. A summary that was generated with GC disabled.
754
- * 3. A summary that was generated before GC even existed.
755
- */
756
- const baseSnapshotData = await this.baseSnapshotDataP;
757
- if (baseSnapshotData === undefined) {
758
- return;
759
- }
760
- this.updateStateFromSnapshotData(baseSnapshotData, currentReferenceTimestampMs);
761
- });
762
-
763
- // Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
764
- // used to initialize the GC state of all the nodes in the container.
765
- this.baseGCDetailsP = new LazyPromise<IGarbageCollectionDetailsBase>(async () => {
766
- const baseSnapshotData = await this.baseSnapshotDataP;
767
- if (baseSnapshotData === undefined) {
768
- return {};
769
- }
770
-
771
- const gcNodes: { [id: string]: string[]; } = {};
772
- for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
773
- gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
774
- }
775
- // Run GC on the nodes in the base summary to get the routes used in each node in the container.
776
- // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
777
- // each node in the summary.
778
- const usedRoutes = runGarbageCollection(gcNodes, ["/"]).referencedNodeIds;
779
-
780
- return { gcData: { gcNodes }, usedRoutes };
781
- });
782
-
783
- // Log all the GC options and the state determined by the garbage collector. This is interesting only for the
784
- // summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
785
- if (this.isSummarizerClient) {
786
- this.mc.logger.sendTelemetryEvent({
787
- eventName: "GarbageCollectorLoaded",
788
- gcConfigs: JSON.stringify(this.configs),
789
- });
790
- }
791
- }
792
-
793
- /**
794
- * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
795
- * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
796
- */
797
- public async initializeBaseState(): Promise<void> {
798
- const baseSnapshotData = await this.baseSnapshotDataP;
799
- /**
800
- * The base snapshot data will not be present if the container is loaded from:
801
- * 1. The first summary created by the detached container.
802
- * 2. A summary that was generated with GC disabled.
803
- * 3. A summary that was generated before GC even existed.
804
- */
805
- if (baseSnapshotData === undefined) {
806
- return;
807
- }
808
-
809
- // Initialize the deleted nodes from the snapshot. This is done irrespective of whether sweep is enabled or not
810
- // to identify deleted nodes' usage.
811
- if (baseSnapshotData.deletedNodes !== undefined) {
812
- this.deletedNodes = new Set(baseSnapshotData.deletedNodes);
813
- }
814
-
815
- // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
816
- // tombstone routes.
817
- if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
818
- this.tombstones = Array.from(baseSnapshotData.tombstones);
819
- this.runtime.updateTombstonedRoutes(this.tombstones);
820
- }
821
- }
822
-
823
- /**
824
- * Update state from the given snapshot data. This is done during load and during refreshing state from a snapshot.
825
- * All current tracking is reset and updated from the data in the snapshot.
826
- * @param snapshotData - The snapshot data to update state from. If this is undefined, all GC state and tracking
827
- * is reset.
828
- * @param currentReferenceTimestampMs - The current reference timestamp for marking unreferenced nodes' unreferenced
829
- * timestamp.
830
- */
831
- private updateStateFromSnapshotData(
832
- snapshotData: IGarbageCollectionSnapshotData | undefined,
833
- currentReferenceTimestampMs: number,
834
- ) {
835
- /**
836
- * Note: "newReferencesSinceLastRun" is not reset here. This is done because there may be references since the
837
- * snapshot that we are updating state from. For example, this client may have processed ops till seq#1000 and
838
- * its refreshing state from a summary that happened at seq#900. In this case, there may be references between
839
- * seq#901 and seq#1000 that we don't want to reset.
840
- * Unfortunately, there is no way to track the seq# of ops that add references, so we choose to not reset any
841
- * references here. This should be fine because, in the worst case, we may end up updating the unreferenced
842
- * timestamp of a node which will delay its deletion. Although not ideal, this will only happen in rare
843
- * scenarios, so it should be okay.
844
- */
845
-
846
- // Clear all existing unreferenced state tracking.
847
- for (const [, nodeStateTracker] of this.unreferencedNodesState) {
848
- nodeStateTracker.stopTracking();
849
- };
850
- this.unreferencedNodesState.clear();
851
-
852
- // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
853
- // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
854
- // tombstones.
855
- // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
856
- // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
857
- if (this.shouldRunSweep) {
858
- const snapshotDeletedNodes = snapshotData?.tombstones ? new Set(snapshotData.tombstones) : undefined;
859
- // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
860
- // delete them.
861
- if (snapshotDeletedNodes !== undefined) {
862
- const newDeletedNodes: string[] = [];
863
- for (const nodeId of snapshotDeletedNodes) {
864
- if (!this.deletedNodes.has(nodeId)) {
865
- newDeletedNodes.push(nodeId);
866
- }
867
- }
868
- if (newDeletedNodes.length > 0) {
869
- // Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
870
- }
871
- }
872
- } else if (this.tombstoneMode) {
873
- // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
874
- // notify the runtime to update its state as well.
875
- this.tombstones = snapshotData?.tombstones ? Array.from(snapshotData.tombstones) : [];
876
- this.runtime.updateTombstonedRoutes(this.tombstones);
877
- }
878
-
879
- // If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
880
- if (snapshotData === undefined) {
881
- this.gcDataFromLastRun = undefined;
882
- this.latestSummaryData = undefined;
883
- return;
884
- }
885
-
886
- // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
887
- // to the GC data from the snapshot data.
888
- const gcNodes: { [id: string]: string[]; } = {};
889
- for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
890
- if (nodeData.unreferencedTimestampMs !== undefined) {
891
- this.unreferencedNodesState.set(
892
- nodeId,
893
- new UnreferencedStateTracker(
894
- nodeData.unreferencedTimestampMs,
895
- this.inactiveTimeoutMs,
896
- currentReferenceTimestampMs,
897
- this.sweepTimeoutMs,
898
- ),
899
- );
900
- }
901
- gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
902
- }
903
- this.gcDataFromLastRun = { gcNodes };
904
-
905
- // If tracking state across summaries, update latest summary data from the snapshot's GC data.
906
- if (this.trackGCState) {
907
- this.latestSummaryData = {
908
- serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
909
- serializedTombstones: JSON.stringify(snapshotData.tombstones),
910
- serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
911
- };
912
- }
913
- }
914
-
915
- /**
916
- * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
917
- * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
918
- * @param connected - Whether the runtime connected / disconnected.
919
- * @param clientId - The clientId of this runtime.
920
- */
921
- public setConnectionState(connected: boolean, clientId?: string | undefined): void {
922
- /**
923
- * For all clients, initialize the base state when the container becomes active, i.e., it transitions
924
- * to "write" mode. This will ensure that the container's own join op is processed and there is a recent
925
- * reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
926
- * could affect the GC state will have been processed.
927
- *
928
- * If GC is up-to-date for the client and the summarizing client, there will be an doubling of both
929
- * InactiveObject_Loaded and SweepReady_Loaded errors, as there will be one from the sending client and one from
930
- * the receiving summarizer client.
931
- *
932
- * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
933
- * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
934
- */
935
- if (this.activeConnection() && this.shouldRunGC) {
936
- this.initializeGCStateFromBaseSnapshotP.catch((error) => {});
937
- }
938
- }
939
-
940
- /**
941
- * Runs garbage collection and updates the reference / used state of the nodes in the container.
942
- * @returns stats of the GC run or undefined if GC did not run.
943
- */
944
- public async collectGarbage(
945
- options: {
946
- /** Logger to use for logging GC events */
947
- logger?: ITelemetryLogger;
948
- /** True to run GC sweep phase after the mark phase */
949
- runSweep?: boolean;
950
- /** True to generate full GC data */
951
- fullGC?: boolean;
952
- },
953
- ): Promise<IGCStats | undefined> {
954
- const fullGC = options.fullGC ?? (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
955
- const logger = options.logger
956
- ? ChildLogger.create(options.logger, undefined, { all: { completedGCRuns: () => this.completedRuns } })
957
- : this.mc.logger;
958
-
959
- /**
960
- * If there is no current reference timestamp, skip running GC. We need the current timestamp to track
961
- * how long objects have been unreferenced and if they should be deleted.
962
- *
963
- * Note that the only scenario where GC is called and there is no reference timestamp is when no ops have ever
964
- * been processed for this container and it is in read mode. In this scenario, there is no point in running GC
965
- * anyway because references in the container do not change without any ops, i.e., there is nothing to collect.
966
- */
967
- const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
968
- if (currentReferenceTimestampMs === undefined) {
969
- // Log an event so we can evaluate how often we run into this scenario.
970
- logger.sendErrorEvent({
971
- eventName: "CollectGarbageCalledWithoutTimestamp",
972
- gcConfigs: JSON.stringify(this.configs),
973
- });
974
- return undefined;
975
- }
976
-
977
- return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
978
- await this.runPreGCSteps();
979
-
980
- // Get the runtime's GC data and run GC on the reference graph in it.
981
- const gcData = await this.runtime.getGCData(fullGC);
982
- const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
983
-
984
- const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
985
- event.end({ ...gcStats, timestamp: currentReferenceTimestampMs });
986
- this.completedRuns++;
987
- return gcStats;
988
- }, { end: true, cancel: "error" });
989
- }
990
-
991
- private async runPreGCSteps() {
992
- // Ensure that state has been initialized from the base snapshot data.
993
- await this.initializeGCStateFromBaseSnapshotP;
994
- // Let the runtime update its pending state before GC runs.
995
- await this.runtime.updateStateBeforeGC();
996
- }
997
-
998
- private async runPostGCSteps(
999
- gcData: IGarbageCollectionData,
1000
- gcResult: IGCResult,
1001
- logger: ITelemetryLogger,
1002
- currentReferenceTimestampMs: number,
1003
- ): Promise<IGCStats> {
1004
- // Generate statistics from the current run. This is done before updating the current state because it
1005
- // generates some of its data based on previous state of the system.
1006
- const gcStats = this.generateStats(gcResult);
1007
-
1008
- // Update the state since the last GC run. There can be nodes that were referenced between the last and
1009
- // the current run. We need to identify than and update their unreferenced state if needed.
1010
- this.updateStateSinceLastRun(gcData, logger);
1011
-
1012
- // Update the current state and update the runtime of all routes or ids that used as per the GC run.
1013
- this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
1014
- this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
1015
-
1016
- // Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
1017
- // delete these objects here instead.
1018
- this.logSweepEvents(logger, currentReferenceTimestampMs);
1019
-
1020
- // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
1021
- // involving access to deleted data.
1022
- if (this.testMode) {
1023
- this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
1024
- } else if (this.tombstoneMode) {
1025
- // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
1026
- // involving access to "deleted" data without actually deleting the data from summaries.
1027
- // Note: we will not tombstone in test mode.
1028
- this.runtime.updateTombstonedRoutes(this.tombstones);
1029
- }
1030
-
1031
- // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
1032
- // updates its state so that we don't send false positives based on intermediate state. For example, we may get
1033
- // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
1034
- await this.logUnreferencedEvents(logger);
1035
-
1036
- return gcStats;
1037
- }
1038
-
1039
- /**
1040
- * Summarizes the GC data and returns it as a summary tree.
1041
- * We current write the entire GC state in a single blob. This can be modified later to write multiple
1042
- * blobs. All the blob keys should start with `gcBlobPrefix`.
1043
- */
1044
- public summarize(
1045
- fullTree: boolean,
1046
- trackState: boolean,
1047
- telemetryContext?: ITelemetryContext,
1048
- ): ISummarizeResult | undefined {
1049
- if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
1050
- return;
1051
- }
1052
-
1053
- const gcState: IGarbageCollectionState = { gcNodes: {} };
1054
- for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
1055
- gcState.gcNodes[nodeId] = {
1056
- outboundRoutes,
1057
- unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
1058
- };
1059
- }
1060
-
1061
- const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
1062
- // Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
1063
- // to identify deleted nodes' usage.
1064
- const serializedDeletedNodes = this.deletedNodes.size > 0
1065
- ? JSON.stringify(Array.from(this.deletedNodes).sort())
1066
- : undefined;
1067
- // If running in tombstone mode, serialize and write tombstones, if any.
1068
- const serializedTombstones = this.tombstoneMode
1069
- ? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
1070
- : undefined;
1071
-
1072
- /**
1073
- * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
1074
- * write summary handle instead of summary tree for GC.
1075
- * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
1076
- * for each of these that did not change, write a summary handle.
1077
- */
1078
- if (this.trackGCState) {
1079
- this.pendingSummaryData = { serializedGCState, serializedTombstones, serializedDeletedNodes };
1080
- if (trackState && !fullTree && this.latestSummaryData !== undefined) {
1081
- // If nothing changed since last summary, send a summary handle for the entire GC data.
1082
- if (this.latestSummaryData.serializedGCState === serializedGCState
1083
- && this.latestSummaryData.serializedTombstones === serializedTombstones) {
1084
- const stats = mergeStats();
1085
- stats.handleNodeCount++;
1086
- return {
1087
- summary: {
1088
- type: SummaryType.Handle,
1089
- handle: `/${gcTreeKey}`,
1090
- handleType: SummaryType.Tree,
1091
- },
1092
- stats,
1093
- };
1094
- }
1095
-
1096
- // If some state changed, build a GC summary tree.
1097
- return this.buildGCSummaryTree(
1098
- serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
1099
- }
1100
- }
1101
- // If not tracking GC state, build a GC summary tree without any summary handles.
1102
- return this.buildGCSummaryTree(
1103
- serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
1104
- }
1105
-
1106
- /**
1107
- * Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
1108
- * If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
1109
- * If trackState is true, only states that changed are written. Rest are written as handles.
1110
- * @param serializedGCState - The GC state serialized as string.
1111
- * @param serializedTombstones - The tombstone state serialized as string.
1112
- * @param serializedDeletedNodes - Deleted nodes serialized as string.
1113
- * @param trackState - Whether we are tracking GC state across summaries.
1114
- * @returns the GC summary tree.
1115
- */
1116
- private buildGCSummaryTree(
1117
- serializedGCState: string,
1118
- serializedTombstones: string | undefined,
1119
- serializedDeletedNodes: string | undefined,
1120
- trackState: boolean,
1121
- ): ISummaryTreeWithStats {
1122
- const gcStateBlobKey = `${gcBlobPrefix}_root`;
1123
- const builder = new SummaryTreeBuilder();
1124
-
1125
- // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
1126
- if (this.latestSummaryData?.serializedGCState === serializedGCState && trackState) {
1127
- builder.addHandle(gcStateBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcStateBlobKey}`);
1128
- } else {
1129
- builder.addBlob(gcStateBlobKey, serializedGCState);
1130
- }
1131
-
1132
- // If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
1133
- // summary blob.
1134
- if (serializedTombstones !== undefined) {
1135
- if (this.latestSummaryData?.serializedTombstones === serializedTombstones && trackState) {
1136
- builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
1137
- } else {
1138
- builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
1139
- }
1140
- }
1141
-
1142
- // If there are no deleted nodes, return the summary tree.
1143
- if (serializedDeletedNodes === undefined) {
1144
- return builder.getSummaryTree();
1145
- }
1146
-
1147
- // If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
1148
- if (this.latestSummaryData?.serializedDeletedNodes === serializedDeletedNodes && trackState) {
1149
- builder.addHandle(gcDeletedBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcDeletedBlobKey}`);
1150
- } else {
1151
- builder.addBlob(gcDeletedBlobKey, serializedDeletedNodes);
1152
- }
1153
- return builder.getSummaryTree();
1154
- }
1155
-
1156
- public getMetadata(): IGCMetadata {
1157
- return {
1158
- /**
1159
- * If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
1160
- * into the metadata blob. If GC is disabled, the gcFeature is 0.
1161
- */
1162
- gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
1163
- sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
1164
- sweepEnabled: this.sweepEnabled,
1165
- sweepTimeoutMs: this.sweepTimeoutMs,
1166
- };
1167
- }
1168
-
1169
- /**
1170
- * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
1171
- * in the container.
1172
- */
1173
- public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
1174
- return this.baseGCDetailsP;
1175
- }
1176
-
1177
- /**
1178
- * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
1179
- * is downloaded and should be used to update the state.
1180
- */
1181
- public async refreshLatestSummary(
1182
- result: RefreshSummaryResult,
1183
- proposalHandle: string | undefined,
1184
- summaryRefSeq: number,
1185
- readAndParseBlob: ReadAndParseBlob,
1186
- ): Promise<void> {
1187
- // If the latest summary was updated and the summary was tracked, this client is the one that generated this
1188
- // summary. So, update wasGCRunInLatestSummary.
1189
- // Note that this has to be updated if GC did not run too. Otherwise, `gcStateNeedsReset` will always return
1190
- // true in scenarios where GC is disabled but enabled in the snapshot we loaded from.
1191
- if (result.latestSummaryUpdated && result.wasSummaryTracked) {
1192
- this.wasGCRunInLatestSummary = this.shouldRunGC;
1193
- }
1194
-
1195
- if (!result.latestSummaryUpdated || !this.shouldRunGC) {
1196
- return;
1197
- }
1198
-
1199
- // If the summary was tracked by this client, it was the one that generated the summary in the first place.
1200
- // Update latest state from pending.
1201
- if (result.wasSummaryTracked) {
1202
- this.latestSummaryGCVersion = this.currentGCVersion;
1203
- if (this.trackGCState) {
1204
- this.latestSummaryData = this.pendingSummaryData;
1205
- this.pendingSummaryData = undefined;
1206
- }
1207
- return;
1208
- }
1209
-
1210
- // If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
1211
- const snapshot = result.snapshot;
1212
- const metadataBlobId = snapshot.blobs[metadataBlobName];
1213
- if (metadataBlobId) {
1214
- const metadata = await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId);
1215
- this.latestSummaryGCVersion = getGCVersion(metadata);
1216
- }
1217
-
1218
- // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
1219
- // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
1220
- const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
1221
- if (currentReferenceTimestampMs === undefined) {
1222
- throw DataProcessingError.create(
1223
- "No reference timestamp when updating GC state from snapshot",
1224
- "refreshLatestSummary",
1225
- undefined,
1226
- { proposalHandle, summaryRefSeq, details: JSON.stringify(this.configs) },
1227
- );
1228
- }
1229
- const gcSnapshotTree = snapshot.trees[gcTreeKey];
1230
- // If GC ran in the container that generated this snapshot, it will have a GC tree.
1231
- this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
1232
- let latestGCData: IGarbageCollectionSnapshotData | undefined;
1233
- if (gcSnapshotTree !== undefined) {
1234
- latestGCData = await getGCDataFromSnapshot(
1235
- gcSnapshotTree,
1236
- readAndParseBlob,
1237
- );
1238
- }
1239
- this.updateStateFromSnapshotData(latestGCData, currentReferenceTimestampMs);
1240
- this.pendingSummaryData = undefined;
1241
- }
1242
-
1243
- /**
1244
- * Called when a node with the given id is updated. If the node is inactive, log an error.
1245
- * @param nodePath - The id of the node that changed.
1246
- * @param reason - Whether the node was loaded or changed.
1247
- * @param timestampMs - The timestamp when the node changed.
1248
- * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
1249
- * @param requestHeaders - If the node was loaded via request path, the headers in the request.
1250
- */
1251
- public nodeUpdated(
1252
- nodePath: string,
1253
- reason: "Loaded" | "Changed",
1254
- timestampMs?: number,
1255
- packagePath?: readonly string[],
1256
- requestHeaders?: IRequestHeader,
1257
- ) {
1258
- if (!this.shouldRunGC) {
1259
- return;
1260
- }
1261
-
1262
- const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
1263
- if (nodeStateTracker && nodeStateTracker.state !== UnreferencedState.Active) {
1264
- this.inactiveNodeUsed(
1265
- reason,
1266
- nodePath,
1267
- nodeStateTracker,
1268
- undefined /* fromNodeId */,
1269
- packagePath,
1270
- timestampMs,
1271
- requestHeaders,
1272
- );
1273
- }
1274
- }
1275
-
1276
- /**
1277
- * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
1278
- * referenced between summaries so that their unreferenced timestamp can be reset.
1279
- *
1280
- * @param fromNodePath - The node from which the reference is added.
1281
- * @param toNodePath - The node to which the reference is added.
1282
- */
1283
- public addedOutboundReference(fromNodePath: string, toNodePath: string) {
1284
- if (!this.shouldRunGC) {
1285
- return;
1286
- }
1287
-
1288
- const outboundRoutes = this.newReferencesSinceLastRun.get(fromNodePath) ?? [];
1289
- outboundRoutes.push(toNodePath);
1290
- this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
1291
-
1292
- const nodeStateTracker = this.unreferencedNodesState.get(toNodePath);
1293
- if (nodeStateTracker && nodeStateTracker.state !== UnreferencedState.Active) {
1294
- this.inactiveNodeUsed("Revived", toNodePath, nodeStateTracker, fromNodePath);
1295
- }
1296
-
1297
- if (this.tombstones.includes(toNodePath)) {
1298
- const nodeType = this.runtime.getNodeType(toNodePath)
1299
-
1300
- let eventName = "GC_Tombstone_SubDatastore_Revived";
1301
- if (nodeType === GCNodeType.DataStore) {
1302
- eventName = "GC_Tombstone_Datastore_Revived";
1303
- } else if (nodeType === GCNodeType.Blob) {
1304
- eventName = "GC_Tombstone_Blob_Revived";
1305
- }
1306
-
1307
- sendGCTombstoneEvent(
1308
- this.mc,
1309
- {
1310
- eventName,
1311
- category: "generic",
1312
- isSummarizerClient: this.isSummarizerClient,
1313
- url: trimLeadingSlashes(toNodePath),
1314
- nodeType,
1315
- },
1316
- undefined /* packagePath */,
1317
- );
1318
- }
1319
- }
1320
-
1321
- /**
1322
- * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
1323
- * cases where objects are used after they are deleted and throw / log errors accordingly.
1324
- */
1325
- public isNodeDeleted(nodePath: string): boolean {
1326
- return this.deletedNodes.has(nodePath);
1327
- }
1328
-
1329
- public dispose(): void {
1330
- this.sessionExpiryTimer?.clear();
1331
- this.sessionExpiryTimer = undefined;
1332
- }
1333
-
1334
- /**
1335
- * Updates the state of the system as per the current GC run. It does the following:
1336
- * 1. Sets up the current GC state as per the gcData.
1337
- * 2. Starts tracking for nodes that have become unreferenced in this run.
1338
- * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
1339
- * @param gcData - The data representing the reference graph on which GC is run.
1340
- * @param gcResult - The result of the GC run on the gcData.
1341
- * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
1342
- */
1343
- private updateCurrentState(
1344
- gcData: IGarbageCollectionData,
1345
- gcResult: IGCResult,
1346
- currentReferenceTimestampMs: number,
1347
- ) {
1348
- this.gcDataFromLastRun = cloneGCData(gcData);
1349
- this.tombstones = [];
1350
- this.newReferencesSinceLastRun.clear();
1351
-
1352
- // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
1353
- for (const nodeId of gcResult.referencedNodeIds) {
1354
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
1355
- if (nodeStateTracker !== undefined) {
1356
- // Stop tracking so as to clear out any running timers.
1357
- nodeStateTracker.stopTracking();
1358
- // Delete the node as we don't need to track it any more.
1359
- this.unreferencedNodesState.delete(nodeId);
1360
- }
1361
- }
1362
-
1363
- /**
1364
- * If a node became unreferenced in this run, start tracking it.
1365
- * If a node was already unreferenced, update its tracking information. Since the current reference time is
1366
- * from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
1367
- */
1368
- for (const nodeId of gcResult.deletedNodeIds) {
1369
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
1370
- if (nodeStateTracker === undefined) {
1371
- this.unreferencedNodesState.set(
1372
- nodeId,
1373
- new UnreferencedStateTracker(
1374
- currentReferenceTimestampMs,
1375
- this.inactiveTimeoutMs,
1376
- currentReferenceTimestampMs,
1377
- this.sweepTimeoutMs,
1378
- ),
1379
- );
1380
- } else {
1381
- nodeStateTracker.updateTracking(currentReferenceTimestampMs);
1382
- if (this.tombstoneMode && nodeStateTracker.state === UnreferencedState.SweepReady) {
1383
- const nodeType = this.runtime.getNodeType(nodeId);
1384
- if (nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) {
1385
- this.tombstones.push(nodeId);
1386
- }
1387
- }
1388
- }
1389
- }
1390
- }
1391
-
1392
- /**
1393
- * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
1394
- * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
1395
- * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
1396
- * these objects while there can be in-memory referenced to it:
1397
- * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
1398
- * added, the object may have been accessed and in-memory reference to it added.
1399
- * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
1400
- * unreferenced, they could have been accessed and in-memory reference to them added.
1401
- *
1402
- * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
1403
- * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
1404
- */
1405
- private updateStateSinceLastRun(currentGCData: IGarbageCollectionData, logger: ITelemetryLogger) {
1406
- // If we haven't run GC before there is nothing to do.
1407
- if (this.gcDataFromLastRun === undefined) {
1408
- return;
1409
- }
1410
-
1411
- // Find any references that haven't been identified correctly.
1412
- const missingExplicitReferences = this.findMissingExplicitReferences(
1413
- currentGCData,
1414
- this.gcDataFromLastRun,
1415
- this.newReferencesSinceLastRun,
1416
- );
1417
-
1418
- if (missingExplicitReferences.length > 0) {
1419
- missingExplicitReferences.forEach((missingExplicitReference) => {
1420
- logger.sendErrorEvent({
1421
- eventName: "gcUnknownOutboundReferences",
1422
- gcNodeId: missingExplicitReference[0],
1423
- gcRoutes: JSON.stringify(missingExplicitReference[1]),
1424
- });
1425
- });
1426
- }
1427
-
1428
- // No references were added since the last run so we don't have to update reference states of any unreferenced
1429
- // nodes
1430
- if (this.newReferencesSinceLastRun.size === 0) {
1431
- return;
1432
- }
1433
-
1434
- /**
1435
- * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
1436
- * edges that have been added since then. To do this, combine the GC data from the last run and the current
1437
- * run, and then add the references since last run.
1438
- *
1439
- * Note on why we need to combine the data from previous run, current run and all references in between -
1440
- * 1. We need data from last run because some of its references may have been deleted since then. If those
1441
- * references added new outbound references before they were deleted, we need to detect them.
1442
- *
1443
- * 2. We need new outbound references since last run because some of them may have been deleted later. If those
1444
- * references added new outbound references before they were deleted, we need to detect them.
1445
- *
1446
- * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
1447
- * - We don't require DDSes handles to be stored in a referenced DDS.
1448
- * - A new data store may have "root" DDSes already created and we don't detect them today.
1449
- */
1450
- const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
1451
- const newOutboundRoutesSinceLastRun: string[] = [];
1452
- this.newReferencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
1453
- if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
1454
- gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
1455
- } else {
1456
- gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
1457
- }
1458
- newOutboundRoutesSinceLastRun.push(...outboundRoutes);
1459
- });
1460
-
1461
- /**
1462
- * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
1463
- * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
1464
- * unreferenced, stop tracking them and remove from unreferenced list.
1465
- * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
1466
- * unreferenced and add unreferenced state.
1467
- */
1468
- const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/", ...newOutboundRoutesSinceLastRun]);
1469
- for (const nodeId of gcResult.referencedNodeIds) {
1470
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
1471
- if (nodeStateTracker !== undefined) {
1472
- // Stop tracking so as to clear out any running timers.
1473
- nodeStateTracker.stopTracking();
1474
- // Delete the unreferenced state as we don't need to track it any more.
1475
- this.unreferencedNodesState.delete(nodeId);
1476
- }
1477
- }
1478
- }
1479
-
1480
- /**
1481
- * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
1482
- * The principle is that every new reference or outbound route must be notified to GC via the
1483
- * addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
1484
- *
1485
- * In more simple terms:
1486
- * Missing Explicit References = Current References - Previous References - Explicitly Added References;
1487
- *
1488
- * @param currentGCData - The GC data (reference graph) from the current GC run.
1489
- * @param previousGCData - The GC data (reference graph) from the previous GC run.
1490
- * @param explicitReferences - New references added explicity between the previous and the current run.
1491
- * @returns - a list of missing explicit references
1492
- */
1493
- private findMissingExplicitReferences(
1494
- currentGCData: IGarbageCollectionData,
1495
- previousGCData: IGarbageCollectionData,
1496
- explicitReferences: Map<string, string[]>,
1497
- ): [string, string[]][] {
1498
- assert(
1499
- previousGCData !== undefined,
1500
- 0x2b7, /* "Can't validate correctness without GC data from last run" */
1501
- );
1502
-
1503
- const currentGraph = Object.entries(currentGCData.gcNodes);
1504
- const missingExplicitReferences: [string, string[]][] = [];
1505
- currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
1506
- const previousRoutes = previousGCData.gcNodes[nodeId] ?? [];
1507
- const explicitRoutes = explicitReferences.get(nodeId) ?? [];
1508
- const missingExplicitRoutes: string[] = [];
1509
-
1510
- /**
1511
- * 1. For routes in the current GC data, routes that were not present in previous GC data and did not have
1512
- * explicit references should be added to missing explicit routes list.
1513
- * 2. Only include data store and blob routes since GC only works for these two.
1514
- * Note: Due to a bug with de-duped blobs, only adding data store routes for now.
1515
- * 3. Ignore DDS routes to their parent datastores since those were added implicitly. So, there won't be
1516
- * explicit routes to them.
1517
- */
1518
- currentOutboundRoutes.forEach((route) => {
1519
- const nodeType = this.runtime.getNodeType(route);
1520
- if ((nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob)
1521
- && !nodeId.startsWith(route)
1522
- && (!previousRoutes.includes(route) && !explicitRoutes.includes(route))) {
1523
- missingExplicitRoutes.push(route);
1524
- }
1525
- });
1526
- if (missingExplicitRoutes.length > 0) {
1527
- missingExplicitReferences.push([nodeId, missingExplicitRoutes]);
1528
- }
1529
- });
1530
-
1531
- // Ideally missingExplicitReferences should always have a size 0
1532
- return missingExplicitReferences;
1533
- }
1534
-
1535
- /**
1536
- * Generates the stats of a garbage collection run from the given results of the run.
1537
- * @param gcResult - The result of a GC run.
1538
- * @returns the GC stats of the GC run.
1539
- */
1540
- private generateStats(gcResult: IGCResult): IGCStats {
1541
- const gcStats: IGCStats = {
1542
- nodeCount: 0,
1543
- dataStoreCount: 0,
1544
- attachmentBlobCount: 0,
1545
- unrefNodeCount: 0,
1546
- unrefDataStoreCount: 0,
1547
- unrefAttachmentBlobCount: 0,
1548
- updatedNodeCount: 0,
1549
- updatedDataStoreCount: 0,
1550
- updatedAttachmentBlobCount: 0,
1551
- };
1552
-
1553
- const updateNodeStats = (nodeId: string, referenced: boolean) => {
1554
- gcStats.nodeCount++;
1555
- // If there is no previous GC data, every node's state is generated and is considered as updated.
1556
- // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
1557
- const stateUpdated = this.gcDataFromLastRun === undefined ||
1558
- this.unreferencedNodesState.has(nodeId) === referenced;
1559
- if (stateUpdated) {
1560
- gcStats.updatedNodeCount++;
1561
- }
1562
- if (!referenced) {
1563
- gcStats.unrefNodeCount++;
1564
- }
1565
-
1566
- if (this.runtime.getNodeType(nodeId) === GCNodeType.DataStore) {
1567
- gcStats.dataStoreCount++;
1568
- if (stateUpdated) {
1569
- gcStats.updatedDataStoreCount++;
1570
- }
1571
- if (!referenced) {
1572
- gcStats.unrefDataStoreCount++;
1573
- }
1574
- }
1575
- if (this.runtime.getNodeType(nodeId) === GCNodeType.Blob) {
1576
- gcStats.attachmentBlobCount++;
1577
- if (stateUpdated) {
1578
- gcStats.updatedAttachmentBlobCount++;
1579
- }
1580
- if (!referenced) {
1581
- gcStats.unrefAttachmentBlobCount++;
1582
- }
1583
- }
1584
- };
1585
-
1586
- for (const nodeId of gcResult.referencedNodeIds) {
1587
- updateNodeStats(nodeId, true /* referenced */);
1588
- }
1589
-
1590
- for (const nodeId of gcResult.deletedNodeIds) {
1591
- updateNodeStats(nodeId, false /* referenced */);
1592
- }
1593
-
1594
- return gcStats;
1595
- }
1596
-
1597
- /**
1598
- * For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
1599
- * this will give us a view into how much deleted content a container has.
1600
- */
1601
- private logSweepEvents(logger: ITelemetryLogger, currentReferenceTimestampMs: number) {
1602
- if (this.mc.config.getBoolean(disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
1603
- return;
1604
- }
1605
-
1606
- this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
1607
- if (nodeStateTracker.state !== UnreferencedState.SweepReady) {
1608
- return;
1609
- }
1610
-
1611
- const nodeType = this.runtime.getNodeType(nodeId);
1612
- if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
1613
- return;
1614
- }
1615
-
1616
- // Log deleted event for each node only once to reduce noise in telemetry.
1617
- const uniqueEventId = `Deleted-${nodeId}`;
1618
- if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
1619
- return;
1620
- }
1621
- this.loggedUnreferencedEvents.add(uniqueEventId);
1622
- logger.sendTelemetryEvent({
1623
- eventName: "GCObjectDeleted",
1624
- id: nodeId,
1625
- type: nodeType,
1626
- age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
1627
- timeout: this.sweepTimeoutMs,
1628
- completedGCRuns: this.completedRuns,
1629
- lastSummaryTime: this.getLastSummaryTimestampMs(),
1630
- });
1631
- });
1632
- }
1633
-
1634
- /**
1635
- * Called when an inactive node is used after. Queue up an event that will be logged next time GC runs.
1636
- */
1637
- private inactiveNodeUsed(
1638
- usageType: "Changed" | "Loaded" | "Revived",
1639
- nodeId: string,
1640
- nodeStateTracker: UnreferencedStateTracker,
1641
- fromNodeId?: string,
1642
- packagePath?: readonly string[],
1643
- currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs(),
1644
- requestHeaders?: IRequestHeader,
1645
- ) {
1646
- // If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
1647
- // logging as nothing interesting would have happened worth logging.
1648
- // If the node is active, skip logging.
1649
- if (currentReferenceTimestampMs === undefined || nodeStateTracker.state === UnreferencedState.Active) {
1650
- return;
1651
- }
1652
-
1653
- // We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
1654
- // as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
1655
- const nodeType = this.runtime.getNodeType(nodeId);
1656
- if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
1657
- return;
1658
- }
1659
-
1660
- const state = nodeStateTracker.state;
1661
- const uniqueEventId = `${state}-${nodeId}-${usageType}`;
1662
- if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
1663
- return;
1664
- }
1665
- this.loggedUnreferencedEvents.add(uniqueEventId);
1666
-
1667
- const propsToLog = {
1668
- id: nodeId,
1669
- type: nodeType,
1670
- unrefTime: nodeStateTracker.unreferencedTimestampMs,
1671
- age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
1672
- timeout: nodeStateTracker.state === UnreferencedState.Inactive
1673
- ? this.inactiveTimeoutMs
1674
- : this.sweepTimeoutMs,
1675
- completedGCRuns: this.completedRuns,
1676
- lastSummaryTime: this.getLastSummaryTimestampMs(),
1677
- ...this.createContainerMetadata,
1678
- externalRequest: requestHeaders?.[RuntimeHeaders.externalRequest],
1679
- viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
1680
- fromId: fromNodeId,
1681
- };
1682
-
1683
- // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
1684
- // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
1685
- // but it's a good signal nonetheless and we can consume it with a grain of salt.
1686
- // Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
1687
- // SweepReady errors are usages of Objects that will be deleted by GC Sweep!
1688
- if (this.isSummarizerClient) {
1689
- this.pendingEventsQueue.push({ ...propsToLog, usageType, state });
1690
- } else {
1691
- // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
1692
- // summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
1693
- // Events generated:
1694
- // InactiveObject_Loaded, SweepReadyObject_Loaded
1695
- if (usageType === "Loaded") {
1696
- const event = {
1697
- ...propsToLog,
1698
- eventName: `${state}Object_${usageType}`,
1699
- pkg: packagePathToTelemetryProperty(packagePath),
1700
- stack: generateStack(),
1701
- };
1702
-
1703
- // Do not log the inactive object x events as error events as they are not the best signal for
1704
- // detecting something wrong with GC either from the partner or from the runtime itself.
1705
- if (state === UnreferencedState.Inactive) {
1706
- this.mc.logger.sendTelemetryEvent(event);
1707
- } else {
1708
- this.mc.logger.sendErrorEvent(event);
1709
- }
1710
- }
1711
-
1712
- // If SweepReady Usage Detection is enabled, the handler may close the interactive container.
1713
- // Once Sweep is fully implemented, this will be removed since the objects will be gone
1714
- // and errors will arise elsewhere in the runtime
1715
- if (state === UnreferencedState.SweepReady) {
1716
- this.sweepReadyUsageHandler.usageDetectedInInteractiveClient({ ...propsToLog, usageType });
1717
- }
1718
- }
1719
- }
1720
-
1721
- private async logUnreferencedEvents(logger: ITelemetryLogger) {
1722
- // Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
1723
- // summary time they are then logged.
1724
- // Events generated:
1725
- // InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
1726
- // SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
1727
- for (const eventProps of this.pendingEventsQueue) {
1728
- const { usageType, state, ...propsToLog } = eventProps;
1729
- /**
1730
- * Revived event is logged only if the node is active. If the node is not active, the reference to it was
1731
- * from another unreferenced node and this scenario is not interesting to log.
1732
- * Loaded and Changed events are logged only if the node is not active. If the node is active, it was
1733
- * revived and a Revived event will be logged for it.
1734
- */
1735
- const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
1736
- const active = nodeStateTracker === undefined || nodeStateTracker.state === UnreferencedState.Active;
1737
- if ((usageType === "Revived") === active) {
1738
- const pkg = await this.getNodePackagePath(eventProps.id);
1739
- const fromPkg = eventProps.fromId ? await this.getNodePackagePath(eventProps.fromId) : undefined;
1740
- const event = {
1741
- ...propsToLog,
1742
- eventName: `${state}Object_${usageType}`,
1743
- pkg: pkg ? { value: pkg.join("/"), tag: TelemetryDataTag.CodeArtifact } : undefined,
1744
- fromPkg: fromPkg ? { value: fromPkg.join("/"), tag: TelemetryDataTag.CodeArtifact } : undefined,
1745
- }
1746
-
1747
- if (state === UnreferencedState.Inactive) {
1748
- logger.sendTelemetryEvent(event);
1749
- } else {
1750
- logger.sendErrorEvent(event);
1751
- }
1752
- }
1753
- }
1754
- this.pendingEventsQueue = [];
1755
- }
361
+ public static create(createParams: IGarbageCollectorCreateParams): IGarbageCollector {
362
+ return new GarbageCollector(createParams);
363
+ }
364
+
365
+ /**
366
+ * Tells whether the GC state needs to be reset in the next summary. We need to do this if:
367
+ *
368
+ * 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
369
+ *
370
+ * 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
371
+ *
372
+ * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
373
+ *
374
+ * 4. The GC version in the latest summary is different from the current GC version. This can happen if:
375
+ *
376
+ * 4.1. The summary this client loaded with has data from a different GC version.
377
+ *
378
+ * 4.2. This client's latest summary was updated from a snapshot that has a different GC version.
379
+ */
380
+ public get summaryStateNeedsReset(): boolean {
381
+ return (
382
+ this.gcStateNeedsReset ||
383
+ (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion)
384
+ );
385
+ }
386
+
387
+ /**
388
+ * Tracks if GC is enabled for this document. This is specified during document creation and doesn't change
389
+ * throughout its lifetime.
390
+ */
391
+ private readonly gcEnabled: boolean;
392
+ /**
393
+ * Tracks if sweep phase is enabled for this document. This is specified during document creation and doesn't change
394
+ * throughout its lifetime.
395
+ */
396
+ private readonly sweepEnabled: boolean;
397
+
398
+ /**
399
+ * Tracks if GC should run or not. Even if GC is enabled for a document (see gcEnabled), it can be explicitly
400
+ * disabled via runtime options or feature flags.
401
+ */
402
+ public readonly shouldRunGC: boolean;
403
+ /**
404
+ * Tracks if sweep phase should run or not. Even if the sweep phase is enabled for a document (see sweepEnabled), it
405
+ * can be explicitly disabled via feature flags. It also won't run if session expiry is not enabled.
406
+ */
407
+ private readonly shouldRunSweep: boolean;
408
+
409
+ public readonly trackGCState: boolean;
410
+
411
+ private readonly testMode: boolean;
412
+ private readonly tombstoneMode: boolean;
413
+ private readonly mc: MonitoringContext;
414
+
415
+ /**
416
+ * Tells whether the GC state needs to be reset. This can happen under 3 conditions:
417
+ *
418
+ * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
419
+ * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
420
+ *
421
+ * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
422
+ * a document and the first time GC is enabled after is was disabled before.
423
+ *
424
+ * 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
425
+ *
426
+ * Note that the state will be reset only once for the first summary generated after this returns true. After that,
427
+ * this will return false.
428
+ */
429
+ private get gcStateNeedsReset(): boolean {
430
+ return this.wasGCRunInLatestSummary !== this.shouldRunGC;
431
+ }
432
+ // Tracks whether there was GC was run in latest summary being tracked.
433
+ private wasGCRunInLatestSummary: boolean;
434
+
435
+ // The current GC version that this container is running.
436
+ private readonly currentGCVersion: GCVersion;
437
+ // This is the version of GC data in the latest summary being tracked.
438
+ private latestSummaryGCVersion: GCVersion;
439
+
440
+ // Feature Support info persisted to this container's summary
441
+ private readonly persistedGcFeatureMatrix: GCFeatureMatrix | undefined;
442
+
443
+ // Keeps track of the GC state from the last run.
444
+ private gcDataFromLastRun: IGarbageCollectionData | undefined;
445
+ // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
446
+ // outbound routes from that node.
447
+ private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
448
+ // A list of nodes that have been tombstoned.
449
+ private tombstones: string[] = [];
450
+ // A list of nodes that have been deleted during sweep phase.
451
+ private deletedNodes: Set<string> = new Set();
452
+
453
+ /**
454
+ * Keeps track of the GC data from the latest summary successfully submitted to and acked from the server.
455
+ */
456
+ private latestSummaryData: IGCSummaryTrackingData | undefined;
457
+ /**
458
+ * Keeps track of the GC data from the last summary submitted to the server but not yet acked.
459
+ */
460
+ private pendingSummaryData: IGCSummaryTrackingData | undefined;
461
+
462
+ // Promise when resolved returns the GC data data in the base snapshot.
463
+ private readonly baseSnapshotDataP: Promise<IGarbageCollectionSnapshotData | undefined>;
464
+ // Promise when resolved initializes the GC state from the data in the base snapshot.
465
+ private readonly initializeGCStateFromBaseSnapshotP: Promise<void>;
466
+ // The GC details generated from the base snapshot.
467
+ private readonly baseGCDetailsP: Promise<IGarbageCollectionDetailsBase>;
468
+ // Map of node ids to their unreferenced state tracker.
469
+ private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();
470
+ // The Timer responsible for closing the container when the session has expired
471
+ private sessionExpiryTimer: Timer | undefined;
472
+
473
+ // Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
474
+ // per event per node.
475
+ private readonly loggedUnreferencedEvents: Set<string> = new Set();
476
+ // Queue for unreferenced events that should be logged the next time GC runs.
477
+ private pendingEventsQueue: IUnreferencedEventProps[] = [];
478
+
479
+ // The number of times GC has successfully completed on this instance of GarbageCollector.
480
+ private completedRuns = 0;
481
+
482
+ private readonly runtime: IGarbageCollectionRuntime;
483
+ private readonly createContainerMetadata: ICreateContainerMetadata;
484
+ private readonly gcOptions: IGCRuntimeOptions;
485
+ private readonly isSummarizerClient: boolean;
486
+
487
+ /** The time in ms to expire a session for a client for gc. */
488
+ private readonly sessionExpiryTimeoutMs: number | undefined;
489
+ /** The time after which an unreferenced node is inactive. */
490
+ private readonly inactiveTimeoutMs: number;
491
+ /** The time after which an unreferenced node is ready to be swept. */
492
+ private readonly sweepTimeoutMs: number | undefined;
493
+
494
+ /** For a given node path, returns the node's package path. */
495
+ private readonly getNodePackagePath: (
496
+ nodePath: string,
497
+ ) => Promise<readonly string[] | undefined>;
498
+ /** Returns the timestamp of the last summary generated for this container. */
499
+ private readonly getLastSummaryTimestampMs: () => number | undefined;
500
+ /** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
501
+ private readonly activeConnection: () => boolean;
502
+
503
+ /** Returns a list of all the configurations for garbage collection. */
504
+ private get configs() {
505
+ return {
506
+ gcEnabled: this.gcEnabled,
507
+ sweepEnabled: this.sweepEnabled,
508
+ runGC: this.shouldRunGC,
509
+ runSweep: this.shouldRunSweep,
510
+ testMode: this.testMode,
511
+ tombstoneMode: this.tombstoneMode,
512
+ sessionExpiry: this.sessionExpiryTimeoutMs,
513
+ sweepTimeout: this.sweepTimeoutMs,
514
+ inactiveTimeout: this.inactiveTimeoutMs,
515
+ trackGCState: this.trackGCState,
516
+ ...this.gcOptions,
517
+ };
518
+ }
519
+
520
+ /** Handler to respond to when a SweepReady object is used */
521
+ private readonly sweepReadyUsageHandler: SweepReadyUsageDetectionHandler;
522
+
523
+ protected constructor(createParams: IGarbageCollectorCreateParams) {
524
+ this.runtime = createParams.runtime;
525
+ this.isSummarizerClient = createParams.isSummarizerClient;
526
+ this.gcOptions = createParams.gcOptions;
527
+ this.createContainerMetadata = createParams.createContainerMetadata;
528
+ this.getNodePackagePath = createParams.getNodePackagePath;
529
+ this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
530
+ this.activeConnection = createParams.activeConnection;
531
+
532
+ const baseSnapshot = createParams.baseSnapshot;
533
+ const metadata = createParams.metadata;
534
+ const readAndParseBlob = createParams.readAndParseBlob;
535
+
536
+ this.mc = loggerToMonitoringContext(
537
+ ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
538
+ all: { completedGCRuns: () => this.completedRuns },
539
+ }),
540
+ );
541
+
542
+ // If version upgrade is not enabled, fall back to the stable GC version.
543
+ this.currentGCVersion =
544
+ this.mc.config.getBoolean(gcVersionUpgradeToV2Key) === true
545
+ ? currentGCVersion
546
+ : stableGCVersion;
547
+
548
+ this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(
549
+ createParams.getContainerDiagnosticId(),
550
+ this.mc,
551
+ this.runtime.closeFn,
552
+ );
553
+
554
+ let prevSummaryGCVersion: number | undefined;
555
+
556
+ /**
557
+ * Sweep timeout is the time after which unreferenced content can be swept.
558
+ * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
559
+ *
560
+ * The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
561
+ * The buffer is added to account for any clock skew or other edge cases.
562
+ * We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
563
+ */
564
+ function computeSweepTimeout(sessionExpiryTimeoutMs: number | undefined) {
565
+ const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
566
+ const bufferMs = oneDayMs;
567
+ return (
568
+ sessionExpiryTimeoutMs &&
569
+ sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs
570
+ );
571
+ }
572
+
573
+ /**
574
+ * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
575
+ * 1. Whether running GC mark phase is allowed or not.
576
+ * 2. Whether running GC sweep phase is allowed or not.
577
+ * 3. Whether GC session expiry is enabled or not.
578
+ * For existing containers, we get this information from the metadata blob of its summary.
579
+ */
580
+ if (createParams.existing) {
581
+ prevSummaryGCVersion = getGCVersion(metadata);
582
+ // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
583
+ // other existing documents, GC is enabled.
584
+ this.gcEnabled = prevSummaryGCVersion > 0;
585
+ this.sweepEnabled = metadata?.sweepEnabled ?? false;
586
+ this.sessionExpiryTimeoutMs = metadata?.sessionExpiryTimeoutMs;
587
+ this.sweepTimeoutMs =
588
+ metadata?.sweepTimeoutMs ?? computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
589
+ this.persistedGcFeatureMatrix = metadata?.gcFeatureMatrix;
590
+ } else {
591
+ // Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
592
+ // scenario but explicitly failing makes it clearer and promotes correct usage.
593
+ if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
594
+ throw new UsageError(
595
+ "GC sweep phase cannot be enabled without enabling GC mark phase",
596
+ );
597
+ }
598
+
599
+ // This Test Override only applies for new containers
600
+ const testOverrideSweepTimeoutMs = this.mc.config.getNumber(
601
+ "Fluid.GarbageCollection.TestOverride.SweepTimeoutMs",
602
+ );
603
+
604
+ // For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
605
+ // flag in GC options to false.
606
+ this.gcEnabled = this.gcOptions.gcAllowed !== false;
607
+ // The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
608
+ this.sweepEnabled = this.gcOptions.sweepAllowed === true;
609
+
610
+ // Set the Session Expiry only if the flag is enabled and GC is enabled.
611
+ if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
612
+ this.sessionExpiryTimeoutMs =
613
+ this.gcOptions.sessionExpiryTimeoutMs ?? defaultSessionExpiryDurationMs;
614
+ }
615
+ this.sweepTimeoutMs =
616
+ testOverrideSweepTimeoutMs ?? computeSweepTimeout(this.sessionExpiryTimeoutMs);
617
+ if (this.gcOptions[gcTombstoneGenerationOptionName] !== undefined) {
618
+ this.persistedGcFeatureMatrix = {
619
+ tombstoneGeneration: this.gcOptions[gcTombstoneGenerationOptionName],
620
+ };
621
+ }
622
+ }
623
+
624
+ // If session expiry is enabled, we need to close the container when the session expiry timeout expires.
625
+ if (this.sessionExpiryTimeoutMs !== undefined) {
626
+ // If Test Override config is set, override Session Expiry timeout.
627
+ const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber(
628
+ "Fluid.GarbageCollection.TestOverride.SessionExpiryMs",
629
+ );
630
+ const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.sessionExpiryTimeoutMs;
631
+
632
+ this.sessionExpiryTimer = new Timer(timeoutMs, () => {
633
+ this.runtime.closeFn(
634
+ new ClientSessionExpiredError(`Client session expired.`, timeoutMs),
635
+ );
636
+ });
637
+ this.sessionExpiryTimer.start();
638
+ }
639
+
640
+ // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
641
+ // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
642
+ this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;
643
+
644
+ /**
645
+ * Whether GC should run or not. The following conditions have to be met to run sweep:
646
+ *
647
+ * 1. GC should be enabled for this container.
648
+ *
649
+ * 2. GC should not be disabled via disableGC GC option.
650
+ *
651
+ * These conditions can be overridden via runGCKey feature flag.
652
+ */
653
+ this.shouldRunGC =
654
+ this.mc.config.getBoolean(runGCKey) ??
655
+ // GC must be enabled for the document.
656
+ (this.gcEnabled &&
657
+ // GC must not be disabled via GC options.
658
+ !this.gcOptions.disableGC);
659
+
660
+ /**
661
+ * Whether sweep should run or not. The following conditions have to be met to run sweep:
662
+ *
663
+ * 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
664
+ * 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
665
+ * 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
666
+ * the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
667
+ * 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
668
+ * feature flag.
669
+ */
670
+ this.shouldRunSweep =
671
+ this.shouldRunGC &&
672
+ this.sweepTimeoutMs !== undefined &&
673
+ (this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
674
+
675
+ this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
676
+
677
+ // Override inactive timeout if test config or gc options to override it is set.
678
+ this.inactiveTimeoutMs =
679
+ this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs") ??
680
+ this.gcOptions.inactiveTimeoutMs ??
681
+ defaultInactiveTimeoutMs;
682
+
683
+ // Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
684
+ if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
685
+ throw new UsageError("inactive timeout should not be greater than the sweep timeout");
686
+ }
687
+
688
+ // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
689
+ this.testMode =
690
+ this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
691
+ // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
692
+ // via feature flags.
693
+ this.tombstoneMode =
694
+ !this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
695
+
696
+ // If GC ran in the container that generated the base snapshot, it will have a GC tree.
697
+ this.wasGCRunInLatestSummary = baseSnapshot?.trees[gcTreeKey] !== undefined;
698
+
699
+ // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
700
+ // it involves fetching blobs from storage which is expensive.
701
+ this.baseSnapshotDataP = new LazyPromise<IGarbageCollectionSnapshotData | undefined>(
702
+ async () => {
703
+ if (baseSnapshot === undefined) {
704
+ return undefined;
705
+ }
706
+
707
+ try {
708
+ // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
709
+ const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
710
+ if (gcSnapshotTree !== undefined) {
711
+ return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
712
+ }
713
+
714
+ // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
715
+ // consolidate into IGarbageCollectionState format.
716
+ // Add a node for the root node that is not present in older snapshot format.
717
+ const gcState: IGarbageCollectionState = {
718
+ gcNodes: { "/": { outboundRoutes: [] } },
719
+ };
720
+ const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
721
+ assert(
722
+ dataStoreSnapshotTree !== undefined,
723
+ 0x2a8 /* "Expected data store snapshot tree in base snapshot" */,
724
+ );
725
+ for (const [dsId, dsSnapshotTree] of Object.entries(
726
+ dataStoreSnapshotTree.trees,
727
+ )) {
728
+ const blobId = dsSnapshotTree.blobs[gcTreeKey];
729
+ if (blobId === undefined) {
730
+ continue;
731
+ }
732
+
733
+ const gcSummaryDetails =
734
+ await readAndParseBlob<IGarbageCollectionSummaryDetailsLegacy>(blobId);
735
+ // If there are no nodes for this data store, skip it.
736
+ if (gcSummaryDetails.gcData?.gcNodes === undefined) {
737
+ continue;
738
+ }
739
+
740
+ const dsRootId = `/${dsId}`;
741
+ // Since we used to write GC data at data store level, we won't have an entry for the root ("/").
742
+ // Construct that entry by adding root data store ids to its outbound routes.
743
+ const initialSnapshotDetails =
744
+ await readAndParseBlob<ReadFluidDataStoreAttributes>(
745
+ dsSnapshotTree.blobs[dataStoreAttributesBlobName],
746
+ );
747
+ if (initialSnapshotDetails.isRootDataStore) {
748
+ gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
749
+ }
750
+
751
+ for (const [id, outboundRoutes] of Object.entries(
752
+ gcSummaryDetails.gcData.gcNodes,
753
+ )) {
754
+ // Prefix the data store id to the GC node ids to make them relative to the root from being
755
+ // relative to the data store. Similar to how its done in DataStore::getGCData.
756
+ const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
757
+ gcState.gcNodes[rootId] = {
758
+ outboundRoutes: Array.from(outboundRoutes),
759
+ };
760
+ }
761
+ assert(
762
+ gcState.gcNodes[dsRootId] !== undefined,
763
+ 0x2a9 /* GC nodes for data store not in GC blob */,
764
+ );
765
+ gcState.gcNodes[dsRootId].unreferencedTimestampMs =
766
+ gcSummaryDetails.unrefTimestamp;
767
+ }
768
+ // If there is only one node (root node just added above), either GC is disabled or we are loading from
769
+ // the first summary generated by detached container. In both cases, GC was not run - return undefined.
770
+ return Object.keys(gcState.gcNodes).length === 1
771
+ ? undefined
772
+ : { gcState, tombstones: undefined, deletedNodes: undefined };
773
+ } catch (error) {
774
+ const dpe = DataProcessingError.wrapIfUnrecognized(
775
+ error,
776
+ "FailedToInitializeGC",
777
+ );
778
+ dpe.addTelemetryProperties({ gcConfigs: JSON.stringify(this.configs) });
779
+ throw dpe;
780
+ }
781
+ },
782
+ );
783
+
784
+ /**
785
+ * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
786
+ * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
787
+ * GC state and updates their inactive or sweep ready state.
788
+ */
789
+ this.initializeGCStateFromBaseSnapshotP = new LazyPromise<void>(async () => {
790
+ /**
791
+ * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
792
+ * how long objects have been unreferenced and if they can be deleted.
793
+ *
794
+ * Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
795
+ * for this container and it is in read mode. In this scenario, there is no point in running GC anyway
796
+ * because references in the container do not change without any ops, i.e., there is nothing to collect.
797
+ */
798
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
799
+ if (currentReferenceTimestampMs === undefined) {
800
+ // Log an event so we can evaluate how often we run into this scenario.
801
+ this.mc.logger.sendErrorEvent({
802
+ eventName: "GarbageCollectorInitializedWithoutTimestamp",
803
+ gcConfigs: JSON.stringify(this.configs),
804
+ });
805
+ return;
806
+ }
807
+ /**
808
+ * The base snapshot data will not be present if the container is loaded from:
809
+ * 1. The first summary created by the detached container.
810
+ * 2. A summary that was generated with GC disabled.
811
+ * 3. A summary that was generated before GC even existed.
812
+ */
813
+ const baseSnapshotData = await this.baseSnapshotDataP;
814
+ if (baseSnapshotData === undefined) {
815
+ return;
816
+ }
817
+ this.updateStateFromSnapshotData(baseSnapshotData, currentReferenceTimestampMs);
818
+ });
819
+
820
+ // Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
821
+ // used to initialize the GC state of all the nodes in the container.
822
+ this.baseGCDetailsP = new LazyPromise<IGarbageCollectionDetailsBase>(async () => {
823
+ const baseSnapshotData = await this.baseSnapshotDataP;
824
+ if (baseSnapshotData === undefined) {
825
+ return {};
826
+ }
827
+
828
+ const gcNodes: { [id: string]: string[] } = {};
829
+ for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
830
+ gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
831
+ }
832
+ // Run GC on the nodes in the base summary to get the routes used in each node in the container.
833
+ // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
834
+ // each node in the summary.
835
+ const usedRoutes = runGarbageCollection(gcNodes, ["/"]).referencedNodeIds;
836
+
837
+ return { gcData: { gcNodes }, usedRoutes };
838
+ });
839
+
840
+ // Log all the GC options and the state determined by the garbage collector. This is interesting only for the
841
+ // summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
842
+ if (this.isSummarizerClient) {
843
+ this.mc.logger.sendTelemetryEvent({
844
+ eventName: "GarbageCollectorLoaded",
845
+ gcConfigs: JSON.stringify(this.configs),
846
+ });
847
+ }
848
+ }
849
+
850
+ /**
851
+ * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
852
+ * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
853
+ */
854
+ public async initializeBaseState(): Promise<void> {
855
+ const baseSnapshotData = await this.baseSnapshotDataP;
856
+ /**
857
+ * The base snapshot data will not be present if the container is loaded from:
858
+ * 1. The first summary created by the detached container.
859
+ * 2. A summary that was generated with GC disabled.
860
+ * 3. A summary that was generated before GC even existed.
861
+ */
862
+ if (baseSnapshotData === undefined) {
863
+ return;
864
+ }
865
+
866
+ // Initialize the deleted nodes from the snapshot. This is done irrespective of whether sweep is enabled or not
867
+ // to identify deleted nodes' usage.
868
+ if (baseSnapshotData.deletedNodes !== undefined) {
869
+ this.deletedNodes = new Set(baseSnapshotData.deletedNodes);
870
+ }
871
+
872
+ // If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
873
+ // tombstone routes.
874
+ if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
875
+ // Create a copy since we are writing from a source we don't control
876
+ this.tombstones = Array.from(baseSnapshotData.tombstones);
877
+ this.runtime.updateTombstonedRoutes(this.tombstones);
878
+ }
879
+ }
880
+
881
+ /**
882
+ * Update state from the given snapshot data. This is done during load and during refreshing state from a snapshot.
883
+ * All current tracking is reset and updated from the data in the snapshot.
884
+ * @param snapshotData - The snapshot data to update state from. If this is undefined, all GC state and tracking
885
+ * is reset.
886
+ * @param currentReferenceTimestampMs - The current reference timestamp for marking unreferenced nodes' unreferenced
887
+ * timestamp.
888
+ */
889
+ private updateStateFromSnapshotData(
890
+ snapshotData: IGarbageCollectionSnapshotData | undefined,
891
+ currentReferenceTimestampMs: number,
892
+ ) {
893
+ /**
894
+ * Note: "newReferencesSinceLastRun" is not reset here. This is done because there may be references since the
895
+ * snapshot that we are updating state from. For example, this client may have processed ops till seq#1000 and
896
+ * its refreshing state from a summary that happened at seq#900. In this case, there may be references between
897
+ * seq#901 and seq#1000 that we don't want to reset.
898
+ * Unfortunately, there is no way to track the seq# of ops that add references, so we choose to not reset any
899
+ * references here. This should be fine because, in the worst case, we may end up updating the unreferenced
900
+ * timestamp of a node which will delay its deletion. Although not ideal, this will only happen in rare
901
+ * scenarios, so it should be okay.
902
+ */
903
+
904
+ // Clear all existing unreferenced state tracking.
905
+ for (const [, nodeStateTracker] of this.unreferencedNodesState) {
906
+ nodeStateTracker.stopTracking();
907
+ }
908
+ this.unreferencedNodesState.clear();
909
+
910
+ // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
911
+ // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
912
+ // tombstones.
913
+ // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
914
+ // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
915
+ if (this.shouldRunSweep) {
916
+ const snapshotDeletedNodes = snapshotData?.deletedNodes
917
+ ? new Set(snapshotData.deletedNodes)
918
+ : undefined;
919
+ // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
920
+ // delete them.
921
+ if (snapshotDeletedNodes !== undefined) {
922
+ const newDeletedNodes: string[] = [];
923
+ for (const nodeId of snapshotDeletedNodes) {
924
+ if (!this.deletedNodes.has(nodeId)) {
925
+ newDeletedNodes.push(nodeId);
926
+ }
927
+ }
928
+ if (newDeletedNodes.length > 0) {
929
+ // Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
930
+ }
931
+ }
932
+ } else if (this.tombstoneMode) {
933
+ // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
934
+ // notify the runtime to update its state as well.
935
+ this.tombstones = snapshotData?.tombstones ? Array.from(snapshotData.tombstones) : [];
936
+ this.runtime.updateTombstonedRoutes(this.tombstones);
937
+ }
938
+
939
+ // If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
940
+ if (snapshotData === undefined) {
941
+ this.gcDataFromLastRun = undefined;
942
+ this.latestSummaryData = undefined;
943
+ return;
944
+ }
945
+
946
+ // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
947
+ // to the GC data from the snapshot data.
948
+ const gcNodes: { [id: string]: string[] } = {};
949
+ for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
950
+ if (nodeData.unreferencedTimestampMs !== undefined) {
951
+ this.unreferencedNodesState.set(
952
+ nodeId,
953
+ new UnreferencedStateTracker(
954
+ nodeData.unreferencedTimestampMs,
955
+ this.inactiveTimeoutMs,
956
+ currentReferenceTimestampMs,
957
+ this.sweepTimeoutMs,
958
+ ),
959
+ );
960
+ }
961
+ gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
962
+ }
963
+ this.gcDataFromLastRun = { gcNodes };
964
+
965
+ // If tracking state across summaries, update latest summary data from the snapshot's GC data.
966
+ if (this.trackGCState) {
967
+ this.latestSummaryData = {
968
+ serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
969
+ serializedTombstones: JSON.stringify(snapshotData.tombstones),
970
+ serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
971
+ };
972
+ }
973
+ }
974
+
975
+ /**
976
+ * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
977
+ * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
978
+ * @param connected - Whether the runtime connected / disconnected.
979
+ * @param clientId - The clientId of this runtime.
980
+ */
981
+ public setConnectionState(connected: boolean, clientId?: string | undefined): void {
982
+ /**
983
+ * For all clients, initialize the base state when the container becomes active, i.e., it transitions
984
+ * to "write" mode. This will ensure that the container's own join op is processed and there is a recent
985
+ * reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
986
+ * could affect the GC state will have been processed.
987
+ *
988
+ * If GC is up-to-date for the client and the summarizing client, there will be an doubling of both
989
+ * InactiveObject_Loaded and SweepReady_Loaded errors, as there will be one from the sending client and one from
990
+ * the receiving summarizer client.
991
+ *
992
+ * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
993
+ * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
994
+ */
995
+ if (this.activeConnection() && this.shouldRunGC) {
996
+ this.initializeGCStateFromBaseSnapshotP.catch((error) => {});
997
+ }
998
+ }
999
+
1000
+ /**
1001
+ * Runs garbage collection and updates the reference / used state of the nodes in the container.
1002
+ * @returns stats of the GC run or undefined if GC did not run.
1003
+ */
1004
+ public async collectGarbage(options: {
1005
+ /** Logger to use for logging GC events */
1006
+ logger?: ITelemetryLogger;
1007
+ /** True to run GC sweep phase after the mark phase */
1008
+ runSweep?: boolean;
1009
+ /** True to generate full GC data */
1010
+ fullGC?: boolean;
1011
+ }): Promise<IGCStats | undefined> {
1012
+ const fullGC =
1013
+ options.fullGC ?? (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
1014
+ const logger = options.logger
1015
+ ? ChildLogger.create(options.logger, undefined, {
1016
+ all: { completedGCRuns: () => this.completedRuns },
1017
+ })
1018
+ : this.mc.logger;
1019
+
1020
+ /**
1021
+ * If there is no current reference timestamp, skip running GC. We need the current timestamp to track
1022
+ * how long objects have been unreferenced and if they should be deleted.
1023
+ *
1024
+ * Note that the only scenario where GC is called and there is no reference timestamp is when no ops have ever
1025
+ * been processed for this container and it is in read mode. In this scenario, there is no point in running GC
1026
+ * anyway because references in the container do not change without any ops, i.e., there is nothing to collect.
1027
+ */
1028
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
1029
+ if (currentReferenceTimestampMs === undefined) {
1030
+ // Log an event so we can evaluate how often we run into this scenario.
1031
+ logger.sendErrorEvent({
1032
+ eventName: "CollectGarbageCalledWithoutTimestamp",
1033
+ gcConfigs: JSON.stringify(this.configs),
1034
+ });
1035
+ return undefined;
1036
+ }
1037
+
1038
+ return PerformanceEvent.timedExecAsync(
1039
+ logger,
1040
+ { eventName: "GarbageCollection" },
1041
+ async (event) => {
1042
+ await this.runPreGCSteps();
1043
+
1044
+ // Get the runtime's GC data and run GC on the reference graph in it.
1045
+ const gcData = await this.runtime.getGCData(fullGC);
1046
+ const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
1047
+
1048
+ const gcStats = await this.runPostGCSteps(
1049
+ gcData,
1050
+ gcResult,
1051
+ logger,
1052
+ currentReferenceTimestampMs,
1053
+ );
1054
+ event.end({ ...gcStats, timestamp: currentReferenceTimestampMs });
1055
+ this.completedRuns++;
1056
+ return gcStats;
1057
+ },
1058
+ { end: true, cancel: "error" },
1059
+ );
1060
+ }
1061
+
1062
+ private async runPreGCSteps() {
1063
+ // Ensure that state has been initialized from the base snapshot data.
1064
+ await this.initializeGCStateFromBaseSnapshotP;
1065
+ // Let the runtime update its pending state before GC runs.
1066
+ await this.runtime.updateStateBeforeGC();
1067
+ }
1068
+
1069
+ private async runPostGCSteps(
1070
+ gcData: IGarbageCollectionData,
1071
+ gcResult: IGCResult,
1072
+ logger: ITelemetryLogger,
1073
+ currentReferenceTimestampMs: number,
1074
+ ): Promise<IGCStats> {
1075
+ // Generate statistics from the current run. This is done before updating the current state because it
1076
+ // generates some of its data based on previous state of the system.
1077
+ const gcStats = this.generateStats(gcResult);
1078
+
1079
+ // Update the current mark state and update the runtime of all used routes or ids that used as per the GC run.
1080
+ const sweepReadyNodes = this.updateMarkPhase(
1081
+ gcData,
1082
+ gcResult,
1083
+ currentReferenceTimestampMs,
1084
+ logger,
1085
+ );
1086
+ this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
1087
+
1088
+ // Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
1089
+ // delete these objects here instead.
1090
+ this.logSweepEvents(logger, currentReferenceTimestampMs);
1091
+
1092
+ let updatedGCData: IGarbageCollectionData = gcData;
1093
+
1094
+ if (this.shouldRunSweep) {
1095
+ updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
1096
+ } else if (this.testMode) {
1097
+ // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
1098
+ // involving access to deleted data.
1099
+ this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
1100
+ } else if (this.tombstoneMode) {
1101
+ this.tombstones = sweepReadyNodes;
1102
+ // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
1103
+ // involving access to "deleted" data without actually deleting the data from summaries.
1104
+ // Note: we will not tombstone in test mode.
1105
+ this.runtime.updateTombstonedRoutes(this.tombstones);
1106
+ }
1107
+
1108
+ this.gcDataFromLastRun = cloneGCData(updatedGCData);
1109
+
1110
+ // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
1111
+ // updates its state so that we don't send false positives based on intermediate state. For example, we may get
1112
+ // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
1113
+ await this.logUnreferencedEvents(logger);
1114
+
1115
+ return gcStats;
1116
+ }
1117
+
1118
+ /**
1119
+ * Summarizes the GC data and returns it as a summary tree.
1120
+ * We current write the entire GC state in a single blob. This can be modified later to write multiple
1121
+ * blobs. All the blob keys should start with `gcBlobPrefix`.
1122
+ */
1123
+ public summarize(
1124
+ fullTree: boolean,
1125
+ trackState: boolean,
1126
+ telemetryContext?: ITelemetryContext,
1127
+ ): ISummarizeResult | undefined {
1128
+ if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
1129
+ return;
1130
+ }
1131
+
1132
+ const gcState: IGarbageCollectionState = { gcNodes: {} };
1133
+ for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
1134
+ gcState.gcNodes[nodeId] = {
1135
+ outboundRoutes,
1136
+ unreferencedTimestampMs:
1137
+ this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
1138
+ };
1139
+ }
1140
+
1141
+ const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
1142
+ // Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
1143
+ // to identify deleted nodes' usage.
1144
+ const serializedDeletedNodes =
1145
+ this.deletedNodes.size > 0
1146
+ ? JSON.stringify(Array.from(this.deletedNodes).sort())
1147
+ : undefined;
1148
+ // If running in tombstone mode, serialize and write tombstones, if any.
1149
+ const serializedTombstones = this.tombstoneMode
1150
+ ? this.tombstones.length > 0
1151
+ ? JSON.stringify(this.tombstones.sort())
1152
+ : undefined
1153
+ : undefined;
1154
+
1155
+ /**
1156
+ * Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
1157
+ * write summary handle instead of summary tree for GC.
1158
+ * Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
1159
+ * for each of these that did not change, write a summary handle.
1160
+ */
1161
+ if (this.trackGCState) {
1162
+ this.pendingSummaryData = {
1163
+ serializedGCState,
1164
+ serializedTombstones,
1165
+ serializedDeletedNodes,
1166
+ };
1167
+ if (trackState && !fullTree && this.latestSummaryData !== undefined) {
1168
+ // If nothing changed since last summary, send a summary handle for the entire GC data.
1169
+ if (
1170
+ this.latestSummaryData.serializedGCState === serializedGCState &&
1171
+ this.latestSummaryData.serializedTombstones === serializedTombstones
1172
+ ) {
1173
+ const stats = mergeStats();
1174
+ stats.handleNodeCount++;
1175
+ return {
1176
+ summary: {
1177
+ type: SummaryType.Handle,
1178
+ handle: `/${gcTreeKey}`,
1179
+ handleType: SummaryType.Tree,
1180
+ },
1181
+ stats,
1182
+ };
1183
+ }
1184
+
1185
+ // If some state changed, build a GC summary tree.
1186
+ return this.buildGCSummaryTree(
1187
+ serializedGCState,
1188
+ serializedTombstones,
1189
+ serializedDeletedNodes,
1190
+ true /* trackState */,
1191
+ );
1192
+ }
1193
+ }
1194
+ // If not tracking GC state, build a GC summary tree without any summary handles.
1195
+ return this.buildGCSummaryTree(
1196
+ serializedGCState,
1197
+ serializedTombstones,
1198
+ serializedDeletedNodes,
1199
+ false /* trackState */,
1200
+ );
1201
+ }
1202
+
1203
+ /**
1204
+ * Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
1205
+ * If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
1206
+ * If trackState is true, only states that changed are written. Rest are written as handles.
1207
+ * @param serializedGCState - The GC state serialized as string.
1208
+ * @param serializedTombstones - The tombstone state serialized as string.
1209
+ * @param serializedDeletedNodes - Deleted nodes serialized as string.
1210
+ * @param trackState - Whether we are tracking GC state across summaries.
1211
+ * @returns the GC summary tree.
1212
+ */
1213
+ private buildGCSummaryTree(
1214
+ serializedGCState: string,
1215
+ serializedTombstones: string | undefined,
1216
+ serializedDeletedNodes: string | undefined,
1217
+ trackState: boolean,
1218
+ ): ISummaryTreeWithStats {
1219
+ const gcStateBlobKey = `${gcBlobPrefix}_root`;
1220
+ const builder = new SummaryTreeBuilder();
1221
+
1222
+ // If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
1223
+ if (this.latestSummaryData?.serializedGCState === serializedGCState && trackState) {
1224
+ builder.addHandle(gcStateBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcStateBlobKey}`);
1225
+ } else {
1226
+ builder.addBlob(gcStateBlobKey, serializedGCState);
1227
+ }
1228
+
1229
+ // If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
1230
+ // summary blob.
1231
+ if (serializedTombstones !== undefined) {
1232
+ if (
1233
+ this.latestSummaryData?.serializedTombstones === serializedTombstones &&
1234
+ trackState
1235
+ ) {
1236
+ builder.addHandle(
1237
+ gcTombstoneBlobKey,
1238
+ SummaryType.Blob,
1239
+ `/${gcTreeKey}/${gcTombstoneBlobKey}`,
1240
+ );
1241
+ } else {
1242
+ builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
1243
+ }
1244
+ }
1245
+
1246
+ // If there are no deleted nodes, return the summary tree.
1247
+ if (serializedDeletedNodes === undefined) {
1248
+ return builder.getSummaryTree();
1249
+ }
1250
+
1251
+ // If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
1252
+ if (
1253
+ this.latestSummaryData?.serializedDeletedNodes === serializedDeletedNodes &&
1254
+ trackState
1255
+ ) {
1256
+ builder.addHandle(
1257
+ gcDeletedBlobKey,
1258
+ SummaryType.Blob,
1259
+ `/${gcTreeKey}/${gcDeletedBlobKey}`,
1260
+ );
1261
+ } else {
1262
+ builder.addBlob(gcDeletedBlobKey, serializedDeletedNodes);
1263
+ }
1264
+ return builder.getSummaryTree();
1265
+ }
1266
+
1267
+ public getMetadata(): IGCMetadata {
1268
+ return {
1269
+ /**
1270
+ * If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
1271
+ * into the metadata blob. If GC is disabled, the gcFeature is 0.
1272
+ */
1273
+ gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
1274
+ gcFeatureMatrix: this.persistedGcFeatureMatrix,
1275
+ sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
1276
+ sweepEnabled: this.sweepEnabled,
1277
+ sweepTimeoutMs: this.sweepTimeoutMs,
1278
+ };
1279
+ }
1280
+
1281
+ /**
1282
+ * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
1283
+ * in the container.
1284
+ */
1285
+ public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
1286
+ return this.baseGCDetailsP;
1287
+ }
1288
+
1289
+ /**
1290
+ * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
1291
+ * is downloaded and should be used to update the state.
1292
+ */
1293
+ public async refreshLatestSummary(
1294
+ proposalHandle: string | undefined,
1295
+ result: RefreshSummaryResult,
1296
+ readAndParseBlob: ReadAndParseBlob,
1297
+ ): Promise<void> {
1298
+ // If the latest summary was updated and the summary was tracked, this client is the one that generated this
1299
+ // summary. So, update wasGCRunInLatestSummary.
1300
+ // Note that this has to be updated if GC did not run too. Otherwise, `gcStateNeedsReset` will always return
1301
+ // true in scenarios where GC is disabled but enabled in the snapshot we loaded from.
1302
+ if (result.latestSummaryUpdated && result.wasSummaryTracked) {
1303
+ this.wasGCRunInLatestSummary = this.shouldRunGC;
1304
+ }
1305
+
1306
+ if (!result.latestSummaryUpdated || !this.shouldRunGC) {
1307
+ return;
1308
+ }
1309
+
1310
+ // If the summary was tracked by this client, it was the one that generated the summary in the first place.
1311
+ // Update latest state from pending.
1312
+ if (result.wasSummaryTracked) {
1313
+ this.latestSummaryGCVersion = this.currentGCVersion;
1314
+ if (this.trackGCState) {
1315
+ this.latestSummaryData = this.pendingSummaryData;
1316
+ this.pendingSummaryData = undefined;
1317
+ }
1318
+ return;
1319
+ }
1320
+
1321
+ // If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
1322
+ const snapshotTree = result.snapshotTree;
1323
+ const metadataBlobId = snapshotTree.blobs[metadataBlobName];
1324
+ if (metadataBlobId) {
1325
+ const metadata = await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId);
1326
+ this.latestSummaryGCVersion = getGCVersion(metadata);
1327
+ }
1328
+
1329
+ // The current reference timestamp should be available if we are refreshing state from a snapshot. There has
1330
+ // to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
1331
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
1332
+ if (currentReferenceTimestampMs === undefined) {
1333
+ throw DataProcessingError.create(
1334
+ "No reference timestamp when updating GC state from snapshot",
1335
+ "refreshLatestSummary",
1336
+ undefined,
1337
+ {
1338
+ proposalHandle,
1339
+ summaryRefSeq: result.summaryRefSeq,
1340
+ details: JSON.stringify(this.configs),
1341
+ },
1342
+ );
1343
+ }
1344
+ const gcSnapshotTree = snapshotTree.trees[gcTreeKey];
1345
+ // If GC ran in the container that generated this snapshot, it will have a GC tree.
1346
+ this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
1347
+ let latestGCData: IGarbageCollectionSnapshotData | undefined;
1348
+ if (gcSnapshotTree !== undefined) {
1349
+ latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
1350
+ }
1351
+ this.updateStateFromSnapshotData(latestGCData, currentReferenceTimestampMs);
1352
+ this.pendingSummaryData = undefined;
1353
+ }
1354
+
1355
+ /**
1356
+ * Called when a node with the given id is updated. If the node is inactive, log an error.
1357
+ * @param nodePath - The id of the node that changed.
1358
+ * @param reason - Whether the node was loaded or changed.
1359
+ * @param timestampMs - The timestamp when the node changed.
1360
+ * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
1361
+ * @param requestHeaders - If the node was loaded via request path, the headers in the request.
1362
+ */
1363
+ public nodeUpdated(
1364
+ nodePath: string,
1365
+ reason: "Loaded" | "Changed",
1366
+ timestampMs?: number,
1367
+ packagePath?: readonly string[],
1368
+ requestHeaders?: IRequestHeader,
1369
+ ) {
1370
+ if (!this.shouldRunGC) {
1371
+ return;
1372
+ }
1373
+
1374
+ const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
1375
+ if (nodeStateTracker && nodeStateTracker.state !== UnreferencedState.Active) {
1376
+ this.inactiveNodeUsed(
1377
+ reason,
1378
+ nodePath,
1379
+ nodeStateTracker,
1380
+ undefined /* fromNodeId */,
1381
+ packagePath,
1382
+ timestampMs,
1383
+ requestHeaders,
1384
+ );
1385
+ }
1386
+ }
1387
+
1388
+ /**
1389
+ * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
1390
+ * referenced between summaries so that their unreferenced timestamp can be reset.
1391
+ *
1392
+ * @param fromNodePath - The node from which the reference is added.
1393
+ * @param toNodePath - The node to which the reference is added.
1394
+ */
1395
+ public addedOutboundReference(fromNodePath: string, toNodePath: string) {
1396
+ if (!this.shouldRunGC) {
1397
+ return;
1398
+ }
1399
+
1400
+ const outboundRoutes = this.newReferencesSinceLastRun.get(fromNodePath) ?? [];
1401
+ outboundRoutes.push(toNodePath);
1402
+ this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
1403
+
1404
+ const nodeStateTracker = this.unreferencedNodesState.get(toNodePath);
1405
+ if (nodeStateTracker && nodeStateTracker.state !== UnreferencedState.Active) {
1406
+ this.inactiveNodeUsed("Revived", toNodePath, nodeStateTracker, fromNodePath);
1407
+ }
1408
+
1409
+ if (this.tombstones.includes(toNodePath)) {
1410
+ const nodeType = this.runtime.getNodeType(toNodePath);
1411
+
1412
+ let eventName = "GC_Tombstone_SubDatastore_Revived";
1413
+ if (nodeType === GCNodeType.DataStore) {
1414
+ eventName = "GC_Tombstone_Datastore_Revived";
1415
+ } else if (nodeType === GCNodeType.Blob) {
1416
+ eventName = "GC_Tombstone_Blob_Revived";
1417
+ }
1418
+
1419
+ sendGCUnexpectedUsageEvent(
1420
+ this.mc,
1421
+ {
1422
+ eventName,
1423
+ category: "generic",
1424
+ url: trimLeadingSlashes(toNodePath),
1425
+ nodeType,
1426
+ gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
1427
+ },
1428
+ undefined /* packagePath */,
1429
+ );
1430
+ }
1431
+ }
1432
+
1433
+ /**
1434
+ * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
1435
+ * cases where objects are used after they are deleted and throw / log errors accordingly.
1436
+ */
1437
+ public isNodeDeleted(nodePath: string): boolean {
1438
+ return this.deletedNodes.has(nodePath);
1439
+ }
1440
+
1441
+ public dispose(): void {
1442
+ this.sessionExpiryTimer?.clear();
1443
+ this.sessionExpiryTimer = undefined;
1444
+ }
1445
+
1446
+ /**
1447
+ * Updates the state of the system as per the current GC run. It does the following:
1448
+ * 1. Sets up the current GC state as per the gcData.
1449
+ * 2. Starts tracking for nodes that have become unreferenced in this run.
1450
+ * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
1451
+ * @param gcData - The data representing the reference graph on which GC is run.
1452
+ * @param gcResult - The result of the GC run on the gcData.
1453
+ * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
1454
+ * @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
1455
+ */
1456
+ private updateMarkPhase(
1457
+ gcData: IGarbageCollectionData,
1458
+ gcResult: IGCResult,
1459
+ currentReferenceTimestampMs: number,
1460
+ logger: ITelemetryLogger,
1461
+ ) {
1462
+ // Get references from the current GC run + references between previous and current run and then update each
1463
+ // node's state
1464
+ const allNodesReferencedBetweenGCs =
1465
+ this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
1466
+ gcResult.referencedNodeIds;
1467
+ this.newReferencesSinceLastRun.clear();
1468
+
1469
+ // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
1470
+ for (const nodeId of allNodesReferencedBetweenGCs) {
1471
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
1472
+ if (nodeStateTracker !== undefined) {
1473
+ // Stop tracking so as to clear out any running timers.
1474
+ nodeStateTracker.stopTracking();
1475
+ // Delete the node as we don't need to track it any more.
1476
+ this.unreferencedNodesState.delete(nodeId);
1477
+ }
1478
+ }
1479
+
1480
+ /**
1481
+ * If a node became unreferenced in this run, start tracking it.
1482
+ * If a node was already unreferenced, update its tracking information. Since the current reference time is
1483
+ * from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
1484
+ *
1485
+ * If a node is sweep ready, store and then return it.
1486
+ */
1487
+ const sweepReadyNodes: string[] = [];
1488
+ for (const nodeId of gcResult.deletedNodeIds) {
1489
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
1490
+ if (nodeStateTracker === undefined) {
1491
+ this.unreferencedNodesState.set(
1492
+ nodeId,
1493
+ new UnreferencedStateTracker(
1494
+ currentReferenceTimestampMs,
1495
+ this.inactiveTimeoutMs,
1496
+ currentReferenceTimestampMs,
1497
+ this.sweepTimeoutMs,
1498
+ ),
1499
+ );
1500
+ } else {
1501
+ nodeStateTracker.updateTracking(currentReferenceTimestampMs);
1502
+ if (nodeStateTracker.state === UnreferencedState.SweepReady) {
1503
+ sweepReadyNodes.push(nodeId);
1504
+ }
1505
+ }
1506
+ }
1507
+
1508
+ return sweepReadyNodes;
1509
+ }
1510
+
1511
+ /**
1512
+ * Deletes nodes from both the runtime and garbage collection
1513
+ * @param sweepReadyNodes - nodes that are ready to be deleted
1514
+ */
1515
+ private runSweepPhase(sweepReadyNodes: string[], gcData: IGarbageCollectionData) {
1516
+ // TODO: GC:Validation - validate that removed routes are not double deleted
1517
+ // TODO: GC:Validation - validate that the child routes of removed routes are deleted as well
1518
+ const sweptRoutes = this.runtime.deleteUnusedNodes(sweepReadyNodes);
1519
+ const updatedGCData = this.deleteSweptRoutes(sweptRoutes, gcData);
1520
+
1521
+ for (const nodeId of sweptRoutes) {
1522
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
1523
+ // TODO: GC:Validation - assert that the nodeStateTracker is defined
1524
+ if (nodeStateTracker !== undefined) {
1525
+ // Stop tracking so as to clear out any running timers.
1526
+ nodeStateTracker.stopTracking();
1527
+ // Delete the node as we don't need to track it any more.
1528
+ this.unreferencedNodesState.delete(nodeId);
1529
+ }
1530
+ // TODO: GC:Validation - assert that the deleted node is not a duplicate
1531
+ this.deletedNodes.add(nodeId);
1532
+ }
1533
+
1534
+ return updatedGCData;
1535
+ }
1536
+
1537
+ /**
1538
+ * @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
1539
+ */
1540
+ private deleteSweptRoutes(
1541
+ sweptRoutes: string[],
1542
+ gcData: IGarbageCollectionData,
1543
+ ): IGarbageCollectionData {
1544
+ const sweptRoutesSet = new Set<string>(sweptRoutes);
1545
+ const gcNodes: { [id: string]: string[] } = {};
1546
+ for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
1547
+ if (!sweptRoutesSet.has(id)) {
1548
+ gcNodes[id] = Array.from(outboundRoutes);
1549
+ }
1550
+ }
1551
+
1552
+ // TODO: GC:Validation - assert that the nodeId is in gcData
1553
+
1554
+ return {
1555
+ gcNodes,
1556
+ };
1557
+ }
1558
+
1559
+ /**
1560
+ * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
1561
+ * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
1562
+ * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
1563
+ * these objects while there can be in-memory referenced to it:
1564
+ * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
1565
+ * added, the object may have been accessed and in-memory reference to it added.
1566
+ * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
1567
+ * unreferenced, they could have been accessed and in-memory reference to them added.
1568
+ *
1569
+ * This function identifies nodes that were referenced since the last run.
1570
+ * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
1571
+ *
1572
+ * @returns - a list of all nodes referenced from the last local summary until now.
1573
+ */
1574
+ private findAllNodesReferencedBetweenGCs(
1575
+ currentGCData: IGarbageCollectionData,
1576
+ previousGCData: IGarbageCollectionData | undefined,
1577
+ logger: ITelemetryLogger,
1578
+ ): string[] | undefined {
1579
+ // If we haven't run GC before there is nothing to do.
1580
+ // No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
1581
+ if (previousGCData === undefined) {
1582
+ return undefined;
1583
+ }
1584
+
1585
+ // Find any references that haven't been identified correctly.
1586
+ const missingExplicitReferences = this.findMissingExplicitReferences(
1587
+ currentGCData,
1588
+ previousGCData,
1589
+ this.newReferencesSinceLastRun,
1590
+ );
1591
+
1592
+ if (missingExplicitReferences.length > 0) {
1593
+ missingExplicitReferences.forEach((missingExplicitReference) => {
1594
+ logger.sendErrorEvent({
1595
+ eventName: "gcUnknownOutboundReferences",
1596
+ gcNodeId: missingExplicitReference[0],
1597
+ gcRoutes: JSON.stringify(missingExplicitReference[1]),
1598
+ });
1599
+ });
1600
+ }
1601
+
1602
+ // No references were added since the last run so we don't have to update reference states of any unreferenced
1603
+ // nodes. There is no in between state at this point.
1604
+ if (this.newReferencesSinceLastRun.size === 0) {
1605
+ return undefined;
1606
+ }
1607
+
1608
+ /**
1609
+ * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
1610
+ * edges that have been added since then. To do this, combine the GC data from the last run and the current
1611
+ * run, and then add the references since last run.
1612
+ *
1613
+ * Note on why we need to combine the data from previous run, current run and all references in between -
1614
+ * 1. We need data from last run because some of its references may have been deleted since then. If those
1615
+ * references added new outbound references before they were deleted, we need to detect them.
1616
+ *
1617
+ * 2. We need new outbound references since last run because some of them may have been deleted later. If those
1618
+ * references added new outbound references before they were deleted, we need to detect them.
1619
+ *
1620
+ * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
1621
+ * - We don't require DDSes handles to be stored in a referenced DDS.
1622
+ * - A new data store may have "root" DDSes already created and we don't detect them today.
1623
+ */
1624
+ const gcDataSuperSet = concatGarbageCollectionData(previousGCData, currentGCData);
1625
+ const newOutboundRoutesSinceLastRun: string[] = [];
1626
+ this.newReferencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
1627
+ if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
1628
+ gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
1629
+ } else {
1630
+ gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
1631
+ }
1632
+ newOutboundRoutesSinceLastRun.push(...outboundRoutes);
1633
+ });
1634
+
1635
+ /**
1636
+ * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
1637
+ * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
1638
+ * unreferenced, stop tracking them and remove from unreferenced list.
1639
+ * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
1640
+ * unreferenced and add unreferenced state.
1641
+ */
1642
+ const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [
1643
+ "/",
1644
+ ...newOutboundRoutesSinceLastRun,
1645
+ ]);
1646
+ return gcResult.referencedNodeIds;
1647
+ }
1648
+
1649
+ /**
1650
+ * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
1651
+ * The principle is that every new reference or outbound route must be notified to GC via the
1652
+ * addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
1653
+ *
1654
+ * In more simple terms:
1655
+ * Missing Explicit References = Current References - Previous References - Explicitly Added References;
1656
+ *
1657
+ * @param currentGCData - The GC data (reference graph) from the current GC run.
1658
+ * @param previousGCData - The GC data (reference graph) from the previous GC run.
1659
+ * @param explicitReferences - New references added explicity between the previous and the current run.
1660
+ * @returns - a list of missing explicit references
1661
+ */
1662
+ private findMissingExplicitReferences(
1663
+ currentGCData: IGarbageCollectionData,
1664
+ previousGCData: IGarbageCollectionData,
1665
+ explicitReferences: Map<string, string[]>,
1666
+ ): [string, string[]][] {
1667
+ assert(
1668
+ previousGCData !== undefined,
1669
+ 0x2b7 /* "Can't validate correctness without GC data from last run" */,
1670
+ );
1671
+
1672
+ const currentGraph = Object.entries(currentGCData.gcNodes);
1673
+ const missingExplicitReferences: [string, string[]][] = [];
1674
+ currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
1675
+ const previousRoutes = previousGCData.gcNodes[nodeId] ?? [];
1676
+ const explicitRoutes = explicitReferences.get(nodeId) ?? [];
1677
+ const missingExplicitRoutes: string[] = [];
1678
+
1679
+ /**
1680
+ * 1. For routes in the current GC data, routes that were not present in previous GC data and did not have
1681
+ * explicit references should be added to missing explicit routes list.
1682
+ * 2. Only include data store and blob routes since GC only works for these two.
1683
+ * Note: Due to a bug with de-duped blobs, only adding data store routes for now.
1684
+ * 3. Ignore DDS routes to their parent datastores since those were added implicitly. So, there won't be
1685
+ * explicit routes to them.
1686
+ */
1687
+ currentOutboundRoutes.forEach((route) => {
1688
+ const nodeType = this.runtime.getNodeType(route);
1689
+ if (
1690
+ (nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) &&
1691
+ !nodeId.startsWith(route) &&
1692
+ !previousRoutes.includes(route) &&
1693
+ !explicitRoutes.includes(route)
1694
+ ) {
1695
+ missingExplicitRoutes.push(route);
1696
+ }
1697
+ });
1698
+ if (missingExplicitRoutes.length > 0) {
1699
+ missingExplicitReferences.push([nodeId, missingExplicitRoutes]);
1700
+ }
1701
+ });
1702
+
1703
+ // Ideally missingExplicitReferences should always have a size 0
1704
+ return missingExplicitReferences;
1705
+ }
1706
+
1707
+ /**
1708
+ * Generates the stats of a garbage collection run from the given results of the run.
1709
+ * @param gcResult - The result of a GC run.
1710
+ * @returns the GC stats of the GC run.
1711
+ */
1712
+ private generateStats(gcResult: IGCResult): IGCStats {
1713
+ const gcStats: IGCStats = {
1714
+ nodeCount: 0,
1715
+ dataStoreCount: 0,
1716
+ attachmentBlobCount: 0,
1717
+ unrefNodeCount: 0,
1718
+ unrefDataStoreCount: 0,
1719
+ unrefAttachmentBlobCount: 0,
1720
+ updatedNodeCount: 0,
1721
+ updatedDataStoreCount: 0,
1722
+ updatedAttachmentBlobCount: 0,
1723
+ };
1724
+
1725
+ const updateNodeStats = (nodeId: string, referenced: boolean) => {
1726
+ gcStats.nodeCount++;
1727
+ // If there is no previous GC data, every node's state is generated and is considered as updated.
1728
+ // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
1729
+ const stateUpdated =
1730
+ this.gcDataFromLastRun === undefined ||
1731
+ this.unreferencedNodesState.has(nodeId) === referenced;
1732
+ if (stateUpdated) {
1733
+ gcStats.updatedNodeCount++;
1734
+ }
1735
+ if (!referenced) {
1736
+ gcStats.unrefNodeCount++;
1737
+ }
1738
+
1739
+ if (this.runtime.getNodeType(nodeId) === GCNodeType.DataStore) {
1740
+ gcStats.dataStoreCount++;
1741
+ if (stateUpdated) {
1742
+ gcStats.updatedDataStoreCount++;
1743
+ }
1744
+ if (!referenced) {
1745
+ gcStats.unrefDataStoreCount++;
1746
+ }
1747
+ }
1748
+ if (this.runtime.getNodeType(nodeId) === GCNodeType.Blob) {
1749
+ gcStats.attachmentBlobCount++;
1750
+ if (stateUpdated) {
1751
+ gcStats.updatedAttachmentBlobCount++;
1752
+ }
1753
+ if (!referenced) {
1754
+ gcStats.unrefAttachmentBlobCount++;
1755
+ }
1756
+ }
1757
+ };
1758
+
1759
+ for (const nodeId of gcResult.referencedNodeIds) {
1760
+ updateNodeStats(nodeId, true /* referenced */);
1761
+ }
1762
+
1763
+ for (const nodeId of gcResult.deletedNodeIds) {
1764
+ updateNodeStats(nodeId, false /* referenced */);
1765
+ }
1766
+
1767
+ return gcStats;
1768
+ }
1769
+
1770
+ /**
1771
+ * For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
1772
+ * this will give us a view into how much deleted content a container has.
1773
+ */
1774
+ private logSweepEvents(logger: ITelemetryLogger, currentReferenceTimestampMs: number) {
1775
+ if (
1776
+ this.mc.config.getBoolean(disableSweepLogKey) === true ||
1777
+ this.sweepTimeoutMs === undefined
1778
+ ) {
1779
+ return;
1780
+ }
1781
+
1782
+ this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
1783
+ if (nodeStateTracker.state !== UnreferencedState.SweepReady) {
1784
+ return;
1785
+ }
1786
+
1787
+ const nodeType = this.runtime.getNodeType(nodeId);
1788
+ if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
1789
+ return;
1790
+ }
1791
+
1792
+ // Log deleted event for each node only once to reduce noise in telemetry.
1793
+ const uniqueEventId = `Deleted-${nodeId}`;
1794
+ if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
1795
+ return;
1796
+ }
1797
+ this.loggedUnreferencedEvents.add(uniqueEventId);
1798
+ logger.sendTelemetryEvent({
1799
+ eventName: "GCObjectDeleted",
1800
+ id: nodeId,
1801
+ type: nodeType,
1802
+ age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
1803
+ timeout: this.sweepTimeoutMs,
1804
+ completedGCRuns: this.completedRuns,
1805
+ lastSummaryTime: this.getLastSummaryTimestampMs(),
1806
+ });
1807
+ });
1808
+ }
1809
+
1810
+ /**
1811
+ * Called when an inactive node is used after. Queue up an event that will be logged next time GC runs.
1812
+ */
1813
+ private inactiveNodeUsed(
1814
+ usageType: "Changed" | "Loaded" | "Revived",
1815
+ nodeId: string,
1816
+ nodeStateTracker: UnreferencedStateTracker,
1817
+ fromNodeId?: string,
1818
+ packagePath?: readonly string[],
1819
+ currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs(),
1820
+ requestHeaders?: IRequestHeader,
1821
+ ) {
1822
+ // If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
1823
+ // logging as nothing interesting would have happened worth logging.
1824
+ // If the node is active, skip logging.
1825
+ if (
1826
+ currentReferenceTimestampMs === undefined ||
1827
+ nodeStateTracker.state === UnreferencedState.Active
1828
+ ) {
1829
+ return;
1830
+ }
1831
+
1832
+ // We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
1833
+ // as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
1834
+ const nodeType = this.runtime.getNodeType(nodeId);
1835
+ if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
1836
+ return;
1837
+ }
1838
+
1839
+ const state = nodeStateTracker.state;
1840
+ const uniqueEventId = `${state}-${nodeId}-${usageType}`;
1841
+ if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
1842
+ return;
1843
+ }
1844
+ this.loggedUnreferencedEvents.add(uniqueEventId);
1845
+
1846
+ const propsToLog = {
1847
+ id: nodeId,
1848
+ type: nodeType,
1849
+ unrefTime: nodeStateTracker.unreferencedTimestampMs,
1850
+ age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
1851
+ timeout:
1852
+ nodeStateTracker.state === UnreferencedState.Inactive
1853
+ ? this.inactiveTimeoutMs
1854
+ : this.sweepTimeoutMs,
1855
+ completedGCRuns: this.completedRuns,
1856
+ lastSummaryTime: this.getLastSummaryTimestampMs(),
1857
+ ...this.createContainerMetadata,
1858
+ externalRequest: requestHeaders?.[RuntimeHeaders.externalRequest],
1859
+ viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
1860
+ fromId: fromNodeId,
1861
+ };
1862
+
1863
+ // For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
1864
+ // For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
1865
+ // but it's a good signal nonetheless and we can consume it with a grain of salt.
1866
+ // Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
1867
+ // SweepReady errors are usages of Objects that will be deleted by GC Sweep!
1868
+ if (this.isSummarizerClient) {
1869
+ this.pendingEventsQueue.push({ ...propsToLog, usageType, state });
1870
+ } else {
1871
+ // For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
1872
+ // summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
1873
+ // Events generated:
1874
+ // InactiveObject_Loaded, SweepReadyObject_Loaded
1875
+ if (usageType === "Loaded") {
1876
+ const event = {
1877
+ ...propsToLog,
1878
+ eventName: `${state}Object_${usageType}`,
1879
+ pkg: packagePathToTelemetryProperty(packagePath),
1880
+ stack: generateStack(),
1881
+ };
1882
+
1883
+ // Do not log the inactive object x events as error events as they are not the best signal for
1884
+ // detecting something wrong with GC either from the partner or from the runtime itself.
1885
+ if (state === UnreferencedState.Inactive) {
1886
+ this.mc.logger.sendTelemetryEvent(event);
1887
+ } else {
1888
+ this.mc.logger.sendErrorEvent(event);
1889
+ }
1890
+ }
1891
+
1892
+ // If SweepReady Usage Detection is enabled, the handler may close the interactive container.
1893
+ // Once Sweep is fully implemented, this will be removed since the objects will be gone
1894
+ // and errors will arise elsewhere in the runtime
1895
+ if (state === UnreferencedState.SweepReady) {
1896
+ this.sweepReadyUsageHandler.usageDetectedInInteractiveClient({
1897
+ ...propsToLog,
1898
+ usageType,
1899
+ });
1900
+ }
1901
+ }
1902
+ }
1903
+
1904
+ private async logUnreferencedEvents(logger: ITelemetryLogger) {
1905
+ // Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
1906
+ // summary time they are then logged.
1907
+ // Events generated:
1908
+ // InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
1909
+ // SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
1910
+ for (const eventProps of this.pendingEventsQueue) {
1911
+ const { usageType, state, ...propsToLog } = eventProps;
1912
+ /**
1913
+ * Revived event is logged only if the node is active. If the node is not active, the reference to it was
1914
+ * from another unreferenced node and this scenario is not interesting to log.
1915
+ * Loaded and Changed events are logged only if the node is not active. If the node is active, it was
1916
+ * revived and a Revived event will be logged for it.
1917
+ */
1918
+ const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
1919
+ const active =
1920
+ nodeStateTracker === undefined ||
1921
+ nodeStateTracker.state === UnreferencedState.Active;
1922
+ if ((usageType === "Revived") === active) {
1923
+ const pkg = await this.getNodePackagePath(eventProps.id);
1924
+ const fromPkg = eventProps.fromId
1925
+ ? await this.getNodePackagePath(eventProps.fromId)
1926
+ : undefined;
1927
+ const event = {
1928
+ ...propsToLog,
1929
+ eventName: `${state}Object_${usageType}`,
1930
+ pkg: pkg
1931
+ ? { value: pkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
1932
+ : undefined,
1933
+ fromPkg: fromPkg
1934
+ ? { value: fromPkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
1935
+ : undefined,
1936
+ };
1937
+
1938
+ if (state === UnreferencedState.Inactive) {
1939
+ logger.sendTelemetryEvent(event);
1940
+ } else {
1941
+ logger.sendErrorEvent(event);
1942
+ }
1943
+ }
1944
+ }
1945
+ this.pendingEventsQueue = [];
1946
+ }
1756
1947
  }
1757
1948
 
1758
1949
  function generateSortedGCState(gcState: IGarbageCollectionState): IGarbageCollectionState {
1759
- const sortableArray: [string, IGarbageCollectionNodeData][] = Object.entries(gcState.gcNodes);
1760
- sortableArray.sort(([a], [b]) => a.localeCompare(b));
1761
- const sortedGCState: IGarbageCollectionState = { gcNodes: {} };
1762
- for (const [nodeId, nodeData] of sortableArray) {
1763
- nodeData.outboundRoutes.sort();
1764
- sortedGCState.gcNodes[nodeId] = nodeData;
1765
- }
1766
- return sortedGCState;
1950
+ const sortableArray: [string, IGarbageCollectionNodeData][] = Object.entries(gcState.gcNodes);
1951
+ sortableArray.sort(([a], [b]) => a.localeCompare(b));
1952
+ const sortedGCState: IGarbageCollectionState = { gcNodes: {} };
1953
+ for (const [nodeId, nodeData] of sortableArray) {
1954
+ nodeData.outboundRoutes.sort();
1955
+ sortedGCState.gcNodes[nodeId] = nodeData;
1956
+ }
1957
+ return sortedGCState;
1767
1958
  }
1768
1959
 
1769
1960
  /** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
1770
1961
  class TimerWithNoDefaultTimeout extends Timer {
1771
- constructor(
1772
- private readonly callback: () => void,
1773
- ) {
1774
- // The default timeout/handlers will never be used since start/restart pass overrides below
1775
- super(0, () => { throw new Error("DefaultHandler should not be used"); });
1776
- }
1777
-
1778
- start(timeoutMs: number) {
1779
- super.start(timeoutMs, this.callback);
1780
- }
1781
-
1782
- restart(timeoutMs: number): void {
1783
- super.restart(timeoutMs, this.callback);
1784
- }
1962
+ constructor(private readonly callback: () => void) {
1963
+ // The default timeout/handlers will never be used since start/restart pass overrides below
1964
+ super(0, () => {
1965
+ throw new Error("DefaultHandler should not be used");
1966
+ });
1967
+ }
1968
+
1969
+ start(timeoutMs: number) {
1970
+ super.start(timeoutMs, this.callback);
1971
+ }
1972
+
1973
+ restart(timeoutMs: number): void {
1974
+ super.restart(timeoutMs, this.callback);
1975
+ }
1785
1976
  }