@fluidframework/container-runtime 2.0.0-internal.7.3.0 → 2.0.0-internal.7.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/api-extractor-lint.json +13 -0
  3. package/api-extractor.json +9 -1
  4. package/api-report/container-runtime.api.md +124 -107
  5. package/dist/blobManager.d.ts +4 -4
  6. package/dist/blobManager.d.ts.map +1 -1
  7. package/dist/blobManager.js.map +1 -1
  8. package/dist/container-runtime-alpha.d.ts +1473 -0
  9. package/dist/container-runtime-beta.d.ts +300 -0
  10. package/dist/container-runtime-public.d.ts +300 -0
  11. package/dist/container-runtime-untrimmed.d.ts +1836 -0
  12. package/dist/containerRuntime.d.ts +30 -30
  13. package/dist/containerRuntime.d.ts.map +1 -1
  14. package/dist/containerRuntime.js +62 -40
  15. package/dist/containerRuntime.js.map +1 -1
  16. package/dist/dataStoreRegistry.d.ts +1 -1
  17. package/dist/dataStoreRegistry.js +1 -1
  18. package/dist/dataStoreRegistry.js.map +1 -1
  19. package/dist/dataStores.d.ts +10 -15
  20. package/dist/dataStores.d.ts.map +1 -1
  21. package/dist/dataStores.js +77 -40
  22. package/dist/dataStores.js.map +1 -1
  23. package/dist/gc/garbageCollection.d.ts +41 -13
  24. package/dist/gc/garbageCollection.d.ts.map +1 -1
  25. package/dist/gc/garbageCollection.js +215 -78
  26. package/dist/gc/garbageCollection.js.map +1 -1
  27. package/dist/gc/gcConfigs.d.ts.map +1 -1
  28. package/dist/gc/gcConfigs.js +34 -37
  29. package/dist/gc/gcConfigs.js.map +1 -1
  30. package/dist/gc/gcDefinitions.d.ts +121 -46
  31. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  32. package/dist/gc/gcDefinitions.js +26 -18
  33. package/dist/gc/gcDefinitions.js.map +1 -1
  34. package/dist/gc/gcHelpers.d.ts +18 -25
  35. package/dist/gc/gcHelpers.d.ts.map +1 -1
  36. package/dist/gc/gcHelpers.js +29 -45
  37. package/dist/gc/gcHelpers.js.map +1 -1
  38. package/dist/gc/gcTelemetry.d.ts +0 -5
  39. package/dist/gc/gcTelemetry.d.ts.map +1 -1
  40. package/dist/gc/gcTelemetry.js +14 -42
  41. package/dist/gc/gcTelemetry.js.map +1 -1
  42. package/dist/gc/gcUnreferencedStateTracker.d.ts +11 -5
  43. package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
  44. package/dist/gc/gcUnreferencedStateTracker.js +43 -19
  45. package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
  46. package/dist/gc/index.d.ts +1 -1
  47. package/dist/gc/index.d.ts.map +1 -1
  48. package/dist/gc/index.js +4 -5
  49. package/dist/gc/index.js.map +1 -1
  50. package/dist/index.d.ts +14 -2
  51. package/dist/index.d.ts.map +1 -1
  52. package/dist/index.js +16 -5
  53. package/dist/index.js.map +1 -1
  54. package/dist/messageTypes.d.ts +15 -7
  55. package/dist/messageTypes.d.ts.map +1 -1
  56. package/dist/messageTypes.js +6 -1
  57. package/dist/messageTypes.js.map +1 -1
  58. package/dist/opLifecycle/definitions.d.ts +1 -1
  59. package/dist/opLifecycle/definitions.js.map +1 -1
  60. package/dist/packageVersion.d.ts +1 -1
  61. package/dist/packageVersion.js +1 -1
  62. package/dist/packageVersion.js.map +1 -1
  63. package/dist/pendingStateManager.d.ts +1 -0
  64. package/dist/pendingStateManager.d.ts.map +1 -1
  65. package/dist/pendingStateManager.js +1 -0
  66. package/dist/pendingStateManager.js.map +1 -1
  67. package/dist/summary/orderedClientElection.d.ts +1 -1
  68. package/dist/summary/orderedClientElection.js.map +1 -1
  69. package/dist/summary/runWhileConnectedCoordinator.d.ts +2 -2
  70. package/dist/summary/runWhileConnectedCoordinator.js +1 -1
  71. package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
  72. package/dist/summary/summarizer.d.ts +1 -1
  73. package/dist/summary/summarizer.js +1 -1
  74. package/dist/summary/summarizer.js.map +1 -1
  75. package/dist/summary/summarizerTypes.d.ts +30 -30
  76. package/dist/summary/summarizerTypes.js.map +1 -1
  77. package/dist/summary/summaryCollection.d.ts +10 -10
  78. package/dist/summary/summaryCollection.js +1 -1
  79. package/dist/summary/summaryCollection.js.map +1 -1
  80. package/dist/summary/summaryFormat.d.ts +3 -3
  81. package/dist/summary/summaryFormat.js.map +1 -1
  82. package/lib/blobManager.d.ts +4 -4
  83. package/lib/blobManager.d.ts.map +1 -1
  84. package/lib/blobManager.js.map +1 -1
  85. package/lib/container-runtime-alpha.d.ts +1473 -0
  86. package/lib/container-runtime-beta.d.ts +300 -0
  87. package/lib/container-runtime-public.d.ts +300 -0
  88. package/lib/container-runtime-untrimmed.d.ts +1836 -0
  89. package/lib/containerRuntime.d.ts +30 -30
  90. package/lib/containerRuntime.d.ts.map +1 -1
  91. package/lib/containerRuntime.js +64 -42
  92. package/lib/containerRuntime.js.map +1 -1
  93. package/lib/dataStoreRegistry.d.ts +1 -1
  94. package/lib/dataStoreRegistry.js +1 -1
  95. package/lib/dataStoreRegistry.js.map +1 -1
  96. package/lib/dataStores.d.ts +10 -15
  97. package/lib/dataStores.d.ts.map +1 -1
  98. package/lib/dataStores.js +80 -43
  99. package/lib/dataStores.js.map +1 -1
  100. package/lib/gc/garbageCollection.d.ts +41 -13
  101. package/lib/gc/garbageCollection.d.ts.map +1 -1
  102. package/lib/gc/garbageCollection.js +217 -80
  103. package/lib/gc/garbageCollection.js.map +1 -1
  104. package/lib/gc/gcConfigs.d.ts.map +1 -1
  105. package/lib/gc/gcConfigs.js +37 -40
  106. package/lib/gc/gcConfigs.js.map +1 -1
  107. package/lib/gc/gcDefinitions.d.ts +121 -46
  108. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  109. package/lib/gc/gcDefinitions.js +25 -17
  110. package/lib/gc/gcDefinitions.js.map +1 -1
  111. package/lib/gc/gcHelpers.d.ts +18 -25
  112. package/lib/gc/gcHelpers.d.ts.map +1 -1
  113. package/lib/gc/gcHelpers.js +27 -43
  114. package/lib/gc/gcHelpers.js.map +1 -1
  115. package/lib/gc/gcTelemetry.d.ts +0 -5
  116. package/lib/gc/gcTelemetry.d.ts.map +1 -1
  117. package/lib/gc/gcTelemetry.js +15 -43
  118. package/lib/gc/gcTelemetry.js.map +1 -1
  119. package/lib/gc/gcUnreferencedStateTracker.d.ts +11 -5
  120. package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
  121. package/lib/gc/gcUnreferencedStateTracker.js +43 -19
  122. package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
  123. package/lib/gc/index.d.ts +1 -1
  124. package/lib/gc/index.d.ts.map +1 -1
  125. package/lib/gc/index.js +1 -1
  126. package/lib/gc/index.js.map +1 -1
  127. package/lib/index.d.ts +14 -2
  128. package/lib/index.d.ts.map +1 -1
  129. package/lib/index.js +15 -1
  130. package/lib/index.js.map +1 -1
  131. package/lib/messageTypes.d.ts +15 -7
  132. package/lib/messageTypes.d.ts.map +1 -1
  133. package/lib/messageTypes.js +6 -1
  134. package/lib/messageTypes.js.map +1 -1
  135. package/lib/opLifecycle/definitions.d.ts +1 -1
  136. package/lib/opLifecycle/definitions.js.map +1 -1
  137. package/lib/packageVersion.d.ts +1 -1
  138. package/lib/packageVersion.js +1 -1
  139. package/lib/packageVersion.js.map +1 -1
  140. package/lib/pendingStateManager.d.ts +1 -0
  141. package/lib/pendingStateManager.d.ts.map +1 -1
  142. package/lib/pendingStateManager.js +1 -0
  143. package/lib/pendingStateManager.js.map +1 -1
  144. package/lib/summary/orderedClientElection.d.ts +1 -1
  145. package/lib/summary/orderedClientElection.js.map +1 -1
  146. package/lib/summary/runWhileConnectedCoordinator.d.ts +2 -2
  147. package/lib/summary/runWhileConnectedCoordinator.js +1 -1
  148. package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
  149. package/lib/summary/summarizer.d.ts +1 -1
  150. package/lib/summary/summarizer.js +1 -1
  151. package/lib/summary/summarizer.js.map +1 -1
  152. package/lib/summary/summarizerTypes.d.ts +30 -30
  153. package/lib/summary/summarizerTypes.js.map +1 -1
  154. package/lib/summary/summaryCollection.d.ts +10 -10
  155. package/lib/summary/summaryCollection.js +1 -1
  156. package/lib/summary/summaryCollection.js.map +1 -1
  157. package/lib/summary/summaryFormat.d.ts +3 -3
  158. package/lib/summary/summaryFormat.js.map +1 -1
  159. package/package.json +42 -19
  160. package/src/blobManager.ts +5 -5
  161. package/src/containerRuntime.ts +86 -56
  162. package/src/dataStoreRegistry.ts +1 -1
  163. package/src/dataStores.ts +140 -69
  164. package/src/gc/garbageCollection.md +14 -15
  165. package/src/gc/garbageCollection.ts +256 -96
  166. package/src/gc/gcConfigs.ts +50 -52
  167. package/src/gc/gcDefinitions.ts +137 -52
  168. package/src/gc/gcHelpers.ts +31 -52
  169. package/src/gc/gcTelemetry.ts +16 -57
  170. package/src/gc/gcUnreferencedStateTracker.ts +61 -22
  171. package/src/gc/index.ts +6 -4
  172. package/src/index.ts +19 -1
  173. package/src/messageTypes.ts +19 -4
  174. package/src/opLifecycle/definitions.ts +1 -1
  175. package/src/packageVersion.ts +1 -1
  176. package/src/pendingStateManager.ts +1 -0
  177. package/src/summary/orderedClientElection.ts +1 -1
  178. package/src/summary/runWhileConnectedCoordinator.ts +2 -2
  179. package/src/summary/summarizer.ts +1 -1
  180. package/src/summary/summarizerTypes.ts +30 -30
  181. package/src/summary/summaryCollection.ts +10 -10
  182. package/src/summary/summaryFormat.ts +3 -3
  183. package/dist/id-compressor/appendOnlySortedMap.d.ts +0 -124
  184. package/dist/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
  185. package/dist/id-compressor/appendOnlySortedMap.js +0 -318
  186. package/dist/id-compressor/appendOnlySortedMap.js.map +0 -1
  187. package/dist/id-compressor/finalSpace.d.ts +0 -29
  188. package/dist/id-compressor/finalSpace.d.ts.map +0 -1
  189. package/dist/id-compressor/finalSpace.js +0 -62
  190. package/dist/id-compressor/finalSpace.js.map +0 -1
  191. package/dist/id-compressor/idCompressor.d.ts +0 -54
  192. package/dist/id-compressor/idCompressor.d.ts.map +0 -1
  193. package/dist/id-compressor/idCompressor.js +0 -495
  194. package/dist/id-compressor/idCompressor.js.map +0 -1
  195. package/dist/id-compressor/identifiers.d.ts +0 -32
  196. package/dist/id-compressor/identifiers.d.ts.map +0 -1
  197. package/dist/id-compressor/identifiers.js +0 -15
  198. package/dist/id-compressor/identifiers.js.map +0 -1
  199. package/dist/id-compressor/index.d.ts +0 -13
  200. package/dist/id-compressor/index.d.ts.map +0 -1
  201. package/dist/id-compressor/index.js +0 -32
  202. package/dist/id-compressor/index.js.map +0 -1
  203. package/dist/id-compressor/persistanceUtilities.d.ts +0 -22
  204. package/dist/id-compressor/persistanceUtilities.d.ts.map +0 -1
  205. package/dist/id-compressor/persistanceUtilities.js +0 -43
  206. package/dist/id-compressor/persistanceUtilities.js.map +0 -1
  207. package/dist/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
  208. package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
  209. package/dist/id-compressor/sessionSpaceNormalizer.js +0 -80
  210. package/dist/id-compressor/sessionSpaceNormalizer.js.map +0 -1
  211. package/dist/id-compressor/sessions.d.ts +0 -115
  212. package/dist/id-compressor/sessions.d.ts.map +0 -1
  213. package/dist/id-compressor/sessions.js +0 -305
  214. package/dist/id-compressor/sessions.js.map +0 -1
  215. package/dist/id-compressor/utilities.d.ts +0 -52
  216. package/dist/id-compressor/utilities.d.ts.map +0 -1
  217. package/dist/id-compressor/utilities.js +0 -169
  218. package/dist/id-compressor/utilities.js.map +0 -1
  219. package/lib/id-compressor/appendOnlySortedMap.d.ts +0 -124
  220. package/lib/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
  221. package/lib/id-compressor/appendOnlySortedMap.js +0 -314
  222. package/lib/id-compressor/appendOnlySortedMap.js.map +0 -1
  223. package/lib/id-compressor/finalSpace.d.ts +0 -29
  224. package/lib/id-compressor/finalSpace.d.ts.map +0 -1
  225. package/lib/id-compressor/finalSpace.js +0 -58
  226. package/lib/id-compressor/finalSpace.js.map +0 -1
  227. package/lib/id-compressor/idCompressor.d.ts +0 -54
  228. package/lib/id-compressor/idCompressor.d.ts.map +0 -1
  229. package/lib/id-compressor/idCompressor.js +0 -491
  230. package/lib/id-compressor/idCompressor.js.map +0 -1
  231. package/lib/id-compressor/identifiers.d.ts +0 -32
  232. package/lib/id-compressor/identifiers.d.ts.map +0 -1
  233. package/lib/id-compressor/identifiers.js +0 -11
  234. package/lib/id-compressor/identifiers.js.map +0 -1
  235. package/lib/id-compressor/index.d.ts +0 -13
  236. package/lib/id-compressor/index.d.ts.map +0 -1
  237. package/lib/id-compressor/index.js +0 -13
  238. package/lib/id-compressor/index.js.map +0 -1
  239. package/lib/id-compressor/persistanceUtilities.d.ts +0 -22
  240. package/lib/id-compressor/persistanceUtilities.d.ts.map +0 -1
  241. package/lib/id-compressor/persistanceUtilities.js +0 -34
  242. package/lib/id-compressor/persistanceUtilities.js.map +0 -1
  243. package/lib/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
  244. package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
  245. package/lib/id-compressor/sessionSpaceNormalizer.js +0 -76
  246. package/lib/id-compressor/sessionSpaceNormalizer.js.map +0 -1
  247. package/lib/id-compressor/sessions.d.ts +0 -115
  248. package/lib/id-compressor/sessions.d.ts.map +0 -1
  249. package/lib/id-compressor/sessions.js +0 -290
  250. package/lib/id-compressor/sessions.js.map +0 -1
  251. package/lib/id-compressor/utilities.d.ts +0 -52
  252. package/lib/id-compressor/utilities.d.ts.map +0 -1
  253. package/lib/id-compressor/utilities.js +0 -151
  254. package/lib/id-compressor/utilities.js.map +0 -1
  255. package/src/id-compressor/README.md +0 -3
  256. package/src/id-compressor/appendOnlySortedMap.ts +0 -366
  257. package/src/id-compressor/finalSpace.ts +0 -67
  258. package/src/id-compressor/idCompressor.ts +0 -630
  259. package/src/id-compressor/identifiers.ts +0 -42
  260. package/src/id-compressor/index.ts +0 -26
  261. package/src/id-compressor/persistanceUtilities.ts +0 -58
  262. package/src/id-compressor/sessionSpaceNormalizer.ts +0 -83
  263. package/src/id-compressor/sessions.ts +0 -405
  264. package/src/id-compressor/utilities.ts +0 -190
@@ -6,6 +6,7 @@ import { IRequest } from "@fluidframework/core-interfaces";
6
6
  import { IGarbageCollectionDetailsBase, ISummarizeResult, ITelemetryContext } from "@fluidframework/runtime-definitions";
7
7
  import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
8
8
  import { RuntimeHeaderData } from "../containerRuntime";
9
+ import { ContainerRuntimeGCMessage } from "../messageTypes";
9
10
  import { IRefreshSummaryResult } from "../summary";
10
11
  import { IGarbageCollector, IGarbageCollectorCreateParams, IGCStats, IGCMetadata } from "./gcDefinitions";
11
12
  /**
@@ -61,6 +62,7 @@ export declare class GarbageCollector implements IGarbageCollector {
61
62
  private readonly getLastSummaryTimestampMs;
62
63
  /** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
63
64
  private readonly activeConnection;
65
+ private readonly submitMessage;
64
66
  get summaryStateNeedsReset(): boolean;
65
67
  /** Returns the count of data stores whose GC state updated since the last summary. */
66
68
  get updatedDSCountSinceLastSummary(): number;
@@ -81,7 +83,7 @@ export declare class GarbageCollector implements IGarbageCollector {
81
83
  private updateStateFromSnapshotData;
82
84
  /**
83
85
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
84
- * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
86
+ * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
85
87
  * @param connected - Whether the runtime connected / disconnected.
86
88
  * @param clientId - The clientId of this runtime.
87
89
  */
@@ -105,10 +107,16 @@ export declare class GarbageCollector implements IGarbageCollector {
105
107
  }, telemetryContext?: ITelemetryContext): Promise<IGCStats | undefined>;
106
108
  /**
107
109
  * Runs garbage collection. It does the following:
110
+ *
108
111
  * 1. It generates / analyzes the runtime's reference graph.
109
- * 2. Generates stats for the GC run based on previous / current GC state.
112
+ *
113
+ * 2. Generates mark phase stats.
114
+ *
110
115
  * 3. Runs Mark phase.
116
+ *
111
117
  * 4. Runs Sweep phase.
118
+ *
119
+ * 5. Generates sweep phase stats.
112
120
  */
113
121
  private runGC;
114
122
  /**
@@ -123,19 +131,19 @@ export declare class GarbageCollector implements IGarbageCollector {
123
131
  * @param gcResult - The result of the GC run on the gcData.
124
132
  * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
125
133
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
126
- * @returns A list of sweep ready nodes, i.e., nodes that ready to be deleted.
134
+ * @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
127
135
  */
128
136
  private runMarkPhase;
129
137
  /**
130
138
  * Runs the GC Sweep phase. It does the following:
131
- * 1. Calls the runtime to delete nodes that are sweep ready.
132
- * 2. Clears tracking for deleted nodes.
139
+ *
140
+ * 1. Marks tombstone-ready nodes as tombstones.
141
+ *
142
+ * 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
133
143
  *
134
144
  * @param gcResult - The result of the GC run on the gcData.
135
- * @param sweepReadyNodes - List of nodes that are sweep ready.
136
- * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
137
- * @param logger - The logger to be used to log any telemetry.
138
- * @returns A list of nodes that have been deleted.
145
+ * @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
146
+ * @param sweepReadyNodes - List of nodes that are sweep-ready.
139
147
  */
140
148
  private runSweepPhase;
141
149
  /**
@@ -165,6 +173,18 @@ export declare class GarbageCollector implements IGarbageCollector {
165
173
  * Called to refresh the latest summary state. This happens when either a pending summary is acked.
166
174
  */
167
175
  refreshLatestSummary(result: IRefreshSummaryResult): Promise<void>;
176
+ /**
177
+ * Process a GC message.
178
+ * @param message - The GC message from the container runtime.
179
+ * @param local - Whether it was send by this client.
180
+ */
181
+ processMessage(message: ContainerRuntimeGCMessage, local: boolean): void;
182
+ /**
183
+ * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
184
+ * tracking for nodes that are actually deleted by the runtime.
185
+ * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
186
+ */
187
+ private deleteSweepReadyNodes;
168
188
  /**
169
189
  * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
170
190
  * or throw an error if failing on incorrect usage is configured.
@@ -191,10 +211,18 @@ export declare class GarbageCollector implements IGarbageCollector {
191
211
  isNodeDeleted(nodePath: string): boolean;
192
212
  dispose(): void;
193
213
  /**
194
- * Generates the stats of a garbage collection run from the given results of the run.
195
- * @param gcResult - The result of a GC run.
196
- * @returns the GC stats of the GC run.
214
+ * Generates the stats of a garbage collection mark phase run.
215
+ * @param gcResult - The result of the current GC run.
216
+ * @returns the stats of the mark phase run.
217
+ */
218
+ private getMarkPhaseStats;
219
+ /**
220
+ * Generates the stats of a garbage collection sweep phase run.
221
+ * @param deletedNodes - The nodes that have been deleted until this run.
222
+ * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
223
+ * @param markPhaseStats - The stats of the mark phase run.
224
+ * @returns the stats of the sweep phase run.
197
225
  */
198
- private generateStats;
226
+ private getSweepPhaseStats;
199
227
  }
200
228
  //# sourceMappingURL=garbageCollection.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEnD,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAEX,MAAM,iBAAiB,CAAC;AAQzB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IACrE,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgB;IAEjD,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IAoKjE;;;OAGG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjD;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA4EnC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAoEhC;;;;;;OAMG;YACW,KAAK;IA0CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAgDpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IAgErB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAyD/B;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAqBtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,aAAa;CAyDrB"}
1
+ {"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAwB,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEnD,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAMX,MAAM,iBAAiB,CAAC;AAazB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IACrE,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgB;IAEjD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IAsKjE;;;OAGG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjD;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA6EnC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAwEhC;;;;;;;;;;;;OAYG;YACW,KAAK;IA2CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAqDpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IAgErB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;OAIG;IACI,cAAc,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAyBxE;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAqC7B;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAyD/B;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAqBtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0DzB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;CAkD1B"}
@@ -11,6 +11,7 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
11
11
  const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
12
12
  const containerRuntime_1 = require("../containerRuntime");
13
13
  const error_1 = require("../error");
14
+ const messageTypes_1 = require("../messageTypes");
14
15
  const gcConfigs_1 = require("./gcConfigs");
15
16
  const gcDefinitions_1 = require("./gcDefinitions");
16
17
  const gcHelpers_1 = require("./gcHelpers");
@@ -49,7 +50,7 @@ class GarbageCollector {
49
50
  }
50
51
  /** If false, loading or using a Tombstoned object should merely log, not fail */
51
52
  get tombstoneEnforcementAllowed() {
52
- return this.configs.tombstoneEnforcementAllowed;
53
+ return this.configs.sweepEnabled;
53
54
  }
54
55
  /** If true, throw an error when a tombstone data store is retrieved */
55
56
  get throwOnTombstoneLoad() {
@@ -83,6 +84,7 @@ class GarbageCollector {
83
84
  this.getNodePackagePath = createParams.getNodePackagePath;
84
85
  this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
85
86
  this.activeConnection = createParams.activeConnection;
87
+ this.submitMessage = createParams.submitMessage;
86
88
  const baseSnapshot = createParams.baseSnapshot;
87
89
  const readAndParseBlob = createParams.readAndParseBlob;
88
90
  this.mc = (0, telemetry_utils_1.createChildMonitoringContext)({
@@ -144,7 +146,7 @@ class GarbageCollector {
144
146
  /**
145
147
  * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
146
148
  * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
147
- * GC state and updates their inactive or sweep ready state.
149
+ * GC state and updates their inactive or sweep-ready state.
148
150
  */
149
151
  this.initializeGCStateFromBaseSnapshotP = new core_utils_1.LazyPromise(async () => {
150
152
  /**
@@ -200,6 +202,7 @@ class GarbageCollector {
200
202
  eventName: "GarbageCollectorLoaded",
201
203
  gcConfigs: JSON.stringify(this.configs),
202
204
  gcOptions: JSON.stringify(createParams.gcOptions),
205
+ ...createParams.createContainerMetadata,
203
206
  });
204
207
  }
205
208
  /**
@@ -293,7 +296,7 @@ class GarbageCollector {
293
296
  const gcNodes = {};
294
297
  for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
295
298
  if (nodeData.unreferencedTimestampMs !== undefined) {
296
- this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
299
+ this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
297
300
  }
298
301
  gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
299
302
  }
@@ -301,7 +304,7 @@ class GarbageCollector {
301
304
  }
302
305
  /**
303
306
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
304
- * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
307
+ * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
305
308
  * @param connected - Whether the runtime connected / disconnected.
306
309
  * @param clientId - The clientId of this runtime.
307
310
  */
@@ -317,7 +320,7 @@ class GarbageCollector {
317
320
  * the receiving summarizer client.
318
321
  *
319
322
  * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
320
- * sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
323
+ * sweep in phases and we want to track when inactive and sweep-ready objects are used in any client.
321
324
  */
322
325
  if (this.activeConnection() && this.configs.shouldRunGC) {
323
326
  this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
@@ -375,7 +378,11 @@ class GarbageCollector {
375
378
  await this.runtime.updateStateBeforeGC();
376
379
  /** GC step */
377
380
  const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
378
- event.end({ ...gcStats, timestamp: currentReferenceTimestampMs });
381
+ event.end({
382
+ ...gcStats,
383
+ timestamp: currentReferenceTimestampMs,
384
+ sweep: this.configs.shouldRunSweep,
385
+ });
379
386
  /** Post-GC steps */
380
387
  // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
381
388
  // updates its state so that we don't send false positives based on intermediate state. For example, we may get
@@ -390,10 +397,16 @@ class GarbageCollector {
390
397
  }
391
398
  /**
392
399
  * Runs garbage collection. It does the following:
400
+ *
393
401
  * 1. It generates / analyzes the runtime's reference graph.
394
- * 2. Generates stats for the GC run based on previous / current GC state.
402
+ *
403
+ * 2. Generates mark phase stats.
404
+ *
395
405
  * 3. Runs Mark phase.
406
+ *
396
407
  * 4. Runs Sweep phase.
408
+ *
409
+ * 5. Generates sweep phase stats.
397
410
  */
398
411
  async runGC(fullGC, currentReferenceTimestampMs, logger) {
399
412
  // 1. Generate / analyze the runtime's reference graph.
@@ -403,17 +416,20 @@ class GarbageCollector {
403
416
  // Get all referenced nodes - References in this run + references between the previous and current runs.
404
417
  const allReferencedNodeIds = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
405
418
  gcResult.referencedNodeIds;
406
- // 2. Generate stats based on the previous / current GC state.
407
- // Must happen before running Mark / Sweep phase because previous GC state will be updated in these stages.
408
- const gcStats = this.generateStats(gcResult);
419
+ // 2. Get the mark phase stats based on the previous / current GC state.
420
+ // This is done before running mark phase because we need the previous GC state before it is updated.
421
+ const markPhaseStats = this.getMarkPhaseStats(gcResult);
409
422
  // 3. Run the Mark phase.
410
- // It will mark nodes as referenced / unreferenced and return a list of node ids that are ready to be swept.
411
- const sweepReadyNodeIds = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
423
+ // It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
424
+ const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
412
425
  // 4. Run the Sweep phase.
413
- // It will delete sweep ready nodes and return a list of deleted node ids.
414
- const deletedNodeIds = this.runSweepPhase(gcResult, sweepReadyNodeIds, currentReferenceTimestampMs, logger);
415
- this.gcDataFromLastRun = (0, gcHelpers_1.cloneGCData)(gcData, (id) => deletedNodeIds.includes(id) /* filter out deleted nodes */);
416
- return gcStats;
426
+ // It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
427
+ // sweep op. All clients, including this one, will delete these nodes once it processes the op.
428
+ this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
429
+ this.gcDataFromLastRun = (0, gcHelpers_1.cloneGCData)(gcData);
430
+ // 5. Get the sweep phase stats.
431
+ const sweepPhaseStats = this.getSweepPhaseStats(this.deletedNodes, sweepReadyNodeIds, markPhaseStats);
432
+ return { ...markPhaseStats, ...sweepPhaseStats };
417
433
  }
418
434
  /**
419
435
  * Runs the GC Mark phase. It does the following:
@@ -427,7 +443,7 @@ class GarbageCollector {
427
443
  * @param gcResult - The result of the GC run on the gcData.
428
444
  * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
429
445
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
430
- * @returns A list of sweep ready nodes, i.e., nodes that ready to be deleted.
446
+ * @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
431
447
  */
432
448
  runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
433
449
  // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
@@ -441,83 +457,94 @@ class GarbageCollector {
441
457
  }
442
458
  }
443
459
  // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
444
- const sweepReadyNodeIds = [];
460
+ const tombstoneReadyNodeIds = new Set();
461
+ const sweepReadyNodeIds = new Set();
445
462
  for (const nodeId of gcResult.deletedNodeIds) {
446
463
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
447
464
  if (nodeStateTracker === undefined) {
448
- this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
465
+ this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
449
466
  }
450
467
  else {
451
468
  // If a node was already unreferenced, update its tracking information. Since the current reference time
452
469
  // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
453
470
  nodeStateTracker.updateTracking(currentReferenceTimestampMs);
454
- // If a node is sweep ready, store it so it can be returned.
471
+ // If a node is tombstone or sweep-ready, store it so it can be returned.
472
+ if (nodeStateTracker.state === gcDefinitions_1.UnreferencedState.TombstoneReady) {
473
+ tombstoneReadyNodeIds.add(nodeId);
474
+ }
455
475
  if (nodeStateTracker.state === gcDefinitions_1.UnreferencedState.SweepReady) {
456
- sweepReadyNodeIds.push(nodeId);
476
+ sweepReadyNodeIds.add(nodeId);
457
477
  }
458
478
  }
459
479
  }
460
480
  // 3. Call the runtime to update referenced nodes in this run.
461
481
  this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
462
- return sweepReadyNodeIds;
482
+ return { tombstoneReadyNodeIds, sweepReadyNodeIds };
463
483
  }
464
484
  /**
465
485
  * Runs the GC Sweep phase. It does the following:
466
- * 1. Calls the runtime to delete nodes that are sweep ready.
467
- * 2. Clears tracking for deleted nodes.
486
+ *
487
+ * 1. Marks tombstone-ready nodes as tombstones.
488
+ *
489
+ * 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
468
490
  *
469
491
  * @param gcResult - The result of the GC run on the gcData.
470
- * @param sweepReadyNodes - List of nodes that are sweep ready.
471
- * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
472
- * @param logger - The logger to be used to log any telemetry.
473
- * @returns A list of nodes that have been deleted.
492
+ * @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
493
+ * @param sweepReadyNodes - List of nodes that are sweep-ready.
474
494
  */
475
- runSweepPhase(gcResult, sweepReadyNodes, currentReferenceTimestampMs, logger) {
476
- // Log events for objects that are ready to be deleted by sweep. This will give us data on sweep when
477
- // its not enabled.
478
- this.telemetryTracker.logSweepEvents(logger, currentReferenceTimestampMs, this.unreferencedNodesState, this.completedRuns, this.getLastSummaryTimestampMs());
495
+ runSweepPhase(gcResult, tombstoneReadyNodes, sweepReadyNodes) {
479
496
  /**
480
- * Currently, there are 3 modes for sweep:
481
- * Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
482
- * Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted.
483
- * Sweep mode - Sweep ready modes are deleted.
497
+ * Under "Test Mode", unreferenced nodes are immediately deleted without waiting for them to be sweep-ready.
484
498
  *
485
- * These modes serve as staging for applications that want to enable sweep by providing an incremental
486
- * way to test and validate sweep works as expected.
499
+ * Otherwise, depending on how long it's been since the node was unreferenced, it will either be
500
+ * marked as Tombstone, or deleted by Sweep.
487
501
  */
488
502
  if (this.configs.testMode) {
489
503
  // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
490
504
  this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
491
- return [];
505
+ return;
492
506
  }
507
+ // If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
508
+ // This is important because a container may never load during a node's Sweep Grace Period,
509
+ // so that node would directly become sweep-ready skipping over tombstone-ready state,
510
+ // but should be Tombstoned since Sweep is disabled.
511
+ const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
512
+ ? {
513
+ nodesToTombstone: [...tombstoneReadyNodes],
514
+ nodesToDelete: [...sweepReadyNodes],
515
+ }
516
+ : {
517
+ nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
518
+ nodesToDelete: [],
519
+ };
493
520
  if (this.configs.tombstoneMode) {
494
- this.tombstones = sweepReadyNodes;
495
- // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
496
- // involving access to "deleted" data without actually deleting the data from summaries.
521
+ this.tombstones = nodesToTombstone;
522
+ // If we are running in GC tombstone mode, update tombstoned routes.
497
523
  this.runtime.updateTombstonedRoutes(this.tombstones);
498
- return [];
499
- }
500
- if (!this.configs.shouldRunSweep) {
501
- return [];
502
524
  }
503
- // 1. Call the runtime to delete sweep ready nodes. The runtime returns a list of nodes it deleted.
504
- // TODO: GC:Validation - validate that removed routes are not double delete and that the child routes of
505
- // removed routes are deleted as well.
506
- const deletedNodeIds = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
507
- // 2. Clear unreferenced state tracking for deleted nodes.
508
- for (const nodeId of deletedNodeIds) {
509
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
510
- // TODO: GC:Validation - assert that the nodeStateTracker is defined
511
- if (nodeStateTracker !== undefined) {
512
- // Stop tracking so as to clear out any running timers.
513
- nodeStateTracker.stopTracking();
514
- // Delete the node as we don't need to track it any more.
515
- this.unreferencedNodesState.delete(nodeId);
516
- }
517
- // TODO: GC:Validation - assert that the deleted node is not a duplicate
518
- this.deletedNodes.add(nodeId);
525
+ if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
526
+ // Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
527
+ // to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
528
+ // local state when processing the op.
529
+ const sweepReadyDSAndBlobs = nodesToDelete.filter((nodeId) => {
530
+ const nodeType = this.runtime.getNodeType(nodeId);
531
+ return nodeType === gcDefinitions_1.GCNodeType.DataStore || nodeType === gcDefinitions_1.GCNodeType.Blob;
532
+ });
533
+ const contents = {
534
+ type: gcDefinitions_1.GarbageCollectionMessageType.Sweep,
535
+ deletedNodeIds: sweepReadyDSAndBlobs,
536
+ };
537
+ // Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
538
+ // is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
539
+ // accidental usage. The clients will sync without the delete op too but it may take longer.
540
+ const containerGCMessage = {
541
+ type: messageTypes_1.ContainerMessageType.GC,
542
+ contents,
543
+ compatDetails: { behavior: "Ignore" },
544
+ };
545
+ this.submitMessage(containerGCMessage);
546
+ return;
519
547
  }
520
- return deletedNodeIds;
521
548
  }
522
549
  /**
523
550
  * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
@@ -627,6 +654,65 @@ class GarbageCollector {
627
654
  async refreshLatestSummary(result) {
628
655
  return this.summaryStateTracker.refreshLatestSummary(result);
629
656
  }
657
+ /**
658
+ * Process a GC message.
659
+ * @param message - The GC message from the container runtime.
660
+ * @param local - Whether it was send by this client.
661
+ */
662
+ processMessage(message, local) {
663
+ switch (message.contents.type) {
664
+ case "Sweep": {
665
+ // Delete the nodes whose ids are present in the contents.
666
+ this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
667
+ break;
668
+ }
669
+ default: {
670
+ if (!(0, gcHelpers_1.compatBehaviorAllowsGCMessageType)(message.contents.type, message.compatDetails?.behavior)) {
671
+ const error = telemetry_utils_1.DataProcessingError.create(`Garbage collection message of unknown type ${message.contents.type}`, "processMessage");
672
+ throw error;
673
+ }
674
+ break;
675
+ }
676
+ }
677
+ }
678
+ /**
679
+ * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
680
+ * tracking for nodes that are actually deleted by the runtime.
681
+ * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
682
+ */
683
+ deleteSweepReadyNodes(sweepReadyNodeIds) {
684
+ // Use a set for lookup because its much faster than array or map.
685
+ const sweepReadyNodesSet = new Set(sweepReadyNodeIds);
686
+ // The ids in the sweep-ready nodes do not contain DDS node ids. This is an optimization to reduce the size
687
+ // of the GC op. Since GC applies to data store only, all its DDSes are deleted along with it. So, get the
688
+ // DDS nodes ID from the unreferenced nodes state.
689
+ const allSweepReadyNodeIds = Array.from(sweepReadyNodeIds);
690
+ for (const [id] of this.unreferencedNodesState) {
691
+ // Ignore data store nodes since they would already be in the list.
692
+ const pathParts = id.split("/");
693
+ if (pathParts.length <= 2) {
694
+ continue;
695
+ }
696
+ // Get the data store id part. Note that this may include blobs but that's okay since the part would just
697
+ // be "_blobs" and it won't be found.
698
+ const dsId = `/${pathParts[1]}`;
699
+ if (sweepReadyNodesSet.has(dsId)) {
700
+ allSweepReadyNodeIds.push(id);
701
+ }
702
+ }
703
+ const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
704
+ // Clear unreferenced state tracking for deleted nodes.
705
+ for (const nodeId of deletedNodeIds) {
706
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
707
+ if (nodeStateTracker !== undefined) {
708
+ // Stop tracking so as to clear out any running timers.
709
+ nodeStateTracker.stopTracking();
710
+ // Delete the node as we don't need to track it any more.
711
+ this.unreferencedNodesState.delete(nodeId);
712
+ }
713
+ this.deletedNodes.add(nodeId);
714
+ }
715
+ }
630
716
  /**
631
717
  * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
632
718
  * or throw an error if failing on incorrect usage is configured.
@@ -715,12 +801,12 @@ class GarbageCollector {
715
801
  this.sessionExpiryTimer = undefined;
716
802
  }
717
803
  /**
718
- * Generates the stats of a garbage collection run from the given results of the run.
719
- * @param gcResult - The result of a GC run.
720
- * @returns the GC stats of the GC run.
804
+ * Generates the stats of a garbage collection mark phase run.
805
+ * @param gcResult - The result of the current GC run.
806
+ * @returns the stats of the mark phase run.
721
807
  */
722
- generateStats(gcResult) {
723
- const gcStats = {
808
+ getMarkPhaseStats(gcResult) {
809
+ const markPhaseStats = {
724
810
  nodeCount: 0,
725
811
  dataStoreCount: 0,
726
812
  attachmentBlobCount: 0,
@@ -732,33 +818,33 @@ class GarbageCollector {
732
818
  updatedAttachmentBlobCount: 0,
733
819
  };
734
820
  const updateNodeStats = (nodeId, referenced) => {
735
- gcStats.nodeCount++;
821
+ markPhaseStats.nodeCount++;
736
822
  // If there is no previous GC data, every node's state is generated and is considered as updated.
737
823
  // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
738
824
  const stateUpdated = this.gcDataFromLastRun === undefined ||
739
825
  this.unreferencedNodesState.has(nodeId) === referenced;
740
826
  if (stateUpdated) {
741
- gcStats.updatedNodeCount++;
827
+ markPhaseStats.updatedNodeCount++;
742
828
  }
743
829
  if (!referenced) {
744
- gcStats.unrefNodeCount++;
830
+ markPhaseStats.unrefNodeCount++;
745
831
  }
746
832
  if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.DataStore) {
747
- gcStats.dataStoreCount++;
833
+ markPhaseStats.dataStoreCount++;
748
834
  if (stateUpdated) {
749
- gcStats.updatedDataStoreCount++;
835
+ markPhaseStats.updatedDataStoreCount++;
750
836
  }
751
837
  if (!referenced) {
752
- gcStats.unrefDataStoreCount++;
838
+ markPhaseStats.unrefDataStoreCount++;
753
839
  }
754
840
  }
755
841
  if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.Blob) {
756
- gcStats.attachmentBlobCount++;
842
+ markPhaseStats.attachmentBlobCount++;
757
843
  if (stateUpdated) {
758
- gcStats.updatedAttachmentBlobCount++;
844
+ markPhaseStats.updatedAttachmentBlobCount++;
759
845
  }
760
846
  if (!referenced) {
761
- gcStats.unrefAttachmentBlobCount++;
847
+ markPhaseStats.unrefAttachmentBlobCount++;
762
848
  }
763
849
  }
764
850
  };
@@ -768,7 +854,58 @@ class GarbageCollector {
768
854
  for (const nodeId of gcResult.deletedNodeIds) {
769
855
  updateNodeStats(nodeId, false /* referenced */);
770
856
  }
771
- return gcStats;
857
+ return markPhaseStats;
858
+ }
859
+ /**
860
+ * Generates the stats of a garbage collection sweep phase run.
861
+ * @param deletedNodes - The nodes that have been deleted until this run.
862
+ * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
863
+ * @param markPhaseStats - The stats of the mark phase run.
864
+ * @returns the stats of the sweep phase run.
865
+ */
866
+ getSweepPhaseStats(deletedNodes, sweepReadyNodes, markPhaseStats) {
867
+ // Initialize the life time node counts to the mark phase node counts. If sweep is not enabled,
868
+ // these will be the life time node count for this container.
869
+ const sweepPhaseStats = {
870
+ lifetimeNodeCount: markPhaseStats.nodeCount,
871
+ lifetimeDataStoreCount: markPhaseStats.dataStoreCount,
872
+ lifetimeAttachmentBlobCount: markPhaseStats.attachmentBlobCount,
873
+ deletedNodeCount: 0,
874
+ deletedDataStoreCount: 0,
875
+ deletedAttachmentBlobCount: 0,
876
+ };
877
+ for (const nodeId of deletedNodes) {
878
+ sweepPhaseStats.deletedNodeCount++;
879
+ const nodeType = this.runtime.getNodeType(nodeId);
880
+ if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
881
+ sweepPhaseStats.deletedDataStoreCount++;
882
+ }
883
+ else if (nodeType === gcDefinitions_1.GCNodeType.Blob) {
884
+ sweepPhaseStats.deletedAttachmentBlobCount++;
885
+ }
886
+ }
887
+ // If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
888
+ // deleted in previous runs. So, add the deleted node counts to life time stats.
889
+ sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
890
+ sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
891
+ sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
892
+ if (this.configs.shouldRunSweep) {
893
+ return sweepPhaseStats;
894
+ }
895
+ // If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
896
+ // is the final state the node will be in.
897
+ // If sweep is enabled, this will happen in the run after the GC op round trips back.
898
+ for (const nodeId of sweepReadyNodes) {
899
+ sweepPhaseStats.deletedNodeCount++;
900
+ const nodeType = this.runtime.getNodeType(nodeId);
901
+ if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
902
+ sweepPhaseStats.deletedDataStoreCount++;
903
+ }
904
+ else if (nodeType === gcDefinitions_1.GCNodeType.Blob) {
905
+ sweepPhaseStats.deletedAttachmentBlobCount++;
906
+ }
907
+ }
908
+ return sweepPhaseStats;
772
909
  }
773
910
  }
774
911
  exports.GarbageCollector = GarbageCollector;