@fluidframework/container-runtime 2.0.0-internal.7.2.2 → 2.0.0-internal.7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (302) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +1 -2
  3. package/api-extractor-lint.json +13 -0
  4. package/api-extractor.json +9 -1
  5. package/api-report/container-runtime.api.md +124 -107
  6. package/dist/batchTracker.d.ts +1 -0
  7. package/dist/batchTracker.d.ts.map +1 -1
  8. package/dist/blobManager.d.ts +4 -4
  9. package/dist/blobManager.d.ts.map +1 -1
  10. package/dist/blobManager.js.map +1 -1
  11. package/dist/connectionTelemetry.js +1 -1
  12. package/dist/connectionTelemetry.js.map +1 -1
  13. package/dist/container-runtime-alpha.d.ts +1473 -0
  14. package/dist/container-runtime-beta.d.ts +300 -0
  15. package/dist/container-runtime-public.d.ts +300 -0
  16. package/dist/container-runtime-untrimmed.d.ts +1836 -0
  17. package/dist/containerRuntime.d.ts +34 -40
  18. package/dist/containerRuntime.d.ts.map +1 -1
  19. package/dist/containerRuntime.js +79 -67
  20. package/dist/containerRuntime.js.map +1 -1
  21. package/dist/dataStoreRegistry.d.ts +1 -1
  22. package/dist/dataStoreRegistry.js +1 -1
  23. package/dist/dataStoreRegistry.js.map +1 -1
  24. package/dist/dataStores.d.ts +10 -15
  25. package/dist/dataStores.d.ts.map +1 -1
  26. package/dist/dataStores.js +77 -40
  27. package/dist/dataStores.js.map +1 -1
  28. package/dist/gc/garbageCollection.d.ts +41 -13
  29. package/dist/gc/garbageCollection.d.ts.map +1 -1
  30. package/dist/gc/garbageCollection.js +215 -78
  31. package/dist/gc/garbageCollection.js.map +1 -1
  32. package/dist/gc/gcConfigs.d.ts.map +1 -1
  33. package/dist/gc/gcConfigs.js +34 -37
  34. package/dist/gc/gcConfigs.js.map +1 -1
  35. package/dist/gc/gcDefinitions.d.ts +121 -46
  36. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  37. package/dist/gc/gcDefinitions.js +26 -18
  38. package/dist/gc/gcDefinitions.js.map +1 -1
  39. package/dist/gc/gcHelpers.d.ts +18 -25
  40. package/dist/gc/gcHelpers.d.ts.map +1 -1
  41. package/dist/gc/gcHelpers.js +29 -45
  42. package/dist/gc/gcHelpers.js.map +1 -1
  43. package/dist/gc/gcTelemetry.d.ts +0 -5
  44. package/dist/gc/gcTelemetry.d.ts.map +1 -1
  45. package/dist/gc/gcTelemetry.js +14 -42
  46. package/dist/gc/gcTelemetry.js.map +1 -1
  47. package/dist/gc/gcUnreferencedStateTracker.d.ts +11 -5
  48. package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
  49. package/dist/gc/gcUnreferencedStateTracker.js +43 -19
  50. package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
  51. package/dist/gc/index.d.ts +1 -1
  52. package/dist/gc/index.d.ts.map +1 -1
  53. package/dist/gc/index.js +4 -5
  54. package/dist/gc/index.js.map +1 -1
  55. package/dist/index.d.ts +14 -2
  56. package/dist/index.d.ts.map +1 -1
  57. package/dist/index.js +16 -5
  58. package/dist/index.js.map +1 -1
  59. package/dist/messageTypes.d.ts +16 -11
  60. package/dist/messageTypes.d.ts.map +1 -1
  61. package/dist/messageTypes.js +6 -1
  62. package/dist/messageTypes.js.map +1 -1
  63. package/dist/metadata.d.ts +6 -0
  64. package/dist/metadata.d.ts.map +1 -1
  65. package/dist/metadata.js.map +1 -1
  66. package/dist/opLifecycle/definitions.d.ts +1 -1
  67. package/dist/opLifecycle/definitions.js.map +1 -1
  68. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  69. package/dist/opLifecycle/opGroupingManager.js +10 -1
  70. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  71. package/dist/opLifecycle/outbox.d.ts +2 -0
  72. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  73. package/dist/opLifecycle/outbox.js +21 -0
  74. package/dist/opLifecycle/outbox.js.map +1 -1
  75. package/dist/packageVersion.d.ts +1 -1
  76. package/dist/packageVersion.js +1 -1
  77. package/dist/packageVersion.js.map +1 -1
  78. package/dist/pendingStateManager.d.ts +1 -1
  79. package/dist/pendingStateManager.d.ts.map +1 -1
  80. package/dist/pendingStateManager.js +2 -11
  81. package/dist/pendingStateManager.js.map +1 -1
  82. package/dist/scheduleManager.d.ts +1 -0
  83. package/dist/scheduleManager.d.ts.map +1 -1
  84. package/dist/summary/orderedClientElection.d.ts +1 -1
  85. package/dist/summary/orderedClientElection.js.map +1 -1
  86. package/dist/summary/runWhileConnectedCoordinator.d.ts +2 -2
  87. package/dist/summary/runWhileConnectedCoordinator.js +1 -1
  88. package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
  89. package/dist/summary/summarizer.d.ts +1 -1
  90. package/dist/summary/summarizer.js +1 -1
  91. package/dist/summary/summarizer.js.map +1 -1
  92. package/dist/summary/summarizerTypes.d.ts +30 -30
  93. package/dist/summary/summarizerTypes.js.map +1 -1
  94. package/dist/summary/summaryCollection.d.ts +10 -10
  95. package/dist/summary/summaryCollection.js +1 -1
  96. package/dist/summary/summaryCollection.js.map +1 -1
  97. package/dist/summary/summaryFormat.d.ts +3 -3
  98. package/dist/summary/summaryFormat.js.map +1 -1
  99. package/dist/tsdoc-metadata.json +1 -1
  100. package/lib/batchTracker.d.ts +1 -0
  101. package/lib/batchTracker.d.ts.map +1 -1
  102. package/lib/blobManager.d.ts +4 -4
  103. package/lib/blobManager.d.ts.map +1 -1
  104. package/lib/blobManager.js.map +1 -1
  105. package/lib/connectionTelemetry.js +1 -1
  106. package/lib/connectionTelemetry.js.map +1 -1
  107. package/lib/container-runtime-alpha.d.ts +1473 -0
  108. package/lib/container-runtime-beta.d.ts +300 -0
  109. package/lib/container-runtime-public.d.ts +300 -0
  110. package/lib/container-runtime-untrimmed.d.ts +1836 -0
  111. package/lib/containerRuntime.d.ts +34 -40
  112. package/lib/containerRuntime.d.ts.map +1 -1
  113. package/lib/containerRuntime.js +81 -69
  114. package/lib/containerRuntime.js.map +1 -1
  115. package/lib/dataStoreRegistry.d.ts +1 -1
  116. package/lib/dataStoreRegistry.js +1 -1
  117. package/lib/dataStoreRegistry.js.map +1 -1
  118. package/lib/dataStores.d.ts +10 -15
  119. package/lib/dataStores.d.ts.map +1 -1
  120. package/lib/dataStores.js +80 -43
  121. package/lib/dataStores.js.map +1 -1
  122. package/lib/gc/garbageCollection.d.ts +41 -13
  123. package/lib/gc/garbageCollection.d.ts.map +1 -1
  124. package/lib/gc/garbageCollection.js +217 -80
  125. package/lib/gc/garbageCollection.js.map +1 -1
  126. package/lib/gc/gcConfigs.d.ts.map +1 -1
  127. package/lib/gc/gcConfigs.js +37 -40
  128. package/lib/gc/gcConfigs.js.map +1 -1
  129. package/lib/gc/gcDefinitions.d.ts +121 -46
  130. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  131. package/lib/gc/gcDefinitions.js +25 -17
  132. package/lib/gc/gcDefinitions.js.map +1 -1
  133. package/lib/gc/gcHelpers.d.ts +18 -25
  134. package/lib/gc/gcHelpers.d.ts.map +1 -1
  135. package/lib/gc/gcHelpers.js +27 -43
  136. package/lib/gc/gcHelpers.js.map +1 -1
  137. package/lib/gc/gcTelemetry.d.ts +0 -5
  138. package/lib/gc/gcTelemetry.d.ts.map +1 -1
  139. package/lib/gc/gcTelemetry.js +15 -43
  140. package/lib/gc/gcTelemetry.js.map +1 -1
  141. package/lib/gc/gcUnreferencedStateTracker.d.ts +11 -5
  142. package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
  143. package/lib/gc/gcUnreferencedStateTracker.js +43 -19
  144. package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
  145. package/lib/gc/index.d.ts +1 -1
  146. package/lib/gc/index.d.ts.map +1 -1
  147. package/lib/gc/index.js +1 -1
  148. package/lib/gc/index.js.map +1 -1
  149. package/lib/index.d.ts +14 -2
  150. package/lib/index.d.ts.map +1 -1
  151. package/lib/index.js +15 -1
  152. package/lib/index.js.map +1 -1
  153. package/lib/messageTypes.d.ts +16 -11
  154. package/lib/messageTypes.d.ts.map +1 -1
  155. package/lib/messageTypes.js +6 -1
  156. package/lib/messageTypes.js.map +1 -1
  157. package/lib/metadata.d.ts +6 -0
  158. package/lib/metadata.d.ts.map +1 -1
  159. package/lib/metadata.js.map +1 -1
  160. package/lib/opLifecycle/definitions.d.ts +1 -1
  161. package/lib/opLifecycle/definitions.js.map +1 -1
  162. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  163. package/lib/opLifecycle/opGroupingManager.js +10 -1
  164. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  165. package/lib/opLifecycle/outbox.d.ts +2 -0
  166. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  167. package/lib/opLifecycle/outbox.js +21 -0
  168. package/lib/opLifecycle/outbox.js.map +1 -1
  169. package/lib/packageVersion.d.ts +1 -1
  170. package/lib/packageVersion.js +1 -1
  171. package/lib/packageVersion.js.map +1 -1
  172. package/lib/pendingStateManager.d.ts +1 -1
  173. package/lib/pendingStateManager.d.ts.map +1 -1
  174. package/lib/pendingStateManager.js +2 -11
  175. package/lib/pendingStateManager.js.map +1 -1
  176. package/lib/scheduleManager.d.ts +1 -0
  177. package/lib/scheduleManager.d.ts.map +1 -1
  178. package/lib/summary/orderedClientElection.d.ts +1 -1
  179. package/lib/summary/orderedClientElection.js.map +1 -1
  180. package/lib/summary/runWhileConnectedCoordinator.d.ts +2 -2
  181. package/lib/summary/runWhileConnectedCoordinator.js +1 -1
  182. package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
  183. package/lib/summary/summarizer.d.ts +1 -1
  184. package/lib/summary/summarizer.js +1 -1
  185. package/lib/summary/summarizer.js.map +1 -1
  186. package/lib/summary/summarizerTypes.d.ts +30 -30
  187. package/lib/summary/summarizerTypes.js.map +1 -1
  188. package/lib/summary/summaryCollection.d.ts +10 -10
  189. package/lib/summary/summaryCollection.js +1 -1
  190. package/lib/summary/summaryCollection.js.map +1 -1
  191. package/lib/summary/summaryFormat.d.ts +3 -3
  192. package/lib/summary/summaryFormat.js.map +1 -1
  193. package/package.json +56 -29
  194. package/src/blobManager.ts +5 -5
  195. package/src/connectionTelemetry.ts +1 -1
  196. package/src/containerRuntime.ts +113 -90
  197. package/src/dataStoreRegistry.ts +1 -1
  198. package/src/dataStores.ts +140 -69
  199. package/src/gc/garbageCollection.md +14 -15
  200. package/src/gc/garbageCollection.ts +256 -96
  201. package/src/gc/gcConfigs.ts +50 -52
  202. package/src/gc/gcDefinitions.ts +137 -52
  203. package/src/gc/gcHelpers.ts +31 -52
  204. package/src/gc/gcTelemetry.ts +16 -57
  205. package/src/gc/gcUnreferencedStateTracker.ts +61 -22
  206. package/src/gc/index.ts +6 -4
  207. package/src/index.ts +19 -1
  208. package/src/messageTypes.ts +21 -11
  209. package/src/metadata.ts +7 -0
  210. package/src/opLifecycle/definitions.ts +1 -1
  211. package/src/opLifecycle/opGroupingManager.ts +10 -1
  212. package/src/opLifecycle/outbox.ts +34 -0
  213. package/src/packageVersion.ts +1 -1
  214. package/src/pendingStateManager.ts +3 -13
  215. package/src/summary/orderedClientElection.ts +1 -1
  216. package/src/summary/runWhileConnectedCoordinator.ts +2 -2
  217. package/src/summary/summarizer.ts +1 -1
  218. package/src/summary/summarizerTypes.ts +30 -30
  219. package/src/summary/summaryCollection.ts +10 -10
  220. package/src/summary/summaryFormat.ts +3 -3
  221. package/dist/id-compressor/appendOnlySortedMap.d.ts +0 -124
  222. package/dist/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
  223. package/dist/id-compressor/appendOnlySortedMap.js +0 -318
  224. package/dist/id-compressor/appendOnlySortedMap.js.map +0 -1
  225. package/dist/id-compressor/finalSpace.d.ts +0 -29
  226. package/dist/id-compressor/finalSpace.d.ts.map +0 -1
  227. package/dist/id-compressor/finalSpace.js +0 -62
  228. package/dist/id-compressor/finalSpace.js.map +0 -1
  229. package/dist/id-compressor/idCompressor.d.ts +0 -54
  230. package/dist/id-compressor/idCompressor.d.ts.map +0 -1
  231. package/dist/id-compressor/idCompressor.js +0 -495
  232. package/dist/id-compressor/idCompressor.js.map +0 -1
  233. package/dist/id-compressor/identifiers.d.ts +0 -32
  234. package/dist/id-compressor/identifiers.d.ts.map +0 -1
  235. package/dist/id-compressor/identifiers.js +0 -15
  236. package/dist/id-compressor/identifiers.js.map +0 -1
  237. package/dist/id-compressor/index.d.ts +0 -13
  238. package/dist/id-compressor/index.d.ts.map +0 -1
  239. package/dist/id-compressor/index.js +0 -32
  240. package/dist/id-compressor/index.js.map +0 -1
  241. package/dist/id-compressor/persistanceUtilities.d.ts +0 -22
  242. package/dist/id-compressor/persistanceUtilities.d.ts.map +0 -1
  243. package/dist/id-compressor/persistanceUtilities.js +0 -43
  244. package/dist/id-compressor/persistanceUtilities.js.map +0 -1
  245. package/dist/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
  246. package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
  247. package/dist/id-compressor/sessionSpaceNormalizer.js +0 -80
  248. package/dist/id-compressor/sessionSpaceNormalizer.js.map +0 -1
  249. package/dist/id-compressor/sessions.d.ts +0 -115
  250. package/dist/id-compressor/sessions.d.ts.map +0 -1
  251. package/dist/id-compressor/sessions.js +0 -305
  252. package/dist/id-compressor/sessions.js.map +0 -1
  253. package/dist/id-compressor/utilities.d.ts +0 -52
  254. package/dist/id-compressor/utilities.d.ts.map +0 -1
  255. package/dist/id-compressor/utilities.js +0 -169
  256. package/dist/id-compressor/utilities.js.map +0 -1
  257. package/lib/id-compressor/appendOnlySortedMap.d.ts +0 -124
  258. package/lib/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
  259. package/lib/id-compressor/appendOnlySortedMap.js +0 -314
  260. package/lib/id-compressor/appendOnlySortedMap.js.map +0 -1
  261. package/lib/id-compressor/finalSpace.d.ts +0 -29
  262. package/lib/id-compressor/finalSpace.d.ts.map +0 -1
  263. package/lib/id-compressor/finalSpace.js +0 -58
  264. package/lib/id-compressor/finalSpace.js.map +0 -1
  265. package/lib/id-compressor/idCompressor.d.ts +0 -54
  266. package/lib/id-compressor/idCompressor.d.ts.map +0 -1
  267. package/lib/id-compressor/idCompressor.js +0 -491
  268. package/lib/id-compressor/idCompressor.js.map +0 -1
  269. package/lib/id-compressor/identifiers.d.ts +0 -32
  270. package/lib/id-compressor/identifiers.d.ts.map +0 -1
  271. package/lib/id-compressor/identifiers.js +0 -11
  272. package/lib/id-compressor/identifiers.js.map +0 -1
  273. package/lib/id-compressor/index.d.ts +0 -13
  274. package/lib/id-compressor/index.d.ts.map +0 -1
  275. package/lib/id-compressor/index.js +0 -13
  276. package/lib/id-compressor/index.js.map +0 -1
  277. package/lib/id-compressor/persistanceUtilities.d.ts +0 -22
  278. package/lib/id-compressor/persistanceUtilities.d.ts.map +0 -1
  279. package/lib/id-compressor/persistanceUtilities.js +0 -34
  280. package/lib/id-compressor/persistanceUtilities.js.map +0 -1
  281. package/lib/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
  282. package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
  283. package/lib/id-compressor/sessionSpaceNormalizer.js +0 -76
  284. package/lib/id-compressor/sessionSpaceNormalizer.js.map +0 -1
  285. package/lib/id-compressor/sessions.d.ts +0 -115
  286. package/lib/id-compressor/sessions.d.ts.map +0 -1
  287. package/lib/id-compressor/sessions.js +0 -290
  288. package/lib/id-compressor/sessions.js.map +0 -1
  289. package/lib/id-compressor/utilities.d.ts +0 -52
  290. package/lib/id-compressor/utilities.d.ts.map +0 -1
  291. package/lib/id-compressor/utilities.js +0 -151
  292. package/lib/id-compressor/utilities.js.map +0 -1
  293. package/src/id-compressor/README.md +0 -3
  294. package/src/id-compressor/appendOnlySortedMap.ts +0 -366
  295. package/src/id-compressor/finalSpace.ts +0 -67
  296. package/src/id-compressor/idCompressor.ts +0 -630
  297. package/src/id-compressor/identifiers.ts +0 -42
  298. package/src/id-compressor/index.ts +0 -26
  299. package/src/id-compressor/persistanceUtilities.ts +0 -58
  300. package/src/id-compressor/sessionSpaceNormalizer.ts +0 -83
  301. package/src/id-compressor/sessions.ts +0 -405
  302. package/src/id-compressor/utilities.ts +0 -190
@@ -14,7 +14,6 @@ import {
14
14
  import { RuntimeHeaderData } from "../containerRuntime";
15
15
  import { ICreateContainerMetadata } from "../summary";
16
16
  import {
17
- disableSweepLogKey,
18
17
  GCNodeType,
19
18
  UnreferencedState,
20
19
  IGarbageCollectorConfigs,
@@ -143,6 +142,21 @@ export class GCTelemetryTracker {
143
142
 
144
143
  const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
145
144
  const nodeType = this.getNodeType(nodeUsageProps.id);
145
+ const timeout = (() => {
146
+ switch (nodeStateTracker?.state) {
147
+ case UnreferencedState.Inactive:
148
+ return this.configs.inactiveTimeoutMs;
149
+ case UnreferencedState.TombstoneReady:
150
+ return this.configs.sweepTimeoutMs;
151
+ case UnreferencedState.SweepReady:
152
+ return (
153
+ this.configs.sweepTimeoutMs &&
154
+ this.configs.sweepTimeoutMs + this.configs.sweepGracePeriodMs
155
+ );
156
+ default:
157
+ return undefined;
158
+ }
159
+ })();
146
160
  const {
147
161
  usageType,
148
162
  currentReferenceTimestampMs,
@@ -160,10 +174,7 @@ export class GCTelemetryTracker {
160
174
  ? nodeUsageProps.currentReferenceTimestampMs -
161
175
  nodeStateTracker.unreferencedTimestampMs
162
176
  : -1,
163
- timeout:
164
- nodeStateTracker?.state === UnreferencedState.Inactive
165
- ? this.configs.inactiveTimeoutMs
166
- : this.configs.sweepTimeoutMs,
177
+ timeout,
167
178
  ...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
168
179
  ...propsToLog,
169
180
  ...this.createContainerMetadata,
@@ -392,58 +403,6 @@ export class GCTelemetryTracker {
392
403
  }
393
404
  this.pendingEventsQueue = [];
394
405
  }
395
-
396
- /**
397
- * For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
398
- * this will give us a view into how much deleted content a container has.
399
- */
400
- public logSweepEvents(
401
- logger: ITelemetryLoggerExt,
402
- currentReferenceTimestampMs: number,
403
- unreferencedNodesState: Map<string, UnreferencedStateTracker>,
404
- completedGCRuns: number,
405
- lastSummaryTime?: number,
406
- ) {
407
- if (
408
- this.mc.config.getBoolean(disableSweepLogKey) === true ||
409
- this.configs.sweepTimeoutMs === undefined
410
- ) {
411
- return;
412
- }
413
-
414
- const deletedNodeIds: string[] = [];
415
- for (const [nodeId, nodeStateTracker] of unreferencedNodesState) {
416
- if (nodeStateTracker.state !== UnreferencedState.SweepReady) {
417
- return;
418
- }
419
-
420
- const nodeType = this.getNodeType(nodeId);
421
- if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
422
- return;
423
- }
424
-
425
- // Log deleted event for each node only once to reduce noise in telemetry.
426
- const uniqueEventId = `Deleted-${nodeId}`;
427
- if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
428
- return;
429
- }
430
- this.loggedUnreferencedEvents.add(uniqueEventId);
431
- deletedNodeIds.push(nodeId);
432
- }
433
-
434
- if (deletedNodeIds.length > 0) {
435
- logger.sendTelemetryEvent({
436
- eventName: "GC_SweepReadyObjects_Delete",
437
- details: JSON.stringify({
438
- timeout: this.configs.sweepTimeoutMs,
439
- completedGCRuns,
440
- lastSummaryTime,
441
- ...this.createContainerMetadata,
442
- }),
443
- ...tagCodeArtifacts({ id: JSON.stringify(deletedNodeIds) }),
444
- });
445
- }
446
- }
447
406
  }
448
407
 
449
408
  /**
@@ -4,6 +4,7 @@
4
4
  */
5
5
 
6
6
  import { assert, Timer } from "@fluidframework/core-utils";
7
+ import { validatePrecondition } from "@fluidframework/telemetry-utils";
7
8
  import { UnreferencedState } from "./gcDefinitions";
8
9
 
9
10
  /** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
@@ -26,7 +27,7 @@ class TimerWithNoDefaultTimeout extends Timer {
26
27
 
27
28
  /**
28
29
  * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
29
- * be deleted by the sweep phase.
30
+ * be tombstoned or deleted by the sweep phase.
30
31
  */
31
32
  export class UnreferencedStateTracker {
32
33
  private _state: UnreferencedState = UnreferencedState.Active;
@@ -36,6 +37,8 @@ export class UnreferencedStateTracker {
36
37
 
37
38
  /** Timer to indicate when an unreferenced object is considered Inactive */
38
39
  private readonly inactiveTimer: TimerWithNoDefaultTimeout;
40
+ /** Timer to indicate when an unreferenced object is Tombstone-Ready */
41
+ private readonly tombstoneTimer: TimerWithNoDefaultTimeout;
39
42
  /** Timer to indicate when an unreferenced object is Sweep-Ready */
40
43
  private readonly sweepTimer: TimerWithNoDefaultTimeout;
41
44
 
@@ -45,32 +48,49 @@ export class UnreferencedStateTracker {
45
48
  private readonly inactiveTimeoutMs: number,
46
49
  /** The current reference timestamp used to track how long this node has been unreferenced for. */
47
50
  currentReferenceTimestampMs: number,
48
- /** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
49
- private readonly sweepTimeoutMs: number | undefined,
51
+ /** The time after which node transitions to TombstoneReady state; undefined if session expiry is disabled. */
52
+ private readonly tombstoneTimeoutMs: number | undefined,
53
+ /** The delay from TombstoneReady to SweepReady (only applies if tombstoneTimeoutMs is defined) */
54
+ private readonly sweepGracePeriodMs: number,
50
55
  ) {
51
- if (this.sweepTimeoutMs !== undefined) {
52
- assert(
53
- this.inactiveTimeoutMs <= this.sweepTimeoutMs,
54
- 0x3b0 /* inactive timeout must not be greater than the sweep timeout */,
55
- );
56
- }
56
+ validatePrecondition(
57
+ this.tombstoneTimeoutMs === undefined ||
58
+ this.tombstoneTimeoutMs >= this.inactiveTimeoutMs,
59
+ "inactiveTimeoutMs must not be greater than the tombstoneTimeoutMs",
60
+ );
57
61
 
58
62
  this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
59
63
  this._state = UnreferencedState.SweepReady;
60
64
  assert(
61
- !this.inactiveTimer.hasTimer,
62
- 0x3b1 /* inactiveTimer still running after sweepTimer fired! */,
65
+ !this.inactiveTimer.hasTimer && !this.tombstoneTimer.hasTimer,
66
+ 0x863 /* inactiveTimer or tombstoneTimer still running after sweepTimer fired! */,
63
67
  );
64
68
  });
65
69
 
70
+ this.tombstoneTimer = new TimerWithNoDefaultTimeout(() => {
71
+ this._state = UnreferencedState.TombstoneReady;
72
+ assert(
73
+ !this.inactiveTimer.hasTimer,
74
+ 0x864 /* inactiveTimer still running after tombstoneTimer fired! */,
75
+ ); // aka 0x3b1
76
+
77
+ if (this.sweepGracePeriodMs > 0) {
78
+ // After the node becomes tombstone ready, start the sweep timer after which the node will be ready for sweep.
79
+ this.sweepTimer.restart(this.sweepGracePeriodMs);
80
+ } else {
81
+ this._state = UnreferencedState.SweepReady;
82
+ }
83
+ });
84
+
66
85
  this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
67
86
  this._state = UnreferencedState.Inactive;
68
87
 
69
- // After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
70
- if (this.sweepTimeoutMs !== undefined) {
71
- this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
88
+ // After the node becomes inactive, start the tombstone timer after which the node will be ready for tombstone.
89
+ if (this.tombstoneTimeoutMs !== undefined) {
90
+ this.tombstoneTimer.restart(this.tombstoneTimeoutMs - this.inactiveTimeoutMs);
72
91
  }
73
92
  });
93
+
74
94
  this.updateTracking(currentReferenceTimestampMs);
75
95
  }
76
96
 
@@ -78,21 +98,39 @@ export class UnreferencedStateTracker {
78
98
  public updateTracking(currentReferenceTimestampMs: number) {
79
99
  const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
80
100
 
81
- // If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
82
- if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
101
+ // Below we will set the appropriate timer (or none). Any running timers are superceded by the new currentReferenceTimestampMs
102
+ this.clearTimers();
103
+
104
+ // If the node has been unreferenced long enough, update the state to SweepReady.
105
+ if (
106
+ this.tombstoneTimeoutMs !== undefined &&
107
+ unreferencedDurationMs >= this.tombstoneTimeoutMs + this.sweepGracePeriodMs
108
+ ) {
83
109
  this._state = UnreferencedState.SweepReady;
84
- this.clearTimers();
85
110
  return;
86
111
  }
87
112
 
88
- // If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
89
- // Also, start a timer for the sweep timeout.
113
+ // If the node has been unreferenced long enough, update the state to TombstoneReady.
114
+ // Also, start a timer for the remainder of the sweep delay.
115
+ if (
116
+ this.tombstoneTimeoutMs !== undefined &&
117
+ unreferencedDurationMs >= this.tombstoneTimeoutMs
118
+ ) {
119
+ this._state = UnreferencedState.TombstoneReady;
120
+
121
+ this.sweepTimer.restart(
122
+ this.tombstoneTimeoutMs + this.sweepGracePeriodMs - unreferencedDurationMs,
123
+ );
124
+ return;
125
+ }
126
+
127
+ // If the node has been unreferenced for long enough, update the state to inactive.
128
+ // Also, start a timer for the remainder of the tombstone timeout.
90
129
  if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
91
130
  this._state = UnreferencedState.Inactive;
92
- this.inactiveTimer.clear();
93
131
 
94
- if (this.sweepTimeoutMs !== undefined) {
95
- this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
132
+ if (this.tombstoneTimeoutMs !== undefined) {
133
+ this.tombstoneTimer.restart(this.tombstoneTimeoutMs - unreferencedDurationMs);
96
134
  }
97
135
  return;
98
136
  }
@@ -103,6 +141,7 @@ export class UnreferencedStateTracker {
103
141
 
104
142
  private clearTimers() {
105
143
  this.inactiveTimer.clear();
144
+ this.tombstoneTimer.clear();
106
145
  this.sweepTimer.clear();
107
146
  }
108
147
 
package/src/gc/index.ts CHANGED
@@ -7,13 +7,12 @@ export { GarbageCollector } from "./garbageCollection";
7
7
  export {
8
8
  nextGCVersion,
9
9
  defaultInactiveTimeoutMs,
10
+ defaultSweepGracePeriodMs,
10
11
  defaultSessionExpiryDurationMs,
11
- disableSweepLogKey,
12
12
  GCNodeType,
13
13
  gcTestModeKey,
14
- gcTombstoneGenerationOptionName,
15
- gcThrowOnTombstoneLoadOptionName,
16
- gcSweepGenerationOptionName,
14
+ gcDisableThrowOnTombstoneLoadOptionName,
15
+ gcGenerationOptionName,
17
16
  GCFeatureMatrix,
18
17
  GCVersion,
19
18
  gcVersionUpgradeToV4Key,
@@ -24,6 +23,8 @@ export {
24
23
  IGCMetadata,
25
24
  IGCResult,
26
25
  IGCRuntimeOptions,
26
+ IMarkPhaseStats,
27
+ ISweepPhaseStats,
27
28
  IGCStats,
28
29
  oneDayMs,
29
30
  runGCKey,
@@ -34,6 +35,7 @@ export {
34
35
  disableDatastoreSweepKey,
35
36
  UnreferencedState,
36
37
  throwOnTombstoneLoadOverrideKey,
38
+ GarbageCollectionMessage,
37
39
  } from "./gcDefinitions";
38
40
  export {
39
41
  cloneGCData,
package/src/index.ts CHANGED
@@ -40,6 +40,8 @@ export {
40
40
  GCFeatureMatrix,
41
41
  GCVersion,
42
42
  IGCRuntimeOptions,
43
+ IMarkPhaseStats,
44
+ ISweepPhaseStats,
43
45
  IGCStats,
44
46
  } from "./gc";
45
47
  export {
@@ -91,5 +93,21 @@ export {
91
93
  IRetriableFailureResult,
92
94
  ISummarizeEventProps,
93
95
  } from "./summary";
94
- export { isStableId, generateStableId, assertIsStableId } from "./id-compressor";
95
96
  export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";
97
+
98
+ // Re-exports for backwards compatibility.
99
+ // Will be removed in the future.
100
+ export {
101
+ /**
102
+ * @deprecated Import from `@fluidframework/id-compressor` instead.
103
+ */
104
+ assertIsStableId,
105
+ /**
106
+ * @deprecated Import from `@fluidframework/id-compressor` instead.
107
+ */
108
+ generateStableId,
109
+ /**
110
+ * @deprecated Import from `@fluidframework/id-compressor` instead.
111
+ */
112
+ isStableId,
113
+ } from "@fluidframework/id-compressor";
@@ -8,14 +8,14 @@ import {
8
8
  IEnvelope,
9
9
  InboundAttachMessage,
10
10
  IAttachMessage,
11
- IdCreationRangeWithStashedState,
12
- IdCreationRange,
13
11
  } from "@fluidframework/runtime-definitions";
12
+ import type { IdCreationRange } from "@fluidframework/id-compressor";
14
13
  import { IDataStoreAliasMessage } from "./dataStore";
14
+ import { GarbageCollectionMessage } from "./gc";
15
15
  import { IChunkedOp } from "./opLifecycle";
16
16
 
17
17
  /**
18
- * @public
18
+ * @internal
19
19
  */
20
20
  export enum ContainerMessageType {
21
21
  // An op to be delivered to store
@@ -42,6 +42,12 @@ export enum ContainerMessageType {
42
42
  * See the [IdCompressor README](./id-compressor/README.md) for more details.
43
43
  */
44
44
  IdAllocation = "idAllocation",
45
+
46
+ /**
47
+ * Garbage collection specific op. This is sent by the summarizer client when GC runs. It's used to synchronize GC
48
+ * state across all clients.
49
+ */
50
+ GC = "GC",
45
51
  }
46
52
 
47
53
  /**
@@ -72,7 +78,8 @@ export interface IContainerRuntimeMessageCompatDetails {
72
78
  * IMPORTANT: when creating one to be serialized, set the properties in the order they appear here.
73
79
  * This way stringified values can be compared.
74
80
  */
75
- interface TypedContainerRuntimeMessage<TType extends ContainerMessageType, TContents> {
81
+ interface TypedContainerRuntimeMessage<TType extends ContainerMessageType, TContents>
82
+ extends Partial<RecentlyAddedContainerRuntimeMessageDetails> {
76
83
  /** Type of the op, within the ContainerRuntime's domain */
77
84
  type: TType;
78
85
  /** Domain-specific contents, interpreted according to the type */
@@ -116,13 +123,13 @@ export type ContainerRuntimeAliasMessage = TypedContainerRuntimeMessage<
116
123
  ContainerMessageType.Alias,
117
124
  IDataStoreAliasMessage
118
125
  >;
119
- export type LocalContainerRuntimeIdAllocationMessage = TypedContainerRuntimeMessage<
120
- ContainerMessageType.IdAllocation,
121
- IdCreationRangeWithStashedState
122
- >;
123
126
  export type ContainerRuntimeIdAllocationMessage = TypedContainerRuntimeMessage<
124
127
  ContainerMessageType.IdAllocation,
125
- IdCreationRange & { stashedState?: never }
128
+ IdCreationRange
129
+ >;
130
+ export type ContainerRuntimeGCMessage = TypedContainerRuntimeMessage<
131
+ ContainerMessageType.GC,
132
+ GarbageCollectionMessage
126
133
  >;
127
134
 
128
135
  /**
@@ -152,6 +159,7 @@ export type InboundContainerRuntimeMessage =
152
159
  | ContainerRuntimeRejoinMessage
153
160
  | ContainerRuntimeAliasMessage
154
161
  | ContainerRuntimeIdAllocationMessage
162
+ | ContainerRuntimeGCMessage
155
163
  // Inbound messages may include unknown types from other clients, so we include that as a special case here
156
164
  | UnknownContainerRuntimeMessage;
157
165
 
@@ -163,7 +171,8 @@ export type LocalContainerRuntimeMessage =
163
171
  | ContainerRuntimeBlobAttachMessage
164
172
  | ContainerRuntimeRejoinMessage
165
173
  | ContainerRuntimeAliasMessage
166
- | LocalContainerRuntimeIdAllocationMessage
174
+ | ContainerRuntimeIdAllocationMessage
175
+ | ContainerRuntimeGCMessage
167
176
  // In rare cases (e.g. related to stashed ops) we could have a local message of an unknown type
168
177
  | UnknownContainerRuntimeMessage;
169
178
 
@@ -175,7 +184,8 @@ export type OutboundContainerRuntimeMessage =
175
184
  | ContainerRuntimeBlobAttachMessage
176
185
  | ContainerRuntimeRejoinMessage
177
186
  | ContainerRuntimeAliasMessage
178
- | ContainerRuntimeIdAllocationMessage;
187
+ | ContainerRuntimeIdAllocationMessage
188
+ | ContainerRuntimeGCMessage;
179
189
 
180
190
  /**
181
191
  * An unpacked ISequencedDocumentMessage with the inner TypedContainerRuntimeMessage type/contents/etc
package/src/metadata.ts CHANGED
@@ -17,3 +17,10 @@ export interface IBlobMetadata {
17
17
  blobId?: string;
18
18
  localId?: string;
19
19
  }
20
+
21
+ /**
22
+ * The IdCompressor needs to know if this is a replayed savedOp as those need to be skipped in stashed ops scenarios.
23
+ */
24
+ export interface IIdAllocationMetadata {
25
+ savedOp?: boolean;
26
+ }
@@ -53,7 +53,7 @@ export interface IBatchCheckpoint {
53
53
  }
54
54
 
55
55
  /**
56
- * @public
56
+ * @internal
57
57
  */
58
58
  export interface IChunkedOp {
59
59
  chunkId: number;
@@ -96,12 +96,21 @@ export class OpGroupingManager {
96
96
  }
97
97
 
98
98
  public ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
99
+ let fakeCsn = 1;
99
100
  if (!isGroupContents(op.contents)) {
101
+ // Align the worlds of what clientSequenceNumber represents when grouped batching is enabled
102
+ if (this.config.groupedBatchingEnabled) {
103
+ return [
104
+ {
105
+ ...op,
106
+ clientSequenceNumber: fakeCsn,
107
+ },
108
+ ];
109
+ }
100
110
  return [op];
101
111
  }
102
112
 
103
113
  const messages = op.contents.contents;
104
- let fakeCsn = 1;
105
114
  return messages.map((subMessage) => ({
106
115
  ...op,
107
116
  clientSequenceNumber: fakeCsn++,
@@ -89,6 +89,7 @@ export class Outbox {
89
89
  private readonly attachFlowBatch: BatchManager;
90
90
  private readonly mainBatch: BatchManager;
91
91
  private readonly blobAttachBatch: BatchManager;
92
+ private readonly idAllocationBatch: BatchManager;
92
93
  private readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;
93
94
  private batchRebasesToReport = 5;
94
95
  private rebasing = false;
@@ -114,6 +115,7 @@ export class Outbox {
114
115
  this.attachFlowBatch = new BatchManager({ hardLimit, softLimit });
115
116
  this.mainBatch = new BatchManager({ hardLimit });
116
117
  this.blobAttachBatch = new BatchManager({ hardLimit });
118
+ this.idAllocationBatch = new BatchManager({ hardLimit });
117
119
  }
118
120
 
119
121
  public get messageCount(): number {
@@ -230,6 +232,37 @@ export class Outbox {
230
232
  }
231
233
  }
232
234
 
235
+ public submitIdAllocation(message: BatchMessage) {
236
+ this.maybeFlushPartialBatch();
237
+
238
+ if (
239
+ !this.idAllocationBatch.push(
240
+ message,
241
+ this.isContextReentrant(),
242
+ this.params.getCurrentSequenceNumbers().clientSequenceNumber,
243
+ )
244
+ ) {
245
+ // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
246
+ // when queue is not empty.
247
+ // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
248
+ this.flushInternal(this.idAllocationBatch);
249
+
250
+ this.addMessageToBatchManager(this.idAllocationBatch, message);
251
+ }
252
+
253
+ // If compression is enabled, we will always successfully receive
254
+ // attach ops and compress then send them at the next JS turn, regardless
255
+ // of the overall size of the accumulated ops in the batch.
256
+ // However, it is more efficient to flush these ops faster, preferably
257
+ // after they reach a size which would benefit from compression.
258
+ if (
259
+ this.idAllocationBatch.contentSizeInBytes >=
260
+ this.params.config.compressionOptions.minimumBatchSizeInBytes
261
+ ) {
262
+ this.flushInternal(this.idAllocationBatch);
263
+ }
264
+ }
265
+
233
266
  private addMessageToBatchManager(batchManager: BatchManager, message: BatchMessage) {
234
267
  if (
235
268
  !batchManager.push(
@@ -258,6 +291,7 @@ export class Outbox {
258
291
  }
259
292
 
260
293
  private flushAll() {
294
+ this.flushInternal(this.idAllocationBatch);
261
295
  this.flushInternal(this.attachFlowBatch);
262
296
  this.flushInternal(this.blobAttachBatch, true /* disableGroupedBatching */);
263
297
  this.flushInternal(this.mainBatch);
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "2.0.0-internal.7.2.2";
9
+ export const pkgVersion = "2.0.0-internal.7.4.0";
@@ -11,7 +11,7 @@ import { ICriticalContainerError } from "@fluidframework/container-definitions";
11
11
  import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
12
12
  import { DataProcessingError, ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
13
13
 
14
- import { ContainerMessageType, InboundSequencedContainerRuntimeMessage } from "./messageTypes";
14
+ import { InboundSequencedContainerRuntimeMessage } from "./messageTypes";
15
15
  import { pkgVersion } from "./packageVersion";
16
16
  import { IBatchMetadata } from "./metadata";
17
17
 
@@ -21,7 +21,6 @@ import { IBatchMetadata } from "./metadata";
21
21
  */
22
22
  export interface IPendingMessage {
23
23
  type: "message";
24
- clientSequenceNumber: number;
25
24
  referenceSequenceNumber: number;
26
25
  content: string;
27
26
  localOpMetadata: unknown;
@@ -128,18 +127,9 @@ export class PendingStateManager implements IDisposable {
128
127
  return {
129
128
  pendingStates: [...this.savedOps, ...this.pendingMessages.toArray()].map(
130
129
  (message) => {
131
- let content = message.content;
132
- const parsedContent = JSON.parse(content);
133
- // IdAllocations need their localOpMetadata stashed in the contents
134
- // of the op to correctly resume the session when processing stashed ops
135
- if (parsedContent.type === ContainerMessageType.IdAllocation) {
136
- parsedContent.contents.stashedState = message.localOpMetadata;
137
- content = JSON.stringify(parsedContent);
138
- }
139
-
140
130
  // delete localOpMetadata since it may not be serializable
141
131
  // and will be regenerated by applyStashedOp()
142
- return { ...message, content, localOpMetadata: undefined };
132
+ return { ...message, localOpMetadata: undefined };
143
133
  },
144
134
  ),
145
135
  };
@@ -176,7 +166,6 @@ export class PendingStateManager implements IDisposable {
176
166
  ) {
177
167
  const pendingMessage: IPendingMessage = {
178
168
  type: "message",
179
- clientSequenceNumber: -1, // dummy value (not to be used anywhere)
180
169
  referenceSequenceNumber,
181
170
  content,
182
171
  localOpMetadata,
@@ -342,6 +331,7 @@ export class PendingStateManager implements IDisposable {
342
331
  /**
343
332
  * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
344
333
  * states in its queue. This includes triggering resubmission of unacked ops.
334
+ * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
345
335
  */
346
336
  public replayPendingStates() {
347
337
  assert(
@@ -227,7 +227,7 @@ export interface IOrderedClientElectionEvents extends IEvent {
227
227
 
228
228
  /**
229
229
  * Serialized state of IOrderedClientElection.
230
- * @public
230
+ * @alpha
231
231
  */
232
232
  export interface ISerializedElection {
233
233
  /** Sequence number at the time of the latest election. */
@@ -12,7 +12,7 @@ import {
12
12
 
13
13
  /**
14
14
  * Similar to AbortController, but using promise instead of events
15
- * @public
15
+ * @internal
16
16
  */
17
17
  export interface ICancellableSummarizerController extends ISummaryCancellationToken {
18
18
  stop(reason: SummarizerStopReason): void;
@@ -21,7 +21,7 @@ export interface ICancellableSummarizerController extends ISummaryCancellationTo
21
21
  /**
22
22
  * Can be useful in testing as well as in places where caller does not use cancellation.
23
23
  * This object implements ISummaryCancellationToken interface but cancellation is never leveraged.
24
- * @public
24
+ * @internal
25
25
  */
26
26
  export const neverCancelledSummaryToken: ISummaryCancellationToken = {
27
27
  cancelled: false,
@@ -69,7 +69,7 @@ export const createSummarizingWarning = (errorMessage: string, logged: boolean)
69
69
  * Summarizer is responsible for coordinating when to generate and send summaries.
70
70
  * It is the main entry point for summary work.
71
71
  * It is created only by summarizing container (i.e. one with clientType === "summarizer")
72
- * @public
72
+ * @internal
73
73
  */
74
74
  export class Summarizer extends TypedEventEmitter<ISummarizerEvents> implements ISummarizer {
75
75
  public get ISummarizer() {