@fluidframework/container-runtime 2.0.0-internal.6.3.2 → 2.0.0-internal.6.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/dist/blobManager.d.ts +3 -2
  3. package/dist/blobManager.d.ts.map +1 -1
  4. package/dist/blobManager.js +29 -25
  5. package/dist/blobManager.js.map +1 -1
  6. package/dist/containerRuntime.d.ts +14 -69
  7. package/dist/containerRuntime.d.ts.map +1 -1
  8. package/dist/containerRuntime.js +155 -184
  9. package/dist/containerRuntime.js.map +1 -1
  10. package/dist/dataStoreContext.d.ts.map +1 -1
  11. package/dist/dataStoreContext.js +3 -1
  12. package/dist/dataStoreContext.js.map +1 -1
  13. package/dist/dataStores.d.ts +1 -1
  14. package/dist/dataStores.js +3 -3
  15. package/dist/dataStores.js.map +1 -1
  16. package/dist/gc/garbageCollection.d.ts +6 -3
  17. package/dist/gc/garbageCollection.d.ts.map +1 -1
  18. package/dist/gc/garbageCollection.js +13 -12
  19. package/dist/gc/garbageCollection.js.map +1 -1
  20. package/dist/gc/gcConfigs.js +1 -1
  21. package/dist/gc/gcConfigs.js.map +1 -1
  22. package/dist/gc/gcDefinitions.d.ts +17 -6
  23. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  24. package/dist/gc/gcDefinitions.js +19 -20
  25. package/dist/gc/gcDefinitions.js.map +1 -1
  26. package/dist/gc/gcHelpers.d.ts +0 -9
  27. package/dist/gc/gcHelpers.d.ts.map +1 -1
  28. package/dist/gc/gcHelpers.js +1 -13
  29. package/dist/gc/gcHelpers.js.map +1 -1
  30. package/dist/gc/gcTelemetry.d.ts.map +1 -1
  31. package/dist/gc/gcTelemetry.js +1 -4
  32. package/dist/gc/gcTelemetry.js.map +1 -1
  33. package/dist/gc/index.d.ts +2 -2
  34. package/dist/gc/index.d.ts.map +1 -1
  35. package/dist/gc/index.js +5 -6
  36. package/dist/gc/index.js.map +1 -1
  37. package/dist/index.d.ts +2 -1
  38. package/dist/index.d.ts.map +1 -1
  39. package/dist/index.js +3 -2
  40. package/dist/index.js.map +1 -1
  41. package/dist/messageTypes.d.ts +134 -0
  42. package/dist/messageTypes.d.ts.map +1 -0
  43. package/dist/messageTypes.js +29 -0
  44. package/dist/messageTypes.js.map +1 -0
  45. package/dist/opLifecycle/definitions.d.ts +2 -1
  46. package/dist/opLifecycle/definitions.d.ts.map +1 -1
  47. package/dist/opLifecycle/definitions.js.map +1 -1
  48. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
  49. package/dist/opLifecycle/opDecompressor.js +0 -4
  50. package/dist/opLifecycle/opDecompressor.js.map +1 -1
  51. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  52. package/dist/opLifecycle/opGroupingManager.js +4 -2
  53. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  54. package/dist/opLifecycle/opSplitter.js +3 -3
  55. package/dist/opLifecycle/opSplitter.js.map +1 -1
  56. package/dist/opLifecycle/remoteMessageProcessor.d.ts +17 -3
  57. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  58. package/dist/opLifecycle/remoteMessageProcessor.js +38 -25
  59. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
  60. package/dist/packageVersion.d.ts +1 -1
  61. package/dist/packageVersion.js +1 -1
  62. package/dist/packageVersion.js.map +1 -1
  63. package/dist/pendingStateManager.d.ts +2 -2
  64. package/dist/pendingStateManager.d.ts.map +1 -1
  65. package/dist/pendingStateManager.js +13 -6
  66. package/dist/pendingStateManager.js.map +1 -1
  67. package/dist/summary/runningSummarizer.d.ts +1 -1
  68. package/dist/summary/runningSummarizer.d.ts.map +1 -1
  69. package/dist/summary/runningSummarizer.js +4 -2
  70. package/dist/summary/runningSummarizer.js.map +1 -1
  71. package/dist/summary/summarizer.d.ts +2 -2
  72. package/dist/summary/summarizer.js +2 -2
  73. package/dist/summary/summarizer.js.map +1 -1
  74. package/dist/summary/summarizerNode/summarizerNode.d.ts +1 -1
  75. package/dist/summary/summarizerNode/summarizerNode.js +1 -1
  76. package/dist/summary/summarizerNode/summarizerNode.js.map +1 -1
  77. package/dist/summary/summarizerTypes.d.ts +2 -1
  78. package/dist/summary/summarizerTypes.d.ts.map +1 -1
  79. package/dist/summary/summarizerTypes.js.map +1 -1
  80. package/dist/summary/summaryFormat.d.ts +3 -0
  81. package/dist/summary/summaryFormat.d.ts.map +1 -1
  82. package/dist/summary/summaryFormat.js +3 -0
  83. package/dist/summary/summaryFormat.js.map +1 -1
  84. package/lib/blobManager.d.ts +3 -2
  85. package/lib/blobManager.d.ts.map +1 -1
  86. package/lib/blobManager.js +30 -26
  87. package/lib/blobManager.js.map +1 -1
  88. package/lib/containerRuntime.d.ts +14 -69
  89. package/lib/containerRuntime.d.ts.map +1 -1
  90. package/lib/containerRuntime.js +121 -150
  91. package/lib/containerRuntime.js.map +1 -1
  92. package/lib/dataStoreContext.d.ts.map +1 -1
  93. package/lib/dataStoreContext.js +3 -1
  94. package/lib/dataStoreContext.js.map +1 -1
  95. package/lib/dataStores.d.ts +1 -1
  96. package/lib/dataStores.js +4 -4
  97. package/lib/dataStores.js.map +1 -1
  98. package/lib/gc/garbageCollection.d.ts +6 -3
  99. package/lib/gc/garbageCollection.d.ts.map +1 -1
  100. package/lib/gc/garbageCollection.js +13 -12
  101. package/lib/gc/garbageCollection.js.map +1 -1
  102. package/lib/gc/gcConfigs.js +2 -2
  103. package/lib/gc/gcConfigs.js.map +1 -1
  104. package/lib/gc/gcDefinitions.d.ts +17 -6
  105. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  106. package/lib/gc/gcDefinitions.js +18 -19
  107. package/lib/gc/gcDefinitions.js.map +1 -1
  108. package/lib/gc/gcHelpers.d.ts +0 -9
  109. package/lib/gc/gcHelpers.d.ts.map +1 -1
  110. package/lib/gc/gcHelpers.js +0 -11
  111. package/lib/gc/gcHelpers.js.map +1 -1
  112. package/lib/gc/gcTelemetry.d.ts.map +1 -1
  113. package/lib/gc/gcTelemetry.js +1 -4
  114. package/lib/gc/gcTelemetry.js.map +1 -1
  115. package/lib/gc/index.d.ts +2 -2
  116. package/lib/gc/index.d.ts.map +1 -1
  117. package/lib/gc/index.js +2 -2
  118. package/lib/gc/index.js.map +1 -1
  119. package/lib/index.d.ts +2 -1
  120. package/lib/index.d.ts.map +1 -1
  121. package/lib/index.js +2 -1
  122. package/lib/index.js.map +1 -1
  123. package/lib/messageTypes.d.ts +134 -0
  124. package/lib/messageTypes.d.ts.map +1 -0
  125. package/lib/messageTypes.js +26 -0
  126. package/lib/messageTypes.js.map +1 -0
  127. package/lib/opLifecycle/definitions.d.ts +2 -1
  128. package/lib/opLifecycle/definitions.d.ts.map +1 -1
  129. package/lib/opLifecycle/definitions.js.map +1 -1
  130. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
  131. package/lib/opLifecycle/opDecompressor.js +0 -4
  132. package/lib/opLifecycle/opDecompressor.js.map +1 -1
  133. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  134. package/lib/opLifecycle/opGroupingManager.js +4 -2
  135. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  136. package/lib/opLifecycle/opSplitter.js +1 -1
  137. package/lib/opLifecycle/opSplitter.js.map +1 -1
  138. package/lib/opLifecycle/remoteMessageProcessor.d.ts +17 -3
  139. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  140. package/lib/opLifecycle/remoteMessageProcessor.js +37 -24
  141. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
  142. package/lib/packageVersion.d.ts +1 -1
  143. package/lib/packageVersion.js +1 -1
  144. package/lib/packageVersion.js.map +1 -1
  145. package/lib/pendingStateManager.d.ts +2 -2
  146. package/lib/pendingStateManager.d.ts.map +1 -1
  147. package/lib/pendingStateManager.js +12 -5
  148. package/lib/pendingStateManager.js.map +1 -1
  149. package/lib/summary/runningSummarizer.d.ts +1 -1
  150. package/lib/summary/runningSummarizer.d.ts.map +1 -1
  151. package/lib/summary/runningSummarizer.js +4 -2
  152. package/lib/summary/runningSummarizer.js.map +1 -1
  153. package/lib/summary/summarizer.d.ts +2 -2
  154. package/lib/summary/summarizer.js +2 -2
  155. package/lib/summary/summarizer.js.map +1 -1
  156. package/lib/summary/summarizerNode/summarizerNode.d.ts +1 -1
  157. package/lib/summary/summarizerNode/summarizerNode.js +1 -1
  158. package/lib/summary/summarizerNode/summarizerNode.js.map +1 -1
  159. package/lib/summary/summarizerTypes.d.ts +2 -1
  160. package/lib/summary/summarizerTypes.d.ts.map +1 -1
  161. package/lib/summary/summarizerTypes.js.map +1 -1
  162. package/lib/summary/summaryFormat.d.ts +3 -0
  163. package/lib/summary/summaryFormat.d.ts.map +1 -1
  164. package/lib/summary/summaryFormat.js +3 -0
  165. package/lib/summary/summaryFormat.js.map +1 -1
  166. package/package.json +16 -16
  167. package/src/blobManager.ts +38 -28
  168. package/src/containerRuntime.ts +181 -245
  169. package/src/dataStoreContext.ts +3 -1
  170. package/src/dataStores.ts +4 -4
  171. package/src/gc/garbageCollection.md +53 -5
  172. package/src/gc/garbageCollection.ts +13 -12
  173. package/src/gc/gcConfigs.ts +3 -3
  174. package/src/gc/gcDefinitions.ts +18 -19
  175. package/src/gc/gcEarlyAdoption.md +145 -0
  176. package/src/gc/gcHelpers.ts +0 -12
  177. package/src/gc/gcTelemetry.ts +1 -4
  178. package/src/gc/index.ts +4 -5
  179. package/src/index.ts +7 -4
  180. package/src/messageTypes.ts +225 -0
  181. package/src/opLifecycle/README.md +40 -40
  182. package/src/opLifecycle/definitions.ts +2 -1
  183. package/src/opLifecycle/opDecompressor.ts +0 -8
  184. package/src/opLifecycle/opGroupingManager.ts +7 -6
  185. package/src/opLifecycle/opSplitter.ts +2 -2
  186. package/src/opLifecycle/remoteMessageProcessor.ts +54 -33
  187. package/src/packageVersion.ts +1 -1
  188. package/src/pendingStateManager.ts +23 -6
  189. package/src/summary/runningSummarizer.ts +4 -2
  190. package/src/summary/summarizer.ts +2 -2
  191. package/src/summary/summarizerNode/summarizerNode.ts +1 -1
  192. package/src/summary/summarizerTypes.ts +2 -1
  193. package/src/summary/summaryFormat.ts +3 -0
@@ -314,7 +314,9 @@ export abstract class FluidDataStoreContext
314
314
  properties: {
315
315
  all: tagCodeArtifacts({
316
316
  fluidDataStoreId: this.id,
317
- fullPackageName: this.pkg?.join("/"),
317
+ // The package name is a getter because `this.pkg` may not be initialized during construction.
318
+ // For data stores loaded from summary, it is initialized during data store realization.
319
+ fullPackageName: () => this.pkg?.join("/"),
318
320
  }),
319
321
  },
320
322
  });
package/src/dataStores.ts CHANGED
@@ -67,7 +67,7 @@ import { StorageServiceWithAttachBlobs } from "./storageServiceWithAttachBlobs";
67
67
  import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
68
68
  import {
69
69
  GCNodeType,
70
- sweepDatastoresKey,
70
+ disableDatastoreSweepKey,
71
71
  throwOnTombstoneLoadKey,
72
72
  sendGCUnexpectedUsageEvent,
73
73
  } from "./gc";
@@ -212,7 +212,7 @@ export class DataStores implements IDisposable {
212
212
 
213
213
  public async waitIfPendingAlias(maybeAlias: string): Promise<AliasResult> {
214
214
  const pendingAliasPromise = this.pendingAliases.get(maybeAlias);
215
- return pendingAliasPromise === undefined ? "Success" : pendingAliasPromise;
215
+ return pendingAliasPromise ?? "Success";
216
216
  }
217
217
 
218
218
  public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
@@ -820,11 +820,11 @@ export class DataStores implements IDisposable {
820
820
  * Delete data stores and its objects that are sweep ready.
821
821
  * @param sweepReadyDataStoreRoutes - The routes of data stores and its objects that are sweep ready and should
822
822
  * be deleted.
823
- * @returns - The routes of data stores and its objects that were deleted.
823
+ * @returns The routes of data stores and its objects that were deleted.
824
824
  */
825
825
  public deleteSweepReadyNodes(sweepReadyDataStoreRoutes: string[]): string[] {
826
826
  // If sweep for data stores is not enabled, return empty list indicating nothing is deleted.
827
- if (this.mc.config.getBoolean(sweepDatastoresKey) !== true) {
827
+ if (this.mc.config.getBoolean(disableDatastoreSweepKey) === true) {
828
828
  return [];
829
829
  }
830
830
  for (const route of sweepReadyDataStoreRoutes) {
@@ -48,12 +48,60 @@ In this phase, the GC algorithm identifies all Fluid objects that are unreferenc
48
48
 
49
49
  Mark phase is enabled by default for a container. It is enabled during creation of the container runtime and remains enabled throughout its lifetime. Basically, this setting is persisted in the summary and cannot be changed.
50
50
 
51
- If you wish to disable this, set the `gcAllowed` option to `false` in `IGCRuntimeOptions`. These options are under `IContainerRuntimeOptions` and are passed to the container runtime during its creation. Note that this will disable GC permanently (including the sweep phase) for the container during its lifetime.
51
+ ### Sweep phase
52
52
 
53
- See `IGCRuntimeOptions` in [containerRuntime.ts](../containerRuntime.ts) for more options to control GC behavior.
53
+ In this phase, the GC algorithm identifies all Fluid objects that have been unreferenced for a specific amount of time (typically 30-40 days) and deletes them.
54
+ Objects are only swept once the GC system is sure that they could never be referenced again by any active clients, i.e., clients that have the object in memory and could reference it.
55
+ The Fluid Runtime enforces a maximum session length (configurable) in order to guarantee an object is safe to delete after sufficient time has elapsed.
54
56
 
55
- ### Sweep phase
57
+ GC sweep phase has not been enabled by default yet. A "soft" version of Sweep called "Tombstone Mode" is enabled by default
58
+ as part of the Mark Phase when Sweep is disabled. In this mode, any object that GC determines is ready to be deleted is
59
+ marked as a "Tombstone", which triggers certain logging events and/or behavior changes if/when that Tombstoned object is
60
+ accessed by the application.
61
+
62
+ Tombstone is intended for use by early adopters of GC and is documented in more detail [here](./gcEarlyAdoption.md).
63
+
64
+ ## GC Configuration
65
+
66
+ The default configuration for GC today is:
67
+
68
+ - GC Mark Phase is **enabled**, including Tombstone Mode
69
+ - Session Expiry is **enabled**
70
+ - GC Sweep Phase is **disabled**
71
+ - Note: Once enabled, Sweep will only run for documents created from that point forward
72
+
73
+ ### Techniques used for configuration
74
+
75
+ There are two ways to configure the Fluid Framework's GC behavior, referred to by name throughout these documents:
76
+
77
+ 1. **"GC Options"**: `ContainerRuntime.loadRuntime` takes an options value of type `IContainerRuntimeOptions`.
78
+ This type includes a sub-object `gcOptions`, for GC-specific options.
79
+ 2. **"Config Settings"**: The `Loader`'s constructor takes in `ILoaderProps`, which includes `configProvider?: IConfigProviderBase`
80
+ This configProvider can be used to inject config settings.
81
+
82
+ Typically GC Options are used for more "official" and stable configuration, whereas Config Settings provide a mechanism
83
+ for apps to override settings easily, e.g. by backing their `IConfigProviderBase` with a configuration/flighting service.
84
+ In cases where a behavior is controlled by both a Config Setting and GC Option, you may experiment at first using Config Settings
85
+ and then later update the passed-in GC Options to finalize the configuration in your code.
86
+
87
+ ### Disabling Mark Phase
88
+
89
+ If you wish to disable Mark Phase for newly-created documents, set the `gcAllowed` GC Option to `false`.
90
+ Note that this will disable GC permanently (including the sweep phase) for the container during its lifetime.
91
+
92
+ Mark Phase can also be disabled just for the session, among other behaviors,
93
+ covered in the [Advanced Configuration](./gcEarlyAdoption.md#more-advanced-configurations) docs.
94
+
95
+ ### Enabling Sweep Phase
96
+
97
+ To enable Sweep Phase for new documents, you must set the `gcSweepGeneration` GC Option to a number, e.g. 0 to start.
98
+ The full semantics of this GC Option are discussed [here](./gcEarlyAdoption.md#more-about-gcsweepgeneration-and-gctombstonegeneration).
99
+ Note that this will disabled Tombstone Mode.
100
+
101
+ A full treatment of Tombstone and Sweep configuration can be found in
102
+ [this companion document geared towards early adopters of GC](./gcEarlyAdoption.md).
56
103
 
57
- In this phase, the GC algorithm identifies all Fluid objects that have been unreferenced for a specific amount of time (typically 30-40 days) and deletes them. Objects are only swept once the GC system is sure that they could never be referenced again by any active clients, i.e., clients that have the object in memory and could reference it.
104
+ ### More Advanced Configuration
58
105
 
59
- GC sweep phase has not been enabled yet. More details will be added here when sweep is enabled.
106
+ For additional behaviors that can be configured (e.g. for testing), please see these
107
+ [Advanced Configuration](./gcEarlyAdoption.md#more-advanced-configurations) docs.
@@ -287,15 +287,13 @@ export class GarbageCollector implements IGarbageCollector {
287
287
  return { gcData: { gcNodes }, usedRoutes };
288
288
  });
289
289
 
290
- // Log all the GC options and the state determined by the garbage collector. This is interesting only for the
291
- // summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
292
- if (this.isSummarizerClient) {
293
- this.mc.logger.sendTelemetryEvent({
294
- eventName: "GarbageCollectorLoaded",
295
- gcConfigs: JSON.stringify(this.configs),
296
- gcOptions: JSON.stringify(createParams.gcOptions),
297
- });
298
- }
290
+ // Log all the GC options and the state determined by the garbage collector.
291
+ // This is useful even for interactive clients since they track unreferenced nodes and log errors.
292
+ this.mc.logger.sendTelemetryEvent({
293
+ eventName: "GarbageCollectorLoaded",
294
+ gcConfigs: JSON.stringify(this.configs),
295
+ gcOptions: JSON.stringify(createParams.gcOptions),
296
+ });
299
297
  }
300
298
 
301
299
  /**
@@ -579,14 +577,17 @@ export class GarbageCollector implements IGarbageCollector {
579
577
 
580
578
  /**
581
579
  * Runs the GC Mark phase. It does the following:
580
+ *
582
581
  * 1. Marks all referenced nodes in this run by clearing tracking for them.
582
+ *
583
583
  * 2. Marks unreferenced nodes in this run by starting tracking for them.
584
+ *
584
585
  * 3. Calls the runtime to update nodes that were marked referenced.
585
586
  *
586
587
  * @param gcResult - The result of the GC run on the gcData.
587
588
  * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
588
589
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
589
- * @returns - A list of sweep ready nodes, i.e., nodes that ready to be deleted.
590
+ * @returns A list of sweep ready nodes, i.e., nodes that ready to be deleted.
590
591
  */
591
592
  private runMarkPhase(
592
593
  gcResult: IGCResult,
@@ -645,7 +646,7 @@ export class GarbageCollector implements IGarbageCollector {
645
646
  * @param sweepReadyNodes - List of nodes that are sweep ready.
646
647
  * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
647
648
  * @param logger - The logger to be used to log any telemetry.
648
- * @returns - A list of nodes that have been deleted.
649
+ * @returns A list of nodes that have been deleted.
649
650
  */
650
651
  private runSweepPhase(
651
652
  gcResult: IGCResult,
@@ -724,7 +725,7 @@ export class GarbageCollector implements IGarbageCollector {
724
725
  * This function identifies nodes that were referenced since the last run.
725
726
  * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
726
727
  *
727
- * @returns - a list of all nodes referenced from the last local summary until now.
728
+ * @returns A list of all nodes referenced from the last local summary until now.
728
729
  */
729
730
  private findAllNodesReferencedBetweenGCs(
730
731
  currentGCData: IGarbageCollectionData,
@@ -6,7 +6,7 @@
6
6
  import { MonitoringContext, UsageError } from "@fluidframework/telemetry-utils";
7
7
  import { IContainerRuntimeMetadata } from "../summary";
8
8
  import {
9
- currentGCVersion,
9
+ nextGCVersion,
10
10
  defaultInactiveTimeoutMs,
11
11
  defaultSessionExpiryDurationMs,
12
12
  disableTombstoneKey,
@@ -15,7 +15,7 @@ import {
15
15
  gcTestModeKey,
16
16
  gcTombstoneGenerationOptionName,
17
17
  GCVersion,
18
- gcVersionUpgradeToV3Key,
18
+ gcVersionUpgradeToV4Key,
19
19
  IGarbageCollectorConfigs,
20
20
  IGCRuntimeOptions,
21
21
  maxSnapshotCacheExpiryMs,
@@ -107,7 +107,7 @@ export function generateGCConfigs(
107
107
 
108
108
  // If version upgrade is not enabled, fall back to the stable GC version.
109
109
  const gcVersionInEffect =
110
- mc.config.getBoolean(gcVersionUpgradeToV3Key) === true ? currentGCVersion : stableGCVersion;
110
+ mc.config.getBoolean(gcVersionUpgradeToV4Key) === true ? nextGCVersion : stableGCVersion;
111
111
 
112
112
  // The GC version is up-to-date if the GC version in effect is at least equal to the GC version in base snapshot.
113
113
  // If it is not up-to-date, there is a newer version of GC out there which is more reliable than this. So, GC
@@ -22,10 +22,10 @@ import {
22
22
 
23
23
  export type GCVersion = number;
24
24
 
25
- /** The stable version of garbage collection in production. */
26
- export const stableGCVersion: GCVersion = 2;
27
- /** The current version of garbage collection. */
28
- export const currentGCVersion: GCVersion = 3;
25
+ /** The stable/default version of GC Data */
26
+ export const stableGCVersion: GCVersion = 3;
27
+ /** The next version of GC Data, to bump to in case we need to regenerate all GC Data across all files. */
28
+ export const nextGCVersion: GCVersion = 4;
29
29
 
30
30
  /**
31
31
  * This undocumented GC Option (on ContainerRuntime Options) allows an app to disable enforcing GC on old documents by incrementing this value
@@ -43,29 +43,28 @@ export const gcTombstoneGenerationOptionName = "gcTombstoneGeneration";
43
43
  */
44
44
  export const gcSweepGenerationOptionName = "gcSweepGeneration";
45
45
 
46
- // Feature gate key to turn GC on / off.
46
+ /** Config key to turn GC on / off. */
47
47
  export const runGCKey = "Fluid.GarbageCollection.RunGC";
48
- // Feature gate key to turn GC sweep on / off.
48
+ /** Config key to turn GC sweep on / off. */
49
49
  export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
50
- // Feature gate key to turn GC test mode on / off.
50
+ /** Config key to turn GC test mode on / off. */
51
51
  export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
52
- // Feature gate key to expire a session after a set period of time.
52
+ /** Config key to expire a session after a set period of time. Defaults to true. */
53
53
  export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
54
- // Feature gate key to turn GC sweep log off.
54
+ /** Config key to turn GC sweep log off. */
55
55
  export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
56
- // Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
56
+ /** Config key to disable the tombstone feature, i.e., tombstone information is not read / written into summary. */
57
57
  export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
58
- // Feature gate to enable throwing an error when tombstone object is loaded (requested).
58
+ /** Config key to enable throwing an error when tombstone object is loaded (requested). */
59
59
  export const throwOnTombstoneLoadKey = "Fluid.GarbageCollection.ThrowOnTombstoneLoad";
60
- // Feature gate to enable throwing an error when tombstone object is used (e.g. outgoing or incoming ops).
60
+ /** Config key to enable throwing an error when tombstone object is used (e.g. outgoing or incoming ops). */
61
61
  export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
62
- // Feature gate to enable GC version upgrade.
63
- export const gcVersionUpgradeToV3Key = "Fluid.GarbageCollection.GCVersionUpgradeToV3";
64
- // Feature gate to enable GC sweep for datastores.
65
- // TODO: Remove Test from the flag when we are confident to turn on sweep
66
- export const sweepDatastoresKey = "Fluid.GarbageCollection.Test.SweepDataStores";
67
- // Feature gate to enable GC sweep for attachment blobs.
68
- export const sweepAttachmentBlobsKey = "Fluid.GarbageCollection.Test.SweepAttachmentBlobs";
62
+ /** Config key to enable GC version upgrade. */
63
+ export const gcVersionUpgradeToV4Key = "Fluid.GarbageCollection.GCVersionUpgradeToV4";
64
+ /** Config key to disable GC sweep for datastores. */
65
+ export const disableDatastoreSweepKey = "Fluid.GarbageCollection.DisableDataStoreSweep";
66
+ /** Config key to disable GC sweep for attachment blobs. */
67
+ export const disableAttachmentBlobSweepKey = "Fluid.GarbageCollection.DisableAttachmentBlobSweep";
69
68
 
70
69
  // One day in milliseconds.
71
70
  export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
@@ -0,0 +1,145 @@
1
+ # Garbage Collection: Advanced configuration for early adopters
2
+
3
+ _For a technical overview of Garbage Collection, start with [GarbageCollection.md](./garbageCollection.md)_
4
+
5
+ GC Sweep is not yet enabled by default, and until that time early adopters have several configuration options available
6
+ for how to enable GC and monitor for any GC-impacting bugs that would need to be mitigated.
7
+
8
+ Please refer to the section [Techniques Used for Configuration](./garbageCollection.md#techniques-used-for-configuration)
9
+ before continuing, to ensure you're familiar with using "GC Options" and "Config Settings" for configuring GC.
10
+
11
+ ## What's on by default
12
+
13
+ GC Mark Phase is enabled by default, which includes marking objects that are ready to be deleted as Tombstones.
14
+ FF will log an informational event if/when a Tombstoned object is loaded - a scenario that would represent data loss if Sweep were enabled.
15
+ The eventName for the Tombstone log ends with `GC_Tombstone_DataStore_Requested`.
16
+
17
+ There's a similar event logged long before an object is Tombstoned. Ater only 7 days (configurable), an unreferenced object is considered
18
+ "Inactive", and FF will log if an Inactive object is loaded as well.
19
+ The eventName for the Inactive log ends with `InactiveObject_Loaded`.
20
+
21
+ ## Getting earlier signals: Shortening the InactiveObject timeout
22
+
23
+ The default timeout for an unreferenced object to become "Inactive" is 7 days. This is intended to be long enough such that
24
+ it's very unlikely to hit a legitimate case where an object is revived within the same session it was deleted (e.g. delete then undo).
25
+ Based on your application's user experience, you may choose to shorten this timeout to get an earlier signal (but beware of false positives).
26
+
27
+ To override the default InactiveObject timeout, use the `inactiveTimeoutMs` GC Option.
28
+ There's also a Config Setting which can be used for testing (if the Config Provider allows overriding via localStorage/sessionStorage):
29
+ `Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs`
30
+
31
+ ## Enabling Tombstone Enforcement
32
+
33
+ By default, GC is marking objects as Tombstoned, but merely logging if they're used after that point.
34
+ You can enable enforcement of Tombstone objects to simulate real Sweep while having the peace of mind
35
+ that the data is not yet deleted from the user's file, and can be recovered.
36
+
37
+ The first step is to prevent the Fluid Framework from loading a Tombstoned object (via `handle.get()` as described previously),
38
+ by using this Config Setting:
39
+
40
+ ```ts
41
+ "Fluid.GarbageCollection.ThrowOnTombstoneLoad": true
42
+ ```
43
+
44
+ Now, even with `Fluid.GarbageCollection.ThrowOnTombstoneLoad` set to true, changes to a Tombstoned object will be allowed.
45
+ This is required for the advanced recovery options to work, explained [below](#advanced-back-door-recovering-and-reviving-tombstoned-objects).
46
+
47
+ To instruct FF to treat Tombstoned objects as if they are truly not present in the document, use this Config Setting:
48
+
49
+ ```ts
50
+ "Fluid.GarbageCollection.ThrowOnTombstoneUsage": true
51
+ ```
52
+
53
+ ### In case of emergency: Bumping the gcTombstoneGeneration
54
+
55
+ GC includes a mechanism for Tombstone by which all new documents may be stamped with a "Generation" number,
56
+ and if set then Tombstone is only enforceable for documents of the latest Generation. This number is specified
57
+ via the `gcTombstoneGeneration` GC Option, and will not change over the lifetime of a given document.
58
+
59
+ In case a bug is released that is found to cause GC errors, a bump to the gcTombstoneGeneration can be incuded
60
+ with the fix, which will prevent any user pain for those potentially affected documents that were exposed to the bug.
61
+
62
+ If gcTombstoneGeneration is unset, Tombstone enforcement will be enabled/disabled as otherwise configured.
63
+ In other words, until you start using this, Tombstone enforcement will apply to all documents.
64
+
65
+ ### Advanced "Back door": Recovering and reviving Tombstoned objects
66
+
67
+ If your application has Tombstone enabled and your users are encountering Tombstones - even at the point where
68
+ Tombstone enforcement is enabled - there is a way to still access these objects to recover them and property
69
+ reference them ("revival"). However, please understand that this is an advanced and unsupported path that may
70
+ be immediately deprecated at any time.
71
+
72
+ As mentioned above, bumping the gcTombstoneGeneration will free up impacted documents, but that's a permanent
73
+ mitigation - those documents will never be exposed to GC Tombstone or Sweep.
74
+
75
+ If there's a particular codepath in your application where objects being loaded may be Tombstoned,
76
+ you may use this advanced "back door" to recover them and then properly reference them, thus restoring the document.
77
+ For this to work, you must disable the `Fluid.GarbageCollection.ThrowOnTombstoneUsage` Config Setting.
78
+
79
+ When a Tombstoned object (via `handle.get()`) fails to load, the 404 response error object has an `underlyingResponseHeaders` with the
80
+ `isTombstoned` flag set to true: i.e. `error.underlyingResponseHeaders?.isTombstoned === true`. In this case,
81
+ you may turn around and use `IContainerRuntime.resolveHandle` with `allowTombstone: true` in `IRequest.headers` to request
82
+ the object again - this time it will succeed.
83
+
84
+ To be very clear once again - This path uses deprecated APIs (`resolveHandle`) and comes with no guarantees of support.
85
+
86
+ ### Tombstones and the Summarizer Client
87
+
88
+ Note: The Summarizer client will _never_ throw on usage or load of a Tombstoned object.
89
+
90
+ ## Enabling Sweep
91
+
92
+ To enable Sweep for the first time, set this GC Option:
93
+
94
+ ```ts
95
+ gcSweepGeneration: 0;
96
+ ```
97
+
98
+ This will enable sweep for all documents moving forward _as well as_ any documents created with `gcTombstoneGeneration: 0`
99
+ (this is a special case in the code).
100
+
101
+ ### A caveat...
102
+
103
+ If you used `gcTombstoneGeneration` **and ever bumped it**, you should skip 0 here to avoid enabling Sweep for old / at-risk documents:
104
+
105
+ ```ts
106
+ gcSweepGeneration: 1;
107
+ ```
108
+
109
+ Remember, you can always bump `gcSweepGeneration` to disable Sweep for all existing documents and start fresh,
110
+ in case a major bug is discovered and fixed.
111
+
112
+ ### More about gcSweepGeneration and gcTombstoneGeneration
113
+
114
+ `gcSweepGeneration` is persisted and immutable in the document, just like `gcTombstoneGeneration`.
115
+ However, behavior differs in a few important ways.
116
+
117
+ For Tombstone, if `gcTombstoneGeneration` is not set, Tombstone enforcement will be **enabled**.
118
+ For Sweep however, if `gcSweepGeneration` is not set, Sweep enforcement will be **disabled**.
119
+
120
+ This means that until the `gcSweepGeneration` GC Option is set, _no existing document will be eligible for Sweep, ever_.
121
+ So all documents created since the most recent bump to the gcSweepGeneration will have Sweep enabled.
122
+ Note that if `gcSweepGeneration` is set and matches, Tombstone Mode is off for the session and `gcTombstoneGeneration` is ignored.
123
+
124
+ And as mentioned above, there is a special case when `gcSweepGeneration === 0`: Any document with `gcTombstoneGeneration: 0` will
125
+ be eligible for Sweep as well. This was done for historical reasons due to circumstances during GC's development.
126
+
127
+ ## More Advanced Configurations
128
+
129
+ There are a handful of other configuration options/settings that can be used to tweak GC's behavior,
130
+ mostly for testing. Please refer to the function [`generateGCConfigs` in gcConfigs.ts](./gcConfigs.ts) and the
131
+ [setting/option names listed in gcDefinitions.ts](./gcDefinitions.ts) for the full story.
132
+
133
+ Examples of available advanced configuration include:
134
+
135
+ - Disabling GC permanently for new files
136
+ - Overriding GC Mark/Sweep enablement for this session:
137
+ - Disabling running GC Mark and/or Sweep phases for this session
138
+ - Forcing GC Mark and/or Sweep to run for this session even if otherwise it would be disabled
139
+ - Disabling Tombstone Mode (don't even mark objects as Tombstones)
140
+ - Disabling the deletion of either DataStores or AttachmentBlobs independent of one another (when Sweep is enabled)
141
+ - Overriding the default Session Expiry for new files (or disabling it altogether, which will also disable Tombstone/Sweep)
142
+ - Overriding the Sweep Timeout, _independent of Session Expiry_, so use with care (for testing purposes only - data loss could occur)
143
+ - Running in "Test Mode", where objects are deleted as soon as they're unreferenced
144
+ - Force "Full GC" to run, which ignores incremental optimizations based on previously computed GC Data
145
+ - Treat InactiveObjects like Tombstones: throw an error on load (with the same back door to follow-up with a successful request)
@@ -12,7 +12,6 @@ import {
12
12
  IGarbageCollectionData,
13
13
  IGarbageCollectionDetailsBase,
14
14
  } from "@fluidframework/runtime-definitions";
15
- import { TelemetryDataTag } from "@fluidframework/telemetry-utils";
16
15
  import { GCFeatureMatrix, GCVersion, IGCMetadata } from "./gcDefinitions";
17
16
  import {
18
17
  IGarbageCollectionNodeData,
@@ -304,14 +303,3 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
304
303
  export function trimLeadingAndTrailingSlashes(str: string) {
305
304
  return str.replace(/^\/+|\/+$/g, "");
306
305
  }
307
-
308
- /**
309
- * Tags the passed value as a CodeArtifact and returns the tagged value.
310
- * @deprecated - Use telemetry-utils tagCodeArtifacts instead
311
- */
312
- export function tagAsCodeArtifact(value: string) {
313
- return {
314
- value,
315
- tag: TelemetryDataTag.CodeArtifact,
316
- };
317
- }
@@ -23,8 +23,6 @@ import {
23
23
  runSweepKey,
24
24
  } from "./gcDefinitions";
25
25
  import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
26
- // eslint-disable-next-line import/no-deprecated
27
- import { tagAsCodeArtifact } from "./gcHelpers";
28
26
 
29
27
  type NodeUsageType = "Changed" | "Loaded" | "Revived";
30
28
 
@@ -184,8 +182,7 @@ export class GCTelemetryTracker {
184
182
  {
185
183
  eventName: `GC_Tombstone_${nodeType}_Revived`,
186
184
  category: "generic",
187
- // eslint-disable-next-line import/no-deprecated
188
- url: tagAsCodeArtifact(id),
185
+ ...tagCodeArtifacts({ url: id }),
189
186
  gcTombstoneEnforcementAllowed: this.gcTombstoneEnforcementAllowed,
190
187
  },
191
188
  undefined /* packagePath */,
package/src/gc/index.ts CHANGED
@@ -5,7 +5,7 @@
5
5
 
6
6
  export { GarbageCollector } from "./garbageCollection";
7
7
  export {
8
- currentGCVersion,
8
+ nextGCVersion,
9
9
  defaultInactiveTimeoutMs,
10
10
  defaultSessionExpiryDurationMs,
11
11
  disableSweepLogKey,
@@ -15,7 +15,7 @@ export {
15
15
  gcSweepGenerationOptionName,
16
16
  GCFeatureMatrix,
17
17
  GCVersion,
18
- gcVersionUpgradeToV3Key,
18
+ gcVersionUpgradeToV4Key,
19
19
  IGarbageCollectionRuntime,
20
20
  IGarbageCollector,
21
21
  IGarbageCollectorConfigs,
@@ -29,8 +29,8 @@ export {
29
29
  runSessionExpiryKey,
30
30
  runSweepKey,
31
31
  stableGCVersion,
32
- sweepAttachmentBlobsKey,
33
- sweepDatastoresKey,
32
+ disableAttachmentBlobSweepKey,
33
+ disableDatastoreSweepKey,
34
34
  throwOnTombstoneLoadKey,
35
35
  throwOnTombstoneUsageKey,
36
36
  UnreferencedState,
@@ -42,7 +42,6 @@ export {
42
42
  shouldAllowGcSweep,
43
43
  trimLeadingAndTrailingSlashes,
44
44
  unpackChildNodesGCDetails,
45
- tagAsCodeArtifact,
46
45
  } from "./gcHelpers";
47
46
  export { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
48
47
  export {
package/src/index.ts CHANGED
@@ -4,10 +4,6 @@
4
4
  */
5
5
 
6
6
  export {
7
- ContainerMessageType,
8
- ContainerRuntimeMessage,
9
- IContainerRuntimeMessageCompatDetails,
10
- CompatModeBehavior,
11
7
  ISummaryRuntimeOptions,
12
8
  ISummaryBaseConfiguration,
13
9
  ISummaryConfigurationHeuristics,
@@ -28,6 +24,13 @@ export {
28
24
  ICompressionRuntimeOptions,
29
25
  CompressionAlgorithms,
30
26
  } from "./containerRuntime";
27
+ export {
28
+ ContainerMessageType,
29
+ ContainerRuntimeMessage,
30
+ IContainerRuntimeMessageCompatDetails,
31
+ CompatModeBehavior,
32
+ RecentlyAddedContainerRuntimeMessageDetails,
33
+ } from "./messageTypes";
31
34
  export { FluidDataStoreRegistry } from "./dataStoreRegistry";
32
35
  export { IGCRuntimeOptions, IGCStats } from "./gc";
33
36
  export {