@fluidframework/container-runtime 0.53.0 → 0.54.0-47413

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/dist/containerRuntime.d.ts +25 -16
  2. package/dist/containerRuntime.d.ts.map +1 -1
  3. package/dist/containerRuntime.js +125 -77
  4. package/dist/containerRuntime.js.map +1 -1
  5. package/dist/dataStoreContext.d.ts +29 -3
  6. package/dist/dataStoreContext.d.ts.map +1 -1
  7. package/dist/dataStoreContext.js +29 -4
  8. package/dist/dataStoreContext.js.map +1 -1
  9. package/dist/dataStores.d.ts +7 -3
  10. package/dist/dataStores.d.ts.map +1 -1
  11. package/dist/dataStores.js +54 -5
  12. package/dist/dataStores.js.map +1 -1
  13. package/dist/garbageCollection.d.ts +22 -2
  14. package/dist/garbageCollection.d.ts.map +1 -1
  15. package/dist/garbageCollection.js +112 -34
  16. package/dist/garbageCollection.js.map +1 -1
  17. package/dist/packageVersion.d.ts +1 -1
  18. package/dist/packageVersion.d.ts.map +1 -1
  19. package/dist/packageVersion.js +1 -1
  20. package/dist/packageVersion.js.map +1 -1
  21. package/dist/runningSummarizer.d.ts +3 -2
  22. package/dist/runningSummarizer.d.ts.map +1 -1
  23. package/dist/runningSummarizer.js +6 -6
  24. package/dist/runningSummarizer.js.map +1 -1
  25. package/dist/summarizer.d.ts +22 -0
  26. package/dist/summarizer.d.ts.map +1 -1
  27. package/dist/summarizer.js +135 -33
  28. package/dist/summarizer.js.map +1 -1
  29. package/dist/summarizerTypes.d.ts +1 -8
  30. package/dist/summarizerTypes.d.ts.map +1 -1
  31. package/dist/summarizerTypes.js.map +1 -1
  32. package/dist/summaryFormat.d.ts +1 -0
  33. package/dist/summaryFormat.d.ts.map +1 -1
  34. package/dist/summaryFormat.js +2 -1
  35. package/dist/summaryFormat.js.map +1 -1
  36. package/dist/summaryManager.d.ts +0 -15
  37. package/dist/summaryManager.d.ts.map +1 -1
  38. package/dist/summaryManager.js +1 -35
  39. package/dist/summaryManager.js.map +1 -1
  40. package/lib/containerRuntime.d.ts +25 -16
  41. package/lib/containerRuntime.d.ts.map +1 -1
  42. package/lib/containerRuntime.js +131 -83
  43. package/lib/containerRuntime.js.map +1 -1
  44. package/lib/dataStoreContext.d.ts +29 -3
  45. package/lib/dataStoreContext.d.ts.map +1 -1
  46. package/lib/dataStoreContext.js +29 -4
  47. package/lib/dataStoreContext.js.map +1 -1
  48. package/lib/dataStores.d.ts +7 -3
  49. package/lib/dataStores.d.ts.map +1 -1
  50. package/lib/dataStores.js +54 -5
  51. package/lib/dataStores.js.map +1 -1
  52. package/lib/garbageCollection.d.ts +22 -2
  53. package/lib/garbageCollection.d.ts.map +1 -1
  54. package/lib/garbageCollection.js +114 -36
  55. package/lib/garbageCollection.js.map +1 -1
  56. package/lib/packageVersion.d.ts +1 -1
  57. package/lib/packageVersion.d.ts.map +1 -1
  58. package/lib/packageVersion.js +1 -1
  59. package/lib/packageVersion.js.map +1 -1
  60. package/lib/runningSummarizer.d.ts +3 -2
  61. package/lib/runningSummarizer.d.ts.map +1 -1
  62. package/lib/runningSummarizer.js +6 -6
  63. package/lib/runningSummarizer.js.map +1 -1
  64. package/lib/summarizer.d.ts +22 -0
  65. package/lib/summarizer.d.ts.map +1 -1
  66. package/lib/summarizer.js +135 -33
  67. package/lib/summarizer.js.map +1 -1
  68. package/lib/summarizerTypes.d.ts +1 -8
  69. package/lib/summarizerTypes.d.ts.map +1 -1
  70. package/lib/summarizerTypes.js.map +1 -1
  71. package/lib/summaryFormat.d.ts +1 -0
  72. package/lib/summaryFormat.d.ts.map +1 -1
  73. package/lib/summaryFormat.js +1 -0
  74. package/lib/summaryFormat.js.map +1 -1
  75. package/lib/summaryManager.d.ts +0 -15
  76. package/lib/summaryManager.d.ts.map +1 -1
  77. package/lib/summaryManager.js +1 -34
  78. package/lib/summaryManager.js.map +1 -1
  79. package/package.json +13 -13
  80. package/src/containerRuntime.ts +176 -93
  81. package/src/dataStoreContext.ts +44 -6
  82. package/src/dataStores.ts +84 -4
  83. package/src/garbageCollection.ts +137 -46
  84. package/src/packageVersion.ts +1 -1
  85. package/src/runningSummarizer.ts +12 -10
  86. package/src/summarizer.ts +154 -38
  87. package/src/summarizerTypes.ts +2 -9
  88. package/src/summaryFormat.ts +1 -0
  89. package/src/summaryManager.ts +2 -49
  90. package/dist/localStorageFeatureGates.d.ts +0 -13
  91. package/dist/localStorageFeatureGates.d.ts.map +0 -1
  92. package/dist/localStorageFeatureGates.js +0 -31
  93. package/dist/localStorageFeatureGates.js.map +0 -1
  94. package/lib/localStorageFeatureGates.d.ts +0 -13
  95. package/lib/localStorageFeatureGates.d.ts.map +0 -1
  96. package/lib/localStorageFeatureGates.js +0 -27
  97. package/lib/localStorageFeatureGates.js.map +0 -1
  98. package/src/localStorageFeatureGates.ts +0 -27
@@ -30,7 +30,7 @@ import { BlobTreeEntry } from "@fluidframework/protocol-base";
30
30
  import {
31
31
  IClientDetails,
32
32
  IDocumentMessage,
33
- IQuorum,
33
+ IQuorumClients,
34
34
  ISequencedDocumentMessage,
35
35
  ISnapshotTree,
36
36
  ITreeEntry,
@@ -67,6 +67,7 @@ import {
67
67
  ThresholdCounter,
68
68
  } from "@fluidframework/telemetry-utils";
69
69
  import { CreateProcessingError } from "@fluidframework/container-utils";
70
+
70
71
  import { ContainerRuntime } from "./containerRuntime";
71
72
  import {
72
73
  dataStoreAttributesBlobName,
@@ -386,7 +387,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
386
387
  this.channel?.processSignal(message, local);
387
388
  }
388
389
 
389
- public getQuorum(): IQuorum {
390
+ public getQuorum(): IQuorumClients {
390
391
  return this._containerRuntime.getQuorum();
391
392
  }
392
393
 
@@ -496,9 +497,19 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
496
497
  }
497
498
  }
498
499
 
500
+ /**
501
+ * Called when a new outbound reference is added to another node. This is used by garbage collection to identify
502
+ * all references added in the system.
503
+ * @param srcHandle - The handle of the node that added the reference.
504
+ * @param outboundHandle - The handle of the outbound node that is referenced.
505
+ */
506
+ public addedGCOutboundReference(srcHandle: IFluidHandle, outboundHandle: IFluidHandle) {
507
+ this._containerRuntime.addedGCOutboundReference(srcHandle, outboundHandle);
508
+ }
509
+
499
510
  /**
500
511
  * Updates the used routes of the channel and its child contexts. The channel must be loaded before calling this.
501
- * It is called in these two scenarions:
512
+ * It is called in these two scenarios:
502
513
  * 1. When the used routes of the data store is updated and the data store is loaded.
503
514
  * 2. When the data store is realized. This updates the channel's used routes as per last GC run.
504
515
  */
@@ -582,7 +593,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
582
593
  try
583
594
  {
584
595
  assert(!this.detachedRuntimeCreation, 0x148 /* "Detached runtime creation on runtime bind" */);
585
- assert(this.channelDeferred !== undefined, 0x149 /* "Undefined channel defferal" */);
596
+ assert(this.channelDeferred !== undefined, 0x149 /* "Undefined channel deferral" */);
586
597
  assert(this.pkg !== undefined, 0x14a /* "Undefined package path" */);
587
598
 
588
599
  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -634,6 +645,13 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
634
645
 
635
646
  protected abstract getInitialSnapshotDetails(): Promise<ISnapshotDetails>;
636
647
 
648
+ /**
649
+ * @deprecated - Sets the datastore as root, for aliasing purposes: #7948
650
+ * This method should not be used outside of the aliasing context.
651
+ * It will be removed, as the source of truth for this flag will be the aliasing blob.
652
+ */
653
+ public abstract setRoot(): void;
654
+
637
655
  public abstract getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails>;
638
656
 
639
657
  public reSubmit(contents: any, localOpMetadata: unknown) {
@@ -679,6 +697,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
679
697
  }
680
698
 
681
699
  export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
700
+ private isRootDataStore: boolean | undefined;
701
+
682
702
  constructor(
683
703
  id: string,
684
704
  private readonly initSnapshotValue: ISnapshotTree | string | undefined,
@@ -750,7 +770,7 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
750
770
  * data stores in older documents are not garbage collected incorrectly. This may lead to additional
751
771
  * roots in the document but they won't break.
752
772
  */
753
- isRootDataStore = attributes.isRootDataStore ?? true;
773
+ isRootDataStore = this.isRootDataStore === true || (attributes.isRootDataStore ?? true);
754
774
 
755
775
  if (hasIsolatedChannels(attributes)) {
756
776
  tree = tree.trees[channelsTreeName];
@@ -782,6 +802,15 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
782
802
  public generateAttachMessage(): IAttachMessage {
783
803
  throw new Error("Cannot attach remote store");
784
804
  }
805
+
806
+ /**
807
+ * @deprecated - Sets the datastore as root, for aliasing purposes: #7948
808
+ * This method should not be used outside of the aliasing context.
809
+ * It will be removed, as the source of truth for this flag will be the aliasing blob.
810
+ */
811
+ public setRoot(): void {
812
+ this.isRootDataStore = true;
813
+ }
785
814
  }
786
815
 
787
816
  /**
@@ -883,7 +912,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
883
912
  // If there is no isRootDataStore in the attributes blob, set it to true. This ensures that data
884
913
  // stores in older documents are not garbage collected incorrectly. This may lead to additional
885
914
  // roots in the document but they won't break.
886
- this.isRootDataStore = attributes.isRootDataStore ?? true;
915
+ this.isRootDataStore = this.isRootDataStore || (attributes.isRootDataStore ?? true);
887
916
  }
888
917
  }
889
918
  assert(this.pkg !== undefined, 0x152 /* "pkg should be available in local data store" */);
@@ -901,6 +930,15 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
901
930
  // Local data store does not have initial summary.
902
931
  return {};
903
932
  }
933
+
934
+ /**
935
+ * @deprecated - Sets the datastore as root, for aliasing purposes: #7948
936
+ * This method should not be used outside of the aliasing context.
937
+ * It will be removed, as the source of truth for this flag will be the aliasing blob.
938
+ */
939
+ public setRoot(): void {
940
+ this.isRootDataStore = true;
941
+ }
904
942
  }
905
943
 
906
944
  /**
package/src/dataStores.ts CHANGED
@@ -50,6 +50,32 @@ import {
50
50
  import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summaryFormat";
51
51
  import { IUsedStateStats } from "./garbageCollection";
52
52
 
53
+ type PendingAliasResolve = (success: boolean) => void;
54
+
55
+ /**
56
+ * Interface for an op to be used for assigning an
57
+ * alias to a datastore
58
+ */
59
+ interface IDataStoreAliasMessage {
60
+ /** The internal id of the datastore */
61
+ readonly internalId: string;
62
+ /** The alias name to be assigned to the datastore */
63
+ readonly alias: string;
64
+ }
65
+
66
+ /**
67
+ * Type guard that returns true if the given alias message is actually an instance of
68
+ * a class which implements @see IDataStoreAliasMessage
69
+ * @param maybeDataStoreAliasMessage - message object to be validated
70
+ * @returns True if the @see IDataStoreAliasMessage is fully implemented, false otherwise
71
+ */
72
+ const isDataStoreAliasMessage = (
73
+ maybeDataStoreAliasMessage: any,
74
+ ): maybeDataStoreAliasMessage is IDataStoreAliasMessage => {
75
+ return typeof maybeDataStoreAliasMessage?.internalId === "string"
76
+ && typeof maybeDataStoreAliasMessage?.alias === "string";
77
+ };
78
+
53
79
  /**
54
80
  * This class encapsulates data store handling. Currently it is only used by the container runtime,
55
81
  * but eventually could be hosted on any channel once we formalize the channel api boundary.
@@ -81,6 +107,7 @@ export class DataStores implements IDisposable {
81
107
  baseLogger: ITelemetryBaseLogger,
82
108
  getDataStoreBaseGCDetails: () => Promise<Map<string, IGarbageCollectionSummaryDetails>>,
83
109
  private readonly dataStoreChanged: (id: string) => void,
110
+ private readonly aliasMap: Map<string, string>,
84
111
  private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
85
112
  ) {
86
113
  this.logger = ChildLogger.create(baseLogger);
@@ -146,6 +173,10 @@ export class DataStores implements IDisposable {
146
173
  };
147
174
  }
148
175
 
176
+ public aliases(): ReadonlyMap<string, string> {
177
+ return this.aliasMap;
178
+ }
179
+
149
180
  public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
150
181
  const attachMessage = message.contents as InboundAttachMessage;
151
182
  // The local object has already been attached
@@ -205,7 +236,6 @@ export class DataStores implements IDisposable {
205
236
  }),
206
237
  pkg);
207
238
 
208
- // Resolve pending gets and store off any new ones
209
239
  this.contexts.addBoundOrRemoted(remotedFluidDataStoreContext);
210
240
 
211
241
  // Equivalent of nextTick() - Prefetch once all current ops have completed
@@ -213,6 +243,55 @@ export class DataStores implements IDisposable {
213
243
  Promise.resolve().then(async () => remotedFluidDataStoreContext.realize());
214
244
  }
215
245
 
246
+ public processAliasMessage(
247
+ message: ISequencedDocumentMessage,
248
+ localOpMetadata: unknown,
249
+ local: boolean,
250
+ ): void {
251
+ const aliasMessage = message.contents as IDataStoreAliasMessage;
252
+ if (!isDataStoreAliasMessage(aliasMessage)) {
253
+ throw new DataCorruptionError(
254
+ "malformedDataStoreAliasMessage",
255
+ {
256
+ ...extractSafePropertiesFromMessage(message),
257
+ },
258
+ );
259
+ }
260
+
261
+ const resolve = localOpMetadata as PendingAliasResolve;
262
+ const aliasResult = this.processAliasMessageCore(aliasMessage);
263
+ if (local) {
264
+ resolve(aliasResult);
265
+ }
266
+ }
267
+
268
+ private processAliasMessageCore(aliasMessage: IDataStoreAliasMessage): boolean {
269
+ const existingMapping = this.aliasMap.get(aliasMessage.alias);
270
+ if (existingMapping !== undefined) {
271
+ return false;
272
+ }
273
+
274
+ // Unlikely scenario, but we may receive an alias OP with the alias value
275
+ // equal to one of the ids supplied to `createRootDataStore` in the past
276
+ const maybeContextWithAliasAsId = this.contexts.get(aliasMessage.alias);
277
+ if (maybeContextWithAliasAsId !== undefined) {
278
+ return false;
279
+ }
280
+
281
+ const currentContext = this.contexts.get(aliasMessage.internalId);
282
+ if (currentContext === undefined) {
283
+ this.logger.sendErrorEvent({
284
+ eventName: "AliasFluidDataStoreNotFound",
285
+ fluidDataStoreId: aliasMessage.internalId,
286
+ });
287
+ return false;
288
+ }
289
+
290
+ this.aliasMap.set(aliasMessage.alias, currentContext.id);
291
+ currentContext.setRoot();
292
+ return true;
293
+ }
294
+
216
295
  public bindFluidDataStore(fluidDataStoreRuntime: IFluidDataStoreChannel): void {
217
296
  const id = fluidDataStoreRuntime.id;
218
297
  const localContext = this.contexts.getUnbound(id);
@@ -304,8 +383,9 @@ export class DataStores implements IDisposable {
304
383
  }
305
384
 
306
385
  public async getDataStore(id: string, wait: boolean): Promise<FluidDataStoreContext> {
307
- const context = await this.contexts.getBoundOrRemoted(id, wait);
386
+ const internalId = this.aliasMap.get(id) ?? id;
308
387
 
388
+ const context = await this.contexts.getBoundOrRemoted(internalId, wait);
309
389
  if (context === undefined) {
310
390
  // The requested data store does not exits. Throw a 404 response exception.
311
391
  const request = { url: id };
@@ -421,8 +501,8 @@ export class DataStores implements IDisposable {
421
501
  /**
422
502
  * Generates data used for garbage collection. It does the following:
423
503
  * 1. Calls into each child data store context to get its GC data.
424
- * 2. Prefixs the child context's id to the GC nodes in the child's GC data. This makes sure that the node can be
425
- * idenfied as belonging to the child.
504
+ * 2. Prefixes the child context's id to the GC nodes in the child's GC data. This makes sure that the node can be
505
+ * identified as belonging to the child.
426
506
  * 3. Adds a GC node for this channel to the nodes received from the children. All these nodes together represent
427
507
  * the GC data of this channel.
428
508
  * @param fullGC - true to bypass optimizations and force full generation of GC data.
@@ -6,16 +6,17 @@
6
6
  import { ITelemetryLogger } from "@fluidframework/common-definitions";
7
7
  import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
8
8
  import {
9
+ cloneGCData,
9
10
  concatGarbageCollectionStates,
10
- unpackChildNodesGCDetails,
11
+ concatGarbageCollectionData,
11
12
  IGCResult,
12
13
  runGarbageCollection,
14
+ unpackChildNodesGCDetails,
13
15
  } from "@fluidframework/garbage-collector";
14
16
  import { ISnapshotTree } from "@fluidframework/protocol-definitions";
15
17
  import {
16
18
  gcBlobKey,
17
19
  IGarbageCollectionData,
18
- IGarbageCollectionNodeData,
19
20
  IGarbageCollectionState,
20
21
  IGarbageCollectionSummaryDetails,
21
22
  ISummaryTreeWithStats,
@@ -25,11 +26,15 @@ import {
25
26
  RefreshSummaryResult,
26
27
  SummaryTreeBuilder,
27
28
  } from "@fluidframework/runtime-utils";
28
- import { ChildLogger, PerformanceEvent } from "@fluidframework/telemetry-utils";
29
+ import {
30
+ ChildLogger,
31
+ loggerToMonitoringContext,
32
+ MonitoringContext,
33
+ PerformanceEvent,
34
+ } from "@fluidframework/telemetry-utils";
29
35
 
30
36
  import { IGCRuntimeOptions } from "./containerRuntime";
31
37
  import { getSummaryForDatastores } from "./dataStores";
32
- import { getLocalStorageFeatureGate } from "./localStorageFeatureGates";
33
38
  import {
34
39
  getGCVersion,
35
40
  GCVersion,
@@ -48,11 +53,11 @@ export const gcTreeKey = "gc";
48
53
  export const gcBlobPrefix = "__gc";
49
54
 
50
55
  // Local storage key to turn GC on / off.
51
- const runGCKey = "FluidRunGC";
56
+ const runGCKey = "Fluid.GarbageCollection.RunGC";
52
57
  // Local storage key to turn GC test mode on / off.
53
- const gcTestModeKey = "FluidGCTestMode";
58
+ const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
54
59
  // Local storage key to turn GC sweep on / off.
55
- const runSweepKey = "FluidRunSweep";
60
+ const runSweepKey = "Fluid.GarbageCollection.RunSweep";
56
61
 
57
62
  const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
58
63
 
@@ -104,6 +109,8 @@ export interface IGarbageCollector {
104
109
  latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;
105
110
  /** Called when a node is changed. Used to detect and log when an inactive node is changed. */
106
111
  nodeChanged(id: string): void;
112
+ /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
113
+ addedOutboundReference(fromNodeId: string, toNodeId: string): void;
107
114
  }
108
115
 
109
116
  /**
@@ -148,9 +155,9 @@ class UnreferencedStateTracker {
148
155
  if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
149
156
  logger.sendErrorEvent({
150
157
  eventName,
151
- unreferencedDuratonMs: currentTimestampMs - this.unreferencedTimestampMs,
152
- deleteTimeoutMs,
153
- inactiveNodeId,
158
+ age: currentTimestampMs - this.unreferencedTimestampMs,
159
+ timeout: deleteTimeoutMs,
160
+ id: inactiveNodeId,
154
161
  });
155
162
  this.inactiveEventsLogged.add(eventName);
156
163
  }
@@ -217,7 +224,7 @@ export class GarbageCollector implements IGarbageCollector {
217
224
  private readonly gcEnabled: boolean;
218
225
  private readonly shouldRunSweep: boolean;
219
226
  private readonly testMode: boolean;
220
- private readonly logger: ITelemetryLogger;
227
+ private readonly mc: MonitoringContext;
221
228
 
222
229
  /**
223
230
  * Tells whether the GC data should be written to the root of the summary tree. We do this under 2 conditions:
@@ -235,8 +242,11 @@ export class GarbageCollector implements IGarbageCollector {
235
242
  // This is the version of GC data in the latest summary being tracked.
236
243
  private latestSummaryGCVersion: GCVersion;
237
244
 
238
- // The current state - each node's GC data and unreferenced timestamp.
239
- private currentGCState: IGarbageCollectionState | undefined;
245
+ // Keeps track of the GC state from the last run.
246
+ private gcDataFromLastRun: IGarbageCollectionData | undefined;
247
+ // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
248
+ // outbound routes from that node.
249
+ private readonly referencesSinceLastRun: Map<string, string[]> = new Map();
240
250
 
241
251
  // Promise when resolved initializes the base state of the nodes from the base summary state.
242
252
  private readonly initializeBaseStateP: Promise<void>;
@@ -260,7 +270,8 @@ export class GarbageCollector implements IGarbageCollector {
260
270
  existing: boolean,
261
271
  metadata?: IContainerRuntimeMetadata,
262
272
  ) {
263
- this.logger = ChildLogger.create(baseLogger, "GarbageCollector");
273
+ this.mc = loggerToMonitoringContext(
274
+ ChildLogger.create(baseLogger, "GarbageCollector"));
264
275
 
265
276
  this.deleteTimeoutMs = this.gcOptions.deleteTimeoutMs ?? defaultDeleteTimeoutMs;
266
277
 
@@ -281,7 +292,7 @@ export class GarbageCollector implements IGarbageCollector {
281
292
  this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;
282
293
 
283
294
  // Whether GC should run or not. Can override with localStorage flag.
284
- this.shouldRunGC = getLocalStorageFeatureGate(runGCKey) ?? (
295
+ this.shouldRunGC = this.mc.config.getBoolean(runGCKey) ?? (
285
296
  // GC must be enabled for the document.
286
297
  this.gcEnabled
287
298
  // GC must not be disabled via GC options.
@@ -291,10 +302,10 @@ export class GarbageCollector implements IGarbageCollector {
291
302
  // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
292
303
  // localStorage flag.
293
304
  this.shouldRunSweep = this.shouldRunGC &&
294
- (getLocalStorageFeatureGate(runSweepKey) ?? gcOptions.runSweep === true);
305
+ (this.mc.config.getBoolean(runSweepKey) ?? gcOptions.runSweep === true);
295
306
 
296
307
  // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
297
- this.testMode = getLocalStorageFeatureGate(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;
308
+ this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;
298
309
 
299
310
  // If `writeDataAtRoot` GC option is true, we should write the GC data into the root of the summary tree. This
300
311
  // GC option is used for testing only. It will be removed once we start writing GC data into root by default.
@@ -302,9 +313,9 @@ export class GarbageCollector implements IGarbageCollector {
302
313
 
303
314
  // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
304
315
  // this once since it involves fetching blobs from storage which is expensive.
305
- const baseSummaryStateP = new LazyPromise<IGarbageCollectionState>(async () => {
316
+ const baseSummaryStateP = new LazyPromise<IGarbageCollectionState | undefined>(async () => {
306
317
  if (baseSnapshot === undefined) {
307
- return { gcNodes: {} };
318
+ return undefined;
308
319
  }
309
320
 
310
321
  // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
@@ -317,6 +328,7 @@ export class GarbageCollector implements IGarbageCollector {
317
328
 
318
329
  // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
319
330
  // consolidate into IGarbageCollectionState format.
331
+ // Add a node for the root node that is not present in older snapshot format.
320
332
  const gcState: IGarbageCollectionState = { gcNodes: { "/": { outboundRoutes: [] } } };
321
333
  const dataStoreSnaphotTree = getSummaryForDatastores(baseSnapshot, metadata);
322
334
  assert(dataStoreSnaphotTree !== undefined,
@@ -353,22 +365,28 @@ export class GarbageCollector implements IGarbageCollector {
353
365
  0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);
354
366
  gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
355
367
  }
356
- return gcState;
368
+
369
+ // If there is only one node (root node just added above), either GC is disabled or we are loading from the
370
+ // very first summary generated by detached container. In both cases, GC was not run - return undefined.
371
+ return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
357
372
  });
358
373
 
359
374
  // Set up the initializer which initializes the base GC state from the base snapshot. Use lazy promise because
360
375
  // we only do this once - the very first time we run GC.
361
376
  this.initializeBaseStateP = new LazyPromise<void>(async () => {
362
- const baseState = await baseSummaryStateP;
377
+ const currentTimestampMs = this.getCurrentTimestampMs();
378
+ const baseState = await baseSummaryStateP;
379
+ if (baseState === undefined) {
380
+ return;
381
+ }
363
382
 
364
- const gcNodes: { [ id: string ]: IGarbageCollectionNodeData } = {};
365
- // Set up tracking for the nodes in the base summary state and add them to GC nodes.
383
+ const gcNodes: { [ id: string ]: string[] } = {};
366
384
  for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
367
385
  const unreferencedTimestampMs = nodeData.unreferencedTimestampMs;
368
386
  if (unreferencedTimestampMs !== undefined) {
369
387
  // Get how long it has been since the node was unreferenced. Start a timeout for the remaining time
370
388
  // left for it to be eligible for deletion.
371
- const unreferencedDurationMs = this.getCurrentTimestampMs() - unreferencedTimestampMs;
389
+ const unreferencedDurationMs = currentTimestampMs - unreferencedTimestampMs;
372
390
  this.unreferencedNodesState.set(
373
391
  nodeId,
374
392
  new UnreferencedStateTracker(
@@ -377,20 +395,20 @@ export class GarbageCollector implements IGarbageCollector {
377
395
  ),
378
396
  );
379
397
  }
380
-
381
- gcNodes[nodeId] = {
382
- outboundRoutes: Array.from(nodeData.outboundRoutes),
383
- unreferencedTimestampMs,
384
- };
398
+ gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
385
399
  }
386
- this.currentGCState = { gcNodes };
400
+ this.gcDataFromLastRun = { gcNodes };
387
401
  });
388
402
 
389
403
  // Get the GC details for each data store from the GC state in the base summary. This is returned in
390
404
  // getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.
391
405
  this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionSummaryDetails>>(async () => {
392
- const gcNodes: { [ id: string ]: string[] } = {};
393
406
  const baseState = await baseSummaryStateP;
407
+ if (baseState === undefined) {
408
+ return new Map();
409
+ }
410
+
411
+ const gcNodes: { [ id: string ]: string[] } = {};
394
412
  for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
395
413
  gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
396
414
  }
@@ -400,7 +418,7 @@ export class GarbageCollector implements IGarbageCollector {
400
418
  const usedRoutes = runGarbageCollection(
401
419
  gcNodes,
402
420
  [ "/" ],
403
- this.logger,
421
+ this.mc.logger,
404
422
  ).referencedNodeIds;
405
423
 
406
424
  const dataStoreGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
@@ -433,7 +451,7 @@ export class GarbageCollector implements IGarbageCollector {
433
451
  },
434
452
  ): Promise<IGCStats> {
435
453
  const {
436
- logger = this.logger,
454
+ logger = this.mc.logger,
437
455
  runSweep = this.shouldRunSweep,
438
456
  fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged,
439
457
  } = options;
@@ -450,6 +468,9 @@ export class GarbageCollector implements IGarbageCollector {
450
468
 
451
469
  // Get the runtime's GC data and run GC on the reference graph in it.
452
470
  const gcData = await this.provider.getGCData(fullGC);
471
+
472
+ this.updateStateSinceLatestRun(gcData);
473
+
453
474
  const gcResult = runGarbageCollection(
454
475
  gcData.gcNodes,
455
476
  [ "/" ],
@@ -489,13 +510,21 @@ export class GarbageCollector implements IGarbageCollector {
489
510
  * We current write the entire GC state in a single blob. This can be modified later to write multiple
490
511
  * blobs. All the blob keys should start with `gcBlobPrefix`.
491
512
  */
492
- public summarize(): ISummaryTreeWithStats | undefined {
493
- if (!this.shouldRunGC || this.currentGCState === undefined) {
513
+ public summarize(): ISummaryTreeWithStats | undefined {
514
+ if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
494
515
  return;
495
516
  }
496
517
 
518
+ const gcState: IGarbageCollectionState = { gcNodes: {} };
519
+ for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
520
+ gcState.gcNodes[nodeId] = {
521
+ outboundRoutes,
522
+ unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
523
+ };
524
+ }
525
+
497
526
  const builder = new SummaryTreeBuilder();
498
- builder.addBlob(`${gcBlobPrefix}_root`, JSON.stringify(this.currentGCState));
527
+ builder.addBlob(`${gcBlobPrefix}_root`, JSON.stringify(gcState));
499
528
  return builder.getSummaryTree();
500
529
  }
501
530
 
@@ -537,7 +566,7 @@ export class GarbageCollector implements IGarbageCollector {
537
566
  // Prefix "/" if needed to make it relative to the root.
538
567
  const nodeId = id.startsWith("/") ? id : `/${id}`;
539
568
  this.unreferencedNodesState.get(nodeId)?.logIfInactive(
540
- this.logger,
569
+ this.mc.logger,
541
570
  "inactiveObjectChanged",
542
571
  this.getCurrentTimestampMs(),
543
572
  this.deleteTimeoutMs,
@@ -545,6 +574,19 @@ export class GarbageCollector implements IGarbageCollector {
545
574
  );
546
575
  }
547
576
 
577
+ /**
578
+ * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
579
+ * referenced between summaries so that their unreferenced timestamp can be reset.
580
+ *
581
+ * @param fromNodeId - The node from which the reference is added.
582
+ * @param toNodeId - The node to which the reference is added.
583
+ */
584
+ public addedOutboundReference(fromNodeId: string, toNodeId: string) {
585
+ const outboundRoutes = this.referencesSinceLastRun.get(fromNodeId) ?? [];
586
+ outboundRoutes.push(toNodeId);
587
+ this.referencesSinceLastRun.set(fromNodeId, outboundRoutes);
588
+ }
589
+
548
590
  /**
549
591
  * Update the latest summary GC version from the metadata blob in the given snapshot.
550
592
  */
@@ -566,15 +608,11 @@ export class GarbageCollector implements IGarbageCollector {
566
608
  * @param currentTimestampMs - The current timestamp to be used for unreferenced nodes' timestamp.
567
609
  */
568
610
  private updateCurrentState(gcData: IGarbageCollectionData, gcResult: IGCResult, currentTimestampMs: number) {
569
- this.currentGCState = { gcNodes: {} };
570
- for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
571
- this.currentGCState.gcNodes[id] = { outboundRoutes: Array.from(outboundRoutes) };
572
- }
611
+ this.gcDataFromLastRun = cloneGCData(gcData);
612
+ this.referencesSinceLastRun.clear();
573
613
 
574
614
  // Iterate through the deleted nodes and start tracking if they became unreferenced in this run.
575
615
  for (const nodeId of gcResult.deletedNodeIds) {
576
- assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2aa /* "Unexpected node when running GC" */);
577
-
578
616
  // The time when the node became unreferenced. This is added to the current GC state.
579
617
  let unreferencedTimestampMs: number = currentTimestampMs;
580
618
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
@@ -587,18 +625,16 @@ export class GarbageCollector implements IGarbageCollector {
587
625
  new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs),
588
626
  );
589
627
  }
590
- this.currentGCState.gcNodes[nodeId].unreferencedTimestampMs = unreferencedTimestampMs;
591
628
  }
592
629
 
593
630
  // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
594
631
  for (const nodeId of gcResult.referencedNodeIds) {
595
- assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2ab /* "Unexpected node when running GC" */);
596
632
  const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
597
633
  if (nodeStateTracker !== undefined) {
598
634
  // If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
599
635
  // log an error as this may mean the deleteTimeoutMs is not long enough.
600
636
  nodeStateTracker.logIfInactive(
601
- this.logger,
637
+ this.mc.logger,
602
638
  "inactiveObjectRevived",
603
639
  currentTimestampMs,
604
640
  this.deleteTimeoutMs,
@@ -611,6 +647,61 @@ export class GarbageCollector implements IGarbageCollector {
611
647
  }
612
648
  }
613
649
  }
650
+
651
+ /**
652
+ * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
653
+ * time. It's possible that nodes transition from `unreferenced -> referenced -> unreferenced` between two runs. The
654
+ * unreferenced timestamp of such nodes needs to be reset as they may have been accessed when they were referenced.
655
+ *
656
+ * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
657
+ * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
658
+ */
659
+ private updateStateSinceLatestRun(currentGCData: IGarbageCollectionData) {
660
+ // If we haven't run GC before or no references were added since the last run, there is nothing to do.
661
+ if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
662
+ return;
663
+ }
664
+
665
+ /**
666
+ * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
667
+ * edges that have been added since then. To do this, combine the GC data from the last run and the current
668
+ * run, and then add the references since last run.
669
+ *
670
+ * Note on why we need to combine the data from previous run, current run and all references in between -
671
+ * 1. We need data from last run because some of its references may have been deleted since then. If those
672
+ * references added new outbound references before getting deleted, we need to detect them.
673
+ * 2. We need new outbound references since last run because some of them may have been deleted later. If those
674
+ * references added new outbound references before getting deleted, we need to detect them.
675
+ * 3. We need data from the current run because currently we may not detect when DDSs are referenced:
676
+ * - We don't require DDSs handles to be stored in a referenced DDS. For this, we need GC at DDS level
677
+ * which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
678
+ * - A new data store may have "root" DDSs already created and we don't detect them today.
679
+ */
680
+ const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
681
+ this.referencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
682
+ if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
683
+ gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
684
+ } else {
685
+ gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
686
+ }
687
+ });
688
+
689
+ /**
690
+ * Run GC on the above reference graph to find all nodes that are referenced. For each one, if they are
691
+ * unreferenced, stop tracking them and remove from unreferenced list.
692
+ * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
693
+ */
694
+ const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/"], this.mc.logger);
695
+ for (const nodeId of gcResult.referencedNodeIds) {
696
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
697
+ if (nodeStateTracker !== undefined) {
698
+ // Stop tracking so as to clear out any running timers.
699
+ nodeStateTracker.stopTracking();
700
+ // Delete the node as we don't need to track it any more.
701
+ this.unreferencedNodesState.delete(nodeId);
702
+ }
703
+ }
704
+ }
614
705
  }
615
706
 
616
707
  /**
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "0.53.0";
9
+ export const pkgVersion = "0.54.0-47413";