@fluidframework/container-runtime 2.0.0-internal.4.3.0 → 2.0.0-internal.4.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +4 -0
  2. package/dist/blobManager.d.ts.map +1 -1
  3. package/dist/blobManager.js +3 -2
  4. package/dist/blobManager.js.map +1 -1
  5. package/dist/containerRuntime.d.ts.map +1 -1
  6. package/dist/containerRuntime.js +13 -7
  7. package/dist/containerRuntime.js.map +1 -1
  8. package/dist/dataStoreContext.d.ts.map +1 -1
  9. package/dist/dataStoreContext.js +1 -2
  10. package/dist/dataStoreContext.js.map +1 -1
  11. package/dist/gc/garbageCollection.d.ts +55 -43
  12. package/dist/gc/garbageCollection.d.ts.map +1 -1
  13. package/dist/gc/garbageCollection.js +219 -203
  14. package/dist/gc/garbageCollection.js.map +1 -1
  15. package/dist/gc/gcConfigs.d.ts.map +1 -1
  16. package/dist/gc/gcConfigs.js +8 -10
  17. package/dist/gc/gcConfigs.js.map +1 -1
  18. package/dist/gc/gcDefinitions.d.ts +2 -0
  19. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  20. package/dist/gc/gcDefinitions.js.map +1 -1
  21. package/dist/gc/gcHelpers.d.ts +11 -1
  22. package/dist/gc/gcHelpers.d.ts.map +1 -1
  23. package/dist/gc/gcHelpers.js +18 -3
  24. package/dist/gc/gcHelpers.js.map +1 -1
  25. package/dist/gc/gcSummaryStateTracker.d.ts +6 -2
  26. package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
  27. package/dist/gc/gcSummaryStateTracker.js +16 -6
  28. package/dist/gc/gcSummaryStateTracker.js.map +1 -1
  29. package/dist/gc/gcTelemetry.d.ts +2 -2
  30. package/dist/gc/gcTelemetry.d.ts.map +1 -1
  31. package/dist/gc/gcTelemetry.js +42 -22
  32. package/dist/gc/gcTelemetry.js.map +1 -1
  33. package/dist/gc/index.d.ts +1 -2
  34. package/dist/gc/index.d.ts.map +1 -1
  35. package/dist/gc/index.js +2 -5
  36. package/dist/gc/index.js.map +1 -1
  37. package/dist/packageVersion.d.ts +1 -1
  38. package/dist/packageVersion.js +1 -1
  39. package/dist/packageVersion.js.map +1 -1
  40. package/dist/scheduleManager.js +15 -4
  41. package/dist/scheduleManager.js.map +1 -1
  42. package/lib/blobManager.d.ts.map +1 -1
  43. package/lib/blobManager.js +3 -2
  44. package/lib/blobManager.js.map +1 -1
  45. package/lib/containerRuntime.d.ts.map +1 -1
  46. package/lib/containerRuntime.js +13 -7
  47. package/lib/containerRuntime.js.map +1 -1
  48. package/lib/dataStoreContext.d.ts.map +1 -1
  49. package/lib/dataStoreContext.js +1 -2
  50. package/lib/dataStoreContext.js.map +1 -1
  51. package/lib/gc/garbageCollection.d.ts +55 -43
  52. package/lib/gc/garbageCollection.d.ts.map +1 -1
  53. package/lib/gc/garbageCollection.js +219 -203
  54. package/lib/gc/garbageCollection.js.map +1 -1
  55. package/lib/gc/gcConfigs.d.ts.map +1 -1
  56. package/lib/gc/gcConfigs.js +8 -10
  57. package/lib/gc/gcConfigs.js.map +1 -1
  58. package/lib/gc/gcDefinitions.d.ts +2 -0
  59. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  60. package/lib/gc/gcDefinitions.js.map +1 -1
  61. package/lib/gc/gcHelpers.d.ts +11 -1
  62. package/lib/gc/gcHelpers.d.ts.map +1 -1
  63. package/lib/gc/gcHelpers.js +16 -2
  64. package/lib/gc/gcHelpers.js.map +1 -1
  65. package/lib/gc/gcSummaryStateTracker.d.ts +6 -2
  66. package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
  67. package/lib/gc/gcSummaryStateTracker.js +16 -6
  68. package/lib/gc/gcSummaryStateTracker.js.map +1 -1
  69. package/lib/gc/gcTelemetry.d.ts +2 -2
  70. package/lib/gc/gcTelemetry.d.ts.map +1 -1
  71. package/lib/gc/gcTelemetry.js +43 -23
  72. package/lib/gc/gcTelemetry.js.map +1 -1
  73. package/lib/gc/index.d.ts +1 -2
  74. package/lib/gc/index.d.ts.map +1 -1
  75. package/lib/gc/index.js +1 -2
  76. package/lib/gc/index.js.map +1 -1
  77. package/lib/packageVersion.d.ts +1 -1
  78. package/lib/packageVersion.js +1 -1
  79. package/lib/packageVersion.js.map +1 -1
  80. package/lib/scheduleManager.js +15 -4
  81. package/lib/scheduleManager.js.map +1 -1
  82. package/package.json +15 -16
  83. package/src/blobManager.ts +3 -2
  84. package/src/containerRuntime.ts +10 -4
  85. package/src/dataStoreContext.ts +1 -2
  86. package/src/gc/garbageCollection.ts +274 -257
  87. package/src/gc/gcConfigs.ts +12 -11
  88. package/src/gc/gcDefinitions.ts +2 -0
  89. package/src/gc/gcHelpers.ts +20 -2
  90. package/src/gc/gcSummaryStateTracker.ts +19 -7
  91. package/src/gc/gcTelemetry.ts +52 -37
  92. package/src/gc/index.ts +1 -5
  93. package/src/packageVersion.ts +1 -1
  94. package/src/scheduleManager.ts +19 -7
  95. package/dist/gc/gcSweepReadyUsageDetection.d.ts +0 -53
  96. package/dist/gc/gcSweepReadyUsageDetection.d.ts.map +0 -1
  97. package/dist/gc/gcSweepReadyUsageDetection.js +0 -130
  98. package/dist/gc/gcSweepReadyUsageDetection.js.map +0 -1
  99. package/lib/gc/gcSweepReadyUsageDetection.d.ts +0 -53
  100. package/lib/gc/gcSweepReadyUsageDetection.d.ts.map +0 -1
  101. package/lib/gc/gcSweepReadyUsageDetection.js +0 -125
  102. package/lib/gc/gcSweepReadyUsageDetection.js.map +0 -1
  103. package/src/gc/gcSweepReadyUsageDetection.ts +0 -145
@@ -91,8 +91,7 @@ export class GarbageCollector {
91
91
  // in the snapshot cannot be interpreted correctly. Set everything to undefined except for
92
92
  // deletedNodes because irrespective of GC versions, these nodes have been deleted and cannot be
93
93
  // brought back. The deletedNodes info is needed to identify when these nodes are used.
94
- if (this.configs.gcVersionInBaseSnapshot !==
95
- this.summaryStateTracker.currentGCVersion) {
94
+ if (this.configs.gcVersionInEffect !== this.configs.gcVersionInBaseSnapshot) {
96
95
  return {
97
96
  gcState: undefined,
98
97
  tombstones: undefined,
@@ -181,6 +180,10 @@ export class GarbageCollector {
181
180
  get summaryStateNeedsReset() {
182
181
  return this.summaryStateTracker.doesSummaryStateNeedReset;
183
182
  }
183
+ /** Returns the count of data stores whose GC state updated since the last summary. */
184
+ get updatedDSCountSinceLastSummary() {
185
+ return this.summaryStateTracker.updatedDSCountSinceLastSummary;
186
+ }
184
187
  /**
185
188
  * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
186
189
  * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
@@ -302,6 +305,13 @@ export class GarbageCollector {
302
305
  this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
303
306
  }
304
307
  }
308
+ /**
309
+ * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
310
+ * in the container.
311
+ */
312
+ async getBaseGCDetails() {
313
+ return this.baseGCDetailsP;
314
+ }
305
315
  /**
306
316
  * Runs garbage collection and updates the reference / used state of the nodes in the container.
307
317
  * @returns stats of the GC run or undefined if GC did not run.
@@ -337,53 +347,224 @@ export class GarbageCollector {
337
347
  return undefined;
338
348
  }
339
349
  return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
340
- await this.runPreGCSteps();
341
- // Get the runtime's GC data and run GC on the reference graph in it.
342
- const gcData = await this.runtime.getGCData(fullGC);
343
- const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
344
- const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
350
+ /** Pre-GC steps */
351
+ // Ensure that state has been initialized from the base snapshot data.
352
+ await this.initializeGCStateFromBaseSnapshotP;
353
+ // Let the runtime update its pending state before GC runs.
354
+ await this.runtime.updateStateBeforeGC();
355
+ /** GC step */
356
+ const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
345
357
  event.end(Object.assign(Object.assign({}, gcStats), { timestamp: currentReferenceTimestampMs }));
358
+ /** Post-GC steps */
359
+ // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
360
+ // updates its state so that we don't send false positives based on intermediate state. For example, we may get
361
+ // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
362
+ await this.telemetryTracker.logPendingEvents(logger);
363
+ // Update the state of summary state tracker from this run's stats.
364
+ this.summaryStateTracker.updateStateFromGCRunStats(gcStats);
365
+ this.newReferencesSinceLastRun.clear();
346
366
  this.completedRuns++;
347
367
  return gcStats;
348
368
  }, { end: true, cancel: "error" });
349
369
  }
350
- async runPreGCSteps() {
351
- // Ensure that state has been initialized from the base snapshot data.
352
- await this.initializeGCStateFromBaseSnapshotP;
353
- // Let the runtime update its pending state before GC runs.
354
- await this.runtime.updateStateBeforeGC();
355
- }
356
- async runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs) {
357
- // Generate statistics from the current run. This is done before updating the current state because it
358
- // generates some of its data based on previous state of the system.
370
+ /**
371
+ * Runs garbage collection. It does the following:
372
+ * 1. It generates / analyzes the runtime's reference graph.
373
+ * 2. Generates stats for the GC run based on previous / current GC state.
374
+ * 3. Runs Mark phase.
375
+ * 4. Runs Sweep phase.
376
+ */
377
+ async runGC(fullGC, currentReferenceTimestampMs, logger) {
378
+ var _a;
379
+ // 1. Generate / analyze the runtime's reference graph.
380
+ // Get the reference graph (gcData) and run GC algorithm to get referenced / unreferenced nodes.
381
+ const gcData = await this.runtime.getGCData(fullGC);
382
+ const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
383
+ // Get all referenced nodes - References in this run + references between the previous and current runs.
384
+ const allReferencedNodeIds = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
385
+ // 2. Generate stats based on the previous / current GC state.
386
+ // Must happen before running Mark / Sweep phase because previous GC state will be updated in these stages.
359
387
  const gcStats = this.generateStats(gcResult);
360
- // Update the current mark state and update the runtime of all used routes or ids that used as per the GC run.
361
- const sweepReadyNodes = this.updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger);
388
+ // 3. Run the Mark phase.
389
+ // It will mark nodes as referenced / unreferenced and return a list of node ids that are ready to be swept.
390
+ const sweepReadyNodeIds = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
391
+ // 4. Run the Sweep phase.
392
+ // It will delete sweep ready nodes and return a list of deleted node ids.
393
+ const deletedNodeIds = this.runSweepPhase(gcResult, sweepReadyNodeIds, currentReferenceTimestampMs, logger);
394
+ this.gcDataFromLastRun = cloneGCData(gcData, (id) => deletedNodeIds.includes(id) /* filter out deleted nodes */);
395
+ return gcStats;
396
+ }
397
+ /**
398
+ * Runs the GC Mark phase. It does the following:
399
+ * 1. Marks all referenced nodes in this run by clearing tracking for them.
400
+ * 2. Marks unreferenced nodes in this run by starting tracking for them.
401
+ * 3. Calls the runtime to update nodes that were marked referenced.
402
+ *
403
+ * @param gcResult - The result of the GC run on the gcData.
404
+ * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
405
+ * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
406
+ * @returns - A list of sweep ready nodes, i.e., nodes that ready to be deleted.
407
+ */
408
+ runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
409
+ // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
410
+ for (const nodeId of allReferencedNodeIds) {
411
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
412
+ if (nodeStateTracker !== undefined) {
413
+ // Stop tracking so as to clear out any running timers.
414
+ nodeStateTracker.stopTracking();
415
+ // Delete the node as we don't need to track it any more.
416
+ this.unreferencedNodesState.delete(nodeId);
417
+ }
418
+ }
419
+ // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
420
+ const sweepReadyNodeIds = [];
421
+ for (const nodeId of gcResult.deletedNodeIds) {
422
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
423
+ if (nodeStateTracker === undefined) {
424
+ this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
425
+ }
426
+ else {
427
+ // If a node was already unreferenced, update its tracking information. Since the current reference time
428
+ // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
429
+ nodeStateTracker.updateTracking(currentReferenceTimestampMs);
430
+ // If a node is sweep ready, store it so it can be returned.
431
+ if (nodeStateTracker.state === UnreferencedState.SweepReady) {
432
+ sweepReadyNodeIds.push(nodeId);
433
+ }
434
+ }
435
+ }
436
+ // 3. Call the runtime to update referenced nodes in this run.
362
437
  this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
363
- // Log events for objects that are ready to be deleted by sweep.
438
+ return sweepReadyNodeIds;
439
+ }
440
+ /**
441
+ * Runs the GC Sweep phase. It does the following:
442
+ * 1. Calls the runtime to delete nodes that are sweep ready.
443
+ * 2. Clears tracking for deleted nodes.
444
+ *
445
+ * @param gcResult - The result of the GC run on the gcData.
446
+ * @param sweepReadyNodes - List of nodes that are sweep ready.
447
+ * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
448
+ * @param logger - The logger to be used to log any telemetry.
449
+ * @returns - A list of nodes that have been deleted.
450
+ */
451
+ runSweepPhase(gcResult, sweepReadyNodes, currentReferenceTimestampMs, logger) {
452
+ // Log events for objects that are ready to be deleted by sweep. This will give us data on sweep when
453
+ // its not enabled.
364
454
  this.telemetryTracker.logSweepEvents(logger, currentReferenceTimestampMs, this.unreferencedNodesState, this.completedRuns, this.getLastSummaryTimestampMs());
365
- let updatedGCData = gcData;
366
- if (this.configs.shouldRunSweep) {
367
- updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
368
- }
369
- else if (this.configs.testMode) {
370
- // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
371
- // involving access to deleted data.
455
+ /**
456
+ * Currently, there are 3 modes for sweep:
457
+ * Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
458
+ * Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted.
459
+ * Sweep mode - Sweep ready modes are deleted.
460
+ *
461
+ * These modes serve as staging for applications that want to enable sweep by providing an incremental
462
+ * way to test and validate sweep works as expected.
463
+ */
464
+ if (this.configs.testMode) {
465
+ // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
372
466
  this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
467
+ return [];
373
468
  }
374
- else if (this.configs.tombstoneMode) {
469
+ if (this.configs.tombstoneMode) {
375
470
  this.tombstones = sweepReadyNodes;
376
471
  // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
377
472
  // involving access to "deleted" data without actually deleting the data from summaries.
378
- // Note: we will not tombstone in test mode.
379
473
  this.runtime.updateTombstonedRoutes(this.tombstones);
474
+ return [];
380
475
  }
381
- this.gcDataFromLastRun = cloneGCData(updatedGCData);
382
- // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
383
- // updates its state so that we don't send false positives based on intermediate state. For example, we may get
384
- // reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
385
- await this.telemetryTracker.logPendingEvents(logger);
386
- return gcStats;
476
+ if (!this.configs.shouldRunSweep) {
477
+ return [];
478
+ }
479
+ // 1. Call the runtime to delete sweep ready nodes. The runtime returns a list of nodes it deleted.
480
+ // TODO: GC:Validation - validate that removed routes are not double delete and that the child routes of
481
+ // removed routes are deleted as well.
482
+ const deletedNodeIds = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
483
+ // 2. Clear unreferenced state tracking for deleted nodes.
484
+ for (const nodeId of deletedNodeIds) {
485
+ const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
486
+ // TODO: GC:Validation - assert that the nodeStateTracker is defined
487
+ if (nodeStateTracker !== undefined) {
488
+ // Stop tracking so as to clear out any running timers.
489
+ nodeStateTracker.stopTracking();
490
+ // Delete the node as we don't need to track it any more.
491
+ this.unreferencedNodesState.delete(nodeId);
492
+ }
493
+ // TODO: GC:Validation - assert that the deleted node is not a duplicate
494
+ this.deletedNodes.add(nodeId);
495
+ }
496
+ return deletedNodeIds;
497
+ }
498
+ /**
499
+ * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
500
+ * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
501
+ * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
502
+ * these objects while there can be in-memory referenced to it:
503
+ * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
504
+ * added, the object may have been accessed and in-memory reference to it added.
505
+ * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
506
+ * unreferenced, they could have been accessed and in-memory reference to them added.
507
+ *
508
+ * This function identifies nodes that were referenced since the last run.
509
+ * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
510
+ *
511
+ * @returns - a list of all nodes referenced from the last local summary until now.
512
+ */
513
+ findAllNodesReferencedBetweenGCs(currentGCData, previousGCData, logger) {
514
+ // If we haven't run GC before there is nothing to do.
515
+ // No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
516
+ if (previousGCData === undefined) {
517
+ return undefined;
518
+ }
519
+ /**
520
+ * If there are references that were not explicitly notified to GC, log an error because this should never happen.
521
+ * If it does, this may result in the unreferenced timestamps of these nodes not updated when they were referenced.
522
+ */
523
+ this.telemetryTracker.logIfMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun, logger);
524
+ // No references were added since the last run so we don't have to update reference states of any unreferenced
525
+ // nodes. There is no in between state at this point.
526
+ if (this.newReferencesSinceLastRun.size === 0) {
527
+ return undefined;
528
+ }
529
+ /**
530
+ * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
531
+ * edges that have been added since then. To do this, combine the GC data from the last run and the current
532
+ * run, and then add the references since last run.
533
+ *
534
+ * Note on why we need to combine the data from previous run, current run and all references in between -
535
+ * 1. We need data from last run because some of its references may have been deleted since then. If those
536
+ * references added new outbound references before they were deleted, we need to detect them.
537
+ *
538
+ * 2. We need new outbound references since last run because some of them may have been deleted later. If those
539
+ * references added new outbound references before they were deleted, we need to detect them.
540
+ *
541
+ * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
542
+ * - We don't require DDSes handles to be stored in a referenced DDS.
543
+ * - A new data store may have "root" DDSes already created and we don't detect them today.
544
+ */
545
+ const gcDataSuperSet = concatGarbageCollectionData(previousGCData, currentGCData);
546
+ const newOutboundRoutesSinceLastRun = [];
547
+ this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
548
+ if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
549
+ gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
550
+ }
551
+ else {
552
+ gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
553
+ }
554
+ newOutboundRoutesSinceLastRun.push(...outboundRoutes);
555
+ });
556
+ /**
557
+ * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
558
+ * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
559
+ * unreferenced, stop tracking them and remove from unreferenced list.
560
+ * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
561
+ * unreferenced and add unreferenced state.
562
+ */
563
+ const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [
564
+ "/",
565
+ ...newOutboundRoutesSinceLastRun,
566
+ ]);
567
+ return gcResult.referencedNodeIds;
387
568
  }
388
569
  /**
389
570
  * Summarizes the GC data and returns it as a summary tree.
@@ -407,23 +588,16 @@ export class GarbageCollector {
407
588
  getMetadata() {
408
589
  return {
409
590
  /**
410
- * If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
591
+ * If GC is enabled, the GC data is written using the GC version in effect and that is the gcFeature that goes
411
592
  * into the metadata blob. If GC is disabled, the gcFeature is 0.
412
593
  */
413
- gcFeature: this.configs.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
594
+ gcFeature: this.configs.gcEnabled ? this.configs.gcVersionInEffect : 0,
414
595
  gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
415
596
  sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
416
597
  sweepEnabled: false,
417
598
  sweepTimeoutMs: this.configs.sweepTimeoutMs,
418
599
  };
419
600
  }
420
- /**
421
- * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
422
- * in the container.
423
- */
424
- async getBaseGCDetails() {
425
- return this.baseGCDetailsP;
426
- }
427
601
  /**
428
602
  * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
429
603
  * is downloaded and should be used to update the state.
@@ -459,7 +633,7 @@ export class GarbageCollector {
459
633
  return;
460
634
  }
461
635
  this.telemetryTracker.nodeUsed({
462
- nodeId: nodePath,
636
+ id: nodePath,
463
637
  usageType: reason,
464
638
  currentReferenceTimestampMs: timestampMs !== null && timestampMs !== void 0 ? timestampMs : this.runtime.getCurrentReferenceTimestampMs(),
465
639
  packagePath,
@@ -485,7 +659,7 @@ export class GarbageCollector {
485
659
  outboundRoutes.push(toNodePath);
486
660
  this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
487
661
  this.telemetryTracker.nodeUsed({
488
- nodeId: toNodePath,
662
+ id: toNodePath,
489
663
  usageType: "Revived",
490
664
  currentReferenceTimestampMs: this.runtime.getCurrentReferenceTimestampMs(),
491
665
  packagePath: undefined,
@@ -507,164 +681,6 @@ export class GarbageCollector {
507
681
  (_a = this.sessionExpiryTimer) === null || _a === void 0 ? void 0 : _a.clear();
508
682
  this.sessionExpiryTimer = undefined;
509
683
  }
510
- /**
511
- * Updates the state of the system as per the current GC run. It does the following:
512
- * 1. Sets up the current GC state as per the gcData.
513
- * 2. Starts tracking for nodes that have become unreferenced in this run.
514
- * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
515
- * @param gcData - The data representing the reference graph on which GC is run.
516
- * @param gcResult - The result of the GC run on the gcData.
517
- * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
518
- * @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
519
- */
520
- updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger) {
521
- var _a;
522
- // Get references from the current GC run + references between previous and current run and then update each
523
- // node's state
524
- const allNodesReferencedBetweenGCs = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
525
- this.newReferencesSinceLastRun.clear();
526
- // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
527
- for (const nodeId of allNodesReferencedBetweenGCs) {
528
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
529
- if (nodeStateTracker !== undefined) {
530
- // Stop tracking so as to clear out any running timers.
531
- nodeStateTracker.stopTracking();
532
- // Delete the node as we don't need to track it any more.
533
- this.unreferencedNodesState.delete(nodeId);
534
- }
535
- }
536
- /**
537
- * If a node became unreferenced in this run, start tracking it.
538
- * If a node was already unreferenced, update its tracking information. Since the current reference time is
539
- * from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
540
- *
541
- * If a node is sweep ready, store and then return it.
542
- */
543
- const sweepReadyNodes = [];
544
- for (const nodeId of gcResult.deletedNodeIds) {
545
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
546
- if (nodeStateTracker === undefined) {
547
- this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
548
- }
549
- else {
550
- nodeStateTracker.updateTracking(currentReferenceTimestampMs);
551
- if (nodeStateTracker.state === UnreferencedState.SweepReady) {
552
- sweepReadyNodes.push(nodeId);
553
- }
554
- }
555
- }
556
- return sweepReadyNodes;
557
- }
558
- /**
559
- * Deletes nodes from both the runtime and garbage collection
560
- * @param sweepReadyNodes - nodes that are ready to be deleted
561
- */
562
- runSweepPhase(sweepReadyNodes, gcData) {
563
- // TODO: GC:Validation - validate that removed routes are not double deleted
564
- // TODO: GC:Validation - validate that the child routes of removed routes are deleted as well
565
- const sweptRoutes = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
566
- const updatedGCData = this.deleteSweptRoutes(sweptRoutes, gcData);
567
- for (const nodeId of sweptRoutes) {
568
- const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
569
- // TODO: GC:Validation - assert that the nodeStateTracker is defined
570
- if (nodeStateTracker !== undefined) {
571
- // Stop tracking so as to clear out any running timers.
572
- nodeStateTracker.stopTracking();
573
- // Delete the node as we don't need to track it any more.
574
- this.unreferencedNodesState.delete(nodeId);
575
- }
576
- // TODO: GC:Validation - assert that the deleted node is not a duplicate
577
- this.deletedNodes.add(nodeId);
578
- }
579
- return updatedGCData;
580
- }
581
- /**
582
- * @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
583
- */
584
- deleteSweptRoutes(sweptRoutes, gcData) {
585
- const sweptRoutesSet = new Set(sweptRoutes);
586
- const gcNodes = {};
587
- for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
588
- if (!sweptRoutesSet.has(id)) {
589
- gcNodes[id] = Array.from(outboundRoutes);
590
- }
591
- }
592
- // TODO: GC:Validation - assert that the nodeId is in gcData
593
- return {
594
- gcNodes,
595
- };
596
- }
597
- /**
598
- * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
599
- * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
600
- * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
601
- * these objects while there can be in-memory referenced to it:
602
- * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
603
- * added, the object may have been accessed and in-memory reference to it added.
604
- * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
605
- * unreferenced, they could have been accessed and in-memory reference to them added.
606
- *
607
- * This function identifies nodes that were referenced since the last run.
608
- * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
609
- *
610
- * @returns - a list of all nodes referenced from the last local summary until now.
611
- */
612
- findAllNodesReferencedBetweenGCs(currentGCData, previousGCData, logger) {
613
- // If we haven't run GC before there is nothing to do.
614
- // No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
615
- if (previousGCData === undefined) {
616
- return undefined;
617
- }
618
- /**
619
- * If there are references that were not explicitly notified to GC, log an error because this should never happen.
620
- * If it does, this may result in the unreferenced timestamps of these nodes not updated when they were referenced.
621
- */
622
- this.telemetryTracker.logIfMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun, logger);
623
- // No references were added since the last run so we don't have to update reference states of any unreferenced
624
- // nodes. There is no in between state at this point.
625
- if (this.newReferencesSinceLastRun.size === 0) {
626
- return undefined;
627
- }
628
- /**
629
- * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
630
- * edges that have been added since then. To do this, combine the GC data from the last run and the current
631
- * run, and then add the references since last run.
632
- *
633
- * Note on why we need to combine the data from previous run, current run and all references in between -
634
- * 1. We need data from last run because some of its references may have been deleted since then. If those
635
- * references added new outbound references before they were deleted, we need to detect them.
636
- *
637
- * 2. We need new outbound references since last run because some of them may have been deleted later. If those
638
- * references added new outbound references before they were deleted, we need to detect them.
639
- *
640
- * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
641
- * - We don't require DDSes handles to be stored in a referenced DDS.
642
- * - A new data store may have "root" DDSes already created and we don't detect them today.
643
- */
644
- const gcDataSuperSet = concatGarbageCollectionData(previousGCData, currentGCData);
645
- const newOutboundRoutesSinceLastRun = [];
646
- this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
647
- if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
648
- gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
649
- }
650
- else {
651
- gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
652
- }
653
- newOutboundRoutesSinceLastRun.push(...outboundRoutes);
654
- });
655
- /**
656
- * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
657
- * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
658
- * unreferenced, stop tracking them and remove from unreferenced list.
659
- * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
660
- * unreferenced and add unreferenced state.
661
- */
662
- const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [
663
- "/",
664
- ...newOutboundRoutesSinceLastRun,
665
- ]);
666
- return gcResult.referencedNodeIds;
667
- }
668
684
  /**
669
685
  * Generates the stats of a garbage collection run from the given results of the run.
670
686
  * @param gcResult - The result of a GC run.