@fluidframework/container-runtime 0.58.2001 → 0.59.1000-61898

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/dist/blobManager.d.ts +15 -2
  2. package/dist/blobManager.d.ts.map +1 -1
  3. package/dist/blobManager.js +65 -9
  4. package/dist/blobManager.js.map +1 -1
  5. package/dist/connectionTelemetry.d.ts.map +1 -1
  6. package/dist/connectionTelemetry.js +63 -23
  7. package/dist/connectionTelemetry.js.map +1 -1
  8. package/dist/containerRuntime.d.ts +39 -7
  9. package/dist/containerRuntime.d.ts.map +1 -1
  10. package/dist/containerRuntime.js +161 -29
  11. package/dist/containerRuntime.js.map +1 -1
  12. package/dist/dataStore.js +8 -1
  13. package/dist/dataStore.js.map +1 -1
  14. package/dist/dataStoreContext.d.ts +9 -3
  15. package/dist/dataStoreContext.d.ts.map +1 -1
  16. package/dist/dataStoreContext.js +22 -6
  17. package/dist/dataStoreContext.js.map +1 -1
  18. package/dist/dataStores.d.ts +13 -5
  19. package/dist/dataStores.d.ts.map +1 -1
  20. package/dist/dataStores.js +39 -18
  21. package/dist/dataStores.js.map +1 -1
  22. package/dist/deltaScheduler.d.ts +4 -5
  23. package/dist/deltaScheduler.d.ts.map +1 -1
  24. package/dist/deltaScheduler.js +54 -35
  25. package/dist/deltaScheduler.js.map +1 -1
  26. package/dist/garbageCollection.d.ts +31 -27
  27. package/dist/garbageCollection.d.ts.map +1 -1
  28. package/dist/garbageCollection.js +76 -75
  29. package/dist/garbageCollection.js.map +1 -1
  30. package/dist/opTelemetry.d.ts +22 -0
  31. package/dist/opTelemetry.d.ts.map +1 -0
  32. package/dist/opTelemetry.js +59 -0
  33. package/dist/opTelemetry.js.map +1 -0
  34. package/dist/orderedClientElection.d.ts +57 -6
  35. package/dist/orderedClientElection.d.ts.map +1 -1
  36. package/dist/orderedClientElection.js +140 -25
  37. package/dist/orderedClientElection.js.map +1 -1
  38. package/dist/packageVersion.d.ts +1 -1
  39. package/dist/packageVersion.d.ts.map +1 -1
  40. package/dist/packageVersion.js +1 -1
  41. package/dist/packageVersion.js.map +1 -1
  42. package/dist/summarizerClientElection.d.ts +2 -0
  43. package/dist/summarizerClientElection.d.ts.map +1 -1
  44. package/dist/summarizerClientElection.js +7 -2
  45. package/dist/summarizerClientElection.js.map +1 -1
  46. package/dist/summarizerTypes.d.ts +9 -0
  47. package/dist/summarizerTypes.d.ts.map +1 -1
  48. package/dist/summarizerTypes.js.map +1 -1
  49. package/dist/summaryGenerator.d.ts.map +1 -1
  50. package/dist/summaryGenerator.js +1 -1
  51. package/dist/summaryGenerator.js.map +1 -1
  52. package/dist/summaryManager.d.ts.map +1 -1
  53. package/dist/summaryManager.js +14 -3
  54. package/dist/summaryManager.js.map +1 -1
  55. package/lib/blobManager.d.ts +15 -2
  56. package/lib/blobManager.d.ts.map +1 -1
  57. package/lib/blobManager.js +66 -10
  58. package/lib/blobManager.js.map +1 -1
  59. package/lib/connectionTelemetry.d.ts.map +1 -1
  60. package/lib/connectionTelemetry.js +63 -23
  61. package/lib/connectionTelemetry.js.map +1 -1
  62. package/lib/containerRuntime.d.ts +39 -7
  63. package/lib/containerRuntime.d.ts.map +1 -1
  64. package/lib/containerRuntime.js +163 -31
  65. package/lib/containerRuntime.js.map +1 -1
  66. package/lib/dataStore.js +8 -1
  67. package/lib/dataStore.js.map +1 -1
  68. package/lib/dataStoreContext.d.ts +9 -3
  69. package/lib/dataStoreContext.d.ts.map +1 -1
  70. package/lib/dataStoreContext.js +22 -6
  71. package/lib/dataStoreContext.js.map +1 -1
  72. package/lib/dataStores.d.ts +13 -5
  73. package/lib/dataStores.d.ts.map +1 -1
  74. package/lib/dataStores.js +39 -18
  75. package/lib/dataStores.js.map +1 -1
  76. package/lib/deltaScheduler.d.ts +4 -5
  77. package/lib/deltaScheduler.d.ts.map +1 -1
  78. package/lib/deltaScheduler.js +54 -35
  79. package/lib/deltaScheduler.js.map +1 -1
  80. package/lib/garbageCollection.d.ts +31 -27
  81. package/lib/garbageCollection.d.ts.map +1 -1
  82. package/lib/garbageCollection.js +75 -74
  83. package/lib/garbageCollection.js.map +1 -1
  84. package/lib/opTelemetry.d.ts +22 -0
  85. package/lib/opTelemetry.d.ts.map +1 -0
  86. package/lib/opTelemetry.js +55 -0
  87. package/lib/opTelemetry.js.map +1 -0
  88. package/lib/orderedClientElection.d.ts +57 -6
  89. package/lib/orderedClientElection.d.ts.map +1 -1
  90. package/lib/orderedClientElection.js +140 -25
  91. package/lib/orderedClientElection.js.map +1 -1
  92. package/lib/packageVersion.d.ts +1 -1
  93. package/lib/packageVersion.d.ts.map +1 -1
  94. package/lib/packageVersion.js +1 -1
  95. package/lib/packageVersion.js.map +1 -1
  96. package/lib/summarizerClientElection.d.ts +2 -0
  97. package/lib/summarizerClientElection.d.ts.map +1 -1
  98. package/lib/summarizerClientElection.js +7 -2
  99. package/lib/summarizerClientElection.js.map +1 -1
  100. package/lib/summarizerTypes.d.ts +9 -0
  101. package/lib/summarizerTypes.d.ts.map +1 -1
  102. package/lib/summarizerTypes.js.map +1 -1
  103. package/lib/summaryGenerator.d.ts.map +1 -1
  104. package/lib/summaryGenerator.js +1 -1
  105. package/lib/summaryGenerator.js.map +1 -1
  106. package/lib/summaryManager.d.ts.map +1 -1
  107. package/lib/summaryManager.js +14 -3
  108. package/lib/summaryManager.js.map +1 -1
  109. package/package.json +63 -19
  110. package/src/blobManager.ts +78 -11
  111. package/src/connectionTelemetry.ts +110 -19
  112. package/src/containerRuntime.ts +191 -36
  113. package/src/dataStore.ts +7 -1
  114. package/src/dataStoreContext.ts +22 -7
  115. package/src/dataStores.ts +40 -19
  116. package/src/deltaScheduler.ts +65 -39
  117. package/src/garbageCollection.ts +92 -78
  118. package/src/opTelemetry.ts +71 -0
  119. package/src/orderedClientElection.ts +154 -25
  120. package/src/packageVersion.ts +1 -1
  121. package/src/summarizerClientElection.ts +7 -2
  122. package/src/summarizerTypes.ts +9 -0
  123. package/src/summaryGenerator.ts +9 -1
  124. package/src/summaryManager.ts +15 -4
@@ -10,7 +10,6 @@ import {
10
10
  FluidObject,
11
11
  IFluidHandle,
12
12
  IFluidHandleContext,
13
- IFluidObject,
14
13
  IFluidRouter,
15
14
  IRequest,
16
15
  IResponse,
@@ -91,7 +90,6 @@ import {
91
90
  import {
92
91
  addBlobToSummary,
93
92
  addTreeToSummary,
94
- convertToSummaryTree,
95
93
  createRootSummarizerNodeWithGC,
96
94
  IRootSummarizerNodeWithGC,
97
95
  RequestParser,
@@ -100,7 +98,9 @@ import {
100
98
  requestFluidObject,
101
99
  responseToException,
102
100
  seqFromTree,
101
+ calculateStats,
103
102
  } from "@fluidframework/runtime-utils";
103
+ import { GCDataBuilder } from "@fluidframework/garbage-collector";
104
104
  import { v4 as uuid } from "uuid";
105
105
  import { ContainerFluidHandleContext } from "./containerHandleContext";
106
106
  import { FluidDataStoreRegistry } from "./dataStoreRegistry";
@@ -141,6 +141,7 @@ import { formExponentialFn, Throttler } from "./throttler";
141
141
  import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
142
142
  import {
143
143
  GarbageCollector,
144
+ GCNodeType,
144
145
  gcTreeKey,
145
146
  IGarbageCollectionRuntime,
146
147
  IGarbageCollector,
@@ -152,6 +153,7 @@ import {
152
153
  isDataStoreAliasMessage,
153
154
  } from "./dataStore";
154
155
  import { BindBatchTracker } from "./batchTracker";
156
+ import { OpTracker } from "./opTelemetry";
155
157
 
156
158
  export enum ContainerMessageType {
157
159
  // An op to be delivered to store
@@ -277,8 +279,8 @@ export interface ISummaryRuntimeOptions {
277
279
  * Options for container runtime.
278
280
  */
279
281
  export interface IContainerRuntimeOptions {
280
- summaryOptions?: ISummaryRuntimeOptions;
281
- gcOptions?: IGCRuntimeOptions;
282
+ readonly summaryOptions?: ISummaryRuntimeOptions;
283
+ readonly gcOptions?: IGCRuntimeOptions;
282
284
  /**
283
285
  * Affects the behavior while loading the runtime when the data verification check which
284
286
  * compares the DeltaManager sequence number (obtained from protocol in summary) to the
@@ -287,13 +289,20 @@ export interface IContainerRuntimeOptions {
287
289
  * 2. "log" will log an error event to telemetry, but still continue to load.
288
290
  * 3. "bypass" will skip the check entirely. This is not recommended.
289
291
  */
290
- loadSequenceNumberVerification?: "close" | "log" | "bypass";
292
+ readonly loadSequenceNumberVerification?: "close" | "log" | "bypass";
291
293
  /**
292
294
  * Should the runtime use data store aliasing for creating root datastores.
293
295
  * In case of aliasing conflicts, the runtime will raise an exception which does
294
296
  * not effect the status of the container.
295
297
  */
296
- useDataStoreAliasing?: boolean;
298
+ readonly useDataStoreAliasing?: boolean;
299
+ /**
300
+ * Sets the flush mode for the runtime. In Immediate flush mode the runtime will immediately
301
+ * send all operations to the driver layer, while in TurnBased the operations will be buffered
302
+ * and then sent them as a single batch at the end of the turn.
303
+ * By default, flush mode is TurnBased.
304
+ */
305
+ readonly flushMode?: FlushMode;
297
306
  }
298
307
 
299
308
  type IRuntimeMessageMetadata = undefined | {
@@ -347,6 +356,13 @@ const maxOpSizeInBytesKey = "Fluid.ContainerRuntime.MaxOpSizeInBytes";
347
356
  // to not reach the 1MB limits in socket.io and Kafka.
348
357
  const defaultMaxOpSizeInBytes = 768000;
349
358
 
359
+ // By default, the size of the contents for the incoming ops is tracked.
360
+ // However, in certain situations, this may incur a performance hit.
361
+ // The feature-gate below can be used to disable this feature.
362
+ const disableOpTrackingKey = "Fluid.ContainerRuntime.DisableOpTracking";
363
+
364
+ const defaultFlushMode = FlushMode.TurnBased;
365
+
350
366
  export enum RuntimeMessage {
351
367
  FluidDataStoreOp = "component",
352
368
  Attach = "attach",
@@ -394,6 +410,7 @@ class ScheduleManagerCore {
394
410
  private currentBatchClientId: string | undefined;
395
411
  private localPaused = false;
396
412
  private timePaused = 0;
413
+ private batchCount = 0;
397
414
 
398
415
  constructor(
399
416
  private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
@@ -482,14 +499,30 @@ class ScheduleManagerCore {
482
499
  this.deltaManager.inbound.pause();
483
500
  }
484
501
 
485
- private resumeQueue(startBatch: number, endBatch: number) {
502
+ private resumeQueue(startBatch: number, messageEndBatch: ISequencedDocumentMessage) {
503
+ const endBatch = messageEndBatch.sequenceNumber;
504
+ const duration = performance.now() - this.timePaused;
505
+
506
+ this.batchCount++;
507
+ if (this.batchCount % 1000 === 1) {
508
+ this.logger.sendTelemetryEvent({
509
+ eventName: "BatchStats",
510
+ sequenceNumber: endBatch,
511
+ length: endBatch - startBatch + 1,
512
+ msnDistance: endBatch - messageEndBatch.minimumSequenceNumber,
513
+ duration,
514
+ batchCount: this.batchCount,
515
+ interrupted: this.localPaused,
516
+ });
517
+ }
518
+
486
519
  // Return early if no change in value
487
520
  if (!this.localPaused) {
488
521
  return;
489
522
  }
490
523
 
491
524
  this.localPaused = false;
492
- const duration = performance.now() - this.timePaused;
525
+
493
526
  // Random round number - we want to know when batch waiting paused op processing.
494
527
  if (duration > latencyThreshold) {
495
528
  this.logger.sendErrorEvent({
@@ -564,7 +597,7 @@ class ScheduleManagerCore {
564
597
  } else if (batchMetadata === false) {
565
598
  assert(this.pauseSequenceNumber !== undefined, 0x2a0 /* "batch presence was validated above" */);
566
599
  // Batch is complete, we can process it!
567
- this.resumeQueue(this.pauseSequenceNumber, message.sequenceNumber);
600
+ this.resumeQueue(this.pauseSequenceNumber, message);
568
601
  this.pauseSequenceNumber = undefined;
569
602
  this.currentBatchClientId = undefined;
570
603
  } else {
@@ -605,7 +638,7 @@ export class ScheduleManager {
605
638
 
606
639
  // This could be the beginning of a new batch or an individual message.
607
640
  this.emitter.emit("batchBegin", message);
608
- this.deltaScheduler.batchBegin();
641
+ this.deltaScheduler.batchBegin(message);
609
642
 
610
643
  const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
611
644
  if (batch) {
@@ -626,7 +659,7 @@ export class ScheduleManager {
626
659
  this.hitError = true;
627
660
  this.batchClientId = undefined;
628
661
  this.emitter.emit("batchEnd", error, message);
629
- this.deltaScheduler.batchEnd();
662
+ this.deltaScheduler.batchEnd(message);
630
663
  return;
631
664
  }
632
665
 
@@ -636,7 +669,7 @@ export class ScheduleManager {
636
669
  if (this.batchClientId === undefined || batch === false) {
637
670
  this.batchClientId = undefined;
638
671
  this.emitter.emit("batchEnd", undefined, message);
639
- this.deltaScheduler.batchEnd();
672
+ this.deltaScheduler.batchEnd(message);
640
673
  return;
641
674
  }
642
675
  }
@@ -710,6 +743,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
710
743
  gcOptions = {},
711
744
  loadSequenceNumberVerification = "close",
712
745
  useDataStoreAliasing = false,
746
+ flushMode = defaultFlushMode,
713
747
  } = runtimeOptions;
714
748
 
715
749
  // We pack at data store level only. If isolated channels are disabled,
@@ -805,6 +839,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
805
839
  gcOptions,
806
840
  loadSequenceNumberVerification,
807
841
  useDataStoreAliasing,
842
+ flushMode,
808
843
  },
809
844
  containerScope,
810
845
  logger,
@@ -873,7 +908,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
873
908
  return this._flushMode;
874
909
  }
875
910
 
876
- public get scope(): IFluidObject & FluidObject {
911
+ public get scope(): FluidObject {
877
912
  return this.containerScope;
878
913
  }
879
914
 
@@ -909,7 +944,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
909
944
  private readonly defaultMaxConsecutiveReconnects = 15;
910
945
 
911
946
  private _orderSequentiallyCalls: number = 0;
912
- private _flushMode: FlushMode = FlushMode.TurnBased;
947
+ private _flushMode: FlushMode;
913
948
  private needsFlush = false;
914
949
  private flushTrigger = false;
915
950
 
@@ -983,6 +1018,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
983
1018
 
984
1019
  private readonly createContainerMetadata: ICreateContainerMetadata;
985
1020
  private summaryCount: number | undefined;
1021
+ private readonly opTracker: OpTracker;
986
1022
 
987
1023
  private constructor(
988
1024
  private readonly context: IContainerContext,
@@ -1037,16 +1073,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1037
1073
  this.maxConsecutiveReconnects =
1038
1074
  this.mc.config.getNumber(maxConsecutiveReconnectsKey) ?? this.defaultMaxConsecutiveReconnects;
1039
1075
 
1076
+ this._flushMode = runtimeOptions.flushMode;
1040
1077
  this.garbageCollector = GarbageCollector.create(
1041
1078
  this,
1042
1079
  this.runtimeOptions.gcOptions,
1043
- (unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
1044
- (nodePath: string) => this.dataStores.getNodePackagePath(nodePath),
1045
- /**
1046
- * Returns the timestamp of the last message seen by this client. This is used by garbage collector as
1047
- * the current reference timestamp for tracking unreferenced objects.
1048
- */
1049
- () => this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp,
1080
+ (nodePath: string) => this.getGCNodePackagePath(nodePath),
1050
1081
  () => this.messageAtLastSummary?.timestamp,
1051
1082
  context.baseSnapshot,
1052
1083
  async <T>(id: string) => readAndParse<T>(this.storage, id),
@@ -1098,7 +1129,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1098
1129
  ),
1099
1130
  (id: string) => this.summarizerNode.deleteChild(id),
1100
1131
  this.mc.logger,
1101
- async () => this.garbageCollector.getDataStoreBaseGCDetails(),
1132
+ async () => this.garbageCollector.getBaseGCDetails(),
1102
1133
  (path: string, timestampMs: number, packagePath?: readonly string[]) => this.garbageCollector.nodeUpdated(
1103
1134
  path,
1104
1135
  "Changed",
@@ -1277,6 +1308,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1277
1308
 
1278
1309
  ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
1279
1310
  BindBatchTracker(this, this.logger);
1311
+ this.opTracker = new OpTracker(this.deltaManager, this.mc.config.getBoolean(disableOpTrackingKey) === true);
1280
1312
  }
1281
1313
 
1282
1314
  public dispose(error?: Error): void {
@@ -1446,13 +1478,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1446
1478
  const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
1447
1479
  addBlobToSummary(summaryTree, electedSummarizerBlobName, electedSummarizerContent);
1448
1480
  }
1449
- const snapshot = this.blobManager.snapshot();
1450
1481
 
1482
+ const summary = this.blobManager.summarize();
1451
1483
  // Some storage (like git) doesn't allow empty tree, so we can omit it.
1452
1484
  // and the blob manager can handle the tree not existing when loading
1453
- if (snapshot.entries.length !== 0) {
1454
- const blobsTree = convertToSummaryTree(snapshot, false);
1455
- addTreeToSummary(summaryTree, blobsTreeName, blobsTree);
1485
+ if (Object.keys(summary.summary.tree).length > 0) {
1486
+ addTreeToSummary(summaryTree, blobsTreeName, summary);
1456
1487
  }
1457
1488
 
1458
1489
  if (this.garbageCollector.writeDataAtRoot) {
@@ -1700,6 +1731,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1700
1731
  return;
1701
1732
  }
1702
1733
 
1734
+ this.mc.logger.sendTelemetryEvent({
1735
+ eventName: "FlushMode Updated",
1736
+ old: this._flushMode,
1737
+ new: mode,
1738
+ });
1739
+
1703
1740
  // Flush any pending batches if switching to immediate
1704
1741
  if (mode === FlushMode.Immediate) {
1705
1742
  this.flush();
@@ -1791,7 +1828,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1791
1828
  */
1792
1829
  private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
1793
1830
  const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
1794
- fluidDataStore.bindToContext();
1831
+ // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
1832
+ // older versions, we still have to call bindToContext.
1833
+ if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
1834
+ fluidDataStore.makeVisibleAndAttachGraph();
1835
+ } else {
1836
+ fluidDataStore.bindToContext();
1837
+ }
1795
1838
  return fluidDataStore;
1796
1839
  }
1797
1840
 
@@ -1863,7 +1906,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1863
1906
  const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
1864
1907
  Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
1865
1908
  if (isRoot) {
1866
- fluidDataStore.bindToContext();
1909
+ // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
1910
+ // For older versions, we still have to call bindToContext.
1911
+ if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
1912
+ fluidDataStore.makeVisibleAndAttachGraph();
1913
+ } else {
1914
+ fluidDataStore.bindToContext();
1915
+ }
1916
+ this.logger.sendTelemetryEvent({
1917
+ eventName: "Root datastore with props",
1918
+ hasProps: props !== undefined,
1919
+ });
1867
1920
  }
1868
1921
  return channelToDataStore(fluidDataStore, id, this, this.dataStores, this.mc.logger);
1869
1922
  }
@@ -2041,11 +2094,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2041
2094
  gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
2042
2095
  }
2043
2096
 
2044
- const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
2045
- assert(summarizeResult.summary.type === SummaryType.Tree,
2097
+ const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState);
2098
+
2099
+ assert(summary.type === SummaryType.Tree,
2046
2100
  0x12f /* "Container Runtime's summarize should always return a tree" */);
2047
2101
 
2048
- return { ...summarizeResult, gcStats } as IRootSummaryTreeWithStats;
2102
+ return { stats, summary, gcStats };
2049
2103
  }
2050
2104
 
2051
2105
  /**
@@ -2064,7 +2118,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2064
2118
  * @param fullGC - true to bypass optimizations and force full generation of GC data.
2065
2119
  */
2066
2120
  public async getGCData(fullGC?: boolean): Promise<IGarbageCollectionData> {
2067
- return this.dataStores.getGCData(fullGC);
2121
+ const builder = new GCDataBuilder();
2122
+ const dsGCData = await this.dataStores.getGCData(fullGC);
2123
+ builder.addNodes(dsGCData.gcNodes);
2124
+
2125
+ const blobsGCData = this.blobManager.getGCData(fullGC);
2126
+ builder.addNodes(blobsGCData.gcNodes);
2127
+ return builder.getGCData();
2068
2128
  }
2069
2129
 
2070
2130
  /**
@@ -2080,7 +2140,83 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2080
2140
  // always referenced, so the used routes is only self-route (empty string).
2081
2141
  this.summarizerNode.updateUsedRoutes([""]);
2082
2142
 
2083
- return this.dataStores.updateUsedRoutes(usedRoutes, gcTimestamp);
2143
+ const dataStoreUsedRoutes: string[] = [];
2144
+ for (const route of usedRoutes) {
2145
+ if (route.split("/")[1] !== BlobManager.basePath) {
2146
+ dataStoreUsedRoutes.push(route);
2147
+ }
2148
+ }
2149
+
2150
+ return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes, gcTimestamp);
2151
+ }
2152
+
2153
+ /**
2154
+ * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
2155
+ * scenarios with accessing deleted content.
2156
+ * @param unusedRoutes - The routes that are unused in all data stores in this Container.
2157
+ */
2158
+ public deleteUnusedRoutes(unusedRoutes: string[]) {
2159
+ const blobManagerUnusedRoutes: string[] = [];
2160
+ const dataStoreUnusedRoutes: string[] = [];
2161
+ for (const route of unusedRoutes) {
2162
+ if (this.isBlobPath(route)) {
2163
+ blobManagerUnusedRoutes.push(route);
2164
+ } else {
2165
+ dataStoreUnusedRoutes.push(route);
2166
+ }
2167
+ }
2168
+
2169
+ this.blobManager.deleteUnusedRoutes(blobManagerUnusedRoutes);
2170
+ this.dataStores.deleteUnusedRoutes(dataStoreUnusedRoutes);
2171
+ }
2172
+
2173
+ /**
2174
+ * Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
2175
+ */
2176
+ public getCurrentReferenceTimestampMs(): number | undefined {
2177
+ // Use the timestamp of the last message seen by this client as that is server generated. If no messages have
2178
+ // been processed, use the timestamp of the message from the last summary.
2179
+ return this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp;
2180
+ }
2181
+
2182
+ /**
2183
+ * Returns the type of the GC node. Currently, there are nodes that belong to data store and nodes that belong
2184
+ * to the blob manager.
2185
+ */
2186
+ public getNodeType(nodePath: string): GCNodeType {
2187
+ if (this.isBlobPath(nodePath)) {
2188
+ return GCNodeType.Blob;
2189
+ }
2190
+ if (this.dataStores.isDataStoreNode(nodePath)) {
2191
+ return GCNodeType.DataStore;
2192
+ }
2193
+ // Root node ("/") and DDS nodes belong to "Other" node types.
2194
+ return GCNodeType.Other;
2195
+ }
2196
+
2197
+ /**
2198
+ * Called by GC to retrieve the package path of the node with the given path. The node should belong to a
2199
+ * data store or an attachment blob.
2200
+ */
2201
+ public getGCNodePackagePath(nodePath: string): readonly string[] | undefined {
2202
+ // If the node is a blob, return "_blobs" as the package path.
2203
+ if (this.isBlobPath(nodePath)) {
2204
+ return ["_blobs"];
2205
+ }
2206
+ const dataStorePkgPath = this.dataStores.getDataStorePackagePath(nodePath);
2207
+ assert(dataStorePkgPath !== undefined, 0x2d6 /* "Package path requested for unknown node type." */);
2208
+ return dataStorePkgPath;
2209
+ }
2210
+
2211
+ /**
2212
+ * Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
2213
+ */
2214
+ private isBlobPath(path: string): boolean {
2215
+ const pathParts = path.split("/");
2216
+ if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
2217
+ return false;
2218
+ }
2219
+ return true;
2084
2220
  }
2085
2221
 
2086
2222
  /**
@@ -2144,6 +2280,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2144
2280
  await this.deltaManager.inbound.pause();
2145
2281
 
2146
2282
  const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
2283
+ const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
2147
2284
  const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
2148
2285
 
2149
2286
  // We should be here is we haven't processed be here. If we are of if the last message's sequence number
@@ -2188,7 +2325,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2188
2325
 
2189
2326
  let continueResult = checkContinue();
2190
2327
  if (!continueResult.continue) {
2191
- return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error: continueResult.error };
2328
+ return {
2329
+ stage: "base",
2330
+ referenceSequenceNumber: summaryRefSeqNum,
2331
+ minimumSequenceNumber,
2332
+ error: continueResult.error,
2333
+ };
2192
2334
  }
2193
2335
 
2194
2336
  // increment summary count
@@ -2211,7 +2353,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2211
2353
  runGC: this.garbageCollector.shouldRunGC,
2212
2354
  });
2213
2355
  } catch (error) {
2214
- return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error };
2356
+ return {
2357
+ stage: "base",
2358
+ referenceSequenceNumber: summaryRefSeqNum,
2359
+ minimumSequenceNumber,
2360
+ error,
2361
+ };
2215
2362
  }
2216
2363
  const { summary: summaryTree, stats: partialStats } = summarizeResult;
2217
2364
 
@@ -2226,15 +2373,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2226
2373
  assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
2227
2374
  const handleCount = Object.values(dataStoreTree.tree).filter(
2228
2375
  (value) => value.type === SummaryType.Handle).length;
2376
+ const gcSummaryTreeStats = summaryTree.tree[gcTreeKey]
2377
+ ? calculateStats((summaryTree.tree[gcTreeKey] as ISummaryTree))
2378
+ : undefined;
2229
2379
 
2230
2380
  const summaryStats: IGeneratedSummaryStats = {
2231
2381
  dataStoreCount: this.dataStores.size,
2232
2382
  summarizedDataStoreCount: this.dataStores.size - handleCount,
2233
2383
  gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
2384
+ gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
2385
+ gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
2386
+ opsSizesSinceLastSummary: this.opTracker.opsSizeAccumulator,
2387
+ nonSystemOpsSinceLastSummary: this.opTracker.nonSystemOpCount,
2234
2388
  ...partialStats,
2235
2389
  };
2236
2390
  const generateSummaryData = {
2237
2391
  referenceSequenceNumber: summaryRefSeqNum,
2392
+ minimumSequenceNumber,
2238
2393
  summaryTree,
2239
2394
  summaryStats,
2240
2395
  generateDuration: trace.trace().duration,
@@ -2301,7 +2456,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2301
2456
  } as const;
2302
2457
 
2303
2458
  this.summarizerNode.completeSummary(handle);
2304
-
2459
+ this.opTracker.reset();
2305
2460
  return submitData;
2306
2461
  } finally {
2307
2462
  // Cleanup wip summary in case of failure
package/src/dataStore.ts CHANGED
@@ -75,7 +75,13 @@ class DataStore implements IDataStore {
75
75
  alias,
76
76
  };
77
77
 
78
- this.fluidDataStoreChannel.bindToContext();
78
+ // back-compat 0.58.2000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
79
+ // older versions, we still have to call bindToContext.
80
+ if (this.fluidDataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
81
+ this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
82
+ } else {
83
+ this.fluidDataStoreChannel.bindToContext();
84
+ }
79
85
 
80
86
  if (this.runtime.attachState === AttachState.Detached) {
81
87
  const localResult = this.datastores.processAliasMessageCore(message);
@@ -132,7 +132,7 @@ export interface ILocalFluidDataStoreContextProps extends IFluidDataStoreContext
132
132
  readonly pkg: Readonly<string[]> | undefined;
133
133
  readonly snapshotTree: ISnapshotTree | undefined;
134
134
  readonly isRootDataStore: boolean | undefined;
135
- readonly bindChannelFn: (channel: IFluidDataStoreChannel) => void;
135
+ readonly makeLocallyVisibleFn: () => void;
136
136
  /**
137
137
  * @deprecated 0.16 Issue #1635, #3631
138
138
  */
@@ -261,7 +261,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
261
261
  private readonly existing: boolean,
262
262
  private bindState: BindState,
263
263
  public readonly isLocalDataStore: boolean,
264
- bindChannelFn: (channel: IFluidDataStoreChannel) => void,
264
+ private readonly makeLocallyVisibleFn: () => void,
265
265
  ) {
266
266
  super();
267
267
 
@@ -284,7 +284,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
284
284
  assert(this.bindState === BindState.NotBound, 0x13b /* "datastore context is already in bound state" */);
285
285
  this.bindState = BindState.Binding;
286
286
  assert(this.channel !== undefined, 0x13c /* "undefined channel on datastore context" */);
287
- bindChannelFn(this.channel);
287
+ this.makeLocallyVisible();
288
288
  this.bindState = BindState.Bound;
289
289
  };
290
290
 
@@ -627,6 +627,15 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
627
627
  return this._containerRuntime.submitDataStoreSignal(this.id, type, content);
628
628
  }
629
629
 
630
+ /**
631
+ * This is called by the data store channel when it becomes locally visible indicating that it is ready to become
632
+ * globally visible now.
633
+ */
634
+ public makeLocallyVisible() {
635
+ assert(this.channel !== undefined, 0x2cf /* "undefined channel on datastore context" */);
636
+ this.makeLocallyVisibleFn();
637
+ }
638
+
630
639
  protected bindRuntime(channel: IFluidDataStoreChannel) {
631
640
  if (this.channel) {
632
641
  throw new Error("Runtime already bound");
@@ -864,7 +873,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
864
873
  props.snapshotTree !== undefined ? true : false /* existing */,
865
874
  props.snapshotTree ? BindState.Bound : BindState.NotBound,
866
875
  true /* isLocalDataStore */,
867
- props.bindChannelFn,
876
+ props.makeLocallyVisibleFn,
868
877
  );
869
878
 
870
879
  this.snapshotTree = props.snapshotTree;
@@ -993,7 +1002,7 @@ export class LocalDetachedFluidDataStoreContext
993
1002
 
994
1003
  public async attachRuntime(
995
1004
  registry: IProvideFluidDataStoreFactory,
996
- dataStoreRuntime: IFluidDataStoreChannel)
1005
+ dataStoreChannel: IFluidDataStoreChannel)
997
1006
  {
998
1007
  assert(this.detachedRuntimeCreation, 0x154 /* "runtime creation is already attached" */);
999
1008
  assert(this.channelDeferred === undefined, 0x155 /* "channel deferral is already set" */);
@@ -1009,10 +1018,16 @@ export class LocalDetachedFluidDataStoreContext
1009
1018
  this.detachedRuntimeCreation = false;
1010
1019
  this.channelDeferred = new Deferred<IFluidDataStoreChannel>();
1011
1020
 
1012
- super.bindRuntime(dataStoreRuntime);
1021
+ super.bindRuntime(dataStoreChannel);
1013
1022
 
1014
1023
  if (await this.isRoot()) {
1015
- dataStoreRuntime.bindToContext();
1024
+ // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
1025
+ // For older versions, we still have to call bindToContext.
1026
+ if (dataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
1027
+ dataStoreChannel.makeVisibleAndAttachGraph();
1028
+ } else {
1029
+ dataStoreChannel.bindToContext();
1030
+ }
1016
1031
  }
1017
1032
  }
1018
1033