@fluidframework/container-runtime 0.52.1 → 0.53.0-46105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/dist/containerHandleContext.d.ts +0 -1
  2. package/dist/containerHandleContext.d.ts.map +1 -1
  3. package/dist/containerHandleContext.js +0 -1
  4. package/dist/containerHandleContext.js.map +1 -1
  5. package/dist/containerRuntime.d.ts +18 -3
  6. package/dist/containerRuntime.d.ts.map +1 -1
  7. package/dist/containerRuntime.js +84 -42
  8. package/dist/containerRuntime.js.map +1 -1
  9. package/dist/dataStoreContext.d.ts +4 -1
  10. package/dist/dataStoreContext.d.ts.map +1 -1
  11. package/dist/dataStoreContext.js +16 -13
  12. package/dist/dataStoreContext.js.map +1 -1
  13. package/dist/dataStores.d.ts +8 -8
  14. package/dist/dataStores.d.ts.map +1 -1
  15. package/dist/dataStores.js +20 -37
  16. package/dist/dataStores.js.map +1 -1
  17. package/dist/garbageCollection.d.ts +61 -14
  18. package/dist/garbageCollection.d.ts.map +1 -1
  19. package/dist/garbageCollection.js +275 -20
  20. package/dist/garbageCollection.js.map +1 -1
  21. package/dist/index.d.ts +2 -2
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +11 -2
  24. package/dist/index.js.map +1 -1
  25. package/dist/packageVersion.d.ts +1 -1
  26. package/dist/packageVersion.d.ts.map +1 -1
  27. package/dist/packageVersion.js +1 -1
  28. package/dist/packageVersion.js.map +1 -1
  29. package/dist/summarizer.d.ts +1 -3
  30. package/dist/summarizer.d.ts.map +1 -1
  31. package/dist/summarizer.js +0 -12
  32. package/dist/summarizer.js.map +1 -1
  33. package/dist/summarizerTypes.d.ts +2 -2
  34. package/dist/summarizerTypes.d.ts.map +1 -1
  35. package/dist/summarizerTypes.js.map +1 -1
  36. package/dist/summaryFormat.d.ts +9 -1
  37. package/dist/summaryFormat.d.ts.map +1 -1
  38. package/dist/summaryFormat.js.map +1 -1
  39. package/dist/summaryGenerator.d.ts.map +1 -1
  40. package/dist/summaryGenerator.js +1 -3
  41. package/dist/summaryGenerator.js.map +1 -1
  42. package/lib/containerHandleContext.d.ts +0 -1
  43. package/lib/containerHandleContext.d.ts.map +1 -1
  44. package/lib/containerHandleContext.js +0 -1
  45. package/lib/containerHandleContext.js.map +1 -1
  46. package/lib/containerRuntime.d.ts +18 -3
  47. package/lib/containerRuntime.d.ts.map +1 -1
  48. package/lib/containerRuntime.js +84 -43
  49. package/lib/containerRuntime.js.map +1 -1
  50. package/lib/dataStoreContext.d.ts +4 -1
  51. package/lib/dataStoreContext.d.ts.map +1 -1
  52. package/lib/dataStoreContext.js +16 -13
  53. package/lib/dataStoreContext.js.map +1 -1
  54. package/lib/dataStores.d.ts +8 -8
  55. package/lib/dataStores.d.ts.map +1 -1
  56. package/lib/dataStores.js +23 -40
  57. package/lib/dataStores.js.map +1 -1
  58. package/lib/garbageCollection.d.ts +61 -14
  59. package/lib/garbageCollection.d.ts.map +1 -1
  60. package/lib/garbageCollection.js +276 -21
  61. package/lib/garbageCollection.js.map +1 -1
  62. package/lib/index.d.ts +2 -2
  63. package/lib/index.d.ts.map +1 -1
  64. package/lib/index.js +2 -1
  65. package/lib/index.js.map +1 -1
  66. package/lib/packageVersion.d.ts +1 -1
  67. package/lib/packageVersion.d.ts.map +1 -1
  68. package/lib/packageVersion.js +1 -1
  69. package/lib/packageVersion.js.map +1 -1
  70. package/lib/summarizer.d.ts +1 -3
  71. package/lib/summarizer.d.ts.map +1 -1
  72. package/lib/summarizer.js +0 -12
  73. package/lib/summarizer.js.map +1 -1
  74. package/lib/summarizerTypes.d.ts +2 -2
  75. package/lib/summarizerTypes.d.ts.map +1 -1
  76. package/lib/summarizerTypes.js.map +1 -1
  77. package/lib/summaryFormat.d.ts +9 -1
  78. package/lib/summaryFormat.d.ts.map +1 -1
  79. package/lib/summaryFormat.js.map +1 -1
  80. package/lib/summaryGenerator.d.ts.map +1 -1
  81. package/lib/summaryGenerator.js +1 -3
  82. package/lib/summaryGenerator.js.map +1 -1
  83. package/package.json +13 -13
  84. package/src/containerHandleContext.ts +0 -1
  85. package/src/containerRuntime.ts +110 -53
  86. package/src/dataStoreContext.ts +15 -14
  87. package/src/dataStores.ts +32 -50
  88. package/src/garbageCollection.ts +390 -18
  89. package/src/index.ts +20 -2
  90. package/src/packageVersion.ts +1 -1
  91. package/src/summarizer.ts +0 -15
  92. package/src/summarizerTypes.ts +1 -2
  93. package/src/summaryFormat.ts +10 -1
  94. package/src/summaryGenerator.ts +2 -3
@@ -6,15 +6,14 @@
6
6
  import { EventEmitter } from "events";
7
7
  import { ITelemetryGenericEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
8
8
  import {
9
+ FluidObject,
10
+ IFluidConfiguration,
11
+ IFluidHandle,
12
+ IFluidHandleContext,
9
13
  IFluidObject,
10
14
  IFluidRouter,
11
- IFluidHandleContext,
12
- IFluidSerializer,
13
15
  IRequest,
14
16
  IResponse,
15
- IFluidHandle,
16
- IFluidConfiguration,
17
- FluidObject,
18
17
  } from "@fluidframework/core-interfaces";
19
18
  import {
20
19
  IAudience,
@@ -49,10 +48,6 @@ import {
49
48
  import { IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
50
49
  import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
51
50
  import { DataCorruptionError, GenericError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
52
- import {
53
- BlobTreeEntry,
54
- TreeTreeEntry,
55
- } from "@fluidframework/protocol-base";
56
51
  import {
57
52
  IClientDetails,
58
53
  IDocumentMessage,
@@ -90,13 +85,13 @@ import {
90
85
  addTreeToSummary,
91
86
  convertToSummaryTree,
92
87
  createRootSummarizerNodeWithGC,
93
- FluidSerializer,
94
88
  IRootSummarizerNodeWithGC,
95
89
  RequestParser,
96
90
  create404Response,
97
91
  exceptionToResponse,
98
92
  responseToException,
99
93
  seqFromTree,
94
+ convertSummaryTreeToITree,
100
95
  } from "@fluidframework/runtime-utils";
101
96
  import { v4 as uuid } from "uuid";
102
97
  import { ContainerFluidHandleContext } from "./containerHandleContext";
@@ -115,6 +110,7 @@ import {
115
110
  electedSummarizerBlobName,
116
111
  extractSummaryMetadataMessage,
117
112
  IContainerRuntimeMetadata,
113
+ ICreateContainerMetadata,
118
114
  ISummaryMetadataMessage,
119
115
  metadataBlobName,
120
116
  wrapSummaryInChannelsTree,
@@ -137,6 +133,7 @@ import { formExponentialFn, Throttler } from "./throttler";
137
133
  import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
138
134
  import {
139
135
  GarbageCollector,
136
+ gcTreeKey,
140
137
  IGarbageCollectionRuntime,
141
138
  IGarbageCollector,
142
139
  IGCStats,
@@ -584,6 +581,19 @@ export class ScheduleManager {
584
581
  */
585
582
  export const agentSchedulerId = "_scheduler";
586
583
 
584
+ // safely check navigator and get the hardware spec value
585
+ export function getDeviceSpec() {
586
+ try {
587
+ if (typeof navigator === "object" && navigator !== null) {
588
+ return {
589
+ deviceMemory: (navigator as any).deviceMemory,
590
+ hardwareConcurrency: navigator.hardwareConcurrency,
591
+ };
592
+ }
593
+ } catch {
594
+ }
595
+ return {};
596
+ }
587
597
  /**
588
598
  * Represents the runtime of the container. Contains helper functions/state of the container.
589
599
  * It will define the store level mappings.
@@ -805,9 +815,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
805
815
  return this.context.attachState;
806
816
  }
807
817
 
808
- // Back compat: 0.28, can be removed in 0.29
809
- public readonly IFluidSerializer: IFluidSerializer;
810
-
811
818
  public readonly IFluidHandleContext: IFluidHandleContext;
812
819
 
813
820
  // internal logger for ContainerRuntime. Use this.logger for stores, summaries, etc.
@@ -874,6 +881,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
874
881
 
875
882
  private readonly dataStores: DataStores;
876
883
 
884
+ /**
885
+ * True, if GC data should be written at root of the summary tree.
886
+ * False, if data stores should write GC blobs in their summary tree.
887
+ */
888
+ public get writeGCDataAtRoot(): boolean {
889
+ return this.garbageCollector.writeDataAtRoot;
890
+ }
891
+
877
892
  /**
878
893
  * True if generating summaries with isolated channels is
879
894
  * explicitly disabled. This only affects how summaries are written,
@@ -892,6 +907,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
892
907
  return this._summarizer;
893
908
  }
894
909
 
910
+ private readonly createContainerMetadata: ICreateContainerMetadata;
911
+ private summaryCount: number | undefined;
912
+
895
913
  private constructor(
896
914
  private readonly context: IContainerContext,
897
915
  private readonly registry: IFluidDataStoreRegistry,
@@ -907,9 +925,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
907
925
  private _storage?: IDocumentStorageService,
908
926
  ) {
909
927
  super();
910
-
911
928
  this.baseSummaryMessage = metadata?.message;
912
929
 
930
+ // If this is an existing container, we get values from metadata.
931
+ // otherwise, we initialize them.
932
+ if (existing) {
933
+ this.createContainerMetadata = {
934
+ createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
935
+ createContainerTimestamp: metadata?.createContainerTimestamp,
936
+ };
937
+ this.summaryCount = metadata?.summaryCount;
938
+ } else {
939
+ this.createContainerMetadata = {
940
+ createContainerRuntimeVersion: pkgVersion,
941
+ createContainerTimestamp: Date.now(),
942
+ };
943
+ }
944
+
913
945
  // Default to false (enabled).
914
946
  this.disableIsolatedChannels = this.runtimeOptions.summaryOptions.disableIsolatedChannels ?? false;
915
947
 
@@ -917,14 +949,26 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
917
949
  this.chunkMap = new Map<string, string[]>(chunks);
918
950
 
919
951
  this.IFluidHandleContext = new ContainerFluidHandleContext("", this);
920
- this.IFluidSerializer = new FluidSerializer(this.IFluidHandleContext);
921
952
 
922
953
  this._logger = ChildLogger.create(this.logger, "ContainerRuntime");
923
954
 
955
+ /**
956
+ * Function that return the current server timestamp. This is used by the garbage collector to set the
957
+ * time when a node becomes unreferenced.
958
+ * For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
959
+ * we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
960
+ * of this client's connection - https://github.com/microsoft/FluidFramework/issues/8375.
961
+ */
962
+ const getCurrentTimestamp = () => {
963
+ return this.deltaManager.lastMessage?.timestamp ?? Date.now();
964
+ };
924
965
  this.garbageCollector = GarbageCollector.create(
925
966
  this,
926
967
  this.runtimeOptions.gcOptions,
927
968
  (unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
969
+ getCurrentTimestamp,
970
+ context.baseSnapshot,
971
+ async <T>(id: string) => readAndParse<T>(this.storage, id),
928
972
  this._logger,
929
973
  existing,
930
974
  metadata,
@@ -948,8 +992,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
948
992
  throwOnFailure: true,
949
993
  // If GC should not run, let the summarizer node know so that it does not track GC state.
950
994
  gcDisabled: !this.garbageCollector.shouldRunGC,
951
- // The max duration for which objects can be unreferenced before they are eligible for deletion.
952
- maxUnreferencedDurationMs: this.runtimeOptions.gcOptions.maxUnreferencedDurationMs,
953
995
  },
954
996
  );
955
997
 
@@ -974,7 +1016,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
974
1016
  getInitialGCSummaryDetailsFn,
975
1017
  ),
976
1018
  (id: string) => this.summarizerNode.deleteChild(id),
977
- this._logger);
1019
+ this._logger,
1020
+ async () => this.garbageCollector.getDataStoreBaseGCDetails(),
1021
+ (id: string) => this.garbageCollector.nodeChanged(id),
1022
+ );
978
1023
 
979
1024
  this.blobManager = new BlobManager(
980
1025
  this.IFluidHandleContext,
@@ -1002,12 +1047,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1002
1047
  this.clearPartialChunks(clientId);
1003
1048
  });
1004
1049
 
1005
- this.context.quorum.on("addProposal", (proposal) => {
1006
- if (proposal.key === "code" || proposal.key === "code2") {
1007
- this.emit("codeDetailsProposed", proposal.value, proposal);
1008
- }
1009
- });
1010
-
1011
1050
  this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
1012
1051
 
1013
1052
  // Only create a SummaryManager if summaries are enabled and we are not the summarizer client
@@ -1104,7 +1143,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1104
1143
  this.deltaManager.on("readonly", (readonly: boolean) => {
1105
1144
  // we accumulate ops while being in read-only state.
1106
1145
  // once user gets write permissions and we have active connection, flush all pending ops.
1107
- assert(readonly === this.deltaManager.readonly, 0x124 /* "inconsistent readonly property/event state" */);
1146
+ // eslint-disable-next-line max-len
1147
+ assert(readonly === this.deltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
1108
1148
 
1109
1149
  // We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
1110
1150
  // when we either never send an op, or attempted to send it but we know for sure it was not
@@ -1127,6 +1167,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1127
1167
  this.deltaManager.on("op", this.onOp);
1128
1168
  }
1129
1169
 
1170
+ // logging hardware telemetry
1171
+ logger.sendTelemetryEvent({
1172
+ eventName:"DeviceSpec",
1173
+ ...getDeviceSpec(),
1174
+ });
1175
+
1176
+ // logging container load stats
1177
+ this.logger.sendTelemetryEvent({
1178
+ eventName: "ContainerLoadStats",
1179
+ ...this.createContainerMetadata,
1180
+ ...this.dataStores.containerLoadStats,
1181
+ summaryCount: this.summaryCount,
1182
+ summaryFormatVersion: metadata?.summaryFormatVersion,
1183
+ disableIsolatedChannels: metadata?.disableIsolatedChannels,
1184
+ gcVersion: metadata?.gcFeature,
1185
+ });
1186
+
1130
1187
  ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
1131
1188
  }
1132
1189
 
@@ -1251,6 +1308,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1251
1308
 
1252
1309
  private formMetadata(): IContainerRuntimeMetadata {
1253
1310
  return {
1311
+ ...this.createContainerMetadata,
1312
+ summaryCount: this.summaryCount,
1254
1313
  summaryFormatVersion: 1,
1255
1314
  disableIsolatedChannels: this.disableIsolatedChannels || undefined,
1256
1315
  gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
@@ -1287,26 +1346,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1287
1346
  * @deprecated - Use summarize to get summary of the container runtime.
1288
1347
  */
1289
1348
  public async snapshot(): Promise<ITree> {
1290
- if (this.garbageCollector.shouldRunGC) {
1291
- await this.collectGarbage({ logger: this.logger, fullGC: true /* fullGC */ });
1292
- }
1293
-
1294
- const root: ITree = { entries: [] };
1295
- const entries = await this.dataStores.snapshot();
1296
-
1297
- if (this.disableIsolatedChannels) {
1298
- root.entries = root.entries.concat(entries);
1299
- } else {
1300
- root.entries.push(new TreeTreeEntry(channelsTreeName, { entries }));
1301
- }
1302
-
1303
- root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata())));
1304
-
1305
- if (this.chunkMap.size > 0) {
1306
- root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap])));
1307
- }
1308
-
1309
- return root;
1349
+ const summaryResult = await this.summarize({
1350
+ summaryLogger: this.logger,
1351
+ fullTree: true,
1352
+ trackState: false,
1353
+ runGC: this.garbageCollector.shouldRunGC,
1354
+ fullGC: true,
1355
+ });
1356
+ return convertSummaryTreeToITree(summaryResult.summary);
1310
1357
  }
1311
1358
 
1312
1359
  private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) {
@@ -1328,6 +1375,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1328
1375
  const blobsTree = convertToSummaryTree(snapshot, false);
1329
1376
  addTreeToSummary(summaryTree, blobsTreeName, blobsTree);
1330
1377
  }
1378
+
1379
+ if (this.writeGCDataAtRoot) {
1380
+ const gcSummary = this.garbageCollector.summarize();
1381
+ if (gcSummary !== undefined) {
1382
+ addTreeToSummary(summaryTree, gcTreeKey, gcSummary);
1383
+ }
1384
+ }
1331
1385
  }
1332
1386
 
1333
1387
  private replayPendingStates() {
@@ -1777,21 +1831,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1777
1831
  * Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
1778
1832
  * After GC has run, called to notify this container's nodes of routes that are used in it.
1779
1833
  * @param usedRoutes - The routes that are used in all nodes in this Container.
1834
+ * @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
1835
+ * unreferenced as part of this GC run, this should be used to update the time when it happens.
1780
1836
  * @returns the statistics of the used state of the data stores.
1781
1837
  */
1782
- public updateUsedRoutes(usedRoutes: string[]): IUsedStateStats {
1838
+ public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats {
1783
1839
  // Update our summarizer node's used routes. Updating used routes in summarizer node before
1784
1840
  // summarizing is required and asserted by the the summarizer node. We are the root and are
1785
1841
  // always referenced, so the used routes is only self-route (empty string).
1786
1842
  this.summarizerNode.updateUsedRoutes([""]);
1787
1843
 
1788
- return this.dataStores.updateUsedRoutes(
1789
- usedRoutes,
1790
- // For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
1791
- // we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
1792
- // of this client's connection - https://github.com/microsoft/FluidFramework/issues/7152.
1793
- this.deltaManager.lastMessage?.timestamp,
1794
- );
1844
+ return this.dataStores.updateUsedRoutes(usedRoutes, gcTimestamp);
1795
1845
  }
1796
1846
 
1797
1847
  /**
@@ -1883,6 +1933,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1883
1933
  return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error: continueResult.error };
1884
1934
  }
1885
1935
 
1936
+ // increment summary count
1937
+ if (this.summaryCount !== undefined) {
1938
+ this.summaryCount++;
1939
+ } else {
1940
+ this.summaryCount = 1;
1941
+ }
1942
+
1886
1943
  const trace = Trace.start();
1887
1944
  let summarizeResult: ISummaryTreeWithStats;
1888
1945
  try {
@@ -188,6 +188,11 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
188
188
  return this._containerRuntime.disableIsolatedChannels;
189
189
  }
190
190
 
191
+ /** Tells whether GC data will be written at the root of the summary tree. If so, data store should not write it. */
192
+ protected get writeGCDataAtRoot(): boolean {
193
+ return this._containerRuntime.writeGCDataAtRoot;
194
+ }
195
+
191
196
  protected registry: IFluidDataStoreRegistry | undefined;
192
197
 
193
198
  protected detachedRuntimeCreation = false;
@@ -416,8 +421,10 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
416
421
  const attributes = createAttributes(pkg, isRootDataStore, this.disableIsolatedChannels);
417
422
  addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes));
418
423
 
419
- // Add GC details to the summary.
420
- addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
424
+ // Add GC data to the summary if it's not written at the root.
425
+ if (!this.writeGCDataAtRoot) {
426
+ addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
427
+ }
421
428
 
422
429
  // If we are not referenced, mark the summary tree as unreferenced. Also, update unreferenced blob
423
430
  // size in the summary stats with the blobs size of this data store.
@@ -675,6 +682,7 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
675
682
  constructor(
676
683
  id: string,
677
684
  private readonly initSnapshotValue: ISnapshotTree | string | undefined,
685
+ private readonly getBaseSummaryGCDetails: () => Promise<IGarbageCollectionSummaryDetails | undefined>,
678
686
  runtime: ContainerRuntime,
679
687
  storage: IDocumentStorageService,
680
688
  scope: FluidObject,
@@ -760,16 +768,7 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
760
768
  });
761
769
 
762
770
  private readonly gcDetailsInInitialSummaryP = new LazyPromise<IGarbageCollectionSummaryDetails>(async () => {
763
- // If the initial snapshot is undefined or string, the snapshot is in old format and won't have GC details.
764
- if (!(!this.initSnapshotValue || typeof this.initSnapshotValue === "string")
765
- && this.initSnapshotValue.blobs[gcBlobKey] !== undefined) {
766
- return readAndParse<IGarbageCollectionSummaryDetails>(
767
- this.storage,
768
- this.initSnapshotValue.blobs[gcBlobKey],
769
- );
770
- } else {
771
- return {};
772
- }
771
+ return (await this.getBaseSummaryGCDetails()) ?? {};
773
772
  });
774
773
 
775
774
  protected async getInitialSnapshotDetails(): Promise<ISnapshotDetails> {
@@ -850,8 +849,10 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
850
849
  );
851
850
  addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes));
852
851
 
853
- // Add GC details to the summary.
854
- addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
852
+ // Add GC data to the summary if it's not written at the root.
853
+ if (!this.writeGCDataAtRoot) {
854
+ addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
855
+ }
855
856
 
856
857
  // Attach message needs the summary in ITree format. Convert the ISummaryTree into an ITree.
857
858
  const snapshot = convertSummaryTreeToITree(summarizeResult.summary);
package/src/dataStores.ts CHANGED
@@ -8,8 +8,6 @@ import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidfra
8
8
  import {
9
9
  ISequencedDocumentMessage,
10
10
  ISnapshotTree,
11
- ITreeEntry,
12
- SummaryType,
13
11
  } from "@fluidframework/protocol-definitions";
14
12
  import {
15
13
  channelsTreeName,
@@ -21,6 +19,7 @@ import {
21
19
  IFluidDataStoreChannel,
22
20
  IFluidDataStoreContextDetached,
23
21
  IGarbageCollectionData,
22
+ IGarbageCollectionSummaryDetails,
24
23
  IInboundSignalMessage,
25
24
  InboundAttachMessage,
26
25
  ISummarizeResult,
@@ -28,7 +27,6 @@ import {
28
27
  } from "@fluidframework/runtime-definitions";
29
28
  import {
30
29
  convertSnapshotTreeToSummaryTree,
31
- convertSummaryTreeToITree,
32
30
  convertToSummaryTree,
33
31
  create404Response,
34
32
  responseToException,
@@ -37,10 +35,9 @@ import {
37
35
  import { ChildLogger, TelemetryDataTag } from "@fluidframework/telemetry-utils";
38
36
  import { AttachState } from "@fluidframework/container-definitions";
39
37
  import { BlobCacheStorageService, buildSnapshotTree } from "@fluidframework/driver-utils";
40
- import { assert, Lazy } from "@fluidframework/common-utils";
38
+ import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
41
39
  import { v4 as uuid } from "uuid";
42
- import { TreeTreeEntry } from "@fluidframework/protocol-base";
43
- import { GCDataBuilder, getChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
40
+ import { GCDataBuilder, unpackChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
44
41
  import { DataStoreContexts } from "./dataStoreContexts";
45
42
  import { ContainerRuntime } from "./containerRuntime";
46
43
  import {
@@ -67,6 +64,13 @@ export class DataStores implements IDisposable {
67
64
 
68
65
  private readonly disposeOnce = new Lazy<void>(() => this.contexts.dispose());
69
66
 
67
+ public readonly containerLoadStats: {
68
+ // number of dataStores during loadContainer
69
+ readonly containerLoadDataStoreCount: number;
70
+ // number of unreferenced dataStores during loadContainer
71
+ readonly referencedDataStoreCount: number;
72
+ };
73
+
70
74
  constructor(
71
75
  private readonly baseSnapshot: ISnapshotTree | undefined,
72
76
  private readonly runtime: ContainerRuntime,
@@ -75,12 +79,23 @@ export class DataStores implements IDisposable {
75
79
  (id: string, createParam: CreateChildSummarizerNodeParam) => CreateChildSummarizerNodeFn,
76
80
  private readonly deleteChildSummarizerNodeFn: (id: string) => void,
77
81
  baseLogger: ITelemetryBaseLogger,
82
+ getDataStoreBaseGCDetails: () => Promise<Map<string, IGarbageCollectionSummaryDetails>>,
83
+ private readonly dataStoreChanged: (id: string) => void,
78
84
  private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
79
85
  ) {
80
86
  this.logger = ChildLogger.create(baseLogger);
87
+
88
+ const baseDataStoresGCDetailsP = new LazyPromise(async () => {
89
+ return getDataStoreBaseGCDetails();
90
+ });
91
+ // Returns the base summary GC details for the data store with the given id.
92
+ const dataStoreBaseGCDetails = async (dataStoreId: string) => {
93
+ const baseGCDetails = await baseDataStoresGCDetailsP;
94
+ return baseGCDetails.get(dataStoreId);
95
+ };
96
+
81
97
  // Extract stores stored inside the snapshot
82
98
  const fluidDataStores = new Map<string, ISnapshotTree>();
83
-
84
99
  if (baseSnapshot) {
85
100
  for (const [key, value] of Object.entries(baseSnapshot.trees)) {
86
101
  fluidDataStores.set(key, value);
@@ -101,6 +116,7 @@ export class DataStores implements IDisposable {
101
116
  dataStoreContext = new RemotedFluidDataStoreContext(
102
117
  key,
103
118
  value,
119
+ async () => dataStoreBaseGCDetails(key),
104
120
  this.runtime,
105
121
  this.runtime.storage,
106
122
  this.runtime.scope,
@@ -124,11 +140,10 @@ export class DataStores implements IDisposable {
124
140
  }
125
141
  this.contexts.addBoundOrRemoted(dataStoreContext);
126
142
  }
127
- this.logger.sendTelemetryEvent({
128
- eventName: "ContainerLoadStats",
129
- dataStoreCount: fluidDataStores.size,
143
+ this.containerLoadStats = {
144
+ containerLoadDataStoreCount: fluidDataStores.size,
130
145
  referencedDataStoreCount: fluidDataStores.size - unreferencedDataStoreCount,
131
- });
146
+ };
132
147
  }
133
148
 
134
149
  public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
@@ -170,6 +185,8 @@ export class DataStores implements IDisposable {
170
185
  const remotedFluidDataStoreContext = new RemotedFluidDataStoreContext(
171
186
  attachMessage.id,
172
187
  snapshotTree,
188
+ // New data stores begin with empty GC details since GC hasn't run on them yet.
189
+ async () => { return {}; },
173
190
  this.runtime,
174
191
  new BlobCacheStorageService(this.runtime.storage, flatBlobs),
175
192
  this.runtime.scope,
@@ -281,6 +298,9 @@ export class DataStores implements IDisposable {
281
298
  const context = this.contexts.get(envelope.address);
282
299
  assert(!!context, 0x162 /* "There should be a store context for the op" */);
283
300
  context.process(transformed, local, localMessageMetadata);
301
+
302
+ // Notify that a data store changed. This is used to detect if a deleted data store is being used.
303
+ this.dataStoreChanged(envelope.address);
284
304
  }
285
305
 
286
306
  public async getDataStore(id: string, wait: boolean): Promise<FluidDataStoreContext> {
@@ -339,44 +359,6 @@ export class DataStores implements IDisposable {
339
359
  }
340
360
  }
341
361
 
342
- /**
343
- * Notifies this object to take the snapshot of the container.
344
- * @deprecated - Use summarize to get summary of the container runtime.
345
- */
346
- public async snapshot(): Promise<ITreeEntry[]> {
347
- // Iterate over each store and ask it to snapshot
348
- const fluidDataStoreSnapshotsP = Array.from(this.contexts).map(async ([fluidDataStoreId, value]) => {
349
- const summaryTree = await value.summarize(true /* fullTree */, false /* trackState */);
350
- assert(
351
- summaryTree.summary.type === SummaryType.Tree,
352
- 0x164 /* "summarize should always return a tree when fullTree is true" */);
353
- // back-compat summary - Remove this once snapshot is removed.
354
- const snapshot = convertSummaryTreeToITree(summaryTree.summary);
355
-
356
- // If ID exists then previous commit is still valid
357
- return {
358
- fluidDataStoreId,
359
- snapshot,
360
- };
361
- });
362
-
363
- const entries: ITreeEntry[] = [];
364
-
365
- // Add in module references to the store snapshots
366
- const fluidDataStoreSnapshots = await Promise.all(fluidDataStoreSnapshotsP);
367
-
368
- // Sort for better diffing of snapshots (in replay tool, used to find bugs in snapshotting logic)
369
- fluidDataStoreSnapshots.sort((a, b) => a?.fluidDataStoreId.localeCompare(b.fluidDataStoreId));
370
-
371
- for (const fluidDataStoreSnapshot of fluidDataStoreSnapshots) {
372
- entries.push(new TreeTreeEntry(
373
- fluidDataStoreSnapshot.fluidDataStoreId,
374
- fluidDataStoreSnapshot.snapshot,
375
- ));
376
- }
377
- return entries;
378
- }
379
-
380
362
  public get size(): number {
381
363
  return this.contexts.size;
382
364
  }
@@ -474,7 +456,7 @@ export class DataStores implements IDisposable {
474
456
  */
475
457
  public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats {
476
458
  // Get a map of data store ids to routes used in it.
477
- const usedDataStoreRoutes = getChildNodesUsedRoutes(usedRoutes);
459
+ const usedDataStoreRoutes = unpackChildNodesUsedRoutes(usedRoutes);
478
460
 
479
461
  // Verify that the used routes are correct.
480
462
  for (const [id] of usedDataStoreRoutes) {