@fluidframework/container-runtime 2.0.0-dev.4.2.0.153917 → 2.0.0-dev.4.3.0.158678

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/dist/containerRuntime.d.ts +33 -3
  2. package/dist/containerRuntime.d.ts.map +1 -1
  3. package/dist/containerRuntime.js +187 -58
  4. package/dist/containerRuntime.js.map +1 -1
  5. package/dist/dataStoreContext.d.ts +2 -1
  6. package/dist/dataStoreContext.d.ts.map +1 -1
  7. package/dist/dataStoreContext.js +3 -0
  8. package/dist/dataStoreContext.js.map +1 -1
  9. package/dist/dataStores.d.ts +5 -5
  10. package/dist/dataStores.d.ts.map +1 -1
  11. package/dist/dataStores.js +3 -6
  12. package/dist/dataStores.js.map +1 -1
  13. package/dist/gc/garbageCollection.d.ts.map +1 -1
  14. package/dist/gc/garbageCollection.js +5 -5
  15. package/dist/gc/garbageCollection.js.map +1 -1
  16. package/dist/gc/gcConfigs.d.ts.map +1 -1
  17. package/dist/gc/gcConfigs.js +1 -3
  18. package/dist/gc/gcConfigs.js.map +1 -1
  19. package/dist/gc/gcDefinitions.js +1 -1
  20. package/dist/gc/gcDefinitions.js.map +1 -1
  21. package/dist/id-compressor/appendOnlySortedMap.d.ts +146 -0
  22. package/dist/id-compressor/appendOnlySortedMap.d.ts.map +1 -0
  23. package/dist/id-compressor/appendOnlySortedMap.js +360 -0
  24. package/dist/id-compressor/appendOnlySortedMap.js.map +1 -0
  25. package/dist/id-compressor/idCompressor.d.ts +279 -0
  26. package/dist/id-compressor/idCompressor.d.ts.map +1 -0
  27. package/dist/id-compressor/idCompressor.js +1258 -0
  28. package/dist/id-compressor/idCompressor.js.map +1 -0
  29. package/dist/id-compressor/idRange.d.ts +11 -0
  30. package/dist/id-compressor/idRange.d.ts.map +1 -0
  31. package/dist/id-compressor/idRange.js +29 -0
  32. package/dist/id-compressor/idRange.js.map +1 -0
  33. package/dist/id-compressor/index.d.ts +14 -0
  34. package/dist/id-compressor/index.d.ts.map +1 -0
  35. package/dist/id-compressor/index.js +38 -0
  36. package/dist/id-compressor/index.js.map +1 -0
  37. package/dist/id-compressor/numericUuid.d.ts +59 -0
  38. package/dist/id-compressor/numericUuid.d.ts.map +1 -0
  39. package/dist/id-compressor/numericUuid.js +325 -0
  40. package/dist/id-compressor/numericUuid.js.map +1 -0
  41. package/dist/id-compressor/sessionIdNormalizer.d.ts +138 -0
  42. package/dist/id-compressor/sessionIdNormalizer.d.ts.map +1 -0
  43. package/dist/id-compressor/sessionIdNormalizer.js +488 -0
  44. package/dist/id-compressor/sessionIdNormalizer.js.map +1 -0
  45. package/dist/id-compressor/utils.d.ts +57 -0
  46. package/dist/id-compressor/utils.d.ts.map +1 -0
  47. package/dist/id-compressor/utils.js +90 -0
  48. package/dist/id-compressor/utils.js.map +1 -0
  49. package/dist/id-compressor/uuidUtilities.d.ts +30 -0
  50. package/dist/id-compressor/uuidUtilities.d.ts.map +1 -0
  51. package/dist/id-compressor/uuidUtilities.js +106 -0
  52. package/dist/id-compressor/uuidUtilities.js.map +1 -0
  53. package/dist/index.d.ts +1 -0
  54. package/dist/index.d.ts.map +1 -1
  55. package/dist/index.js +5 -1
  56. package/dist/index.js.map +1 -1
  57. package/dist/opLifecycle/batchManager.d.ts +9 -2
  58. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  59. package/dist/opLifecycle/batchManager.js +21 -2
  60. package/dist/opLifecycle/batchManager.js.map +1 -1
  61. package/dist/opLifecycle/index.d.ts +1 -1
  62. package/dist/opLifecycle/index.d.ts.map +1 -1
  63. package/dist/opLifecycle/index.js.map +1 -1
  64. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  65. package/dist/opLifecycle/opGroupingManager.js +5 -0
  66. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  67. package/dist/opLifecycle/outbox.d.ts +2 -2
  68. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  69. package/dist/opLifecycle/outbox.js +34 -22
  70. package/dist/opLifecycle/outbox.js.map +1 -1
  71. package/dist/packageVersion.d.ts +1 -1
  72. package/dist/packageVersion.js +1 -1
  73. package/dist/packageVersion.js.map +1 -1
  74. package/dist/pendingStateManager.d.ts +1 -1
  75. package/dist/pendingStateManager.d.ts.map +1 -1
  76. package/dist/pendingStateManager.js +11 -3
  77. package/dist/pendingStateManager.js.map +1 -1
  78. package/dist/summary/index.d.ts +1 -1
  79. package/dist/summary/index.d.ts.map +1 -1
  80. package/dist/summary/index.js +2 -1
  81. package/dist/summary/index.js.map +1 -1
  82. package/dist/summary/orderedClientElection.d.ts +1 -0
  83. package/dist/summary/orderedClientElection.d.ts.map +1 -1
  84. package/dist/summary/orderedClientElection.js +17 -1
  85. package/dist/summary/orderedClientElection.js.map +1 -1
  86. package/dist/summary/runningSummarizer.d.ts +0 -1
  87. package/dist/summary/runningSummarizer.d.ts.map +1 -1
  88. package/dist/summary/runningSummarizer.js +1 -17
  89. package/dist/summary/runningSummarizer.js.map +1 -1
  90. package/dist/summary/summaryFormat.d.ts +3 -0
  91. package/dist/summary/summaryFormat.d.ts.map +1 -1
  92. package/dist/summary/summaryFormat.js +3 -1
  93. package/dist/summary/summaryFormat.js.map +1 -1
  94. package/dist/summary/summaryManager.d.ts.map +1 -1
  95. package/dist/summary/summaryManager.js +2 -0
  96. package/dist/summary/summaryManager.js.map +1 -1
  97. package/lib/containerRuntime.d.ts +33 -3
  98. package/lib/containerRuntime.d.ts.map +1 -1
  99. package/lib/containerRuntime.js +170 -60
  100. package/lib/containerRuntime.js.map +1 -1
  101. package/lib/dataStoreContext.d.ts +2 -1
  102. package/lib/dataStoreContext.d.ts.map +1 -1
  103. package/lib/dataStoreContext.js +3 -0
  104. package/lib/dataStoreContext.js.map +1 -1
  105. package/lib/dataStores.d.ts +5 -5
  106. package/lib/dataStores.d.ts.map +1 -1
  107. package/lib/dataStores.js +3 -6
  108. package/lib/dataStores.js.map +1 -1
  109. package/lib/gc/garbageCollection.d.ts.map +1 -1
  110. package/lib/gc/garbageCollection.js +5 -5
  111. package/lib/gc/garbageCollection.js.map +1 -1
  112. package/lib/gc/gcConfigs.d.ts.map +1 -1
  113. package/lib/gc/gcConfigs.js +1 -3
  114. package/lib/gc/gcConfigs.js.map +1 -1
  115. package/lib/gc/gcDefinitions.js +1 -1
  116. package/lib/gc/gcDefinitions.js.map +1 -1
  117. package/lib/id-compressor/appendOnlySortedMap.d.ts +146 -0
  118. package/lib/id-compressor/appendOnlySortedMap.d.ts.map +1 -0
  119. package/lib/id-compressor/appendOnlySortedMap.js +355 -0
  120. package/lib/id-compressor/appendOnlySortedMap.js.map +1 -0
  121. package/lib/id-compressor/idCompressor.d.ts +279 -0
  122. package/lib/id-compressor/idCompressor.d.ts.map +1 -0
  123. package/lib/id-compressor/idCompressor.js +1248 -0
  124. package/lib/id-compressor/idCompressor.js.map +1 -0
  125. package/lib/id-compressor/idRange.d.ts +11 -0
  126. package/lib/id-compressor/idRange.d.ts.map +1 -0
  127. package/lib/id-compressor/idRange.js +25 -0
  128. package/lib/id-compressor/idRange.js.map +1 -0
  129. package/lib/id-compressor/index.d.ts +14 -0
  130. package/lib/id-compressor/index.d.ts.map +1 -0
  131. package/lib/id-compressor/index.js +14 -0
  132. package/lib/id-compressor/index.js.map +1 -0
  133. package/lib/id-compressor/numericUuid.d.ts +59 -0
  134. package/lib/id-compressor/numericUuid.d.ts.map +1 -0
  135. package/lib/id-compressor/numericUuid.js +315 -0
  136. package/lib/id-compressor/numericUuid.js.map +1 -0
  137. package/lib/id-compressor/sessionIdNormalizer.d.ts +138 -0
  138. package/lib/id-compressor/sessionIdNormalizer.d.ts.map +1 -0
  139. package/lib/id-compressor/sessionIdNormalizer.js +484 -0
  140. package/lib/id-compressor/sessionIdNormalizer.js.map +1 -0
  141. package/lib/id-compressor/utils.d.ts +57 -0
  142. package/lib/id-compressor/utils.d.ts.map +1 -0
  143. package/lib/id-compressor/utils.js +79 -0
  144. package/lib/id-compressor/utils.js.map +1 -0
  145. package/lib/id-compressor/uuidUtilities.d.ts +30 -0
  146. package/lib/id-compressor/uuidUtilities.d.ts.map +1 -0
  147. package/lib/id-compressor/uuidUtilities.js +98 -0
  148. package/lib/id-compressor/uuidUtilities.js.map +1 -0
  149. package/lib/index.d.ts +1 -0
  150. package/lib/index.d.ts.map +1 -1
  151. package/lib/index.js +1 -0
  152. package/lib/index.js.map +1 -1
  153. package/lib/opLifecycle/batchManager.d.ts +9 -2
  154. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  155. package/lib/opLifecycle/batchManager.js +19 -1
  156. package/lib/opLifecycle/batchManager.js.map +1 -1
  157. package/lib/opLifecycle/index.d.ts +1 -1
  158. package/lib/opLifecycle/index.d.ts.map +1 -1
  159. package/lib/opLifecycle/index.js.map +1 -1
  160. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  161. package/lib/opLifecycle/opGroupingManager.js +5 -0
  162. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  163. package/lib/opLifecycle/outbox.d.ts +2 -2
  164. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  165. package/lib/opLifecycle/outbox.js +35 -23
  166. package/lib/opLifecycle/outbox.js.map +1 -1
  167. package/lib/packageVersion.d.ts +1 -1
  168. package/lib/packageVersion.js +1 -1
  169. package/lib/packageVersion.js.map +1 -1
  170. package/lib/pendingStateManager.d.ts +1 -1
  171. package/lib/pendingStateManager.d.ts.map +1 -1
  172. package/lib/pendingStateManager.js +11 -3
  173. package/lib/pendingStateManager.js.map +1 -1
  174. package/lib/summary/index.d.ts +1 -1
  175. package/lib/summary/index.d.ts.map +1 -1
  176. package/lib/summary/index.js +1 -1
  177. package/lib/summary/index.js.map +1 -1
  178. package/lib/summary/orderedClientElection.d.ts +1 -0
  179. package/lib/summary/orderedClientElection.d.ts.map +1 -1
  180. package/lib/summary/orderedClientElection.js +17 -1
  181. package/lib/summary/orderedClientElection.js.map +1 -1
  182. package/lib/summary/runningSummarizer.d.ts +0 -1
  183. package/lib/summary/runningSummarizer.d.ts.map +1 -1
  184. package/lib/summary/runningSummarizer.js +1 -17
  185. package/lib/summary/runningSummarizer.js.map +1 -1
  186. package/lib/summary/summaryFormat.d.ts +3 -0
  187. package/lib/summary/summaryFormat.d.ts.map +1 -1
  188. package/lib/summary/summaryFormat.js +2 -0
  189. package/lib/summary/summaryFormat.js.map +1 -1
  190. package/lib/summary/summaryManager.d.ts.map +1 -1
  191. package/lib/summary/summaryManager.js +2 -0
  192. package/lib/summary/summaryManager.js.map +1 -1
  193. package/package.json +27 -18
  194. package/src/containerRuntime.ts +256 -88
  195. package/src/dataStoreContext.ts +6 -0
  196. package/src/dataStores.ts +4 -7
  197. package/src/gc/garbageCollection.ts +7 -6
  198. package/src/gc/gcConfigs.ts +1 -3
  199. package/src/gc/gcDefinitions.ts +1 -1
  200. package/src/id-compressor/README.md +3 -0
  201. package/src/id-compressor/appendOnlySortedMap.ts +427 -0
  202. package/src/id-compressor/idCompressor.ts +1854 -0
  203. package/src/id-compressor/idRange.ts +35 -0
  204. package/src/id-compressor/index.ts +35 -0
  205. package/src/id-compressor/numericUuid.ts +383 -0
  206. package/src/id-compressor/sessionIdNormalizer.ts +609 -0
  207. package/src/id-compressor/utils.ts +114 -0
  208. package/src/id-compressor/uuidUtilities.ts +123 -0
  209. package/src/index.ts +1 -0
  210. package/src/opLifecycle/README.md +13 -0
  211. package/src/opLifecycle/batchManager.ts +35 -2
  212. package/src/opLifecycle/index.ts +1 -1
  213. package/src/opLifecycle/opGroupingManager.ts +5 -1
  214. package/src/opLifecycle/outbox.ts +57 -23
  215. package/src/packageVersion.ts +1 -1
  216. package/src/pendingStateManager.ts +21 -7
  217. package/src/summary/index.ts +1 -0
  218. package/src/summary/orderedClientElection.ts +15 -2
  219. package/src/summary/runningSummarizer.ts +3 -24
  220. package/src/summary/summaryFormat.ts +4 -0
  221. package/src/summary/summaryManager.ts +2 -0
@@ -31,6 +31,7 @@ import {
31
31
  } from "@fluidframework/container-runtime-definitions";
32
32
  import {
33
33
  assert,
34
+ delay,
34
35
  LazyPromise,
35
36
  Trace,
36
37
  TypedEventEmitter,
@@ -88,9 +89,14 @@ import {
88
89
  CreateChildSummarizerNodeParam,
89
90
  SummarizeInternalFn,
90
91
  channelsTreeName,
91
- IAttachMessage,
92
92
  IDataStore,
93
93
  ITelemetryContext,
94
+ SerializedIdCompressorWithNoSession,
95
+ IIdCompressor,
96
+ IIdCompressorCore,
97
+ IdCreationRange,
98
+ IdCreationRangeWithStashedState,
99
+ IAttachMessage,
94
100
  } from "@fluidframework/runtime-definitions";
95
101
  import {
96
102
  addBlobToSummary,
@@ -123,6 +129,7 @@ import {
123
129
  extractSummaryMetadataMessage,
124
130
  IContainerRuntimeMetadata,
125
131
  ICreateContainerMetadata,
132
+ idCompressorBlobName,
126
133
  IFetchSnapshotResult,
127
134
  IRootSummarizerNodeWithGC,
128
135
  ISummaryMetadataMessage,
@@ -190,6 +197,13 @@ export enum ContainerMessageType {
190
197
 
191
198
  // Sets the alias of a root data store
192
199
  Alias = "alias",
200
+
201
+ /**
202
+ * An op containing an IdRange of Ids allocated using the runtime's IdCompressor since
203
+ * the last allocation op was sent.
204
+ * See the [IdCompressor README](./id-compressor/README.md) for more details.
205
+ */
206
+ IdAllocation = "idAllocation",
193
207
  }
194
208
 
195
209
  export interface ContainerRuntimeMessage {
@@ -393,6 +407,13 @@ export interface IContainerRuntimeOptions {
393
407
  * @experimental Not ready for use.
394
408
  */
395
409
  readonly chunkSizeInBytes?: number;
410
+
411
+ /**
412
+ * Enable the IdCompressor in the runtime.
413
+ * @experimental Not ready for use.
414
+ */
415
+ readonly enableRuntimeIdCompressor?: boolean;
416
+
396
417
  /**
397
418
  * If enabled, the runtime will block all attempts to send an op inside the
398
419
  * {@link ContainerRuntime#ensureNoDataModelChanges} callback. The callback is used by
@@ -508,6 +529,13 @@ const defaultCompressionConfig = {
508
529
 
509
530
  const defaultChunkSizeInBytes = 204800;
510
531
 
532
+ /**
533
+ * Instead of refreshing from latest because we do not have 100% confidence in the state
534
+ * of the current system, we should close the summarizer and let it recover.
535
+ * This delay's goal is to prevent tight restart loops
536
+ */
537
+ const defaultCloseSummarizerDelayMs = 10000; // 10 seconds
538
+
511
539
  /**
512
540
  * @deprecated - use ContainerRuntimeMessage instead
513
541
  */
@@ -653,6 +681,7 @@ export class ContainerRuntime
653
681
  flushMode = defaultFlushMode,
654
682
  compressionOptions = defaultCompressionConfig,
655
683
  maxBatchSizeInBytes = defaultMaxBatchSizeInBytes,
684
+ enableRuntimeIdCompressor = false,
656
685
  chunkSizeInBytes = defaultChunkSizeInBytes,
657
686
  enableOpReentryCheck = false,
658
687
  enableGroupedBatching = false,
@@ -673,12 +702,14 @@ export class ContainerRuntime
673
702
  }
674
703
  };
675
704
 
676
- const [chunks, metadata, electedSummarizerData, aliases] = await Promise.all([
677
- tryFetchBlob<[string, string[]][]>(chunksBlobName),
678
- tryFetchBlob<IContainerRuntimeMetadata>(metadataBlobName),
679
- tryFetchBlob<ISerializedElection>(electedSummarizerBlobName),
680
- tryFetchBlob<[string, string][]>(aliasBlobName),
681
- ]);
705
+ const [chunks, metadata, electedSummarizerData, aliases, serializedIdCompressor] =
706
+ await Promise.all([
707
+ tryFetchBlob<[string, string[]][]>(chunksBlobName),
708
+ tryFetchBlob<IContainerRuntimeMetadata>(metadataBlobName),
709
+ tryFetchBlob<ISerializedElection>(electedSummarizerBlobName),
710
+ tryFetchBlob<[string, string][]>(aliasBlobName),
711
+ tryFetchBlob<SerializedIdCompressorWithNoSession>(idCompressorBlobName),
712
+ ]);
682
713
 
683
714
  const loadExisting = existing === true || context.existing === true;
684
715
 
@@ -723,6 +754,17 @@ export class ContainerRuntime
723
754
  }
724
755
  }
725
756
 
757
+ const idCompressorEnabled =
758
+ metadata?.idCompressorEnabled ?? runtimeOptions.enableRuntimeIdCompressor ?? false;
759
+ let idCompressor: (IIdCompressor & IIdCompressorCore) | undefined;
760
+ if (idCompressorEnabled) {
761
+ const { IdCompressor, createSessionId } = await import("./id-compressor");
762
+ idCompressor =
763
+ serializedIdCompressor !== undefined
764
+ ? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
765
+ : new IdCompressor(createSessionId(), logger);
766
+ }
767
+
726
768
  const runtime = new containerRuntimeCtor(
727
769
  context,
728
770
  registry,
@@ -738,6 +780,7 @@ export class ContainerRuntime
738
780
  compressionOptions,
739
781
  maxBatchSizeInBytes,
740
782
  chunkSizeInBytes,
783
+ enableRuntimeIdCompressor,
741
784
  enableOpReentryCheck,
742
785
  enableGroupedBatching,
743
786
  },
@@ -746,6 +789,7 @@ export class ContainerRuntime
746
789
  loadExisting,
747
790
  blobManagerSnapshot,
748
791
  context.storage,
792
+ idCompressor,
749
793
  requestHandler,
750
794
  undefined, // summaryConfiguration
751
795
  initializeEntryPoint,
@@ -816,6 +860,8 @@ export class ContainerRuntime
816
860
  return this.context.attachState;
817
861
  }
818
862
 
863
+ public idCompressor: (IIdCompressor & IIdCompressorCore) | undefined;
864
+
819
865
  public get IFluidHandleContext(): IFluidHandleContext {
820
866
  return this.handleContext;
821
867
  }
@@ -910,6 +956,8 @@ export class ContainerRuntime
910
956
  private emitDirtyDocumentEvent = true;
911
957
  private readonly enableOpReentryCheck: boolean;
912
958
  private readonly disableAttachReorder: boolean | undefined;
959
+ private readonly summaryStateUpdateMethod: string | undefined;
960
+ private readonly closeSummarizerDelayMs: number;
913
961
 
914
962
  private readonly defaultTelemetrySignalSampleCount = 100;
915
963
  private _perfSignalData: IPerfSignalReport = {
@@ -989,6 +1037,11 @@ export class ContainerRuntime
989
1037
  */
990
1038
  private readonly telemetryDocumentId: string;
991
1039
 
1040
+ /**
1041
+ * If true, the runtime has access to an IdCompressor
1042
+ */
1043
+ private readonly idCompressorEnabled: boolean;
1044
+
992
1045
  /**
993
1046
  * @internal
994
1047
  */
@@ -1005,6 +1058,7 @@ export class ContainerRuntime
1005
1058
  existing: boolean,
1006
1059
  blobManagerSnapshot: IBlobManagerLoadInfo,
1007
1060
  private readonly _storage: IDocumentStorageService,
1061
+ idCompressor: (IIdCompressor & IIdCompressorCore) | undefined,
1008
1062
  private readonly requestHandler?: (
1009
1063
  request: IRequest,
1010
1064
  runtime: IContainerRuntime,
@@ -1022,6 +1076,8 @@ export class ContainerRuntime
1022
1076
  this.innerDeltaManager = context.deltaManager;
1023
1077
  this.deltaManager = new DeltaManagerSummarizerProxy(context.deltaManager);
1024
1078
 
1079
+ this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
1080
+
1025
1081
  let loadSummaryNumber: number;
1026
1082
  // Get the container creation metadata. For new container, we initialize these. For existing containers,
1027
1083
  // get the values from the metadata blob.
@@ -1033,12 +1089,20 @@ export class ContainerRuntime
1033
1089
  // summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
1034
1090
  // the count is reset to 0.
1035
1091
  loadSummaryNumber = metadata?.summaryNumber ?? 0;
1092
+
1093
+ // Enabling the IdCompressor is a one-way operation and we only want to
1094
+ // allow new containers to turn it on
1095
+ this.idCompressorEnabled = metadata?.idCompressorEnabled ?? false;
1036
1096
  } else {
1037
1097
  this.createContainerMetadata = {
1038
1098
  createContainerRuntimeVersion: pkgVersion,
1039
1099
  createContainerTimestamp: Date.now(),
1040
1100
  };
1041
1101
  loadSummaryNumber = 0;
1102
+
1103
+ this.idCompressorEnabled =
1104
+ this.mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled") ??
1105
+ idCompressor !== undefined;
1042
1106
  }
1043
1107
  this.nextSummaryNumber = loadSummaryNumber + 1;
1044
1108
 
@@ -1051,8 +1115,6 @@ export class ContainerRuntime
1051
1115
  this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName] /* current */,
1052
1116
  );
1053
1117
 
1054
- this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
1055
-
1056
1118
  this.mc.logger.sendTelemetryEvent({
1057
1119
  eventName: "GCFeatureMatrix",
1058
1120
  metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
@@ -1106,6 +1168,10 @@ export class ContainerRuntime
1106
1168
  this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
1107
1169
  this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
1108
1170
 
1171
+ if (this.idCompressorEnabled) {
1172
+ this.idCompressor = idCompressor;
1173
+ }
1174
+
1109
1175
  this.maxConsecutiveReconnects =
1110
1176
  this.mc.config.getNumber(maxConsecutiveReconnectsKey) ??
1111
1177
  this.defaultMaxConsecutiveReconnects;
@@ -1212,15 +1278,9 @@ export class ContainerRuntime
1212
1278
  () => this.storage,
1213
1279
  (localId: string, blobId?: string) => {
1214
1280
  if (!this.disposed) {
1215
- // eslint-disable-next-line @typescript-eslint/no-floating-promises
1216
- Promise.resolve().then(() => {
1217
- // Blob attaches need to be in their own batch (grouped batching would hide metadata)
1218
- this.flush();
1219
- this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
1220
- localId,
1221
- blobId,
1222
- });
1223
- this.flush();
1281
+ this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
1282
+ localId,
1283
+ blobId,
1224
1284
  });
1225
1285
  }
1226
1286
  },
@@ -1278,12 +1338,24 @@ export class ContainerRuntime
1278
1338
  },
1279
1339
  logger: this.mc.logger,
1280
1340
  groupingManager: opGroupingManager,
1341
+ getCurrentSequenceNumbers: () => ({
1342
+ referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
1343
+ clientSequenceNumber: this._processedClientSequenceNumber,
1344
+ }),
1281
1345
  });
1282
1346
 
1283
1347
  this.context.quorum.on("removeMember", (clientId: string) => {
1284
1348
  this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
1285
1349
  });
1286
1350
 
1351
+ this.summaryStateUpdateMethod = this.mc.config.getString(
1352
+ "Fluid.ContainerRuntime.Test.SummaryStateUpdateMethod",
1353
+ );
1354
+ const closeSummarizerDelayOverride = this.mc.config.getNumber(
1355
+ "Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs",
1356
+ );
1357
+ this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
1358
+
1287
1359
  this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
1288
1360
 
1289
1361
  this.dirtyContainer =
@@ -1424,6 +1496,9 @@ export class ContainerRuntime
1424
1496
  disableChunking,
1425
1497
  disableAttachReorder: this.disableAttachReorder,
1426
1498
  disablePartialFlush,
1499
+ idCompressorEnabled: this.idCompressorEnabled,
1500
+ summaryStateUpdateMethod: this.summaryStateUpdateMethod,
1501
+ closeSummarizerDelayOverride,
1427
1502
  }),
1428
1503
  telemetryDocumentId: this.telemetryDocumentId,
1429
1504
  groupedBatchingEnabled: this.groupedBatchingEnabled,
@@ -1603,6 +1678,7 @@ export class ContainerRuntime
1603
1678
  extractSummaryMetadataMessage(this.deltaManager.lastMessage) ??
1604
1679
  this.messageAtLastSummary,
1605
1680
  telemetryDocumentId: this.telemetryDocumentId,
1681
+ idCompressorEnabled: this.idCompressorEnabled ? true : undefined,
1606
1682
  };
1607
1683
  addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
1608
1684
  }
@@ -1615,6 +1691,15 @@ export class ContainerRuntime
1615
1691
  ) {
1616
1692
  this.addMetadataToSummary(summaryTree);
1617
1693
 
1694
+ if (this.idCompressorEnabled) {
1695
+ assert(
1696
+ this.idCompressor !== undefined,
1697
+ 0x67a /* IdCompressor should be defined if enabled */,
1698
+ );
1699
+ const idCompressorState = JSON.stringify(this.idCompressor.serialize(false));
1700
+ addBlobToSummary(summaryTree, idCompressorBlobName, idCompressorState);
1701
+ }
1702
+
1618
1703
  if (this.remoteMessageProcessor.partialMessages.size > 0) {
1619
1704
  const content = JSON.stringify([...this.remoteMessageProcessor.partialMessages]);
1620
1705
  addBlobToSummary(summaryTree, chunksBlobName, content);
@@ -1713,15 +1798,32 @@ export class ContainerRuntime
1713
1798
  this.updateDocumentDirtyState(newState);
1714
1799
  }
1715
1800
 
1716
- private async applyStashedOp(
1717
- type: ContainerMessageType,
1718
- op: ISequencedDocumentMessage,
1719
- ): Promise<unknown> {
1801
+ /**
1802
+ * Updates the runtime's IdCompressor with the stashed state present in the given op. This is a bit of a
1803
+ * hack and is unnecessarily expensive. As it stands, every locally stashed op (all ops that get stored in
1804
+ * the PendingStateManager) will store their serialized representation locally until ack'd. Upon receiving
1805
+ * this stashed state, the IdCompressor blindly deserializes to the stashed state and assumes the session.
1806
+ * Technically only the last stashed state is needed to do this correctly, but we would have to write some
1807
+ * more hacky code to modify the batch before it gets sent out.
1808
+ * @param content - An IdAllocationOp with "stashedState", which is a representation of un-ack'd local state.
1809
+ */
1810
+ private async applyStashedIdAllocationOp(op: IdCreationRangeWithStashedState) {
1811
+ const { IdCompressor } = await import("./id-compressor");
1812
+ this.idCompressor = IdCompressor.deserialize(op.stashedState);
1813
+ }
1814
+
1815
+ private async applyStashedOp(type: ContainerMessageType, contents: unknown): Promise<unknown> {
1720
1816
  switch (type) {
1721
1817
  case ContainerMessageType.FluidDataStoreOp:
1722
- return this.dataStores.applyStashedOp(op);
1818
+ return this.dataStores.applyStashedOp(contents as IEnvelope);
1723
1819
  case ContainerMessageType.Attach:
1724
- return this.dataStores.applyStashedAttachOp(op as unknown as IAttachMessage);
1820
+ return this.dataStores.applyStashedAttachOp(contents as IAttachMessage);
1821
+ case ContainerMessageType.IdAllocation:
1822
+ assert(
1823
+ this.idCompressor !== undefined,
1824
+ 0x67b /* IdCompressor should be defined if enabled */,
1825
+ );
1826
+ return this.applyStashedIdAllocationOp(contents as IdCreationRangeWithStashedState);
1725
1827
  case ContainerMessageType.Alias:
1726
1828
  case ContainerMessageType.BlobAttach:
1727
1829
  return;
@@ -1846,6 +1948,8 @@ export class ContainerRuntime
1846
1948
  }
1847
1949
  }
1848
1950
 
1951
+ private _processedClientSequenceNumber: number | undefined;
1952
+
1849
1953
  private processCore(
1850
1954
  message: ISequencedDocumentMessage,
1851
1955
  local: boolean,
@@ -1856,6 +1960,8 @@ export class ContainerRuntime
1856
1960
  // messages once a batch has been fully processed.
1857
1961
  this.scheduleManager.beforeOpProcessing(message);
1858
1962
 
1963
+ this._processedClientSequenceNumber = message.clientSequenceNumber;
1964
+
1859
1965
  try {
1860
1966
  let localOpMetadata: unknown;
1861
1967
  if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
@@ -1882,6 +1988,13 @@ export class ContainerRuntime
1882
1988
  case ContainerMessageType.BlobAttach:
1883
1989
  this.blobManager.processBlobAttachOp(message, local);
1884
1990
  break;
1991
+ case ContainerMessageType.IdAllocation:
1992
+ assert(
1993
+ this.idCompressor !== undefined,
1994
+ 0x67c /* IdCompressor should be defined if enabled */,
1995
+ );
1996
+ this.idCompressor.finalizeCreationRange(message.contents as IdCreationRange);
1997
+ break;
1885
1998
  case ContainerMessageType.ChunkedOp:
1886
1999
  case ContainerMessageType.Rejoin:
1887
2000
  break;
@@ -2325,31 +2438,33 @@ export class ContainerRuntime
2325
2438
  runSweep,
2326
2439
  });
2327
2440
 
2328
- let gcStats: IGCStats | undefined;
2329
- if (runGC) {
2330
- gcStats = await this.collectGarbage(
2331
- { logger: summaryLogger, runSweep, fullGC },
2441
+ try {
2442
+ let gcStats: IGCStats | undefined;
2443
+ if (runGC) {
2444
+ gcStats = await this.collectGarbage(
2445
+ { logger: summaryLogger, runSweep, fullGC },
2446
+ telemetryContext,
2447
+ );
2448
+ }
2449
+
2450
+ const { stats, summary } = await this.summarizerNode.summarize(
2451
+ fullTree,
2452
+ trackState,
2332
2453
  telemetryContext,
2333
2454
  );
2334
- }
2335
2455
 
2336
- const { stats, summary } = await this.summarizerNode.summarize(
2337
- fullTree,
2338
- trackState,
2339
- telemetryContext,
2340
- );
2341
-
2342
- this.logger.sendTelemetryEvent({
2343
- eventName: "SummarizeTelemetry",
2344
- details: telemetryContext.serialize(),
2345
- });
2346
-
2347
- assert(
2348
- summary.type === SummaryType.Tree,
2349
- 0x12f /* "Container Runtime's summarize should always return a tree" */,
2350
- );
2456
+ assert(
2457
+ summary.type === SummaryType.Tree,
2458
+ 0x12f /* "Container Runtime's summarize should always return a tree" */,
2459
+ );
2351
2460
 
2352
- return { stats, summary, gcStats };
2461
+ return { stats, summary, gcStats };
2462
+ } finally {
2463
+ this.logger.sendTelemetryEvent({
2464
+ eventName: "SummarizeTelemetry",
2465
+ details: telemetryContext.serialize(),
2466
+ });
2467
+ }
2353
2468
  }
2354
2469
 
2355
2470
  /**
@@ -2831,6 +2946,40 @@ export class ContainerRuntime
2831
2946
  return this.blobManager.createBlob(blob);
2832
2947
  }
2833
2948
 
2949
+ private maybeSubmitIdAllocationOp(type: ContainerMessageType) {
2950
+ if (type !== ContainerMessageType.IdAllocation) {
2951
+ let idAllocationBatchMessage: BatchMessage | undefined;
2952
+ let idRange: IdCreationRange | undefined;
2953
+ if (this.idCompressorEnabled) {
2954
+ assert(
2955
+ this.idCompressor !== undefined,
2956
+ 0x67d /* IdCompressor should be defined if enabled */,
2957
+ );
2958
+ idRange = this.idCompressor.takeNextCreationRange();
2959
+ // Don't include the idRange if there weren't any Ids allocated
2960
+ idRange = idRange?.ids?.first !== undefined ? idRange : undefined;
2961
+ }
2962
+
2963
+ if (idRange !== undefined) {
2964
+ const idAllocationMessage: ContainerRuntimeMessage = {
2965
+ type: ContainerMessageType.IdAllocation,
2966
+ contents: idRange,
2967
+ };
2968
+ idAllocationBatchMessage = {
2969
+ contents: JSON.stringify(idAllocationMessage),
2970
+ deserializedContent: idAllocationMessage,
2971
+ referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
2972
+ metadata: undefined,
2973
+ localOpMetadata: this.idCompressor?.serialize(true),
2974
+ };
2975
+ }
2976
+
2977
+ if (idAllocationBatchMessage !== undefined) {
2978
+ this.outbox.submit(idAllocationBatchMessage);
2979
+ }
2980
+ }
2981
+ }
2982
+
2834
2983
  private submit(
2835
2984
  type: ContainerMessageType,
2836
2985
  contents: any,
@@ -2866,6 +3015,12 @@ export class ContainerRuntime
2866
3015
  };
2867
3016
 
2868
3017
  try {
3018
+ // Submit an IdAllocation op if any Ids have been generated since
3019
+ // the last op was submitted. Don't submit another if it's an IdAllocation
3020
+ // op as that means we're in resubmission flow and we don't want to send
3021
+ // IdRanges out of order.
3022
+ this.maybeSubmitIdAllocationOp(type);
3023
+
2869
3024
  // If this is attach message for new data store, and we are in a batch, send this op out of order
2870
3025
  // Is it safe:
2871
3026
  // Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
@@ -3028,6 +3183,7 @@ export class ContainerRuntime
3028
3183
  break;
3029
3184
  case ContainerMessageType.Attach:
3030
3185
  case ContainerMessageType.Alias:
3186
+ case ContainerMessageType.IdAllocation:
3031
3187
  this.submit(type, content, localOpMetadata);
3032
3188
  break;
3033
3189
  case ContainerMessageType.ChunkedOp:
@@ -3093,6 +3249,25 @@ export class ContainerRuntime
3093
3249
  readAndParseBlob,
3094
3250
  );
3095
3251
 
3252
+ /**
3253
+ * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
3254
+ * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
3255
+ * these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
3256
+ * change that started fetching latest snapshot always.
3257
+ */
3258
+ if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
3259
+ fetchResult = await this.fetchSnapshotFromStorage(
3260
+ summaryLogger,
3261
+ {
3262
+ eventName: "RefreshLatestSummaryAckFetchBackCompat",
3263
+ ackHandle,
3264
+ targetSequenceNumber: summaryRefSeq,
3265
+ },
3266
+ readAndParseBlob,
3267
+ ackHandle,
3268
+ );
3269
+ }
3270
+
3096
3271
  /**
3097
3272
  * If the fetched snapshot is older than the one for which the ack was received, close the container.
3098
3273
  * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -3104,32 +3279,18 @@ export class ContainerRuntime
3104
3279
  * state.
3105
3280
  */
3106
3281
  if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
3107
- /* before failing, let's try to retrieve the latest snapshot for that specific ackHandle */
3108
- fetchResult = await this.fetchSnapshotFromStorage(
3109
- summaryLogger,
3282
+ const error = DataProcessingError.create(
3283
+ "Fetched snapshot is older than the received ack",
3284
+ "RefreshLatestSummaryAck",
3285
+ undefined /* sequencedMessage */,
3110
3286
  {
3111
- eventName: "RefreshLatestSummaryAckFetch",
3112
3287
  ackHandle,
3113
- targetSequenceNumber: summaryRefSeq,
3288
+ summaryRefSeq,
3289
+ fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
3114
3290
  },
3115
- readAndParseBlob,
3116
- ackHandle,
3117
3291
  );
3118
-
3119
- if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
3120
- const error = DataProcessingError.create(
3121
- "Fetched snapshot is older than the received ack",
3122
- "RefreshLatestSummaryAck",
3123
- undefined /* sequencedMessage */,
3124
- {
3125
- ackHandle,
3126
- summaryRefSeq,
3127
- fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
3128
- },
3129
- );
3130
- this.closeFn(error);
3131
- throw error;
3132
- }
3292
+ this.closeFn(error);
3293
+ throw error;
3133
3294
  }
3134
3295
 
3135
3296
  // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
@@ -3211,27 +3372,7 @@ export class ContainerRuntime
3211
3372
  readAndParseBlob: ReadAndParseBlob,
3212
3373
  versionId: string | null,
3213
3374
  ): Promise<{ snapshotTree: ISnapshotTree; versionId: string; latestSnapshotRefSeq: number }> {
3214
- const recoveryMethod = this.mc.config.getString(
3215
- "Fluid.ContainerRuntime.Test.SummarizationRecoveryMethod",
3216
- );
3217
- if (recoveryMethod === "restart") {
3218
- const error = new GenericError("Restarting summarizer instead of refreshing");
3219
- this.mc.logger.sendTelemetryEvent(
3220
- {
3221
- ...event,
3222
- eventName: "ClosingSummarizerOnSummaryStale",
3223
- codePath: event.eventName,
3224
- message: "Stopping fetch from storage",
3225
- versionId: versionId != null ? versionId : undefined,
3226
- },
3227
- error,
3228
- );
3229
- this._summarizer?.stop("latestSummaryStateStale");
3230
- this.closeFn();
3231
- throw error;
3232
- }
3233
-
3234
- return PerformanceEvent.timedExecAsync(
3375
+ const snapshotResults = await PerformanceEvent.timedExecAsync(
3235
3376
  logger,
3236
3377
  event,
3237
3378
  async (perfEvent: {
@@ -3277,6 +3418,33 @@ export class ContainerRuntime
3277
3418
  };
3278
3419
  },
3279
3420
  );
3421
+
3422
+ // We choose to close the summarizer after the snapshot cache is updated to avoid
3423
+ // situations which the main client (which is likely to be re-elected as the leader again)
3424
+ // loads the summarizer from cache.
3425
+ if (this.summaryStateUpdateMethod === "restart") {
3426
+ const error = new GenericError("Restarting summarizer instead of refreshing");
3427
+
3428
+ this.mc.logger.sendTelemetryEvent(
3429
+ {
3430
+ ...event,
3431
+ eventName: "ClosingSummarizerOnSummaryStale",
3432
+ codePath: event.eventName,
3433
+ message: "Stopping fetch from storage",
3434
+ versionId: versionId != null ? versionId : undefined,
3435
+ closeSummarizerDelayMs: this.closeSummarizerDelayMs,
3436
+ },
3437
+ error,
3438
+ );
3439
+
3440
+ // Delay 10 seconds before restarting summarizer to prevent the summarizer from restarting too frequently.
3441
+ await delay(this.closeSummarizerDelayMs);
3442
+ this._summarizer?.stop("latestSummaryStateStale");
3443
+ this.closeFn();
3444
+ throw error;
3445
+ }
3446
+
3447
+ return snapshotResults;
3280
3448
  }
3281
3449
 
3282
3450
  public notifyAttaching() {} // do nothing (deprecated method)
@@ -48,6 +48,8 @@ import {
48
48
  ISummarizerNodeWithGC,
49
49
  SummarizeInternalFn,
50
50
  ITelemetryContext,
51
+ IIdCompressor,
52
+ IIdCompressorCore,
51
53
  VisibilityState,
52
54
  } from "@fluidframework/runtime-definitions";
53
55
  import {
@@ -193,6 +195,10 @@ export abstract class FluidDataStoreContext
193
195
  return this._baseSnapshot;
194
196
  }
195
197
 
198
+ public get idCompressor(): (IIdCompressorCore & IIdCompressor) | undefined {
199
+ return this._containerRuntime.idCompressor;
200
+ }
201
+
196
202
  private _disposed = false;
197
203
  public get disposed() {
198
204
  return this._disposed;
package/src/dataStores.ts CHANGED
@@ -116,7 +116,7 @@ export class DataStores implements IDisposable {
116
116
  constructor(
117
117
  private readonly baseSnapshot: ISnapshotTree | undefined,
118
118
  private readonly runtime: ContainerRuntime,
119
- private readonly submitAttachFn: (attachContent: any) => void,
119
+ private readonly submitAttachFn: (attachContent: IAttachMessage) => void,
120
120
  private readonly getCreateChildSummarizerNodeFn: (
121
121
  id: string,
122
122
  createParam: CreateChildSummarizerNodeParam,
@@ -399,22 +399,19 @@ export class DataStores implements IDisposable {
399
399
  }
400
400
  public readonly dispose = () => this.disposeOnce.value;
401
401
 
402
- public resubmitDataStoreOp(content: any, localOpMetadata: unknown) {
403
- const envelope = content as IEnvelope;
402
+ public resubmitDataStoreOp(envelope: IEnvelope, localOpMetadata: unknown) {
404
403
  const context = this.contexts.get(envelope.address);
405
404
  assert(!!context, 0x160 /* "There should be a store context for the op" */);
406
405
  context.reSubmit(envelope.contents, localOpMetadata);
407
406
  }
408
407
 
409
- public rollbackDataStoreOp(content: any, localOpMetadata: unknown) {
410
- const envelope = content as IEnvelope;
408
+ public rollbackDataStoreOp(envelope: IEnvelope, localOpMetadata: unknown) {
411
409
  const context = this.contexts.get(envelope.address);
412
410
  assert(!!context, 0x2e8 /* "There should be a store context for the op" */);
413
411
  context.rollback(envelope.contents, localOpMetadata);
414
412
  }
415
413
 
416
- public async applyStashedOp(content: any): Promise<unknown> {
417
- const envelope = content as IEnvelope;
414
+ public async applyStashedOp(envelope: IEnvelope): Promise<unknown> {
418
415
  const context = this.contexts.get(envelope.address);
419
416
  assert(!!context, 0x161 /* "There should be a store context for the op" */);
420
417
  return context.applyStashedOp(envelope.contents);
@@ -465,6 +465,13 @@ export class GarbageCollector implements IGarbageCollector {
465
465
  const fullGC =
466
466
  options.fullGC ??
467
467
  (this.configs.runFullGC === true || this.summaryStateTracker.doesSummaryStateNeedReset);
468
+
469
+ // Add the options that are used to run GC to the telemetry context.
470
+ telemetryContext?.setMultiple("fluid_GC", "Options", {
471
+ fullGC,
472
+ runSweep: options.runSweep,
473
+ });
474
+
468
475
  const logger = options.logger
469
476
  ? ChildLogger.create(options.logger, undefined, {
470
477
  all: { completedGCRuns: () => this.completedRuns },
@@ -489,12 +496,6 @@ export class GarbageCollector implements IGarbageCollector {
489
496
  return undefined;
490
497
  }
491
498
 
492
- // Add the options that are used to run GC to the telemetry context.
493
- telemetryContext?.setMultiple("fluid_GC", "Options", {
494
- fullGC,
495
- runSweep: options.runSweep,
496
- });
497
-
498
499
  return PerformanceEvent.timedExecAsync(
499
500
  logger,
500
501
  { eventName: "GarbageCollection" },