@fluidframework/container-runtime 2.0.0-internal.1.4.4 → 2.0.0-internal.2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/batchManager.d.ts +2 -3
  2. package/dist/batchManager.d.ts.map +1 -1
  3. package/dist/batchManager.js +3 -8
  4. package/dist/batchManager.js.map +1 -1
  5. package/dist/containerRuntime.d.ts +43 -16
  6. package/dist/containerRuntime.d.ts.map +1 -1
  7. package/dist/containerRuntime.js +107 -82
  8. package/dist/containerRuntime.js.map +1 -1
  9. package/dist/dataStoreContext.d.ts +4 -20
  10. package/dist/dataStoreContext.d.ts.map +1 -1
  11. package/dist/dataStoreContext.js +17 -47
  12. package/dist/dataStoreContext.js.map +1 -1
  13. package/dist/dataStores.d.ts +2 -5
  14. package/dist/dataStores.d.ts.map +1 -1
  15. package/dist/dataStores.js +3 -11
  16. package/dist/dataStores.js.map +1 -1
  17. package/dist/garbageCollection.d.ts +1 -10
  18. package/dist/garbageCollection.d.ts.map +1 -1
  19. package/dist/garbageCollection.js +43 -51
  20. package/dist/garbageCollection.js.map +1 -1
  21. package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
  22. package/dist/gcSweepReadyUsageDetection.js +3 -12
  23. package/dist/gcSweepReadyUsageDetection.js.map +1 -1
  24. package/dist/index.d.ts +3 -5
  25. package/dist/index.d.ts.map +1 -1
  26. package/dist/index.js +1 -5
  27. package/dist/index.js.map +1 -1
  28. package/dist/packageVersion.d.ts +1 -1
  29. package/dist/packageVersion.js +1 -1
  30. package/dist/packageVersion.js.map +1 -1
  31. package/dist/pendingStateManager.d.ts +6 -26
  32. package/dist/pendingStateManager.d.ts.map +1 -1
  33. package/dist/pendingStateManager.js +42 -62
  34. package/dist/pendingStateManager.js.map +1 -1
  35. package/dist/scheduleManager.js.map +1 -1
  36. package/dist/summarizer.js +7 -2
  37. package/dist/summarizer.js.map +1 -1
  38. package/dist/summarizerHeuristics.d.ts.map +1 -1
  39. package/dist/summarizerHeuristics.js +0 -3
  40. package/dist/summarizerHeuristics.js.map +1 -1
  41. package/dist/summarizerTypes.d.ts +19 -2
  42. package/dist/summarizerTypes.d.ts.map +1 -1
  43. package/dist/summarizerTypes.js.map +1 -1
  44. package/dist/summaryFormat.d.ts +4 -2
  45. package/dist/summaryFormat.d.ts.map +1 -1
  46. package/dist/summaryFormat.js.map +1 -1
  47. package/dist/summaryManager.d.ts.map +1 -1
  48. package/dist/summaryManager.js +10 -6
  49. package/dist/summaryManager.js.map +1 -1
  50. package/lib/batchManager.d.ts +2 -3
  51. package/lib/batchManager.d.ts.map +1 -1
  52. package/lib/batchManager.js +3 -8
  53. package/lib/batchManager.js.map +1 -1
  54. package/lib/containerRuntime.d.ts +43 -16
  55. package/lib/containerRuntime.d.ts.map +1 -1
  56. package/lib/containerRuntime.js +108 -83
  57. package/lib/containerRuntime.js.map +1 -1
  58. package/lib/dataStoreContext.d.ts +4 -20
  59. package/lib/dataStoreContext.d.ts.map +1 -1
  60. package/lib/dataStoreContext.js +18 -48
  61. package/lib/dataStoreContext.js.map +1 -1
  62. package/lib/dataStores.d.ts +2 -5
  63. package/lib/dataStores.d.ts.map +1 -1
  64. package/lib/dataStores.js +3 -11
  65. package/lib/dataStores.js.map +1 -1
  66. package/lib/garbageCollection.d.ts +1 -10
  67. package/lib/garbageCollection.d.ts.map +1 -1
  68. package/lib/garbageCollection.js +42 -50
  69. package/lib/garbageCollection.js.map +1 -1
  70. package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
  71. package/lib/gcSweepReadyUsageDetection.js +3 -12
  72. package/lib/gcSweepReadyUsageDetection.js.map +1 -1
  73. package/lib/index.d.ts +3 -5
  74. package/lib/index.d.ts.map +1 -1
  75. package/lib/index.js +0 -2
  76. package/lib/index.js.map +1 -1
  77. package/lib/packageVersion.d.ts +1 -1
  78. package/lib/packageVersion.js +1 -1
  79. package/lib/packageVersion.js.map +1 -1
  80. package/lib/pendingStateManager.d.ts +6 -26
  81. package/lib/pendingStateManager.d.ts.map +1 -1
  82. package/lib/pendingStateManager.js +42 -62
  83. package/lib/pendingStateManager.js.map +1 -1
  84. package/lib/scheduleManager.js.map +1 -1
  85. package/lib/summarizer.js +7 -2
  86. package/lib/summarizer.js.map +1 -1
  87. package/lib/summarizerHeuristics.d.ts.map +1 -1
  88. package/lib/summarizerHeuristics.js +0 -3
  89. package/lib/summarizerHeuristics.js.map +1 -1
  90. package/lib/summarizerTypes.d.ts +19 -2
  91. package/lib/summarizerTypes.d.ts.map +1 -1
  92. package/lib/summarizerTypes.js.map +1 -1
  93. package/lib/summaryFormat.d.ts +4 -2
  94. package/lib/summaryFormat.d.ts.map +1 -1
  95. package/lib/summaryFormat.js.map +1 -1
  96. package/lib/summaryManager.d.ts.map +1 -1
  97. package/lib/summaryManager.js +10 -6
  98. package/lib/summaryManager.js.map +1 -1
  99. package/package.json +22 -65
  100. package/src/batchManager.ts +7 -11
  101. package/src/containerRuntime.ts +149 -100
  102. package/src/dataStoreContext.ts +20 -62
  103. package/src/dataStores.ts +2 -10
  104. package/src/garbageCollection.ts +45 -55
  105. package/src/gcSweepReadyUsageDetection.ts +2 -10
  106. package/src/index.ts +2 -3
  107. package/src/packageVersion.ts +1 -1
  108. package/src/pendingStateManager.ts +57 -96
  109. package/src/scheduleManager.ts +1 -0
  110. package/src/summarizer.ts +6 -6
  111. package/src/summarizerHeuristics.ts +0 -3
  112. package/src/summarizerTypes.ts +20 -7
  113. package/src/summaryFormat.ts +4 -2
  114. package/src/summaryManager.ts +18 -7
@@ -34,6 +34,7 @@ import {
34
34
  Trace,
35
35
  TypedEventEmitter,
36
36
  unreachableCase,
37
+ IsoBuffer,
37
38
  } from "@fluidframework/common-utils";
38
39
  import {
39
40
  ChildLogger,
@@ -107,6 +108,7 @@ import {
107
108
  } from "@fluidframework/runtime-utils";
108
109
  import { GCDataBuilder, trimLeadingAndTrailingSlashes } from "@fluidframework/garbage-collector";
109
110
  import { v4 as uuid } from "uuid";
111
+ import { compress, decompress } from "lz4js";
110
112
  import { ContainerFluidHandleContext } from "./containerHandleContext";
111
113
  import { FluidDataStoreRegistry } from "./dataStoreRegistry";
112
114
  import { Summarizer } from "./summarizer";
@@ -147,6 +149,7 @@ import {
147
149
  ISummarizerInternalsProvider,
148
150
  ISummarizerOptions,
149
151
  ISummarizerRuntime,
152
+ IRefreshSummaryAckOptions,
150
153
  } from "./summarizerTypes";
151
154
  import { formExponentialFn, Throttler } from "./throttler";
152
155
  import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
@@ -229,11 +232,6 @@ export interface ISummaryBaseConfiguration {
229
232
 
230
233
  export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfiguration {
231
234
  state: "enabled";
232
- /**
233
- * @deprecated Please move all implementations to {@link ISummaryConfigurationHeuristics.minIdleTime} and
234
- * {@link ISummaryConfigurationHeuristics.maxIdleTime} instead.
235
- */
236
- idleTime?: number;
237
235
  /**
238
236
  * Defines the maximum allowed time, since the last received Ack, before running the summary
239
237
  * with reason maxTime.
@@ -416,6 +414,18 @@ export interface ISummaryRuntimeOptions {
416
414
  summarizerOptions?: Readonly<Partial<ISummarizerOptions>>;
417
415
  }
418
416
 
417
+ /**
418
+ * Options for op compression.
419
+ * @experimental - Not ready for use
420
+ */
421
+ export interface ICompressionRuntimeOptions {
422
+ /**
423
+ * The minimum size the content payload must exceed before it is compressed.
424
+ * Compression is disabled if undefined.
425
+ */
426
+ readonly minimumSize?: number;
427
+ }
428
+
419
429
  /**
420
430
  * Options for container runtime.
421
431
  */
@@ -442,6 +452,22 @@ export interface IContainerRuntimeOptions {
442
452
  * Save enough runtime state to be able to serialize upon request and load to the same state in a new container.
443
453
  */
444
454
  readonly enableOfflineLoad?: boolean;
455
+ /**
456
+ * Enables the runtime to compress ops.
457
+ * @experimental Not ready for use.
458
+ */
459
+ readonly compressionOptions?: ICompressionRuntimeOptions;
460
+ /**
461
+ * If specified, when in FlushMode.TurnBased, if the size of the ops between JS turns exceeds this value,
462
+ * an error will be thrown and the container will close.
463
+ *
464
+ * If unspecified, the limit is 950 * 1024.
465
+ *
466
+ * 'Infinity' will disable any limit.
467
+ *
468
+ * @experimental This config should be driven by the connection with the service and will be moved in the future.
469
+ */
470
+ readonly maxBatchSizeInBytes?: number;
445
471
  }
446
472
 
447
473
  /**
@@ -513,6 +539,12 @@ const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconn
513
539
 
514
540
  const defaultFlushMode = FlushMode.TurnBased;
515
541
 
542
+ // The actual limit is 1Mb (socket.io and Kafka limits)
543
+ // We can't estimate it fully, as we
544
+ // - do not know what properties relay service will add
545
+ // - we do not stringify final op, thus we do not know how much escaping will be added.
546
+ const defaultMaxBatchSizeInBytes = 950 * 1024;
547
+
516
548
  /**
517
549
  * @deprecated - use ContainerRuntimeMessage instead
518
550
  */
@@ -530,22 +562,28 @@ export enum RuntimeMessage {
530
562
  * @deprecated - please use version in driver-utils
531
563
  */
532
564
  export function isRuntimeMessage(message: ISequencedDocumentMessage): boolean {
533
- if ((Object.values(RuntimeMessage) as string[]).includes(message.type)) {
534
- return true;
535
- }
536
- return false;
565
+ return (Object.values(RuntimeMessage) as string[]).includes(message.type);
537
566
  }
538
567
 
539
568
  /**
540
569
  * Unpacks runtime messages
541
570
  *
542
- * @remarks This API makes no promises regarding backward-compatability. This is internal API.
571
+ * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
543
572
  * @param message - message (as it observed in storage / service)
544
573
  * @returns unpacked runtime message
545
574
  *
546
575
  * @internal
547
576
  */
548
577
  export function unpackRuntimeMessage(message: ISequencedDocumentMessage) {
578
+ if (message.metadata?.compressed) {
579
+ const contents = IsoBuffer.from(message.contents.contents, "base64");
580
+ const decompressedMessage = decompress(contents);
581
+ const intoString = new TextDecoder().decode(decompressedMessage);
582
+ const asObj = JSON.parse(intoString);
583
+ message.contents.contents = asObj;
584
+ message.metadata.compressed = false;
585
+ }
586
+
549
587
  if (message.type === MessageType.Operation) {
550
588
  // legacy op format?
551
589
  if (message.contents.address !== undefined && message.contents.type === undefined) {
@@ -635,6 +673,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
635
673
  loadSequenceNumberVerification = "close",
636
674
  flushMode = defaultFlushMode,
637
675
  enableOfflineLoad = false,
676
+ compressionOptions = {},
677
+ maxBatchSizeInBytes = defaultMaxBatchSizeInBytes,
638
678
  } = runtimeOptions;
639
679
 
640
680
  const pendingRuntimeState = context.pendingLocalState as IPendingRuntimeState | undefined;
@@ -710,6 +750,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
710
750
  loadSequenceNumberVerification,
711
751
  flushMode,
712
752
  enableOfflineLoad,
753
+ compressionOptions,
754
+ maxBatchSizeInBytes,
713
755
  },
714
756
  containerScope,
715
757
  logger,
@@ -802,8 +844,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
802
844
  private readonly defaultMaxConsecutiveReconnects = 7;
803
845
 
804
846
  private _orderSequentiallyCalls: number = 0;
805
- private _flushMode: FlushMode;
806
- private flushTrigger = false;
847
+ private readonly _flushMode: FlushMode;
848
+ private flushMicroTaskExists = false;
807
849
 
808
850
  private _connected: boolean;
809
851
 
@@ -811,6 +853,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
811
853
  private baseSnapshotBlobs?: ISerializedBaseSnapshotBlobs;
812
854
 
813
855
  private consecutiveReconnects = 0;
856
+ private compressedOpCount = 0;
814
857
 
815
858
  /**
816
859
  * Used to delay transition to "connected" state while we upload
@@ -851,13 +894,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
851
894
  private readonly scheduleManager: ScheduleManager;
852
895
  private readonly blobManager: BlobManager;
853
896
  private readonly pendingStateManager: PendingStateManager;
854
-
855
- // Provide lower soft limit - we want to have some number of ops to get efficiency in compression & bandwidth usage,
856
- // but at the same time we want to send these ops sooner, to reduce overall latency of processing a batch.
857
- // So there is some ballance here, that depends on compression algorithm and its efficiency working with smaller
858
- // payloads. That number represents final (compressed) bits (once compression is implemented).
859
- private readonly pendingAttachBatch = new BatchManager(64 * 1024);
860
- private readonly pendingBatch = new BatchManager();
897
+ private readonly pendingAttachBatch: BatchManager;
898
+ private readonly pendingBatch: BatchManager;
861
899
 
862
900
  private readonly garbageCollector: IGarbageCollector;
863
901
 
@@ -990,9 +1028,25 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
990
1028
 
991
1029
  this._flushMode = runtimeOptions.flushMode;
992
1030
 
1031
+ // Provide lower soft limit - we want to have some number of ops to get efficiency in compression
1032
+ // & bandwidth usage, but at the same time we want to send these ops sooner, to reduce overall
1033
+ // latency of processing a batch.
1034
+ // So there is some ballance here, that depends on compression algorithm and its efficiency working with smaller
1035
+ // payloads. That number represents final (compressed) bits (once compression is implemented).
1036
+ this.pendingAttachBatch = new BatchManager(runtimeOptions.maxBatchSizeInBytes, 64 * 1024);
1037
+ this.pendingBatch = new BatchManager(runtimeOptions.maxBatchSizeInBytes);
1038
+
993
1039
  const pendingRuntimeState = context.pendingLocalState as IPendingRuntimeState | undefined;
994
1040
  const baseSnapshot: ISnapshotTree | undefined = pendingRuntimeState?.baseSnapshot ?? context.baseSnapshot;
995
1041
 
1042
+ const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
1043
+ if (maxSnapshotCacheDurationMs !== undefined && maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
1044
+ // This is a runtime enforcement of what's already explicit in the policy's type itself,
1045
+ // which dictates the value is either undefined or exactly 5 days in ms.
1046
+ // As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
1047
+ throw new UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
1048
+ }
1049
+
996
1050
  this.garbageCollector = GarbageCollector.create({
997
1051
  runtime: this,
998
1052
  gcOptions: this.runtimeOptions.gcOptions,
@@ -1031,7 +1085,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1031
1085
  );
1032
1086
 
1033
1087
  if (baseSnapshot) {
1034
- this.summarizerNode.loadBaseSummaryWithoutDifferential(baseSnapshot);
1088
+ this.summarizerNode.updateBaseSummaryState(baseSnapshot);
1035
1089
  }
1036
1090
 
1037
1091
  this.dataStores = new DataStores(
@@ -1060,7 +1114,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1060
1114
  packagePath,
1061
1115
  ),
1062
1116
  new Map<string, string>(dataStoreAliasMap),
1063
- this.garbageCollector.writeDataAtRoot,
1064
1117
  );
1065
1118
 
1066
1119
  this.blobManager = new BlobManager(
@@ -1093,11 +1146,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1093
1146
  close: this.closeFn,
1094
1147
  connected: () => this.connected,
1095
1148
  flush: this.flush.bind(this),
1096
- flushMode: () => this.flushMode,
1097
1149
  reSubmit: this.reSubmit.bind(this),
1098
- setFlushMode: (mode) => this.setFlushMode(mode),
1150
+ rollback: this.rollback.bind(this),
1151
+ orderSequentially: this.orderSequentially.bind(this),
1099
1152
  },
1100
- this._flushMode,
1101
1153
  pendingRuntimeState?.pending);
1102
1154
 
1103
1155
  this.context.quorum.on("removeMember", (clientId: string) => {
@@ -1150,7 +1202,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1150
1202
  // if summaries are enabled and we are not the summarizer client.
1151
1203
  const defaultAction = () => {
1152
1204
  if (this.summaryCollection.opsSinceLastAck > this.maxOpsSinceLastSummary) {
1153
- this.logger.sendErrorEvent({ eventName: "SummaryStatus:Behind" });
1205
+ this.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
1154
1206
  // unregister default to no log on every op after falling behind
1155
1207
  // and register summary ack handler to re-register this handler
1156
1208
  // after successful summary
@@ -1441,11 +1493,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1441
1493
  addTreeToSummary(summaryTree, blobsTreeName, blobManagerSummary);
1442
1494
  }
1443
1495
 
1444
- if (this.garbageCollector.writeDataAtRoot) {
1445
- const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
1446
- if (gcSummary !== undefined) {
1447
- addSummarizeResultToSummary(summaryTree, gcTreeKey, gcSummary);
1448
- }
1496
+ const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
1497
+ if (gcSummary !== undefined) {
1498
+ addSummarizeResultToSummary(summaryTree, gcTreeKey, gcSummary);
1449
1499
  }
1450
1500
  }
1451
1501
 
@@ -1772,29 +1822,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1772
1822
  return context.realize();
1773
1823
  }
1774
1824
 
1775
- public setFlushMode(mode: FlushMode): void {
1776
- if (mode === this._flushMode) {
1777
- return;
1778
- }
1779
-
1780
- this.mc.logger.sendTelemetryEvent({
1781
- eventName: "FlushMode Updated",
1782
- old: this._flushMode,
1783
- new: mode,
1784
- });
1785
-
1786
- // Flush any pending batches if switching to immediate
1787
- if (mode === FlushMode.Immediate) {
1788
- this.flush();
1789
- }
1790
-
1791
- this._flushMode = mode;
1792
-
1793
- // Let the PendingStateManager know that FlushMode has been updated.
1794
- this.pendingStateManager.onFlushModeUpdated(mode);
1795
- }
1796
-
1797
- public flush(): void {
1825
+ /**
1826
+ * Flush the pending ops manually.
1827
+ * This method is expected to be called at the end of a batch.
1828
+ */
1829
+ private flush(): void {
1798
1830
  assert(this._orderSequentiallyCalls === 0,
1799
1831
  0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
1800
1832
 
@@ -1830,7 +1862,38 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1830
1862
  if (this.context.submitBatchFn !== undefined) {
1831
1863
  const batchToSend: IBatchMessage[] = [];
1832
1864
  for (const message of batch) {
1833
- batchToSend.push({ contents: message.contents, metadata: message.metadata });
1865
+ let contents = message.contents;
1866
+ let metadata = message.metadata;
1867
+ if (this.runtimeOptions.compressionOptions.minimumSize &&
1868
+ this.runtimeOptions.compressionOptions.minimumSize < message.contents.length) {
1869
+ this.compressedOpCount++;
1870
+ const copiedMessage = { ...message.deserializedContent };
1871
+
1872
+ const compressionStart = Date.now();
1873
+ const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(copiedMessage.contents));
1874
+ const compressedContents = compress(contentsAsBuffer);
1875
+ const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
1876
+ const duration = Date.now() - compressionStart;
1877
+
1878
+ if (this.compressedOpCount % 100) {
1879
+ this.mc.logger.sendPerformanceEvent({
1880
+ eventName: "compressedOp",
1881
+ duration,
1882
+ sizeBeforeCompression: message.contents.length,
1883
+ sizeAfterCompression: compressedContent.length,
1884
+ });
1885
+ }
1886
+
1887
+ copiedMessage.contents = compressedContent;
1888
+ const stringifiedContents = JSON.stringify(copiedMessage);
1889
+
1890
+ if (stringifiedContents.length < message.contents.length) {
1891
+ contents = JSON.stringify(copiedMessage);
1892
+ metadata = { ...message.metadata, compressed: true };
1893
+ }
1894
+ }
1895
+
1896
+ batchToSend.push({ contents, metadata });
1834
1897
  }
1835
1898
  // returns clientSequenceNumber of last message in a batch
1836
1899
  clientSequenceNumber = this.context.submitBatchFn(batchToSend);
@@ -1871,36 +1934,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1871
1934
  }
1872
1935
 
1873
1936
  public orderSequentially(callback: () => void): void {
1874
- // If flush mode is already TurnBased we are either
1875
- // nested in another orderSequentially, or
1876
- // the app is flushing manually, in which
1877
- // case this invocation doesn't own
1878
- // flushing.
1879
- if (this.flushMode === FlushMode.TurnBased) {
1880
- this.trackOrderSequentiallyCalls(callback);
1881
- return;
1882
- }
1883
-
1884
- const savedFlushMode = this.flushMode;
1885
- this.setFlushMode(FlushMode.TurnBased);
1886
-
1887
- try {
1888
- this.trackOrderSequentiallyCalls(callback);
1889
- this.flush();
1890
- } finally {
1891
- this.setFlushMode(savedFlushMode);
1892
- }
1893
- }
1894
-
1895
- private trackOrderSequentiallyCalls(callback: () => void): void {
1896
1937
  let checkpoint: { rollback: (action: (message: BatchMessage) => void) => void; } | undefined;
1938
+
1897
1939
  if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
1898
1940
  // Note: we are not touching this.pendingAttachBatch here, for two reasons:
1899
1941
  // 1. It would not help, as we flush attach ops as they become available.
1900
1942
  // 2. There is no way to undo process of data store creation.
1901
1943
  checkpoint = this.pendingBatch.checkpoint();
1902
1944
  }
1903
-
1904
1945
  try {
1905
1946
  this._orderSequentiallyCalls++;
1906
1947
  callback();
@@ -1931,6 +1972,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1931
1972
  } finally {
1932
1973
  this._orderSequentiallyCalls--;
1933
1974
  }
1975
+
1976
+ if (this.flushMode === FlushMode.Immediate && this._orderSequentiallyCalls === 0) {
1977
+ this.flush();
1978
+ }
1934
1979
  }
1935
1980
 
1936
1981
  public async createDataStore(pkg: string | string[]): Promise<IDataStore> {
@@ -1980,6 +2025,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1980
2025
  return this.connected && !this.deltaManager.readOnlyInfo.readonly;
1981
2026
  }
1982
2027
 
2028
+ /**
2029
+ * Are we in the middle of batching ops together?
2030
+ */
2031
+ private currentlyBatching() {
2032
+ return this.flushMode === FlushMode.TurnBased || this._orderSequentiallyCalls !== 0;
2033
+ }
2034
+
1983
2035
  public getQuorum(): IQuorumClients {
1984
2036
  return this.context.quorum;
1985
2037
  }
@@ -2192,10 +2244,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2192
2244
  * Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
2193
2245
  * After GC has run, called to notify this container's nodes of routes that are used in it.
2194
2246
  * @param usedRoutes - The routes that are used in all nodes in this Container.
2195
- * @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
2196
- * unreferenced as part of this GC run, this should be used to update the time when it happens.
2197
2247
  */
2198
- public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
2248
+ public updateUsedRoutes(usedRoutes: string[]) {
2199
2249
  // Update our summarizer node's used routes. Updating used routes in summarizer node before
2200
2250
  // summarizing is required and asserted by the the summarizer node. We are the root and are
2201
2251
  // always referenced, so the used routes is only self-route (empty string).
@@ -2208,7 +2258,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2208
2258
  }
2209
2259
  }
2210
2260
 
2211
- return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes, gcTimestamp);
2261
+ return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes);
2212
2262
  }
2213
2263
 
2214
2264
  /**
@@ -2680,8 +2730,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2680
2730
  // issue than sending.
2681
2731
  // Please note that this does not change file format, so it can be disabled in the future if this
2682
2732
  // optimization no longer makes sense (for example, batch compression may make it less appealing).
2683
- if (type === ContainerMessageType.Attach &&
2684
- this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
2733
+ if (this.currentlyBatching() && type === ContainerMessageType.Attach &&
2734
+ this.mc.config.getBoolean("Fluid.ContainerRuntime.enableAttachOpReorder") === true) {
2685
2735
  if (!this.pendingAttachBatch.push(message)) {
2686
2736
  // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
2687
2737
  // when queue is not empty.
@@ -2709,18 +2759,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2709
2759
  limit: this.pendingBatch.limit,
2710
2760
  });
2711
2761
  }
2712
- }
2713
-
2714
- if (this._flushMode !== FlushMode.TurnBased) {
2715
- this.flush();
2716
- } else if (!this.flushTrigger) {
2717
- this.flushTrigger = true;
2718
- // Queue a microtask to detect the end of the turn and force a flush.
2719
- // eslint-disable-next-line @typescript-eslint/no-floating-promises
2720
- Promise.resolve().then(() => {
2721
- this.flushTrigger = false;
2762
+ if (!this.currentlyBatching()) {
2722
2763
  this.flush();
2723
- });
2764
+ } else if (!this.flushMicroTaskExists) {
2765
+ this.flushMicroTaskExists = true;
2766
+ // Queue a microtask to detect the end of the turn and force a flush.
2767
+ // eslint-disable-next-line @typescript-eslint/no-floating-promises
2768
+ Promise.resolve().then(() => {
2769
+ this.flushMicroTaskExists = false;
2770
+ this.flush();
2771
+ });
2772
+ }
2724
2773
  }
2725
2774
  } catch (error) {
2726
2775
  this.closeFn(error as GenericError);
@@ -2810,12 +2859,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2810
2859
  }
2811
2860
 
2812
2861
  /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
2813
- public async refreshLatestSummaryAck(
2814
- proposalHandle: string | undefined,
2815
- ackHandle: string,
2816
- summaryRefSeq: number,
2817
- summaryLogger: ITelemetryLogger,
2818
- ) {
2862
+ public async refreshLatestSummaryAck(options: IRefreshSummaryAckOptions) {
2863
+ const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
2819
2864
  const readAndParseBlob = async <T>(id: string) => readAndParse<T>(this.storage, id);
2820
2865
  // The call to fetch the snapshot is very expensive and not always needed.
2821
2866
  // It should only be done by the summarizerNode, if required.
@@ -2931,6 +2976,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2931
2976
  // to close current batch.
2932
2977
  this.flush();
2933
2978
 
2979
+ if (this._orderSequentiallyCalls !== 0) {
2980
+ throw new UsageError("can't get state during orderSequentially");
2981
+ }
2982
+
2934
2983
  const previousPendingState = this.context.pendingLocalState as IPendingRuntimeState | undefined;
2935
2984
  if (previousPendingState) {
2936
2985
  return {
@@ -42,7 +42,6 @@ import {
42
42
  CreateChildSummarizerNodeFn,
43
43
  CreateChildSummarizerNodeParam,
44
44
  FluidDataStoreRegistryEntry,
45
- gcBlobKey,
46
45
  IAttachMessage,
47
46
  IFluidDataStoreChannel,
48
47
  IFluidDataStoreContext,
@@ -51,7 +50,6 @@ import {
51
50
  IFluidDataStoreRegistry,
52
51
  IGarbageCollectionData,
53
52
  IGarbageCollectionDetailsBase,
54
- IGarbageCollectionSummaryDetails,
55
53
  IInboundSignalMessage,
56
54
  IProvideFluidDataStoreFactory,
57
55
  ISummarizeInternalResult,
@@ -117,7 +115,6 @@ export interface IFluidDataStoreContextProps {
117
115
  readonly storage: IDocumentStorageService;
118
116
  readonly scope: FluidObject;
119
117
  readonly createSummarizerNodeFn: CreateChildSummarizerNodeFn;
120
- readonly writeGCDataAtRoot: boolean;
121
118
  readonly pkg?: Readonly<string[]>;
122
119
  }
123
120
 
@@ -135,7 +132,7 @@ export interface ILocalFluidDataStoreContextProps extends IFluidDataStoreContext
135
132
 
136
133
  /** Properties necessary for creating a remote FluidDataStoreContext */
137
134
  export interface IRemoteFluidDataStoreContextProps extends IFluidDataStoreContextProps {
138
- readonly snapshotTree: ISnapshotTree | string | undefined;
135
+ readonly snapshotTree: ISnapshotTree | undefined;
139
136
  readonly getBaseGCDetails: () => Promise<IGarbageCollectionDetailsBase | undefined>;
140
137
  }
141
138
 
@@ -239,15 +236,14 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
239
236
  private readonly thresholdOpsCounter: ThresholdCounter;
240
237
  private static readonly pendingOpsCountThreshold = 1000;
241
238
 
242
- // The used state of this node as per the last GC run. This is used to update the used state of the channel
239
+ // The used routes of this node as per the last GC run. This is used to update the used routes of the channel
243
240
  // if it realizes after GC is run.
244
- private lastUsedState: { usedRoutes: string[]; gcTimestamp?: number; } | undefined;
241
+ private lastUsedRoutes: string[] | undefined;
245
242
 
246
243
  public readonly id: string;
247
244
  private readonly _containerRuntime: ContainerRuntime;
248
245
  public readonly storage: IDocumentStorageService;
249
246
  public readonly scope: FluidObject;
250
- private readonly writeGCDataAtRoot: boolean;
251
247
  protected pkg?: readonly string[];
252
248
 
253
249
  constructor(
@@ -263,7 +259,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
263
259
  this.id = props.id;
264
260
  this.storage = props.storage;
265
261
  this.scope = props.scope;
266
- this.writeGCDataAtRoot = props.writeGCDataAtRoot;
267
262
  this.pkg = props.pkg;
268
263
 
269
264
  // URIs use slashes as delimiters. Handles use URIs.
@@ -474,11 +469,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
474
469
  const attributes = createAttributes(pkg, isRoot);
475
470
  addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes));
476
471
 
477
- // Add GC data to the summary if it's not written at the root.
478
- if (!this.writeGCDataAtRoot) {
479
- addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
480
- }
481
-
482
472
  // If we are not referenced, mark the summary tree as unreferenced. Also, update unreferenced blob
483
473
  // size in the summary stats with the blobs size of this data store.
484
474
  if (!this.summarizerNode.isReferenced()) {
@@ -532,21 +522,17 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
532
522
  * 5. To update the timestamp when this data store or any children are marked as unreferenced.
533
523
  *
534
524
  * @param usedRoutes - The routes that are used in this data store.
535
- * @param gcTimestamp - The time when GC was run that generated these used routes. If any node becomes unreferenced
536
- * as part of this GC run, this should be used to update the time when it happens.
537
525
  */
538
- public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
526
+ public updateUsedRoutes(usedRoutes: string[]) {
539
527
  // Update the used routes in this data store's summarizer node.
540
- this.summarizerNode.updateUsedRoutes(usedRoutes, gcTimestamp);
528
+ this.summarizerNode.updateUsedRoutes(usedRoutes);
541
529
 
542
530
  /**
543
- * If the data store has not been realized yet, we need this used state to update the used state of the channel
544
- * when it realizes. It's safe to keep only the last used state because if something changes because of this GC
545
- * run, the data store will be immediately realized as part of the summary that follows GC. For example, if a
546
- * child's reference state changes, the gcTimestamp has to be used to update its unreferencedTimestamp. Since
547
- * it will result in a change in this data store's used routes, it will be realized to regenerate its summary.
531
+ * Store the used routes to update the channel if the data store is not loaded yet. If the used routes changed
532
+ * since the previous run, the data store will be loaded during summarize since the used state changed. So, it's
533
+ * safe to only store the last used routes.
548
534
  */
549
- this.lastUsedState = { usedRoutes, gcTimestamp };
535
+ this.lastUsedRoutes = usedRoutes;
550
536
 
551
537
  // If we are loaded, call the channel so it can update the used routes of the child contexts.
552
538
  // If we are not loaded, we will update this when we are realized.
@@ -575,16 +561,16 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
575
561
  assert(this.loaded, 0x144 /* "Channel should be loaded when updating used routes" */);
576
562
  assert(this.channel !== undefined, 0x145 /* "Channel should be present when data store is loaded" */);
577
563
 
578
- // If there is no lastUsedState, GC has not run up until this point.
579
- if (this.lastUsedState === undefined) {
564
+ // If there is no lastUsedRoutes, GC has not run up until this point.
565
+ if (this.lastUsedRoutes === undefined) {
580
566
  return;
581
567
  }
582
568
 
583
569
  // Remove the route to this data store, if it exists.
584
- const usedChannelRoutes = this.lastUsedState.usedRoutes.filter(
570
+ const usedChannelRoutes = this.lastUsedRoutes.filter(
585
571
  (id: string) => { return id !== "/" && id !== ""; },
586
572
  );
587
- this.channel.updateUsedRoutes(usedChannelRoutes, this.lastUsedState.gcTimestamp);
573
+ this.channel.updateUsedRoutes(usedChannelRoutes);
588
574
  }
589
575
 
590
576
  /**
@@ -718,11 +704,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
718
704
  this._isInMemoryRoot = true;
719
705
  }
720
706
 
721
- /**
722
- * @deprecated Renamed to `{@link FluidDataStoreContext.getBaseGCDetails}()`.
723
- */
724
- public abstract getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails>;
725
-
726
707
  public abstract getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
727
708
 
728
709
  public reSubmit(contents: any, localOpMetadata: unknown) {
@@ -779,7 +760,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
779
760
  }
780
761
 
781
762
  export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
782
- private readonly initSnapshotValue: ISnapshotTree | string | undefined;
763
+ private readonly initSnapshotValue: ISnapshotTree | undefined;
783
764
  private readonly baseGCDetailsP: Promise<IGarbageCollectionDetailsBase>;
784
765
 
785
766
  constructor(props: IRemoteFluidDataStoreContextProps) {
@@ -797,28 +778,20 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
797
778
  this.baseGCDetailsP = new LazyPromise<IGarbageCollectionDetailsBase>(async () => {
798
779
  return (await props.getBaseGCDetails()) ?? {};
799
780
  });
781
+
782
+ if (props.snapshotTree !== undefined) {
783
+ this.summarizerNode.updateBaseSummaryState(props.snapshotTree);
784
+ }
800
785
  }
801
786
 
802
787
  private readonly initialSnapshotDetailsP = new LazyPromise<ISnapshotDetails>(async () => {
803
- let tree: ISnapshotTree | undefined;
788
+ let tree = this.initSnapshotValue;
804
789
  let isRootDataStore = true;
805
790
 
806
- if (typeof this.initSnapshotValue === "string") {
807
- const commit = (await this.storage.getVersions(this.initSnapshotValue, 1))[0];
808
- tree = await this.storage.getSnapshotTree(commit) ?? undefined;
809
- } else {
810
- tree = this.initSnapshotValue;
811
- }
812
-
813
- const localReadAndParse = async <T>(id: string) => readAndParse<T>(this.storage, id);
814
- if (tree) {
815
- tree = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
816
- }
817
-
818
791
  if (!!tree && tree.blobs[dataStoreAttributesBlobName] !== undefined) {
819
792
  // Need to get through snapshot and use that to populate extraBlobs
820
793
  const attributes =
821
- await localReadAndParse<ReadFluidDataStoreAttributes>(tree.blobs[dataStoreAttributesBlobName]);
794
+ await readAndParse<ReadFluidDataStoreAttributes>(this.storage, tree.blobs[dataStoreAttributesBlobName]);
822
795
 
823
796
  let pkgFromSnapshot: string[];
824
797
  // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot.
@@ -859,13 +832,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
859
832
  return this.initialSnapshotDetailsP;
860
833
  }
861
834
 
862
- /**
863
- * @deprecated Renamed to {@link RemoteFluidDataStoreContext.getBaseGCDetails}.
864
- */
865
- public async getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails> {
866
- return this.getBaseGCDetails();
867
- }
868
-
869
835
  public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
870
836
  return this.baseGCDetailsP;
871
837
  }
@@ -974,14 +940,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
974
940
  };
975
941
  }
976
942
 
977
- /**
978
- * @deprecated Renamed to {@link LocalFluidDataStoreContextBase.getBaseGCDetails}.
979
- */
980
- public async getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails> {
981
- // Local data store does not have initial summary.
982
- return {};
983
- }
984
-
985
943
  public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
986
944
  // Local data store does not have initial summary.
987
945
  return {};