@fluidframework/container-runtime 2.1.0-274160 → 2.1.0-276985

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/README.md +14 -5
  2. package/container-runtime.test-files.tar +0 -0
  3. package/{lib → dist/blobManager}/blobManager.d.ts +9 -28
  4. package/dist/blobManager/blobManager.d.ts.map +1 -0
  5. package/dist/{blobManager.js → blobManager/blobManager.js} +23 -83
  6. package/dist/blobManager/blobManager.js.map +1 -0
  7. package/dist/blobManager/blobManagerSnapSum.d.ts +30 -0
  8. package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -0
  9. package/dist/blobManager/blobManagerSnapSum.js +82 -0
  10. package/dist/blobManager/blobManagerSnapSum.js.map +1 -0
  11. package/dist/blobManager/index.d.ts +7 -0
  12. package/dist/blobManager/index.d.ts.map +1 -0
  13. package/dist/blobManager/index.js +16 -0
  14. package/dist/blobManager/index.js.map +1 -0
  15. package/dist/channelCollection.d.ts +1 -0
  16. package/dist/channelCollection.d.ts.map +1 -1
  17. package/dist/channelCollection.js +23 -13
  18. package/dist/channelCollection.js.map +1 -1
  19. package/dist/containerRuntime.d.ts +20 -7
  20. package/dist/containerRuntime.d.ts.map +1 -1
  21. package/dist/containerRuntime.js +82 -77
  22. package/dist/containerRuntime.js.map +1 -1
  23. package/dist/dataStoreContext.d.ts +1 -0
  24. package/dist/dataStoreContext.d.ts.map +1 -1
  25. package/dist/dataStoreContext.js +7 -2
  26. package/dist/dataStoreContext.js.map +1 -1
  27. package/dist/gc/garbageCollection.js +2 -2
  28. package/dist/gc/garbageCollection.js.map +1 -1
  29. package/dist/gc/gcDefinitions.d.ts +9 -0
  30. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  31. package/dist/gc/gcDefinitions.js +1 -0
  32. package/dist/gc/gcDefinitions.js.map +1 -1
  33. package/dist/index.d.ts +1 -1
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js.map +1 -1
  36. package/dist/legacy.d.ts +1 -1
  37. package/dist/messageTypes.d.ts +1 -0
  38. package/dist/messageTypes.d.ts.map +1 -1
  39. package/dist/messageTypes.js +1 -0
  40. package/dist/messageTypes.js.map +1 -1
  41. package/dist/opLifecycle/batchManager.d.ts +4 -0
  42. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  43. package/dist/opLifecycle/batchManager.js.map +1 -1
  44. package/dist/opLifecycle/outbox.d.ts +8 -4
  45. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  46. package/dist/opLifecycle/outbox.js +18 -16
  47. package/dist/opLifecycle/outbox.js.map +1 -1
  48. package/dist/opLifecycle/remoteMessageProcessor.d.ts +17 -1
  49. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  50. package/dist/opLifecycle/remoteMessageProcessor.js +43 -5
  51. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
  52. package/dist/packageVersion.d.ts +1 -1
  53. package/dist/packageVersion.js +1 -1
  54. package/dist/packageVersion.js.map +1 -1
  55. package/dist/pendingStateManager.d.ts +11 -7
  56. package/dist/pendingStateManager.d.ts.map +1 -1
  57. package/dist/pendingStateManager.js +34 -15
  58. package/dist/pendingStateManager.js.map +1 -1
  59. package/dist/summary/documentSchema.d.ts +8 -0
  60. package/dist/summary/documentSchema.d.ts.map +1 -1
  61. package/dist/summary/documentSchema.js +2 -0
  62. package/dist/summary/documentSchema.js.map +1 -1
  63. package/dist/summary/index.d.ts +1 -1
  64. package/dist/summary/index.d.ts.map +1 -1
  65. package/dist/summary/index.js +1 -2
  66. package/dist/summary/index.js.map +1 -1
  67. package/dist/summary/orderedClientElection.d.ts +1 -0
  68. package/dist/summary/orderedClientElection.d.ts.map +1 -1
  69. package/dist/summary/orderedClientElection.js.map +1 -1
  70. package/dist/summary/runWhileConnectedCoordinator.d.ts +1 -0
  71. package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
  72. package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
  73. package/dist/summary/summarizer.d.ts +1 -0
  74. package/dist/summary/summarizer.d.ts.map +1 -1
  75. package/dist/summary/summarizer.js +1 -0
  76. package/dist/summary/summarizer.js.map +1 -1
  77. package/dist/summary/summarizerTypes.d.ts +29 -0
  78. package/dist/summary/summarizerTypes.d.ts.map +1 -1
  79. package/dist/summary/summarizerTypes.js.map +1 -1
  80. package/dist/summary/summaryCollection.d.ts +10 -0
  81. package/dist/summary/summaryCollection.d.ts.map +1 -1
  82. package/dist/summary/summaryCollection.js +1 -0
  83. package/dist/summary/summaryCollection.js.map +1 -1
  84. package/dist/summary/summaryFormat.d.ts +8 -1
  85. package/dist/summary/summaryFormat.d.ts.map +1 -1
  86. package/dist/summary/summaryFormat.js +3 -3
  87. package/dist/summary/summaryFormat.js.map +1 -1
  88. package/{dist → lib/blobManager}/blobManager.d.ts +9 -28
  89. package/lib/blobManager/blobManager.d.ts.map +1 -0
  90. package/lib/{blobManager.js → blobManager/blobManager.js} +21 -83
  91. package/lib/blobManager/blobManager.js.map +1 -0
  92. package/lib/blobManager/blobManagerSnapSum.d.ts +30 -0
  93. package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -0
  94. package/lib/blobManager/blobManagerSnapSum.js +75 -0
  95. package/lib/blobManager/blobManagerSnapSum.js.map +1 -0
  96. package/lib/blobManager/index.d.ts +7 -0
  97. package/lib/blobManager/index.d.ts.map +1 -0
  98. package/lib/blobManager/index.js +7 -0
  99. package/lib/blobManager/index.js.map +1 -0
  100. package/lib/channelCollection.d.ts +1 -0
  101. package/lib/channelCollection.d.ts.map +1 -1
  102. package/lib/channelCollection.js +23 -13
  103. package/lib/channelCollection.js.map +1 -1
  104. package/lib/containerRuntime.d.ts +20 -7
  105. package/lib/containerRuntime.d.ts.map +1 -1
  106. package/lib/containerRuntime.js +31 -26
  107. package/lib/containerRuntime.js.map +1 -1
  108. package/lib/dataStoreContext.d.ts +1 -0
  109. package/lib/dataStoreContext.d.ts.map +1 -1
  110. package/lib/dataStoreContext.js +7 -2
  111. package/lib/dataStoreContext.js.map +1 -1
  112. package/lib/gc/garbageCollection.js +2 -2
  113. package/lib/gc/garbageCollection.js.map +1 -1
  114. package/lib/gc/gcDefinitions.d.ts +9 -0
  115. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  116. package/lib/gc/gcDefinitions.js +1 -0
  117. package/lib/gc/gcDefinitions.js.map +1 -1
  118. package/lib/index.d.ts +1 -1
  119. package/lib/index.d.ts.map +1 -1
  120. package/lib/index.js.map +1 -1
  121. package/lib/legacy.d.ts +1 -1
  122. package/lib/messageTypes.d.ts +1 -0
  123. package/lib/messageTypes.d.ts.map +1 -1
  124. package/lib/messageTypes.js +1 -0
  125. package/lib/messageTypes.js.map +1 -1
  126. package/lib/opLifecycle/batchManager.d.ts +4 -0
  127. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  128. package/lib/opLifecycle/batchManager.js.map +1 -1
  129. package/lib/opLifecycle/outbox.d.ts +8 -4
  130. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  131. package/lib/opLifecycle/outbox.js +18 -16
  132. package/lib/opLifecycle/outbox.js.map +1 -1
  133. package/lib/opLifecycle/remoteMessageProcessor.d.ts +17 -1
  134. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  135. package/lib/opLifecycle/remoteMessageProcessor.js +41 -3
  136. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
  137. package/lib/packageVersion.d.ts +1 -1
  138. package/lib/packageVersion.js +1 -1
  139. package/lib/packageVersion.js.map +1 -1
  140. package/lib/pendingStateManager.d.ts +11 -7
  141. package/lib/pendingStateManager.d.ts.map +1 -1
  142. package/lib/pendingStateManager.js +35 -16
  143. package/lib/pendingStateManager.js.map +1 -1
  144. package/lib/summary/documentSchema.d.ts +8 -0
  145. package/lib/summary/documentSchema.d.ts.map +1 -1
  146. package/lib/summary/documentSchema.js +2 -0
  147. package/lib/summary/documentSchema.js.map +1 -1
  148. package/lib/summary/index.d.ts +1 -1
  149. package/lib/summary/index.d.ts.map +1 -1
  150. package/lib/summary/index.js +1 -1
  151. package/lib/summary/index.js.map +1 -1
  152. package/lib/summary/orderedClientElection.d.ts +1 -0
  153. package/lib/summary/orderedClientElection.d.ts.map +1 -1
  154. package/lib/summary/orderedClientElection.js.map +1 -1
  155. package/lib/summary/runWhileConnectedCoordinator.d.ts +1 -0
  156. package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -1
  157. package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
  158. package/lib/summary/summarizer.d.ts +1 -0
  159. package/lib/summary/summarizer.d.ts.map +1 -1
  160. package/lib/summary/summarizer.js +1 -0
  161. package/lib/summary/summarizer.js.map +1 -1
  162. package/lib/summary/summarizerTypes.d.ts +29 -0
  163. package/lib/summary/summarizerTypes.d.ts.map +1 -1
  164. package/lib/summary/summarizerTypes.js.map +1 -1
  165. package/lib/summary/summaryCollection.d.ts +10 -0
  166. package/lib/summary/summaryCollection.d.ts.map +1 -1
  167. package/lib/summary/summaryCollection.js +1 -0
  168. package/lib/summary/summaryCollection.js.map +1 -1
  169. package/lib/summary/summaryFormat.d.ts +8 -1
  170. package/lib/summary/summaryFormat.d.ts.map +1 -1
  171. package/lib/summary/summaryFormat.js +1 -1
  172. package/lib/summary/summaryFormat.js.map +1 -1
  173. package/package.json +23 -23
  174. package/src/{blobManager.ts → blobManager/blobManager.ts} +38 -122
  175. package/src/blobManager/blobManagerSnapSum.ts +133 -0
  176. package/src/blobManager/index.ts +19 -0
  177. package/src/channelCollection.ts +23 -13
  178. package/src/containerRuntime.ts +57 -39
  179. package/src/dataStoreContext.ts +8 -2
  180. package/src/gc/garbageCollection.ts +2 -2
  181. package/src/gc/gcDefinitions.ts +9 -0
  182. package/src/index.ts +1 -1
  183. package/src/messageTypes.ts +1 -0
  184. package/src/opLifecycle/batchManager.ts +4 -0
  185. package/src/opLifecycle/outbox.ts +19 -21
  186. package/src/opLifecycle/remoteMessageProcessor.ts +63 -6
  187. package/src/packageVersion.ts +1 -1
  188. package/src/pendingStateManager.ts +43 -20
  189. package/src/summary/documentSchema.ts +8 -0
  190. package/src/summary/index.ts +0 -1
  191. package/src/summary/orderedClientElection.ts +1 -0
  192. package/src/summary/runWhileConnectedCoordinator.ts +1 -0
  193. package/src/summary/summarizer.ts +1 -0
  194. package/src/summary/summarizerTypes.ts +29 -0
  195. package/src/summary/summaryCollection.ts +10 -0
  196. package/src/summary/summaryFormat.ts +9 -1
  197. package/dist/blobManager.d.ts.map +0 -1
  198. package/dist/blobManager.js.map +0 -1
  199. package/lib/blobManager.d.ts.map +0 -1
  200. package/lib/blobManager.js.map +0 -1
  201. /package/api-report/{container-runtime.alpha.api.md → container-runtime.legacy.alpha.api.md} +0 -0
@@ -126,7 +126,15 @@ import {
126
126
  import { v4 as uuid } from "uuid";
127
127
 
128
128
  import { BindBatchTracker } from "./batchTracker.js";
129
- import { BlobManager, IBlobManagerLoadInfo, IPendingBlobs } from "./blobManager.js";
129
+ import {
130
+ BlobManager,
131
+ IPendingBlobs,
132
+ blobManagerBasePath,
133
+ blobsTreeName,
134
+ isBlobPath,
135
+ loadBlobManagerLoadInfo,
136
+ type IBlobManagerLoadInfo,
137
+ } from "./blobManager/index.js";
130
138
  import {
131
139
  ChannelCollection,
132
140
  getSummaryForDatastores,
@@ -211,7 +219,6 @@ import {
211
219
  SummaryCollection,
212
220
  SummaryManager,
213
221
  aliasBlobName,
214
- blobsTreeName,
215
222
  chunksBlobName,
216
223
  createRootSummarizerNodeWithGC,
217
224
  electedSummarizerBlobName,
@@ -241,6 +248,7 @@ function compatBehaviorAllowsMessageType(
241
248
  }
242
249
 
243
250
  /**
251
+ * @legacy
244
252
  * @alpha
245
253
  */
246
254
  export interface ISummaryBaseConfiguration {
@@ -263,6 +271,7 @@ export interface ISummaryBaseConfiguration {
263
271
  }
264
272
 
265
273
  /**
274
+ * @legacy
266
275
  * @alpha
267
276
  */
268
277
  export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfiguration {
@@ -326,6 +335,7 @@ export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfigurati
326
335
  }
327
336
 
328
337
  /**
338
+ * @legacy
329
339
  * @alpha
330
340
  */
331
341
  export interface ISummaryConfigurationDisableSummarizer {
@@ -333,6 +343,7 @@ export interface ISummaryConfigurationDisableSummarizer {
333
343
  }
334
344
 
335
345
  /**
346
+ * @legacy
336
347
  * @alpha
337
348
  */
338
349
  export interface ISummaryConfigurationDisableHeuristics extends ISummaryBaseConfiguration {
@@ -340,6 +351,7 @@ export interface ISummaryConfigurationDisableHeuristics extends ISummaryBaseConf
340
351
  }
341
352
 
342
353
  /**
354
+ * @legacy
343
355
  * @alpha
344
356
  */
345
357
  export type ISummaryConfiguration =
@@ -348,6 +360,7 @@ export type ISummaryConfiguration =
348
360
  | ISummaryConfigurationHeuristics;
349
361
 
350
362
  /**
363
+ * @legacy
351
364
  * @alpha
352
365
  */
353
366
  export const DefaultSummaryConfiguration: ISummaryConfiguration = {
@@ -377,6 +390,7 @@ export const DefaultSummaryConfiguration: ISummaryConfiguration = {
377
390
  };
378
391
 
379
392
  /**
393
+ * @legacy
380
394
  * @alpha
381
395
  */
382
396
  export interface ISummaryRuntimeOptions {
@@ -394,6 +408,7 @@ export interface ISummaryRuntimeOptions {
394
408
 
395
409
  /**
396
410
  * Options for op compression.
411
+ * @legacy
397
412
  * @alpha
398
413
  */
399
414
  export interface ICompressionRuntimeOptions {
@@ -412,6 +427,7 @@ export interface ICompressionRuntimeOptions {
412
427
 
413
428
  /**
414
429
  * Options for container runtime.
430
+ * @legacy
415
431
  * @alpha
416
432
  */
417
433
  export interface IContainerRuntimeOptions {
@@ -490,16 +506,19 @@ export interface IContainerRuntimeOptions {
490
506
 
491
507
  /**
492
508
  * Error responses when requesting a deleted object will have this header set to true
509
+ * @legacy
493
510
  * @alpha
494
511
  */
495
512
  export const DeletedResponseHeaderKey = "wasDeleted";
496
513
  /**
497
514
  * Tombstone error responses will have this header set to true
515
+ * @legacy
498
516
  * @alpha
499
517
  */
500
518
  export const TombstoneResponseHeaderKey = "isTombstoned";
501
519
  /**
502
520
  * Inactive error responses will have this header set to true
521
+ * @legacy
503
522
  * @alpha
504
523
  */
505
524
  export const InactiveResponseHeaderKey = "isInactive";
@@ -525,13 +544,17 @@ export const defaultRuntimeHeaderData: Required<RuntimeHeaderData> = {
525
544
 
526
545
  /**
527
546
  * Available compression algorithms for op compression.
547
+ * @legacy
528
548
  * @alpha
529
549
  */
530
550
  export enum CompressionAlgorithms {
531
551
  lz4 = "lz4",
532
552
  }
533
553
 
534
- /** @alpha */
554
+ /**
555
+ * @legacy
556
+ * @alpha
557
+ */
535
558
  export const disabledCompressionConfig: ICompressionRuntimeOptions = {
536
559
  minimumBatchSizeInBytes: Infinity,
537
560
  compressionAlgorithm: CompressionAlgorithms.lz4,
@@ -643,8 +666,10 @@ export const makeLegacySendBatchFn =
643
666
  deltaManager: Pick<IDeltaManager<unknown, unknown>, "flush">,
644
667
  ) =>
645
668
  (batch: IBatch) => {
669
+ // Default to negative one to match Container.submitBatch behavior
670
+ let clientSequenceNumber: number = -1;
646
671
  for (const message of batch.content) {
647
- submitFn(
672
+ clientSequenceNumber = submitFn(
648
673
  MessageType.Operation,
649
674
  // For back-compat (submitFn only works on deserialized content)
650
675
  message.contents === undefined ? undefined : JSON.parse(message.contents),
@@ -654,26 +679,31 @@ export const makeLegacySendBatchFn =
654
679
  }
655
680
 
656
681
  deltaManager.flush();
682
+
683
+ return clientSequenceNumber;
657
684
  };
658
685
 
659
686
  /** Helper type for type constraints passed through several functions.
687
+ * local - Did this client send the op?
688
+ * savedOp - Is this op being replayed after being serialized (having been sequenced previously)
689
+ * batchStartCsn - The clientSequenceNumber given on submit to the start of this batch
660
690
  * message - The unpacked message. Likely a TypedContainerRuntimeMessage, but could also be a system op
661
691
  * modernRuntimeMessage - Does this appear like a current TypedContainerRuntimeMessage?
662
- * local - Did this client send the op?
663
692
  */
664
- type MessageWithContext =
693
+ type MessageWithContext = {
694
+ local: boolean;
695
+ savedOp?: boolean;
696
+ batchStartCsn: number;
697
+ } & (
665
698
  | {
666
699
  message: InboundSequencedContainerRuntimeMessage;
667
700
  modernRuntimeMessage: true;
668
- local: boolean;
669
- savedOp?: boolean;
670
701
  }
671
702
  | {
672
703
  message: InboundSequencedContainerRuntimeMessageOrSystemMessage;
673
704
  modernRuntimeMessage: false;
674
- local: boolean;
675
- savedOp?: boolean;
676
- };
705
+ }
706
+ );
677
707
 
678
708
  const summarizerRequestUrl = "_summarizer";
679
709
 
@@ -732,6 +762,7 @@ function lastMessageFromMetadata(metadata: IContainerRuntimeMetadata | undefined
732
762
  /**
733
763
  * Represents the runtime of the container. Contains helper functions/state of the container.
734
764
  * It will define the store level mappings.
765
+ * @legacy
735
766
  * @alpha
736
767
  */
737
768
  export class ContainerRuntime
@@ -836,18 +867,7 @@ export class ContainerRuntime
836
867
  ]);
837
868
 
838
869
  // read snapshot blobs needed for BlobManager to load
839
- const blobManagerSnapshot = await BlobManager.load(
840
- context.baseSnapshot?.trees[blobsTreeName],
841
- async (id) => {
842
- // IContainerContext storage api return type still has undefined in 0.39 package version.
843
- // So once we release 0.40 container-defn package we can remove this check.
844
- assert(
845
- context.storage !== undefined,
846
- 0x256 /* "storage undefined in attached container" */,
847
- );
848
- return readAndParse(context.storage, id);
849
- },
850
- );
870
+ const blobManagerSnapshot = await loadBlobManagerLoadInfo(context);
851
871
 
852
872
  const messageAtLastSummary = lastMessageFromMetadata(metadata);
853
873
 
@@ -2200,7 +2220,7 @@ export class ContainerRuntime
2200
2220
  return this.resolveHandle(requestParser.createSubRequest(1));
2201
2221
  }
2202
2222
 
2203
- if (id === BlobManager.basePath && requestParser.isLeaf(2)) {
2223
+ if (id === blobManagerBasePath && requestParser.isLeaf(2)) {
2204
2224
  const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
2205
2225
  return blob
2206
2226
  ? {
@@ -2598,7 +2618,13 @@ export class ContainerRuntime
2598
2618
  // but will not modify the contents object (likely it will replace it on the message).
2599
2619
  const messageCopy = { ...messageArg };
2600
2620
  const savedOp = (messageCopy.metadata as ISavedOpMetadata)?.savedOp;
2601
- for (const message of this.remoteMessageProcessor.process(messageCopy)) {
2621
+ const processResult = this.remoteMessageProcessor.process(messageCopy);
2622
+ if (processResult === undefined) {
2623
+ // This means the incoming message is an incomplete part of a message or batch
2624
+ // and we need to process more messages before the rest of the system can understand it.
2625
+ return;
2626
+ }
2627
+ for (const message of processResult.messages) {
2602
2628
  const msg: MessageWithContext = modernRuntimeMessage
2603
2629
  ? {
2604
2630
  // Cast it since we expect it to be this based on modernRuntimeMessage computation above.
@@ -2608,12 +2634,14 @@ export class ContainerRuntime
2608
2634
  message: message as InboundSequencedContainerRuntimeMessage,
2609
2635
  local,
2610
2636
  modernRuntimeMessage,
2637
+ batchStartCsn: processResult.batchStartCsn,
2611
2638
  }
2612
2639
  : // Unrecognized message will be ignored.
2613
2640
  {
2614
2641
  message,
2615
2642
  local,
2616
2643
  modernRuntimeMessage,
2644
+ batchStartCsn: processResult.batchStartCsn,
2617
2645
  };
2618
2646
  msg.savedOp = savedOp;
2619
2647
 
@@ -2661,6 +2689,7 @@ export class ContainerRuntime
2661
2689
  if (local && messageWithContext.modernRuntimeMessage) {
2662
2690
  localOpMetadata = this.pendingStateManager.processPendingLocalMessage(
2663
2691
  messageWithContext.message,
2692
+ messageWithContext.batchStartCsn,
2664
2693
  );
2665
2694
  }
2666
2695
 
@@ -3339,7 +3368,7 @@ export class ContainerRuntime
3339
3368
  * blob manager.
3340
3369
  */
3341
3370
  public getNodeType(nodePath: string): GCNodeType {
3342
- if (this.isBlobPath(nodePath)) {
3371
+ if (isBlobPath(nodePath)) {
3343
3372
  return GCNodeType.Blob;
3344
3373
  }
3345
3374
  return this.channelCollection.getGCNodeType(nodePath) ?? GCNodeType.Other;
@@ -3358,7 +3387,7 @@ export class ContainerRuntime
3358
3387
 
3359
3388
  switch (this.getNodeType(nodePath)) {
3360
3389
  case GCNodeType.Blob:
3361
- return [BlobManager.basePath];
3390
+ return [blobManagerBasePath];
3362
3391
  case GCNodeType.DataStore:
3363
3392
  case GCNodeType.SubDataStore:
3364
3393
  return this.channelCollection.getDataStorePackagePath(nodePath);
@@ -3367,17 +3396,6 @@ export class ContainerRuntime
3367
3396
  }
3368
3397
  }
3369
3398
 
3370
- /**
3371
- * Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
3372
- */
3373
- private isBlobPath(path: string): boolean {
3374
- const pathParts = path.split("/");
3375
- if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
3376
- return false;
3377
- }
3378
- return true;
3379
- }
3380
-
3381
3399
  /**
3382
3400
  * From a given list of routes, separate and return routes that belong to blob manager and data stores.
3383
3401
  * @param routes - A list of routes that can belong to data stores or blob manager.
@@ -3388,7 +3406,7 @@ export class ContainerRuntime
3388
3406
  const blobManagerRoutes: string[] = [];
3389
3407
  const dataStoreRoutes: string[] = [];
3390
3408
  for (const route of routes) {
3391
- if (this.isBlobPath(route)) {
3409
+ if (isBlobPath(route)) {
3392
3410
  blobManagerRoutes.push(route);
3393
3411
  } else {
3394
3412
  dataStoreRoutes.push(route);
@@ -1058,6 +1058,7 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
1058
1058
  private snapshotFetchRequired: boolean | undefined;
1059
1059
  private readonly runtime: IContainerRuntimeBase;
1060
1060
  private readonly blobContents: Map<string, ArrayBuffer> | undefined;
1061
+ private readonly isSnapshotInISnapshotFormat: boolean | undefined;
1061
1062
 
1062
1063
  constructor(props: IRemoteFluidDataStoreContextProps) {
1063
1064
  super(props, true /* existing */, false /* isLocalDataStore */, () => {
@@ -1068,8 +1069,10 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
1068
1069
  if (isInstanceOfISnapshot(props.snapshot)) {
1069
1070
  this.blobContents = props.snapshot.blobContents;
1070
1071
  this._baseSnapshot = props.snapshot.snapshotTree;
1072
+ this.isSnapshotInISnapshotFormat = true;
1071
1073
  } else {
1072
1074
  this._baseSnapshot = props.snapshot;
1075
+ this.isSnapshotInISnapshotFormat = false;
1073
1076
  }
1074
1077
  if (this._baseSnapshot !== undefined) {
1075
1078
  this.summarizerNode.updateBaseSummaryState(this._baseSnapshot);
@@ -1091,10 +1094,13 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
1091
1094
  private readonly initialSnapshotDetailsP = new LazyPromise<ISnapshotDetails>(async () => {
1092
1095
  // Sequence number of the snapshot.
1093
1096
  let sequenceNumber: number | undefined;
1094
- // Check whether we need to fetch the snapshot first to load.
1097
+ // Check whether we need to fetch the snapshot first to load. The snapshot should be in new format to see
1098
+ // whether we want to evaluate to fetch snapshot or not for loadingGroupId. Otherwise, the snapshot
1099
+ // will contain all the blobs.
1095
1100
  if (
1096
1101
  this.snapshotFetchRequired === undefined &&
1097
- this._baseSnapshot?.groupId !== undefined
1102
+ this._baseSnapshot?.groupId !== undefined &&
1103
+ this.isSnapshotInISnapshotFormat
1098
1104
  ) {
1099
1105
  assert(
1100
1106
  this.blobContents !== undefined,
@@ -26,7 +26,7 @@ import {
26
26
  tagCodeArtifacts,
27
27
  } from "@fluidframework/telemetry-utils/internal";
28
28
 
29
- import { BlobManager } from "../blobManager.js";
29
+ import { blobManagerBasePath } from "../blobManager/index.js";
30
30
  import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey } from "../containerRuntime.js";
31
31
  import { ClientSessionExpiredError } from "../error.js";
32
32
  import { ContainerMessageType, ContainerRuntimeGCMessage } from "../messageTypes.js";
@@ -1272,7 +1272,7 @@ export class GarbageCollector implements IGarbageCollector {
1272
1272
  // be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
1273
1273
  const getDeletedNodeType = (nodeId: string): GCNodeType => {
1274
1274
  const pathParts = nodeId.split("/");
1275
- if (pathParts[1] === BlobManager.basePath) {
1275
+ if (pathParts[1] === blobManagerBasePath) {
1276
1276
  return GCNodeType.Blob;
1277
1277
  }
1278
1278
  if (pathParts.length === 2) {
@@ -24,6 +24,7 @@ import {
24
24
  } from "../summary/index.js";
25
25
 
26
26
  /**
27
+ * @legacy
27
28
  * @alpha
28
29
  */
29
30
  export type GCVersion = number;
@@ -95,6 +96,7 @@ export const defaultSweepGracePeriodMs = 1 * oneDayMs; // 1 day
95
96
 
96
97
  /**
97
98
  * @see IGCMetadata.gcFeatureMatrix and @see gcGenerationOptionName
99
+ * @legacy
98
100
  * @alpha
99
101
  */
100
102
  export type GCFeatureMatrix =
@@ -132,6 +134,7 @@ export interface IGCMetadata_Deprecated {
132
134
  /**
133
135
  * GC-specific metadata to be written into the summary.
134
136
  *
137
+ * @legacy
135
138
  * @alpha
136
139
  */
137
140
  export interface IGCMetadata {
@@ -179,6 +182,7 @@ export interface IGCMetadata {
179
182
 
180
183
  /**
181
184
  * The statistics of the system state after a garbage collection mark phase run.
185
+ * @legacy
182
186
  * @alpha
183
187
  */
184
188
  export interface IMarkPhaseStats {
@@ -204,6 +208,7 @@ export interface IMarkPhaseStats {
204
208
 
205
209
  /**
206
210
  * The statistics of the system state after a garbage collection sweep phase run.
211
+ * @legacy
207
212
  * @alpha
208
213
  */
209
214
  export interface ISweepPhaseStats {
@@ -223,12 +228,14 @@ export interface ISweepPhaseStats {
223
228
 
224
229
  /**
225
230
  * The statistics of the system state after a garbage collection run.
231
+ * @legacy
226
232
  * @alpha
227
233
  */
228
234
  export interface IGCStats extends IMarkPhaseStats, ISweepPhaseStats {}
229
235
 
230
236
  /**
231
237
  * The types of GC nodes in the GC reference graph.
238
+ * @legacy
232
239
  * @alpha
233
240
  */
234
241
  export const GCNodeType = {
@@ -243,6 +250,7 @@ export const GCNodeType = {
243
250
  } as const;
244
251
 
245
252
  /**
253
+ * @legacy
246
254
  * @alpha
247
255
  */
248
256
  export type GCNodeType = (typeof GCNodeType)[keyof typeof GCNodeType];
@@ -421,6 +429,7 @@ export interface IGarbageCollectorCreateParams {
421
429
  }
422
430
 
423
431
  /**
432
+ * @legacy
424
433
  * @alpha
425
434
  */
426
435
  export interface IGCRuntimeOptions {
package/src/index.ts CHANGED
@@ -30,7 +30,7 @@ export {
30
30
  RecentlyAddedContainerRuntimeMessageDetails,
31
31
  UnknownContainerRuntimeMessage,
32
32
  } from "./messageTypes.js";
33
- export { IBlobManagerLoadInfo } from "./blobManager.js";
33
+ export { IBlobManagerLoadInfo } from "./blobManager/index.js";
34
34
  export { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
35
35
  export {
36
36
  detectOutboundReferences,
@@ -17,6 +17,7 @@ import { IChunkedOp } from "./opLifecycle/index.js";
17
17
  import { IDocumentSchemaChangeMessage } from "./summary/index.js";
18
18
 
19
19
  /**
20
+ * @legacy
20
21
  * @alpha
21
22
  */
22
23
  export enum ContainerMessageType {
@@ -56,6 +56,10 @@ export class BatchManager {
56
56
  : this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
57
57
  }
58
58
 
59
+ /**
60
+ * The last-processed CSN when this batch started.
61
+ * This is used to ensure that while the batch is open, no incoming ops are processed.
62
+ */
59
63
  private clientSequenceNumber: number | undefined;
60
64
 
61
65
  constructor(public readonly options: IBatchManagerOptions) {}
@@ -41,7 +41,7 @@ export interface IOutboxParameters {
41
41
  readonly submitBatchFn:
42
42
  | ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)
43
43
  | undefined;
44
- readonly legacySendBatchFn: (batch: IBatch) => void;
44
+ readonly legacySendBatchFn: (batch: IBatch) => number;
45
45
  readonly config: IOutboxConfig;
46
46
  readonly compressor: OpCompressor;
47
47
  readonly splitter: OpSplitter;
@@ -126,10 +126,14 @@ export class Outbox {
126
126
  }
127
127
 
128
128
  /**
129
- * If we detect that the reference sequence number of the incoming message does not match
130
- * what was already in the batch managers, this means that batching has been interrupted so
129
+ * Detect whether batching has been interrupted by an incoming message being processed. In this case,
131
130
  * we will flush the accumulated messages to account for that and create a new batch with the new
132
131
  * message as the first message.
132
+ *
133
+ * @remarks - To detect batch interruption, we compare both the reference sequence number
134
+ * (i.e. last message processed by DeltaManager) and the client sequence number of the
135
+ * last message processed by the ContainerRuntime. In the absence of op reentrancy, this
136
+ * pair will remain stable during a single JS turn during which the batch is being built up.
133
137
  */
134
138
  private maybeFlushPartialBatch() {
135
139
  const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
@@ -254,6 +258,7 @@ export class Outbox {
254
258
  return;
255
259
  }
256
260
 
261
+ let clientSequenceNumber: number | undefined;
257
262
  // Did we disconnect? (i.e. is shouldSend false?)
258
263
  // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
259
264
  // Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
@@ -261,10 +266,10 @@ export class Outbox {
261
266
  const processedBatch = this.compressBatch(
262
267
  shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch,
263
268
  );
264
- this.sendBatch(processedBatch);
269
+ clientSequenceNumber = this.sendBatch(processedBatch);
265
270
  }
266
271
 
267
- this.persistBatch(rawBatch.content);
272
+ this.params.pendingStateManager.onFlushBatch(rawBatch.content, clientSequenceNumber);
268
273
  }
269
274
 
270
275
  /**
@@ -355,11 +360,12 @@ export class Outbox {
355
360
  * Sends the batch object to the container context to be sent over the wire.
356
361
  *
357
362
  * @param batch - batch to be sent
363
+ * @returns the clientSequenceNumber of the start of the batch, or undefined if nothing was sent
358
364
  */
359
365
  private sendBatch(batch: IBatch) {
360
366
  const length = batch.content.length;
361
367
  if (length === 0) {
362
- return;
368
+ return undefined; // Nothing submitted
363
369
  }
364
370
 
365
371
  const socketSize = estimateSocketSize(batch);
@@ -372,6 +378,7 @@ export class Outbox {
372
378
  });
373
379
  }
374
380
 
381
+ let clientSequenceNumber: number;
375
382
  if (this.params.submitBatchFn === undefined) {
376
383
  // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
377
384
  // version that has support for batches (submitBatchFn)
@@ -380,10 +387,10 @@ export class Outbox {
380
387
  0x5a6 /* Compression should not have happened if the loader does not support it */,
381
388
  );
382
389
 
383
- this.params.legacySendBatchFn(batch);
390
+ clientSequenceNumber = this.params.legacySendBatchFn(batch);
384
391
  } else {
385
392
  assert(batch.referenceSequenceNumber !== undefined, 0x58e /* Batch must not be empty */);
386
- this.params.submitBatchFn(
393
+ clientSequenceNumber = this.params.submitBatchFn(
387
394
  batch.content.map((message) => ({
388
395
  contents: message.contents,
389
396
  metadata: message.metadata,
@@ -393,20 +400,11 @@ export class Outbox {
393
400
  batch.referenceSequenceNumber,
394
401
  );
395
402
  }
396
- }
397
403
 
398
- private persistBatch(batch: BatchMessage[]) {
399
- // Let the PendingStateManager know that a message was submitted.
400
- // In future, need to shift toward keeping batch as a whole!
401
- for (const message of batch) {
402
- this.params.pendingStateManager.onSubmitMessage(
403
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
404
- message.contents!,
405
- message.referenceSequenceNumber,
406
- message.localOpMetadata,
407
- message.metadata,
408
- );
409
- }
404
+ // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
405
+ clientSequenceNumber -= length - 1;
406
+ assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
407
+ return clientSequenceNumber;
410
408
  }
411
409
 
412
410
  public checkpoint() {
@@ -3,6 +3,7 @@
3
3
  * Licensed under the MIT License.
4
4
  */
5
5
 
6
+ import { assert } from "@fluidframework/core-utils/internal";
6
7
  import {
7
8
  MessageType,
8
9
  ISequencedDocumentMessage,
@@ -27,6 +28,14 @@ import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
27
28
  * @internal
28
29
  */
29
30
  export class RemoteMessageProcessor {
31
+ /**
32
+ * Client Sequence Number of the first message in the current batch being processed.
33
+ * If undefined, we are expecting the next message to start a new batch.
34
+ *
35
+ * @remarks For chunked batches, this is the CSN of the "representative" chunk (the final chunk)
36
+ */
37
+ private batchStartCsn: number | undefined;
38
+
30
39
  constructor(
31
40
  private readonly opSplitter: OpSplitter,
32
41
  private readonly opDecompressor: OpDecompressor,
@@ -61,17 +70,21 @@ export class RemoteMessageProcessor {
61
70
  * For ops that weren't virtualized (e.g. System ops that the ContainerRuntime will ultimately ignore),
62
71
  * a singleton array [remoteMessageCopy] is returned
63
72
  */
64
- public process(
65
- remoteMessageCopy: ISequencedDocumentMessage,
66
- ): InboundSequencedContainerRuntimeMessageOrSystemMessage[] {
73
+ public process(remoteMessageCopy: ISequencedDocumentMessage):
74
+ | {
75
+ messages: InboundSequencedContainerRuntimeMessageOrSystemMessage[];
76
+ batchStartCsn: number;
77
+ }
78
+ | undefined {
67
79
  let message = remoteMessageCopy;
80
+
68
81
  ensureContentsDeserialized(message);
69
82
 
70
83
  if (isChunkedMessage(message)) {
71
84
  const chunkProcessingResult = this.opSplitter.processChunk(message);
72
85
  // Only continue further if current chunk is the final chunk
73
86
  if (!chunkProcessingResult.isFinalChunk) {
74
- return [];
87
+ return;
75
88
  }
76
89
  // This message will always be compressed
77
90
  message = chunkProcessingResult.message;
@@ -90,12 +103,56 @@ export class RemoteMessageProcessor {
90
103
  }
91
104
 
92
105
  if (isGroupedBatch(message)) {
93
- return this.opGroupingManager.ungroupOp(message).map(unpack);
106
+ // We should be awaiting a new batch (batchStartCsn undefined)
107
+ assert(this.batchStartCsn === undefined, "Grouped batch interrupting another batch");
108
+ return {
109
+ messages: this.opGroupingManager.ungroupOp(message).map(unpack),
110
+ batchStartCsn: message.clientSequenceNumber,
111
+ };
94
112
  }
95
113
 
114
+ const batchStartCsn = this.getAndUpdateBatchStartCsn(message);
115
+
96
116
  // Do a final unpack of runtime messages in case the message was not grouped, compressed, or chunked
97
117
  unpackRuntimeMessage(message);
98
- return [message as InboundSequencedContainerRuntimeMessageOrSystemMessage];
118
+ return {
119
+ messages: [message as InboundSequencedContainerRuntimeMessageOrSystemMessage],
120
+ batchStartCsn,
121
+ };
122
+ }
123
+
124
+ /**
125
+ * Based on pre-existing batch tracking info and the current message's batch metadata,
126
+ * this will return the starting CSN for this message's batch, and will also update
127
+ * the batch tracking info (this.batchStartCsn) based on whether we're still mid-batch.
128
+ */
129
+ private getAndUpdateBatchStartCsn(message: ISequencedDocumentMessage): number {
130
+ const batchMetadataFlag = (message.metadata as { batch: boolean | undefined })?.batch;
131
+ if (this.batchStartCsn === undefined) {
132
+ // We are waiting for a new batch
133
+ assert(batchMetadataFlag !== false, "Unexpected batch end marker");
134
+
135
+ // Start of a new multi-message batch
136
+ if (batchMetadataFlag === true) {
137
+ this.batchStartCsn = message.clientSequenceNumber;
138
+ return this.batchStartCsn;
139
+ }
140
+
141
+ // Single-message batch (Since metadata flag is undefined)
142
+ // IMPORTANT: Leave this.batchStartCsn undefined, we're ready for the next batch now.
143
+ return message.clientSequenceNumber;
144
+ }
145
+
146
+ // We are in the middle or end of an existing multi-message batch. Return the current batchStartCsn
147
+ const batchStartCsn = this.batchStartCsn;
148
+
149
+ assert(batchMetadataFlag !== true, "Unexpected batch start marker");
150
+ if (batchMetadataFlag === false) {
151
+ // Batch end? Then get ready for the next batch to start
152
+ this.batchStartCsn = undefined;
153
+ }
154
+
155
+ return batchStartCsn;
99
156
  }
100
157
  }
101
158
 
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "2.1.0-274160";
9
+ export const pkgVersion = "2.1.0-276985";