@fluidframework/container-runtime 2.33.0-333010 → 2.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/CHANGELOG.md +4 -0
  2. package/api-report/container-runtime.legacy.alpha.api.md +71 -67
  3. package/container-runtime.test-files.tar +0 -0
  4. package/dist/blobManager/blobManager.d.ts +7 -4
  5. package/dist/blobManager/blobManager.d.ts.map +1 -1
  6. package/dist/blobManager/blobManager.js +38 -12
  7. package/dist/blobManager/blobManager.js.map +1 -1
  8. package/dist/channelCollection.d.ts +4 -0
  9. package/dist/channelCollection.d.ts.map +1 -1
  10. package/dist/channelCollection.js +24 -0
  11. package/dist/channelCollection.js.map +1 -1
  12. package/dist/compatUtils.d.ts +74 -0
  13. package/dist/compatUtils.d.ts.map +1 -0
  14. package/dist/compatUtils.js +151 -0
  15. package/dist/compatUtils.js.map +1 -0
  16. package/dist/compressionDefinitions.d.ts +39 -0
  17. package/dist/compressionDefinitions.d.ts.map +1 -0
  18. package/dist/compressionDefinitions.js +30 -0
  19. package/dist/compressionDefinitions.js.map +1 -0
  20. package/dist/containerRuntime.d.ts +78 -52
  21. package/dist/containerRuntime.d.ts.map +1 -1
  22. package/dist/containerRuntime.js +141 -54
  23. package/dist/containerRuntime.js.map +1 -1
  24. package/dist/dataStoreContext.d.ts +3 -0
  25. package/dist/dataStoreContext.d.ts.map +1 -1
  26. package/dist/dataStoreContext.js +122 -66
  27. package/dist/dataStoreContext.js.map +1 -1
  28. package/dist/deltaManagerProxies.d.ts +55 -12
  29. package/dist/deltaManagerProxies.d.ts.map +1 -1
  30. package/dist/deltaManagerProxies.js +63 -55
  31. package/dist/deltaManagerProxies.js.map +1 -1
  32. package/dist/gc/gcDefinitions.d.ts +2 -0
  33. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  34. package/dist/gc/gcDefinitions.js.map +1 -1
  35. package/dist/index.d.ts +4 -2
  36. package/dist/index.d.ts.map +1 -1
  37. package/dist/index.js +3 -2
  38. package/dist/index.js.map +1 -1
  39. package/dist/legacy.d.ts +1 -0
  40. package/dist/opLifecycle/batchManager.d.ts +1 -1
  41. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  42. package/dist/opLifecycle/batchManager.js +4 -1
  43. package/dist/opLifecycle/batchManager.js.map +1 -1
  44. package/dist/opLifecycle/definitions.d.ts +35 -4
  45. package/dist/opLifecycle/definitions.d.ts.map +1 -1
  46. package/dist/opLifecycle/definitions.js.map +1 -1
  47. package/dist/opLifecycle/index.d.ts +1 -1
  48. package/dist/opLifecycle/index.d.ts.map +1 -1
  49. package/dist/opLifecycle/index.js.map +1 -1
  50. package/dist/opLifecycle/opCompressor.js +2 -2
  51. package/dist/opLifecycle/opCompressor.js.map +1 -1
  52. package/dist/opLifecycle/opDecompressor.js +3 -3
  53. package/dist/opLifecycle/opDecompressor.js.map +1 -1
  54. package/dist/opLifecycle/opGroupingManager.d.ts +2 -2
  55. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  56. package/dist/opLifecycle/opGroupingManager.js +1 -2
  57. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  58. package/dist/opLifecycle/opSerialization.d.ts +3 -1
  59. package/dist/opLifecycle/opSerialization.d.ts.map +1 -1
  60. package/dist/opLifecycle/opSerialization.js +4 -2
  61. package/dist/opLifecycle/opSerialization.js.map +1 -1
  62. package/dist/opLifecycle/outbox.d.ts +6 -3
  63. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  64. package/dist/opLifecycle/outbox.js +46 -20
  65. package/dist/opLifecycle/outbox.js.map +1 -1
  66. package/dist/packageVersion.d.ts +1 -1
  67. package/dist/packageVersion.d.ts.map +1 -1
  68. package/dist/packageVersion.js +1 -1
  69. package/dist/packageVersion.js.map +1 -1
  70. package/dist/pendingStateManager.d.ts +36 -7
  71. package/dist/pendingStateManager.d.ts.map +1 -1
  72. package/dist/pendingStateManager.js +83 -16
  73. package/dist/pendingStateManager.js.map +1 -1
  74. package/dist/runtimeLayerCompatState.d.ts.map +1 -1
  75. package/dist/runtimeLayerCompatState.js +1 -1
  76. package/dist/runtimeLayerCompatState.js.map +1 -1
  77. package/dist/summary/documentSchema.d.ts +1 -0
  78. package/dist/summary/documentSchema.d.ts.map +1 -1
  79. package/dist/summary/documentSchema.js +2 -0
  80. package/dist/summary/documentSchema.js.map +1 -1
  81. package/lib/blobManager/blobManager.d.ts +7 -4
  82. package/lib/blobManager/blobManager.d.ts.map +1 -1
  83. package/lib/blobManager/blobManager.js +38 -12
  84. package/lib/blobManager/blobManager.js.map +1 -1
  85. package/lib/channelCollection.d.ts +4 -0
  86. package/lib/channelCollection.d.ts.map +1 -1
  87. package/lib/channelCollection.js +24 -0
  88. package/lib/channelCollection.js.map +1 -1
  89. package/lib/compatUtils.d.ts +74 -0
  90. package/lib/compatUtils.d.ts.map +1 -0
  91. package/lib/compatUtils.js +142 -0
  92. package/lib/compatUtils.js.map +1 -0
  93. package/lib/compressionDefinitions.d.ts +39 -0
  94. package/lib/compressionDefinitions.d.ts.map +1 -0
  95. package/lib/compressionDefinitions.js +27 -0
  96. package/lib/compressionDefinitions.js.map +1 -0
  97. package/lib/containerRuntime.d.ts +78 -52
  98. package/lib/containerRuntime.d.ts.map +1 -1
  99. package/lib/containerRuntime.js +143 -56
  100. package/lib/containerRuntime.js.map +1 -1
  101. package/lib/dataStoreContext.d.ts +3 -0
  102. package/lib/dataStoreContext.d.ts.map +1 -1
  103. package/lib/dataStoreContext.js +57 -1
  104. package/lib/dataStoreContext.js.map +1 -1
  105. package/lib/deltaManagerProxies.d.ts +55 -12
  106. package/lib/deltaManagerProxies.d.ts.map +1 -1
  107. package/lib/deltaManagerProxies.js +63 -55
  108. package/lib/deltaManagerProxies.js.map +1 -1
  109. package/lib/gc/gcDefinitions.d.ts +2 -0
  110. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  111. package/lib/gc/gcDefinitions.js.map +1 -1
  112. package/lib/index.d.ts +4 -2
  113. package/lib/index.d.ts.map +1 -1
  114. package/lib/index.js +2 -1
  115. package/lib/index.js.map +1 -1
  116. package/lib/legacy.d.ts +1 -0
  117. package/lib/opLifecycle/batchManager.d.ts +1 -1
  118. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  119. package/lib/opLifecycle/batchManager.js +4 -1
  120. package/lib/opLifecycle/batchManager.js.map +1 -1
  121. package/lib/opLifecycle/definitions.d.ts +35 -4
  122. package/lib/opLifecycle/definitions.d.ts.map +1 -1
  123. package/lib/opLifecycle/definitions.js.map +1 -1
  124. package/lib/opLifecycle/index.d.ts +1 -1
  125. package/lib/opLifecycle/index.d.ts.map +1 -1
  126. package/lib/opLifecycle/index.js.map +1 -1
  127. package/lib/opLifecycle/opCompressor.js +1 -1
  128. package/lib/opLifecycle/opCompressor.js.map +1 -1
  129. package/lib/opLifecycle/opDecompressor.js +1 -1
  130. package/lib/opLifecycle/opDecompressor.js.map +1 -1
  131. package/lib/opLifecycle/opGroupingManager.d.ts +2 -2
  132. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  133. package/lib/opLifecycle/opGroupingManager.js +1 -2
  134. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  135. package/lib/opLifecycle/opSerialization.d.ts +3 -1
  136. package/lib/opLifecycle/opSerialization.d.ts.map +1 -1
  137. package/lib/opLifecycle/opSerialization.js +4 -2
  138. package/lib/opLifecycle/opSerialization.js.map +1 -1
  139. package/lib/opLifecycle/outbox.d.ts +6 -3
  140. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  141. package/lib/opLifecycle/outbox.js +46 -20
  142. package/lib/opLifecycle/outbox.js.map +1 -1
  143. package/lib/packageVersion.d.ts +1 -1
  144. package/lib/packageVersion.d.ts.map +1 -1
  145. package/lib/packageVersion.js +1 -1
  146. package/lib/packageVersion.js.map +1 -1
  147. package/lib/pendingStateManager.d.ts +36 -7
  148. package/lib/pendingStateManager.d.ts.map +1 -1
  149. package/lib/pendingStateManager.js +84 -17
  150. package/lib/pendingStateManager.js.map +1 -1
  151. package/lib/runtimeLayerCompatState.d.ts.map +1 -1
  152. package/lib/runtimeLayerCompatState.js +2 -2
  153. package/lib/runtimeLayerCompatState.js.map +1 -1
  154. package/lib/summary/documentSchema.d.ts +1 -0
  155. package/lib/summary/documentSchema.d.ts.map +1 -1
  156. package/lib/summary/documentSchema.js +2 -0
  157. package/lib/summary/documentSchema.js.map +1 -1
  158. package/lib/tsdoc-metadata.json +1 -1
  159. package/package.json +21 -20
  160. package/src/blobManager/blobManager.ts +48 -15
  161. package/src/channelCollection.ts +27 -0
  162. package/src/compatUtils.ts +211 -0
  163. package/src/compressionDefinitions.ts +47 -0
  164. package/src/containerRuntime.ts +259 -108
  165. package/src/dataStoreContext.ts +82 -2
  166. package/src/deltaManagerProxies.ts +132 -70
  167. package/src/gc/gcDefinitions.ts +2 -0
  168. package/src/index.ts +5 -3
  169. package/src/opLifecycle/batchManager.ts +5 -4
  170. package/src/opLifecycle/definitions.ts +34 -4
  171. package/src/opLifecycle/index.ts +1 -0
  172. package/src/opLifecycle/opCompressor.ts +1 -1
  173. package/src/opLifecycle/opDecompressor.ts +1 -1
  174. package/src/opLifecycle/opGroupingManager.ts +7 -5
  175. package/src/opLifecycle/opSerialization.ts +6 -2
  176. package/src/opLifecycle/outbox.ts +65 -30
  177. package/src/packageVersion.ts +1 -1
  178. package/src/pendingStateManager.ts +135 -21
  179. package/src/runtimeLayerCompatState.ts +5 -2
  180. package/src/summary/documentSchema.ts +3 -0
@@ -17,7 +17,7 @@ import {
17
17
  type ITelemetryLoggerExt,
18
18
  } from "@fluidframework/telemetry-utils/internal";
19
19
 
20
- import { ICompressionRuntimeOptions } from "../containerRuntime.js";
20
+ import { ICompressionRuntimeOptions } from "../compressionDefinitions.js";
21
21
  import { PendingMessageResubmitData, PendingStateManager } from "../pendingStateManager.js";
22
22
 
23
23
  import {
@@ -36,6 +36,7 @@ import {
36
36
  } from "./definitions.js";
37
37
  import { OpCompressor } from "./opCompressor.js";
38
38
  import { OpGroupingManager } from "./opGroupingManager.js";
39
+ import { serializeOp } from "./opSerialization.js";
39
40
  import { OpSplitter } from "./opSplitter.js";
40
41
 
41
42
  export interface IOutboxConfig {
@@ -111,11 +112,17 @@ export function getLongStack<T>(action: () => T, length: number = 50): T {
111
112
  /**
112
113
  * Convert from local batch to outbound batch, including computing contentSizeInBytes.
113
114
  */
114
- export function localBatchToOutboundBatch(localBatch: LocalBatch): OutboundBatch {
115
+ export function localBatchToOutboundBatch({
116
+ staged: _staged, // Peel this off the incoming batch, it's irrelevant (see Note below)
117
+ ...localBatch
118
+ }: LocalBatch): OutboundBatch {
119
+ // Note: the staged property might be misleading here, in case a pre-staging batch is resubmitted during staging mode.
120
+ // It will be set to true, but the batch was not actually staged.
121
+
115
122
  // Shallow copy each message as we switch types
116
123
  const outboundMessages = localBatch.messages.map<OutboundBatchMessage>(
117
- ({ serializedOp, ...message }) => ({
118
- contents: serializedOp,
124
+ ({ runtimeOp, ...message }) => ({
125
+ contents: serializeOp(runtimeOp),
119
126
  ...message,
120
127
  }),
121
128
  );
@@ -266,7 +273,7 @@ export class Outbox {
266
273
  ? "generic"
267
274
  : "error",
268
275
  eventName: "ReferenceSequenceNumberMismatch",
269
- Data_details: {
276
+ details: {
270
277
  expectedDueToReentrancy,
271
278
  mainReferenceSequenceNumber: mainBatchSeqNums.referenceSequenceNumber,
272
279
  mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
@@ -330,17 +337,19 @@ export class Outbox {
330
337
  * @throws If called from a reentrant context, or if the batch being flushed is too large.
331
338
  * @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
332
339
  * with the given Batch ID, which must be preserved
340
+ * @param resubmittingStagedBatch - If defined, indicates this is a resubmission of a batch that is staged,
341
+ * meaning it should not be sent to the ordering service yet.
333
342
  */
334
- public flush(resubmittingBatchId?: BatchId): void {
343
+ public flush(resubmittingBatchId?: BatchId, resubmittingStagedBatch?: boolean): void {
335
344
  assert(
336
345
  !this.isContextReentrant(),
337
346
  0xb7b /* Flushing must not happen while incoming changes are being processed */,
338
347
  );
339
348
 
340
- this.flushAll(resubmittingBatchId);
349
+ this.flushAll(resubmittingBatchId, resubmittingStagedBatch);
341
350
  }
342
351
 
343
- private flushAll(resubmittingBatchId?: BatchId): void {
352
+ private flushAll(resubmittingBatchId?: BatchId, resubmittingStagedBatch?: boolean): void {
344
353
  const allBatchesEmpty =
345
354
  this.idAllocationBatch.empty && this.blobAttachBatch.empty && this.mainBatch.empty;
346
355
  if (allBatchesEmpty) {
@@ -350,27 +359,35 @@ export class Outbox {
350
359
  // by the rest of the system, including remote clients.
351
360
  // In some cases we *must* resubmit the empty batch (to match up with a non-empty version tracked locally by a container fork), so we do it always.
352
361
  if (resubmittingBatchId) {
353
- this.flushEmptyBatch(resubmittingBatchId);
362
+ this.flushEmptyBatch(resubmittingBatchId, resubmittingStagedBatch === true);
354
363
  }
355
364
  return;
356
365
  }
357
366
 
358
367
  // Don't use resubmittingBatchId for idAllocationBatch.
359
368
  // ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
360
- this.flushInternal(this.idAllocationBatch);
361
- this.flushInternal(
362
- this.blobAttachBatch,
363
- true /* disableGroupedBatching */,
369
+ this.flushInternal({
370
+ batchManager: this.idAllocationBatch,
371
+ // Note: For now, we will never stage ID Allocation messages.
372
+ // They won't contain personal info and no harm in extra allocations in case of discarding the staged changes
373
+ });
374
+ this.flushInternal({
375
+ batchManager: this.blobAttachBatch,
376
+ disableGroupedBatching: true,
364
377
  resubmittingBatchId,
365
- );
366
- this.flushInternal(
367
- this.mainBatch,
368
- false /* disableGroupedBatching */,
378
+ resubmittingStagedBatch,
379
+ });
380
+ this.flushInternal({
381
+ batchManager: this.mainBatch,
369
382
  resubmittingBatchId,
370
- );
383
+ resubmittingStagedBatch,
384
+ });
371
385
  }
372
386
 
373
- private flushEmptyBatch(resubmittingBatchId: BatchId): void {
387
+ private flushEmptyBatch(
388
+ resubmittingBatchId: BatchId,
389
+ resubmittingStagedBatch: boolean,
390
+ ): void {
374
391
  const referenceSequenceNumber =
375
392
  this.params.getCurrentSequenceNumbers().referenceSequenceNumber;
376
393
  assert(
@@ -383,28 +400,42 @@ export class Outbox {
383
400
  referenceSequenceNumber,
384
401
  );
385
402
  let clientSequenceNumber: number | undefined;
386
- if (this.params.shouldSend()) {
403
+ if (this.params.shouldSend() && !resubmittingStagedBatch) {
387
404
  clientSequenceNumber = this.sendBatch(outboundBatch);
388
405
  }
389
406
 
390
407
  // Push the empty batch placeholder to the PendingStateManager
391
- this.params.pendingStateManager.onFlushBatch(
392
- [{ ...placeholderMessage, serializedOp: "", contents: undefined }], // placeholder message - serializedOp will never be used
408
+ this.params.pendingStateManager.onFlushEmptyBatch(
409
+ placeholderMessage,
393
410
  clientSequenceNumber,
411
+ resubmittingStagedBatch,
394
412
  );
395
413
  return;
396
414
  }
397
415
 
398
- private flushInternal(
399
- batchManager: BatchManager,
400
- disableGroupedBatching: boolean = false,
401
- resubmittingBatchId?: BatchId,
402
- ): void {
416
+ private flushInternal(params: {
417
+ batchManager: BatchManager;
418
+ disableGroupedBatching?: boolean;
419
+ resubmittingBatchId?: BatchId; // undefined if not resubmitting
420
+ resubmittingStagedBatch?: boolean; // undefined if not resubmitting
421
+ }): void {
422
+ const {
423
+ batchManager,
424
+ disableGroupedBatching = false,
425
+ resubmittingBatchId,
426
+ resubmittingStagedBatch,
427
+ } = params;
403
428
  if (batchManager.empty) {
404
429
  return;
405
430
  }
406
431
 
407
432
  const rawBatch = batchManager.popBatch(resubmittingBatchId);
433
+
434
+ // When resubmitting, we respect the staged state of the original batch.
435
+ // In this case rawBatch.staged will match the state of inStagingMode when
436
+ // the resubmit occurred, which is not relevant.
437
+ const staged = resubmittingStagedBatch ?? rawBatch.staged === true;
438
+
408
439
  const groupingEnabled =
409
440
  !disableGroupedBatching && this.params.groupingManager.groupedBatchingEnabled();
410
441
  if (
@@ -419,6 +450,9 @@ export class Outbox {
419
450
  // If a batch contains reentrant ops (ops created as a result from processing another op)
420
451
  // it needs to be rebased so that we can ensure consistent reference sequence numbers
421
452
  // and eventual consistency at the DDS level.
453
+ // Note: Since this is happening in the same turn the ops were originally created with,
454
+ // and they haven't gone to PendingStateManager yet, we can just let them respect
455
+ // ContainerRuntime.inStagingMode
422
456
  this.rebase(rawBatch, batchManager);
423
457
  return;
424
458
  }
@@ -427,7 +461,7 @@ export class Outbox {
427
461
  // Did we disconnect? (i.e. is shouldSend false?)
428
462
  // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
429
463
  // Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
430
- if (this.params.shouldSend()) {
464
+ if (this.params.shouldSend() && !staged) {
431
465
  const virtualizedBatch = this.virtualizeBatch(rawBatch, groupingEnabled);
432
466
 
433
467
  clientSequenceNumber = this.sendBatch(virtualizedBatch);
@@ -440,6 +474,7 @@ export class Outbox {
440
474
  this.params.pendingStateManager.onFlushBatch(
441
475
  rawBatch.messages,
442
476
  clientSequenceNumber,
477
+ staged,
443
478
  batchManager.options.ignoreBatchId,
444
479
  );
445
480
  }
@@ -457,7 +492,7 @@ export class Outbox {
457
492
  this.rebasing = true;
458
493
  for (const message of rawBatch.messages) {
459
494
  this.params.reSubmit({
460
- content: message.serializedOp,
495
+ runtimeOp: message.runtimeOp,
461
496
  localOpMetadata: message.localOpMetadata,
462
497
  opMetadata: message.metadata,
463
498
  });
@@ -475,7 +510,7 @@ export class Outbox {
475
510
  this.batchRebasesToReport--;
476
511
  }
477
512
 
478
- this.flushInternal(batchManager);
513
+ this.flushInternal({ batchManager });
479
514
  this.rebasing = false;
480
515
  }
481
516
 
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "2.33.0-333010";
9
+ export const pkgVersion = "2.33.0";
@@ -27,6 +27,8 @@ import {
27
27
  getEffectiveBatchId,
28
28
  BatchStartInfo,
29
29
  InboundMessageResult,
30
+ serializeOp,
31
+ type LocalEmptyBatchPlaceholder,
30
32
  } from "./opLifecycle/index.js";
31
33
 
32
34
  /**
@@ -38,7 +40,15 @@ import {
38
40
  export interface IPendingMessage {
39
41
  type: "message";
40
42
  referenceSequenceNumber: number;
43
+ /**
44
+ * Serialized copy of runtimeOp
45
+ */
41
46
  content: string;
47
+ /**
48
+ * The original runtime op that was submitted to the ContainerRuntime
49
+ * Unless this pending message came from stashed content, in which case this was roundtripped through string
50
+ */
51
+ runtimeOp?: LocalContainerRuntimeMessage | undefined; // Undefined for empty batches and initial messages before parsing
42
52
  localOpMetadata: unknown;
43
53
  opMetadata: Record<string, unknown> | undefined;
44
54
  sequenceNumber?: number;
@@ -65,6 +75,10 @@ export interface IPendingMessage {
65
75
  * If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored
66
76
  */
67
77
  ignoreBatchId?: boolean;
78
+ /**
79
+ * If true, this batch is staged and should not actually be submitted on replayPendingStates.
80
+ */
81
+ staged: boolean;
68
82
  };
69
83
  }
70
84
 
@@ -94,14 +108,17 @@ export interface IPendingLocalState {
94
108
  */
95
109
  export type PendingMessageResubmitData = Pick<
96
110
  IPendingMessage,
97
- "content" | "localOpMetadata" | "opMetadata"
98
- >;
111
+ "runtimeOp" | "localOpMetadata" | "opMetadata"
112
+ > & {
113
+ // Required (it's only missing on IPendingMessage for empty batch, which will be resubmitted as an empty array)
114
+ runtimeOp: LocalContainerRuntimeMessage;
115
+ };
99
116
 
100
117
  export interface IRuntimeStateHandler {
101
118
  connected(): boolean;
102
119
  clientId(): string | undefined;
103
- applyStashedOp(content: string): Promise<unknown>;
104
- reSubmitBatch(batch: PendingMessageResubmitData[], batchId: BatchId): void;
120
+ applyStashedOp(serializedOp: string): Promise<unknown>;
121
+ reSubmitBatch(batch: PendingMessageResubmitData[], batchId: BatchId, staged: boolean): void;
105
122
  isActiveConnection: () => boolean;
106
123
  isAttached: () => boolean;
107
124
  }
@@ -180,10 +197,17 @@ export function findFirstCharacterMismatched(
180
197
  return [index, charA, charB];
181
198
  }
182
199
 
183
- function withoutLocalOpMetadata(message: IPendingMessage): IPendingMessage {
200
+ /**
201
+ * Returns a shallow copy of the given message with the non-serializable properties removed.
202
+ * Note that the runtimeOp's data has already been serialized in the content property.
203
+ */
204
+ function toSerializableForm(
205
+ message: IPendingMessage,
206
+ ): IPendingMessage & { runtimeOp: undefined; localOpMetadata: undefined } {
184
207
  return {
185
208
  ...message,
186
209
  localOpMetadata: undefined,
210
+ runtimeOp: undefined,
187
211
  };
188
212
  }
189
213
 
@@ -273,7 +297,7 @@ export class PendingStateManager implements IDisposable {
273
297
  return {
274
298
  pendingStates: [
275
299
  ...newSavedOps,
276
- ...this.pendingMessages.toArray().map((message) => withoutLocalOpMetadata(message)),
300
+ ...this.pendingMessages.toArray().map((message) => toSerializableForm(message)),
277
301
  ],
278
302
  };
279
303
  }
@@ -296,17 +320,36 @@ export class PendingStateManager implements IDisposable {
296
320
  }
297
321
  public readonly dispose = (): void => this.disposeOnce.value;
298
322
 
323
+ /**
324
+ * We've flushed an empty batch, and need to track it locally until the corresponding
325
+ * ack is processed, to properly track batch IDs
326
+ */
327
+ public onFlushEmptyBatch(
328
+ placeholder: LocalEmptyBatchPlaceholder,
329
+ clientSequenceNumber: number | undefined,
330
+ staged: boolean,
331
+ ): void {
332
+ // We have to cast because runtimeOp doesn't apply for empty batches and is missing on LocalEmptyBatchPlaceholder
333
+ this.onFlushBatch(
334
+ [placeholder satisfies Omit<LocalBatchMessage, "runtimeOp"> as LocalBatchMessage],
335
+ clientSequenceNumber,
336
+ staged,
337
+ );
338
+ }
339
+
299
340
  /**
300
341
  * The given batch has been flushed, and needs to be tracked locally until the corresponding
301
342
  * acks are processed, to ensure it is successfully sent.
302
343
  * @param batch - The batch that was flushed
303
344
  * @param clientSequenceNumber - The CSN of the first message in the batch,
304
345
  * or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
346
+ * @param staged - Indicates whether batch is staged (not to be submitted while runtime is in Staging Mode)
305
347
  * @param ignoreBatchId - Whether to ignore the batchId in the batchStartInfo
306
348
  */
307
349
  public onFlushBatch(
308
350
  batch: LocalBatchMessage[],
309
351
  clientSequenceNumber: number | undefined,
352
+ staged: boolean,
310
353
  ignoreBatchId?: boolean,
311
354
  ): void {
312
355
  // clientId and batchStartCsn are used for generating the batchId so we can detect container forks
@@ -315,6 +358,9 @@ export class PendingStateManager implements IDisposable {
315
358
  // In the case where the batch was not sent, use a random uuid for clientId, and -1 for clientSequenceNumber to indicate this case.
316
359
  // This will guarantee uniqueness of the batchId, and is a suitable fallback since clientId/CSN is only needed if the batch was actually sent/sequenced.
317
360
  const batchWasSent = clientSequenceNumber !== undefined;
361
+ if (batchWasSent) {
362
+ assert(!staged, 0xb84 /* Staged batches should not have been submitted */);
363
+ }
318
364
  const [clientId, batchStartCsn] = batchWasSent
319
365
  ? [this.stateHandler.clientId(), clientSequenceNumber]
320
366
  : [uuid(), -1]; // -1 will indicate not a real clientId/CSN pair
@@ -325,7 +371,7 @@ export class PendingStateManager implements IDisposable {
325
371
 
326
372
  for (const message of batch) {
327
373
  const {
328
- serializedOp: content,
374
+ runtimeOp,
329
375
  referenceSequenceNumber,
330
376
  localOpMetadata,
331
377
  metadata: opMetadata,
@@ -333,11 +379,12 @@ export class PendingStateManager implements IDisposable {
333
379
  const pendingMessage: IPendingMessage = {
334
380
  type: "message",
335
381
  referenceSequenceNumber,
336
- content,
382
+ content: serializeOp(runtimeOp),
383
+ runtimeOp,
337
384
  localOpMetadata,
338
385
  opMetadata,
339
386
  // Note: We only will read this off the first message, but put it on all for simplicity
340
- batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId },
387
+ batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId, staged },
341
388
  };
342
389
  this.pendingMessages.push(pendingMessage);
343
390
  }
@@ -375,6 +422,11 @@ export class PendingStateManager implements IDisposable {
375
422
  const localOpMetadata = await this.stateHandler.applyStashedOp(nextMessage.content);
376
423
  if (this.stateHandler.isAttached()) {
377
424
  nextMessage.localOpMetadata = localOpMetadata;
425
+ // NOTE: This runtimeOp has been roundtripped through string, which is technically lossy.
426
+ // e.g. At this point, handles are in their encoded form.
427
+ nextMessage.runtimeOp = JSON.parse(
428
+ nextMessage.content,
429
+ ) as LocalContainerRuntimeMessage;
378
430
  // then we push onto pendingMessages which will cause PendingStateManager to resubmit when we connect
379
431
  patchbatchInfo(nextMessage); // Back compat
380
432
  this.pendingMessages.push(nextMessage);
@@ -508,7 +560,7 @@ export class PendingStateManager implements IDisposable {
508
560
  );
509
561
 
510
562
  pendingMessage.sequenceNumber = sequenceNumber;
511
- this.savedOps.push(withoutLocalOpMetadata(pendingMessage));
563
+ this.savedOps.push(toSerializableForm(pendingMessage));
512
564
 
513
565
  this.pendingMessages.shift();
514
566
 
@@ -579,6 +631,7 @@ export class PendingStateManager implements IDisposable {
579
631
  pendingMessage !== undefined,
580
632
  0xa21 /* No pending message found as we start processing this remote batch */,
581
633
  );
634
+ assert(!pendingMessage.batchInfo.staged, 0xb85 /* Can't get an ack from a staged batch */);
582
635
 
583
636
  // If this batch became empty on resubmit, batch.messages will be empty (but keyMessage is always set)
584
637
  // and the next pending message should be an empty batch marker.
@@ -630,18 +683,22 @@ export class PendingStateManager implements IDisposable {
630
683
  * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
631
684
  * states in its queue. This includes triggering resubmission of unacked ops.
632
685
  * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
686
+ * @param onlyStagedBatches - If true, only replay staged batches. This is used when we are exiting staging mode and want to rebase and submit the staged batches.
633
687
  */
634
- public replayPendingStates(): void {
688
+ public replayPendingStates(onlyStagedBatches?: boolean): void {
635
689
  assert(
636
690
  this.stateHandler.connected(),
637
691
  0x172 /* "The connection state is not consistent with the runtime" */,
638
692
  );
639
693
 
640
- // This assert suggests we are about to send same ops twice, which will result in data loss.
641
- assert(
642
- this.clientIdFromLastReplay !== this.stateHandler.clientId(),
643
- 0x173 /* "replayPendingStates called twice for same clientId!" */,
644
- );
694
+ // Staged batches have not yet been submitted so check doesn't apply
695
+ if (!onlyStagedBatches) {
696
+ // This assert suggests we are about to send same ops twice, which will result in data loss.
697
+ assert(
698
+ this.clientIdFromLastReplay !== this.stateHandler.clientId(),
699
+ 0x173 /* "replayPendingStates called twice for same clientId!" */,
700
+ );
701
+ }
645
702
  this.clientIdFromLastReplay = this.stateHandler.clientId();
646
703
 
647
704
  assert(
@@ -652,6 +709,8 @@ export class PendingStateManager implements IDisposable {
652
709
  const initialPendingMessagesCount = this.pendingMessages.length;
653
710
  let remainingPendingMessagesCount = this.pendingMessages.length;
654
711
 
712
+ let seenStagedBatch = false;
713
+
655
714
  // Process exactly `pendingMessagesCount` items in the queue as it represents the number of messages that were
656
715
  // pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
657
716
  // which must not be replayed.
@@ -660,18 +719,37 @@ export class PendingStateManager implements IDisposable {
660
719
  let pendingMessage = this.pendingMessages.shift()!;
661
720
  remainingPendingMessagesCount--;
662
721
 
722
+ // Re-queue pre-staging messages if we are only processing staged batches
723
+ if (onlyStagedBatches) {
724
+ if (!pendingMessage.batchInfo.staged) {
725
+ assert(!seenStagedBatch, 0xb86 /* Staged batch was followed by non-staged batch */);
726
+ this.pendingMessages.push(pendingMessage);
727
+ continue;
728
+ }
729
+
730
+ seenStagedBatch = true;
731
+ pendingMessage.batchInfo.staged = false; // Clear staged flag so we can submit
732
+ }
733
+
663
734
  const batchMetadataFlag = asBatchMetadata(pendingMessage.opMetadata)?.batch;
664
735
  assert(batchMetadataFlag !== false, 0x41b /* We cannot process batches in chunks */);
665
736
 
666
737
  // The next message starts a batch (possibly single-message), and we'll need its batchId.
667
738
  const batchId = getEffectiveBatchId(pendingMessage);
668
739
 
740
+ const staged = pendingMessage.batchInfo.staged;
741
+
669
742
  if (asEmptyBatchLocalOpMetadata(pendingMessage.localOpMetadata)?.emptyBatch === true) {
670
743
  // Resubmit no messages, with the batchId. Will result in another empty batch marker.
671
- this.stateHandler.reSubmitBatch([], batchId);
744
+ this.stateHandler.reSubmitBatch([], batchId, staged);
672
745
  continue;
673
746
  }
674
747
 
748
+ assert(
749
+ pendingMessage.runtimeOp !== undefined,
750
+ 0xb87 /* viableOp is only undefined for empty batches */,
751
+ );
752
+
675
753
  /**
676
754
  * We must preserve the distinct batches on resubmit.
677
755
  * Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will
@@ -683,12 +761,13 @@ export class PendingStateManager implements IDisposable {
683
761
  this.stateHandler.reSubmitBatch(
684
762
  [
685
763
  {
686
- content: pendingMessage.content,
764
+ runtimeOp: pendingMessage.runtimeOp,
687
765
  localOpMetadata: pendingMessage.localOpMetadata,
688
766
  opMetadata: pendingMessage.opMetadata,
689
767
  },
690
768
  ],
691
769
  batchId,
770
+ staged,
692
771
  );
693
772
  continue;
694
773
  }
@@ -703,12 +782,17 @@ export class PendingStateManager implements IDisposable {
703
782
 
704
783
  // check is >= because batch end may be last pending message
705
784
  while (remainingPendingMessagesCount >= 0) {
785
+ assert(
786
+ pendingMessage.runtimeOp !== undefined,
787
+ 0xb88 /* viableOp is only undefined for empty batches */,
788
+ );
706
789
  batch.push({
707
- content: pendingMessage.content,
790
+ runtimeOp: pendingMessage.runtimeOp,
708
791
  localOpMetadata: pendingMessage.localOpMetadata,
709
792
  opMetadata: pendingMessage.opMetadata,
710
793
  });
711
794
 
795
+ // End of the batch
712
796
  if (pendingMessage.opMetadata?.batch === false) {
713
797
  break;
714
798
  }
@@ -723,7 +807,7 @@ export class PendingStateManager implements IDisposable {
723
807
  );
724
808
  }
725
809
 
726
- this.stateHandler.reSubmitBatch(batch, batchId);
810
+ this.stateHandler.reSubmitBatch(batch, batchId, staged);
727
811
  }
728
812
 
729
813
  // pending ops should no longer depend on previous sequenced local ops after resubmit
@@ -740,6 +824,36 @@ export class PendingStateManager implements IDisposable {
740
824
  });
741
825
  }
742
826
  }
827
+
828
+ /**
829
+ * Clears the 'staged' flag off all pending messages.
830
+ */
831
+ public clearStagingFlags(): void {
832
+ for (const message of this.pendingMessages.toArray()) {
833
+ if (message.batchInfo.staged) {
834
+ message.batchInfo.staged = false;
835
+ }
836
+ }
837
+ }
838
+
839
+ /**
840
+ * Pops all staged batches, invoking the callback on each one in order (LIFO)
841
+ */
842
+ public popStagedBatches(callback: (stagedMessage: IPendingMessage) => void): void {
843
+ while (!this.pendingMessages.isEmpty()) {
844
+ const stagedMessage = this.pendingMessages.peekBack();
845
+ if (stagedMessage?.batchInfo.staged === true) {
846
+ callback(stagedMessage);
847
+ this.pendingMessages.pop();
848
+ } else {
849
+ break; // no more staged messages
850
+ }
851
+ }
852
+ assert(
853
+ this.pendingMessages.toArray().every((m) => m.batchInfo.staged !== true),
854
+ 0xb89 /* Shouldn't be any more staged messages */,
855
+ );
856
+ }
743
857
  }
744
858
 
745
859
  /**
@@ -751,6 +865,6 @@ function patchbatchInfo(
751
865
  const batchInfo: IPendingMessageFromStash["batchInfo"] = message.batchInfo;
752
866
  if (batchInfo === undefined) {
753
867
  // Using uuid guarantees uniqueness, retaining existing behavior
754
- message.batchInfo = { clientId: uuid(), batchStartCsn: -1, length: -1 };
868
+ message.batchInfo = { clientId: uuid(), batchStartCsn: -1, length: -1, staged: false };
755
869
  }
756
870
  }
@@ -9,7 +9,10 @@ import {
9
9
  type ILayerCompatSupportRequirements,
10
10
  } from "@fluid-internal/client-utils";
11
11
  import type { ICriticalContainerError } from "@fluidframework/container-definitions";
12
- import { encodeHandlesInContainerRuntime } from "@fluidframework/runtime-definitions/internal";
12
+ import {
13
+ encodeHandlesInContainerRuntime,
14
+ notifiesReadOnlyState,
15
+ } from "@fluidframework/runtime-definitions/internal";
13
16
  import { UsageError } from "@fluidframework/telemetry-utils/internal";
14
17
 
15
18
  import { pkgVersion } from "./packageVersion.js";
@@ -66,7 +69,7 @@ export const runtimeCompatDetailsForDataStore: ILayerCompatDetails = {
66
69
  /**
67
70
  * The features supported by the Runtime layer across the Runtime / DataStore boundary.
68
71
  */
69
- supportedFeatures: new Set<string>([encodeHandlesInContainerRuntime]),
72
+ supportedFeatures: new Set<string>([encodeHandlesInContainerRuntime, notifiesReadOnlyState]),
70
73
  };
71
74
 
72
75
  /**
@@ -96,6 +96,7 @@ export interface IDocumentSchemaFeatures {
96
96
  compressionLz4: boolean;
97
97
  idCompressorMode: IdCompressorMode;
98
98
  opGroupingEnabled: boolean;
99
+ createBlobPayloadPending: true | undefined;
99
100
 
100
101
  /**
101
102
  * List of disallowed versions of the runtime.
@@ -227,6 +228,7 @@ const documentSchemaSupportedConfigs = {
227
228
  idCompressorMode: new IdCompressorProperty(["delayed", "on"]),
228
229
  opGroupingEnabled: new TrueOrUndefined(),
229
230
  compressionLz4: new TrueOrUndefined(),
231
+ createBlobPayloadPending: new TrueOrUndefined(),
230
232
  disallowedVersions: new CheckVersions(),
231
233
  };
232
234
 
@@ -482,6 +484,7 @@ export class DocumentsSchemaController {
482
484
  compressionLz4: boolToProp(features.compressionLz4),
483
485
  idCompressorMode: features.idCompressorMode,
484
486
  opGroupingEnabled: boolToProp(features.opGroupingEnabled),
487
+ createBlobPayloadPending: features.createBlobPayloadPending,
485
488
  disallowedVersions: arrayToProp(features.disallowedVersions),
486
489
  },
487
490
  };