@fluidframework/container-runtime 2.3.0-288113 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/api-report/container-runtime.legacy.alpha.api.md +15 -0
  3. package/container-runtime.test-files.tar +0 -0
  4. package/dist/channelCollection.d.ts +1 -1
  5. package/dist/channelCollection.d.ts.map +1 -1
  6. package/dist/channelCollection.js +1 -16
  7. package/dist/channelCollection.js.map +1 -1
  8. package/dist/connectionTelemetry.d.ts +27 -3
  9. package/dist/connectionTelemetry.d.ts.map +1 -1
  10. package/dist/connectionTelemetry.js.map +1 -1
  11. package/dist/containerRuntime.d.ts +68 -13
  12. package/dist/containerRuntime.d.ts.map +1 -1
  13. package/dist/containerRuntime.js +262 -180
  14. package/dist/containerRuntime.js.map +1 -1
  15. package/dist/deltaManagerProxies.d.ts.map +1 -1
  16. package/dist/deltaManagerProxies.js +11 -4
  17. package/dist/deltaManagerProxies.js.map +1 -1
  18. package/dist/gc/garbageCollection.d.ts.map +1 -1
  19. package/dist/gc/garbageCollection.js +0 -2
  20. package/dist/gc/garbageCollection.js.map +1 -1
  21. package/dist/gc/gcHelpers.d.ts.map +1 -1
  22. package/dist/gc/gcHelpers.js +0 -8
  23. package/dist/gc/gcHelpers.js.map +1 -1
  24. package/dist/index.d.ts +1 -1
  25. package/dist/index.d.ts.map +1 -1
  26. package/dist/index.js +2 -1
  27. package/dist/index.js.map +1 -1
  28. package/dist/legacy.d.ts +3 -1
  29. package/dist/messageTypes.d.ts +0 -9
  30. package/dist/messageTypes.d.ts.map +1 -1
  31. package/dist/messageTypes.js.map +1 -1
  32. package/dist/opLifecycle/batchManager.d.ts +9 -0
  33. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  34. package/dist/opLifecycle/batchManager.js +19 -6
  35. package/dist/opLifecycle/batchManager.js.map +1 -1
  36. package/dist/opLifecycle/duplicateBatchDetector.d.ts +32 -0
  37. package/dist/opLifecycle/duplicateBatchDetector.d.ts.map +1 -0
  38. package/dist/opLifecycle/duplicateBatchDetector.js +68 -0
  39. package/dist/opLifecycle/duplicateBatchDetector.js.map +1 -0
  40. package/dist/opLifecycle/index.d.ts +3 -2
  41. package/dist/opLifecycle/index.d.ts.map +1 -1
  42. package/dist/opLifecycle/index.js +4 -1
  43. package/dist/opLifecycle/index.js.map +1 -1
  44. package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
  45. package/dist/opLifecycle/opCompressor.js +0 -4
  46. package/dist/opLifecycle/opCompressor.js.map +1 -1
  47. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  48. package/dist/opLifecycle/opGroupingManager.js +0 -4
  49. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  50. package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
  51. package/dist/opLifecycle/opSplitter.js +1 -6
  52. package/dist/opLifecycle/opSplitter.js.map +1 -1
  53. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  54. package/dist/opLifecycle/outbox.js +1 -4
  55. package/dist/opLifecycle/outbox.js.map +1 -1
  56. package/dist/opLifecycle/remoteMessageProcessor.d.ts +37 -17
  57. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  58. package/dist/opLifecycle/remoteMessageProcessor.js +47 -37
  59. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
  60. package/dist/packageVersion.d.ts +1 -1
  61. package/dist/packageVersion.d.ts.map +1 -1
  62. package/dist/packageVersion.js +1 -1
  63. package/dist/packageVersion.js.map +1 -1
  64. package/dist/pendingStateManager.d.ts +27 -17
  65. package/dist/pendingStateManager.d.ts.map +1 -1
  66. package/dist/pendingStateManager.js +85 -56
  67. package/dist/pendingStateManager.js.map +1 -1
  68. package/dist/scheduleManager.d.ts +2 -4
  69. package/dist/scheduleManager.d.ts.map +1 -1
  70. package/dist/scheduleManager.js +6 -37
  71. package/dist/scheduleManager.js.map +1 -1
  72. package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
  73. package/dist/summary/summarizerNode/summarizerNodeUtils.js +0 -2
  74. package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
  75. package/dist/summary/summaryCollection.d.ts.map +1 -1
  76. package/dist/summary/summaryCollection.js +5 -7
  77. package/dist/summary/summaryCollection.js.map +1 -1
  78. package/dist/summary/summaryFormat.d.ts.map +1 -1
  79. package/dist/summary/summaryFormat.js +1 -4
  80. package/dist/summary/summaryFormat.js.map +1 -1
  81. package/lib/channelCollection.d.ts +1 -1
  82. package/lib/channelCollection.d.ts.map +1 -1
  83. package/lib/channelCollection.js +1 -16
  84. package/lib/channelCollection.js.map +1 -1
  85. package/lib/connectionTelemetry.d.ts +27 -3
  86. package/lib/connectionTelemetry.d.ts.map +1 -1
  87. package/lib/connectionTelemetry.js.map +1 -1
  88. package/lib/containerRuntime.d.ts +68 -13
  89. package/lib/containerRuntime.d.ts.map +1 -1
  90. package/lib/containerRuntime.js +262 -181
  91. package/lib/containerRuntime.js.map +1 -1
  92. package/lib/deltaManagerProxies.d.ts.map +1 -1
  93. package/lib/deltaManagerProxies.js +11 -4
  94. package/lib/deltaManagerProxies.js.map +1 -1
  95. package/lib/gc/garbageCollection.d.ts.map +1 -1
  96. package/lib/gc/garbageCollection.js +0 -2
  97. package/lib/gc/garbageCollection.js.map +1 -1
  98. package/lib/gc/gcHelpers.d.ts.map +1 -1
  99. package/lib/gc/gcHelpers.js +0 -8
  100. package/lib/gc/gcHelpers.js.map +1 -1
  101. package/lib/index.d.ts +1 -1
  102. package/lib/index.d.ts.map +1 -1
  103. package/lib/index.js +1 -1
  104. package/lib/index.js.map +1 -1
  105. package/lib/legacy.d.ts +3 -1
  106. package/lib/messageTypes.d.ts +0 -9
  107. package/lib/messageTypes.d.ts.map +1 -1
  108. package/lib/messageTypes.js.map +1 -1
  109. package/lib/opLifecycle/batchManager.d.ts +9 -0
  110. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  111. package/lib/opLifecycle/batchManager.js +17 -5
  112. package/lib/opLifecycle/batchManager.js.map +1 -1
  113. package/lib/opLifecycle/duplicateBatchDetector.d.ts +32 -0
  114. package/lib/opLifecycle/duplicateBatchDetector.d.ts.map +1 -0
  115. package/lib/opLifecycle/duplicateBatchDetector.js +64 -0
  116. package/lib/opLifecycle/duplicateBatchDetector.js.map +1 -0
  117. package/lib/opLifecycle/index.d.ts +3 -2
  118. package/lib/opLifecycle/index.d.ts.map +1 -1
  119. package/lib/opLifecycle/index.js +2 -1
  120. package/lib/opLifecycle/index.js.map +1 -1
  121. package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
  122. package/lib/opLifecycle/opCompressor.js +0 -4
  123. package/lib/opLifecycle/opCompressor.js.map +1 -1
  124. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  125. package/lib/opLifecycle/opGroupingManager.js +0 -4
  126. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  127. package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
  128. package/lib/opLifecycle/opSplitter.js +1 -6
  129. package/lib/opLifecycle/opSplitter.js.map +1 -1
  130. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  131. package/lib/opLifecycle/outbox.js +1 -4
  132. package/lib/opLifecycle/outbox.js.map +1 -1
  133. package/lib/opLifecycle/remoteMessageProcessor.d.ts +37 -17
  134. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  135. package/lib/opLifecycle/remoteMessageProcessor.js +47 -37
  136. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
  137. package/lib/packageVersion.d.ts +1 -1
  138. package/lib/packageVersion.d.ts.map +1 -1
  139. package/lib/packageVersion.js +1 -1
  140. package/lib/packageVersion.js.map +1 -1
  141. package/lib/pendingStateManager.d.ts +27 -17
  142. package/lib/pendingStateManager.d.ts.map +1 -1
  143. package/lib/pendingStateManager.js +85 -56
  144. package/lib/pendingStateManager.js.map +1 -1
  145. package/lib/scheduleManager.d.ts +2 -4
  146. package/lib/scheduleManager.d.ts.map +1 -1
  147. package/lib/scheduleManager.js +6 -37
  148. package/lib/scheduleManager.js.map +1 -1
  149. package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
  150. package/lib/summary/summarizerNode/summarizerNodeUtils.js +0 -2
  151. package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
  152. package/lib/summary/summaryCollection.d.ts.map +1 -1
  153. package/lib/summary/summaryCollection.js +5 -7
  154. package/lib/summary/summaryCollection.js.map +1 -1
  155. package/lib/summary/summaryFormat.d.ts.map +1 -1
  156. package/lib/summary/summaryFormat.js +1 -4
  157. package/lib/summary/summaryFormat.js.map +1 -1
  158. package/lib/tsdoc-metadata.json +1 -1
  159. package/package.json +50 -24
  160. package/src/channelCollection.ts +7 -21
  161. package/src/connectionTelemetry.ts +33 -3
  162. package/src/containerRuntime.ts +382 -233
  163. package/src/deltaManagerProxies.ts +11 -4
  164. package/src/gc/garbageCollection.ts +1 -3
  165. package/src/gc/gcHelpers.ts +4 -12
  166. package/src/index.ts +2 -0
  167. package/src/messageTypes.ts +0 -10
  168. package/src/opLifecycle/batchManager.ts +29 -7
  169. package/src/opLifecycle/duplicateBatchDetector.ts +78 -0
  170. package/src/opLifecycle/index.ts +4 -1
  171. package/src/opLifecycle/opCompressor.ts +2 -6
  172. package/src/opLifecycle/opGroupingManager.ts +2 -6
  173. package/src/opLifecycle/opSplitter.ts +2 -6
  174. package/src/opLifecycle/outbox.ts +1 -3
  175. package/src/opLifecycle/remoteMessageProcessor.ts +87 -59
  176. package/src/packageVersion.ts +1 -1
  177. package/src/pendingStateManager.ts +114 -66
  178. package/src/scheduleManager.ts +8 -47
  179. package/src/summary/summarizerNode/summarizerNodeUtils.ts +1 -3
  180. package/src/summary/summaryCollection.ts +7 -9
  181. package/src/summary/summaryFormat.ts +1 -3
  182. package/src/summary/summaryFormats.md +11 -9
  183. package/tsconfig.json +1 -0
@@ -208,10 +208,17 @@ export class DeltaManagerSummarizerProxy extends BaseDeltaManagerProxy {
208
208
  export class DeltaManagerPendingOpsProxy extends BaseDeltaManagerProxy {
209
209
  public get minimumSequenceNumber(): number {
210
210
  const minPendingSeqNum = this.pendingStateManager.minimumPendingMessageSequenceNumber;
211
- // There is a chance that minPendingSeqNum is greater than minimum sequence number.
212
- // minPendingSeqNum is based on the pending ops, so it's based on ref seq number.
213
- // Imagine an op has just be sent while there's another client that has been lagging behind,
214
- // it will likely have a ref seq number greater than the minimum seq number.
211
+ /**
212
+ * The reason why the minimum pending sequence number can be less than the delta manager's minimum sequence
213
+ * number (DM's msn) is that when we are processing messages in the container runtime/delta manager, the delta
214
+ * manager's msn can be updated to continually increase. In the meantime, the pending state manager's op which
215
+ * hasn't been sent can still have a lower sequence number than the DM's msn (think about a disconnected
216
+ * scenario). To successfully resubmit that pending op it has to be rebased first by the DDS. The DDS still
217
+ * needs to keep the local data for that op that has a reference sequence number lower than the DM's msn. To
218
+ * achieve this, the msn passed to the DDS needs to be the minimum of the DM's msn and the minimum pending
219
+ * sequence number, so that it can keep the relevant local data to generate the right data for the new op
220
+ * during resubmission.
221
+ */
215
222
  if (
216
223
  minPendingSeqNum !== undefined &&
217
224
  minPendingSeqNum < this.deltaManager.minimumSequenceNumber
@@ -789,9 +789,7 @@ export class GarbageCollector implements IGarbageCollector {
789
789
  if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
790
790
  gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
791
791
  } else {
792
- // Non null asserting here because we are checking if it is undefined above.
793
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
794
- gcDataSuperSet.gcNodes[sourceNodeId]!.push(...outboundRoutes);
792
+ gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
795
793
  }
796
794
  newOutboundRoutesSinceLastRun.push(...outboundRoutes);
797
795
  },
@@ -166,9 +166,7 @@ export function concatGarbageCollectionData(
166
166
  if (combinedGCData.gcNodes[id] === undefined) {
167
167
  combinedGCData.gcNodes[id] = Array.from(routes);
168
168
  } else {
169
- // Non null asserting here since we are checking if combinedGCData.gcNodes[id] is not undefined above.
170
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
171
- const combinedRoutes = [...routes, ...combinedGCData.gcNodes[id]!];
169
+ const combinedRoutes = [...routes, ...combinedGCData.gcNodes[id]];
172
170
  combinedGCData.gcNodes[id] = [...new Set(combinedRoutes)];
173
171
  }
174
172
  }
@@ -189,17 +187,13 @@ export async function getGCDataFromSnapshot(
189
187
  for (const key of Object.keys(gcSnapshotTree.blobs)) {
190
188
  // Update deleted nodes blob.
191
189
  if (key === gcDeletedBlobKey) {
192
- // Non null asserting here, we can change this to Object.entries later
193
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
194
- deletedNodes = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]!);
190
+ deletedNodes = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]);
195
191
  continue;
196
192
  }
197
193
 
198
194
  // Update tombstone blob.
199
195
  if (key === gcTombstoneBlobKey) {
200
- // Non null asserting here, we can change this to Object.entries later
201
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
202
- tombstones = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]!);
196
+ tombstones = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]);
203
197
  continue;
204
198
  }
205
199
 
@@ -302,9 +296,7 @@ function trimLeadingAndTrailingSlashes(str: string) {
302
296
 
303
297
  /** Reformats a request URL to match expected format for a GC node path */
304
298
  export function urlToGCNodePath(url: string): string {
305
- // TODO Why are we non null asserting here
306
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
307
- return `/${trimLeadingAndTrailingSlashes(url.split("?")[0]!)}`;
299
+ return `/${trimLeadingAndTrailingSlashes(url.split("?")[0])}`;
308
300
  }
309
301
 
310
302
  /**
package/src/index.ts CHANGED
@@ -10,6 +10,8 @@ export {
10
10
  ISummaryConfigurationDisableSummarizer,
11
11
  ISummaryConfigurationDisableHeuristics,
12
12
  IContainerRuntimeOptions,
13
+ loadContainerRuntime,
14
+ LoadContainerRuntimeParams,
13
15
  isRuntimeMessage,
14
16
  agentSchedulerId,
15
17
  ContainerRuntime,
@@ -223,16 +223,6 @@ export type InboundSequencedContainerRuntimeMessage = Omit<
223
223
  > &
224
224
  InboundContainerRuntimeMessage;
225
225
 
226
- /** Essentially ISequencedDocumentMessage except that `type` is not `string` to enable narrowing
227
- * as `Exclude<string, InboundContainerRuntimeMessage['type']>` is not supported.
228
- * There should never be a runtime value of "__not_a_...".
229
- * Currently additionally replaces `contents` type until protocol-definitions update is taken with `unknown` instead of `any`.
230
- */
231
- export type InboundSequencedNonContainerRuntimeMessage = Omit<
232
- ISequencedDocumentMessage,
233
- "type" | "contents"
234
- > & { type: "__not_a_container_runtime_message_type__"; contents: unknown };
235
-
236
226
  /** A [loose] InboundSequencedContainerRuntimeMessage that is recent and may contain compat details.
237
227
  * It exists solely to to provide access to those details.
238
228
  */
@@ -6,9 +6,11 @@
6
6
  import { assert } from "@fluidframework/core-utils/internal";
7
7
 
8
8
  import { ICompressionRuntimeOptions } from "../containerRuntime.js";
9
- import { type IBatchMetadata } from "../metadata.js";
9
+ import { asBatchMetadata, type IBatchMetadata } from "../metadata.js";
10
+ import type { IPendingMessage } from "../pendingStateManager.js";
10
11
 
11
12
  import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.js";
13
+ import type { BatchStartInfo } from "./remoteMessageProcessor.js";
12
14
 
13
15
  export interface IBatchManagerOptions {
14
16
  readonly hardLimit: number;
@@ -33,6 +35,30 @@ export function generateBatchId(originalClientId: string, batchStartCsn: number)
33
35
  return `${originalClientId}_[${batchStartCsn}]`;
34
36
  }
35
37
 
38
+ /**
39
+ * Get the effective batch ID for the input argument.
40
+ * Supports either an IPendingMessage or BatchStartInfo.
41
+ * If the batch ID is explicitly present, return it.
42
+ * Otherwise, generate a new batch ID using the client ID and batch start CSN.
43
+ */
44
+ export function getEffectiveBatchId(
45
+ pendingMessageOrBatchStartInfo: IPendingMessage | BatchStartInfo,
46
+ ): string {
47
+ if ("localOpMetadata" in pendingMessageOrBatchStartInfo) {
48
+ const pendingMessage: IPendingMessage = pendingMessageOrBatchStartInfo;
49
+ return (
50
+ asBatchMetadata(pendingMessage.opMetadata)?.batchId ??
51
+ generateBatchId(
52
+ pendingMessage.batchInfo.clientId,
53
+ pendingMessage.batchInfo.batchStartCsn,
54
+ )
55
+ );
56
+ }
57
+
58
+ const batchStart: BatchStartInfo = pendingMessageOrBatchStartInfo;
59
+ return batchStart.batchId ?? generateBatchId(batchStart.clientId, batchStart.batchStartCsn);
60
+ }
61
+
36
62
  /**
37
63
  * Estimated size of the stringification overhead for an op accumulated
38
64
  * from runtime to loader to the service.
@@ -64,9 +90,7 @@ export class BatchManager {
64
90
  private get referenceSequenceNumber(): number | undefined {
65
91
  return this.pendingBatch.length === 0
66
92
  ? undefined
67
- : // Non null asserting here since we are checking the length above
68
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
69
- this.pendingBatch[this.pendingBatch.length - 1]!.referenceSequenceNumber;
93
+ : this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
70
94
  }
71
95
 
72
96
  /**
@@ -138,9 +162,7 @@ export class BatchManager {
138
162
  rollback: (process: (message: BatchMessage) => void) => {
139
163
  for (let i = this.pendingBatch.length; i > startPoint; ) {
140
164
  i--;
141
- // Non null asserting here since we are iterating though pendingBatch
142
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
143
- const message = this.pendingBatch[i]!;
165
+ const message = this.pendingBatch[i];
144
166
  this.batchContentSize -= message.contents?.length ?? 0;
145
167
  process(message);
146
168
  }
@@ -0,0 +1,78 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { assert } from "@fluidframework/core-utils/internal";
7
+
8
+ import { getEffectiveBatchId } from "./batchManager.js";
9
+ import { type BatchStartInfo } from "./remoteMessageProcessor.js";
10
+
11
+ /**
12
+ * This class tracks recent batchIds we've seen, and checks incoming batches for duplicates.
13
+ */
14
+ export class DuplicateBatchDetector {
15
+ /** All batchIds we've seen recently enough (based on MSN) that we need to watch for duplicates */
16
+ private readonly batchIdsAll = new Set<string>();
17
+
18
+ /** We map from sequenceNumber to batchId to find which ones we can stop tracking as MSN advances */
19
+ private readonly batchIdsBySeqNum = new Map<number, string>();
20
+
21
+ /**
22
+ * Records this batch's batchId, and checks if it's a duplicate of a batch we've already seen.
23
+ * If it's a duplicate, also return the sequence number of the other batch for logging.
24
+ *
25
+ * @remarks - We also use the minimumSequenceNumber to clear out old batchIds that are no longer at risk for duplicates.
26
+ */
27
+ public processInboundBatch(
28
+ batchStart: BatchStartInfo,
29
+ ): { duplicate: true; otherSequenceNumber: number } | { duplicate: false } {
30
+ const { sequenceNumber, minimumSequenceNumber } = batchStart.keyMessage;
31
+
32
+ // Glance at this batch's MSN. Any batchIds we're tracking with a lower sequence number are now safe to forget.
33
+ // Why? Because any other client holding the same batch locally would have seen the earlier batch and closed before submitting its duplicate.
34
+ this.clearOldBatchIds(minimumSequenceNumber);
35
+
36
+ // getEffectiveBatchId is only needed in the SUPER rare/surprising case where
37
+ // the original batch (not resubmitted, so no batchId) arrives in parallel with a resubmitted batch.
38
+ // In the presence of typical network conditions, this would not be possible
39
+ // (the original batch should roundtrip WAY before another container could rehydrate, connect, and resubmit)
40
+ const batchId = getEffectiveBatchId(batchStart);
41
+
42
+ // Check this batch against the tracked batchIds to see if it's a duplicate
43
+ if (this.batchIdsAll.has(batchId)) {
44
+ for (const [otherSequenceNumber, otherBatchId] of this.batchIdsBySeqNum.entries()) {
45
+ if (otherBatchId === batchId) {
46
+ return {
47
+ duplicate: true,
48
+ otherSequenceNumber,
49
+ };
50
+ }
51
+ }
52
+ assert(false, 0xa34 /* Should have found the batchId in batchIdBySeqNum map */);
53
+ }
54
+
55
+ // Now we know it's not a duplicate, so add it to the tracked batchIds and return.
56
+ assert(
57
+ !this.batchIdsBySeqNum.has(sequenceNumber),
58
+ 0xa35 /* batchIdsAll and batchIdsBySeqNum should be in sync */,
59
+ );
60
+ this.batchIdsBySeqNum.set(sequenceNumber, batchId);
61
+ this.batchIdsAll.add(batchId);
62
+
63
+ return { duplicate: false };
64
+ }
65
+
66
+ /**
67
+ * Batches that started before the MSN are not at risk for a sequenced duplicate to arrive,
68
+ * since the batch start has been processed by all clients, and local batches are deduped and the forked client would close.
69
+ */
70
+ private clearOldBatchIds(msn: number) {
71
+ this.batchIdsBySeqNum.forEach((batchId, sequenceNumber) => {
72
+ if (sequenceNumber < msn) {
73
+ this.batchIdsBySeqNum.delete(sequenceNumber);
74
+ this.batchIdsAll.delete(batchId);
75
+ }
76
+ });
77
+ }
78
+ }
@@ -8,17 +8,20 @@ export {
8
8
  BatchManager,
9
9
  BatchSequenceNumbers,
10
10
  estimateSocketSize,
11
+ getEffectiveBatchId,
11
12
  generateBatchId,
12
13
  IBatchManagerOptions,
13
14
  } from "./batchManager.js";
14
15
  export { BatchMessage, IBatch, IBatchCheckpoint, IChunkedOp } from "./definitions.js";
16
+ export { DuplicateBatchDetector } from "./duplicateBatchDetector.js";
15
17
  export { Outbox, getLongStack } from "./outbox.js";
16
18
  export { OpCompressor } from "./opCompressor.js";
17
19
  export { OpDecompressor } from "./opDecompressor.js";
18
20
  export { OpSplitter, splitOp, isChunkedMessage } from "./opSplitter.js";
19
21
  export {
20
22
  ensureContentsDeserialized,
21
- InboundBatch,
23
+ InboundMessageResult,
24
+ BatchStartInfo,
22
25
  RemoteMessageProcessor,
23
26
  unpackRuntimeMessage,
24
27
  } from "./remoteMessageProcessor.js";
@@ -47,13 +47,9 @@ export class OpCompressor {
47
47
 
48
48
  const messages: BatchMessage[] = [];
49
49
  messages.push({
50
- // Non null asserting here because of the length assert above
51
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
52
- ...batch.messages[0]!,
50
+ ...batch.messages[0],
53
51
  contents: JSON.stringify({ packedContents: compressedContent }),
54
- // Non null asserting here because of the length assert above
55
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
56
- metadata: batch.messages[0]!.metadata,
52
+ metadata: batch.messages[0].metadata,
57
53
  compression: CompressionAlgorithms.lz4,
58
54
  });
59
55
 
@@ -99,9 +99,7 @@ export class OpGroupingManager {
99
99
  length: batch.messages.length,
100
100
  threshold: this.config.opCountThreshold,
101
101
  reentrant: batch.hasReentrantOps,
102
- // Non null asserting here because of the length check above
103
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
104
- referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
102
+ referenceSequenceNumber: batch.messages[0].referenceSequenceNumber,
105
103
  });
106
104
  }
107
105
  // We expect this will be on the first message, if present at all.
@@ -130,9 +128,7 @@ export class OpGroupingManager {
130
128
  messages: [
131
129
  {
132
130
  metadata: { batchId: groupedBatchId },
133
- // TODO why are we non null asserting here?
134
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
135
- referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
131
+ referenceSequenceNumber: batch.messages[0].referenceSequenceNumber,
136
132
  contents: serializedContent,
137
133
  },
138
134
  ],
@@ -134,9 +134,7 @@ export class OpSplitter {
134
134
  0x516 /* Chunk size needs to be smaller than the max batch size */,
135
135
  );
136
136
 
137
- // Non null asserting here because of the length check above
138
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
139
- const firstMessage = batch.messages[0]!; // we expect this to be the large compressed op, which needs to be split
137
+ const firstMessage = batch.messages[0]; // we expect this to be the large compressed op, which needs to be split
140
138
  assert(
141
139
  (firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,
142
140
  0x518 /* First message in the batch needs to be chunkable */,
@@ -165,9 +163,7 @@ export class OpSplitter {
165
163
  // The last chunk will be part of the new batch and needs to
166
164
  // preserve the batch metadata of the original batch
167
165
  const lastChunk = chunkToBatchMessage(
168
- // Non null asserting here because of the length assert above
169
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
170
- chunks[chunks.length - 1]!,
166
+ chunks[chunks.length - 1],
171
167
  batch.referenceSequenceNumber,
172
168
  { batch: firstMessage.metadata?.batch },
173
169
  );
@@ -446,9 +446,7 @@ export class Outbox {
446
446
  // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
447
447
  // version that has support for batches (submitBatchFn)
448
448
  assert(
449
- // Non null asserting here because of the length check above
450
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
451
- batch.messages[0]!.compression === undefined,
449
+ batch.messages[0].compression === undefined,
452
450
  0x5a6 /* Compression should not have happened if the loader does not support it */,
453
451
  );
454
452
 
@@ -21,26 +21,56 @@ import { OpDecompressor } from "./opDecompressor.js";
21
21
  import { OpGroupingManager, isGroupedBatch } from "./opGroupingManager.js";
22
22
  import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
23
23
 
24
- /** Messages being received as a batch, with details needed to process the batch */
25
- export interface InboundBatch {
26
- /** Messages in this batch */
27
- readonly messages: InboundSequencedContainerRuntimeMessage[];
24
+ /** Info about the batch we learn when we process the first message */
25
+ export interface BatchStartInfo {
28
26
  /** Batch ID, if present */
29
27
  readonly batchId: string | undefined;
30
28
  /** clientId that sent this batch. Used to compute Batch ID if needed */
31
29
  readonly clientId: string;
32
30
  /**
33
- * Client Sequence Number of the first message in the batch.
31
+ * Client Sequence Number of the Grouped Batch message, or the first message in the ungrouped batch.
34
32
  * Used to compute Batch ID if needed
35
33
  *
36
34
  * @remarks For chunked batches, this is the CSN of the "representative" chunk (the final chunk).
37
35
  * For grouped batches, clientSequenceNumber on messages is overwritten, so we track this original value here.
38
36
  */
39
37
  readonly batchStartCsn: number;
40
- /** For an empty batch (with no messages), we need to remember the empty grouped batch's sequence number */
41
- readonly emptyBatchSequenceNumber?: number;
38
+ /**
39
+ * The first message in the batch, or if the batch is empty, the empty grouped batch message.
40
+ * Used for accessing the sequence numbers for the (start of the) batch.
41
+ *
42
+ * @remarks Do not use clientSequenceNumber here, use batchStartCsn instead.
43
+ */
44
+ readonly keyMessage: ISequencedDocumentMessage;
42
45
  }
43
46
 
47
+ /**
48
+ * Result of processing the next inbound message.
49
+ * Depending on the message and configuration of RemoteMessageProcessor, the result may be:
50
+ * - A full batch of messages (including a single-message batch)
51
+ * - The first message of a multi-message batch
52
+ * - The next message in a multi-message batch
53
+ */
54
+ export type InboundMessageResult =
55
+ | {
56
+ type: "fullBatch";
57
+ messages: InboundSequencedContainerRuntimeMessage[];
58
+ batchStart: BatchStartInfo;
59
+ length: number;
60
+ }
61
+ | {
62
+ type: "batchStartingMessage";
63
+ batchStart: BatchStartInfo;
64
+ nextMessage: InboundSequencedContainerRuntimeMessage;
65
+ length?: never;
66
+ }
67
+ | {
68
+ type: "nextBatchMessage";
69
+ batchEnd?: boolean;
70
+ nextMessage: InboundSequencedContainerRuntimeMessage;
71
+ length?: never;
72
+ };
73
+
44
74
  function assertHasClientId(
45
75
  message: ISequencedDocumentMessage,
46
76
  ): asserts message is ISequencedDocumentMessage & { clientId: string } {
@@ -57,12 +87,7 @@ function assertHasClientId(
57
87
  * @internal
58
88
  */
59
89
  export class RemoteMessageProcessor {
60
- /**
61
- * The current batch being received, with details needed to process it.
62
- *
63
- * @remarks If undefined, we are expecting the next message to start a new batch.
64
- */
65
- private batchInProgress: InboundBatch | undefined;
90
+ private batchInProgress: boolean = false;
66
91
 
67
92
  constructor(
68
93
  private readonly opSplitter: OpSplitter,
@@ -100,7 +125,7 @@ export class RemoteMessageProcessor {
100
125
  public process(
101
126
  remoteMessageCopy: ISequencedDocumentMessage,
102
127
  logLegacyCase: (codePath: string) => void,
103
- ): InboundBatch | undefined {
128
+ ): InboundMessageResult | undefined {
104
129
  let message = remoteMessageCopy;
105
130
 
106
131
  assertHasClientId(message);
@@ -129,80 +154,84 @@ export class RemoteMessageProcessor {
129
154
  }
130
155
 
131
156
  if (isGroupedBatch(message)) {
132
- // We should be awaiting a new batch (batchInProgress undefined)
133
- assert(
134
- this.batchInProgress === undefined,
135
- 0x9d3 /* Grouped batch interrupting another batch */,
136
- );
157
+ // We should be awaiting a new batch (batchInProgress false)
158
+ assert(!this.batchInProgress, 0x9d3 /* Grouped batch interrupting another batch */);
137
159
  const batchId = asBatchMetadata(message.metadata)?.batchId;
138
160
  const groupedMessages = this.opGroupingManager.ungroupOp(message).map(unpack);
161
+
139
162
  return {
163
+ type: "fullBatch",
140
164
  messages: groupedMessages, // Will be [] for an empty batch
141
- batchStartCsn: message.clientSequenceNumber,
142
- clientId,
143
- batchId,
144
- // If the batch is empty, we need to return the sequence number aside
145
- emptyBatchSequenceNumber:
146
- groupedMessages.length === 0 ? message.sequenceNumber : undefined,
165
+ batchStart: {
166
+ batchStartCsn: message.clientSequenceNumber,
167
+ clientId,
168
+ batchId,
169
+ keyMessage: groupedMessages[0] ?? message, // For an empty batch, this is the empty grouped batch message. Needed for sequence numbers for this batch
170
+ },
171
+ length: groupedMessages.length, // Will be 0 for an empty batch
147
172
  };
148
173
  }
149
174
 
150
175
  // Do a final unpack of runtime messages in case the message was not grouped, compressed, or chunked
151
176
  unpackRuntimeMessage(message, logLegacyCase);
152
177
 
153
- const { batchEnded } = this.addMessageToBatch(
178
+ return this.getResultBasedOnBatchMetadata(
154
179
  message as InboundSequencedContainerRuntimeMessage & { clientId: string },
155
180
  );
156
-
157
- if (!batchEnded) {
158
- // batch not yet complete
159
- return undefined;
160
- }
161
-
162
- const completedBatch = this.batchInProgress;
163
- this.batchInProgress = undefined;
164
- return completedBatch;
165
181
  }
166
182
 
167
183
  /**
168
- * Add the given message to the current batch, and indicate whether the batch is now complete.
169
- *
170
- * @returns batchEnded: true if the batch is now complete, batchEnded: false if more messages are expected
184
+ * Now that the message has been "unwrapped" as to any virtualization (grouping, compression, chunking),
185
+ * inspect the batch metadata flag and determine what kind of result to return.
171
186
  */
172
- private addMessageToBatch(
187
+ private getResultBasedOnBatchMetadata(
173
188
  message: InboundSequencedContainerRuntimeMessage & { clientId: string },
174
- ): { batchEnded: boolean } {
189
+ ): InboundMessageResult {
175
190
  const batchMetadataFlag = asBatchMetadata(message.metadata)?.batch;
176
- if (this.batchInProgress === undefined) {
191
+ if (!this.batchInProgress) {
177
192
  // We are waiting for a new batch
178
193
  assert(batchMetadataFlag !== false, 0x9d5 /* Unexpected batch end marker */);
179
194
 
180
195
  // Start of a new multi-message batch
181
196
  if (batchMetadataFlag === true) {
182
- this.batchInProgress = {
183
- messages: [message],
184
- batchId: asBatchMetadata(message.metadata)?.batchId,
185
- clientId: message.clientId,
186
- batchStartCsn: message.clientSequenceNumber,
197
+ this.batchInProgress = true;
198
+ return {
199
+ type: "batchStartingMessage",
200
+ batchStart: {
201
+ batchId: asBatchMetadata(message.metadata)?.batchId,
202
+ clientId: message.clientId,
203
+ batchStartCsn: message.clientSequenceNumber,
204
+ keyMessage: message,
205
+ },
206
+ nextMessage: message,
187
207
  };
188
-
189
- return { batchEnded: false };
190
208
  }
191
209
 
192
210
  // Single-message batch (Since metadata flag is undefined)
193
- this.batchInProgress = {
211
+ return {
212
+ type: "fullBatch",
194
213
  messages: [message],
195
- batchStartCsn: message.clientSequenceNumber,
196
- clientId: message.clientId,
197
- batchId: asBatchMetadata(message.metadata)?.batchId,
214
+ batchStart: {
215
+ batchStartCsn: message.clientSequenceNumber,
216
+ clientId: message.clientId,
217
+ batchId: asBatchMetadata(message.metadata)?.batchId,
218
+ keyMessage: message,
219
+ },
220
+ length: 1,
198
221
  };
199
- return { batchEnded: true };
200
222
  }
201
223
  assert(batchMetadataFlag !== true, 0x9d6 /* Unexpected batch start marker */);
202
224
 
203
- this.batchInProgress.messages.push(message);
225
+ // Clear batchInProgress state if the batch is ending
226
+ if (batchMetadataFlag === false) {
227
+ this.batchInProgress = false;
228
+ }
204
229
 
205
- return { batchEnded: batchMetadataFlag === false };
230
+ return {
231
+ type: "nextBatchMessage",
232
+ nextMessage: message,
233
+ batchEnd: batchMetadataFlag === false,
234
+ };
206
235
  }
207
236
  }
208
237
 
@@ -217,10 +246,9 @@ export function ensureContentsDeserialized(
217
246
  hasModernRuntimeMessageEnvelope: boolean,
218
247
  logLegacyCase: (codePath: string) => void,
219
248
  ): void {
220
- // Currently the loader layer is parsing the contents of the message as JSON if it is a string,
221
- // so we never expect to see this case.
222
- // We intend to remove that logic from the Loader, at which point we will have it here.
223
- // Only hasModernRuntimeMessageEnvelope true will be expected to have JSON contents.
249
+ // This should become unconditional once (Loader LTS) DeltaManager.processInboundMessage() stops parsing content (ADO #12052)
250
+ // Note: Until that change is made in the loader, this case will never be hit.
251
+ // Then there will be a long time of needing both cases, until LTS catches up to the change.
224
252
  let didParseJsonContents: boolean;
225
253
  if (typeof mutableMessage.contents === "string" && mutableMessage.contents !== "") {
226
254
  mutableMessage.contents = JSON.parse(mutableMessage.contents);
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "2.3.0-288113";
9
+ export const pkgVersion = "2.3.1";