@fluidframework/container-runtime 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/api-report/container-runtime.legacy.alpha.api.md +15 -0
- package/container-runtime.test-files.tar +0 -0
- package/dist/channelCollection.d.ts +1 -1
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +1 -16
- package/dist/channelCollection.js.map +1 -1
- package/dist/connectionTelemetry.d.ts +27 -3
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +68 -13
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +262 -180
- package/dist/containerRuntime.js.map +1 -1
- package/dist/deltaManagerProxies.d.ts.map +1 -1
- package/dist/deltaManagerProxies.js +11 -4
- package/dist/deltaManagerProxies.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +0 -2
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +0 -8
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/legacy.d.ts +3 -1
- package/dist/messageTypes.d.ts +0 -9
- package/dist/messageTypes.d.ts.map +1 -1
- package/dist/messageTypes.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +9 -0
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +19 -6
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/duplicateBatchDetector.d.ts +32 -0
- package/dist/opLifecycle/duplicateBatchDetector.d.ts.map +1 -0
- package/dist/opLifecycle/duplicateBatchDetector.js +68 -0
- package/dist/opLifecycle/duplicateBatchDetector.js.map +1 -0
- package/dist/opLifecycle/index.d.ts +3 -2
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +4 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +0 -4
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +0 -4
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +1 -6
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +1 -4
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +37 -17
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +47 -37
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +27 -17
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +85 -56
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts +2 -4
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +6 -37
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.js +0 -2
- package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts.map +1 -1
- package/dist/summary/summaryCollection.js +5 -7
- package/dist/summary/summaryCollection.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js +1 -4
- package/dist/summary/summaryFormat.js.map +1 -1
- package/lib/channelCollection.d.ts +1 -1
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +1 -16
- package/lib/channelCollection.js.map +1 -1
- package/lib/connectionTelemetry.d.ts +27 -3
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +68 -13
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +262 -181
- package/lib/containerRuntime.js.map +1 -1
- package/lib/deltaManagerProxies.d.ts.map +1 -1
- package/lib/deltaManagerProxies.js +11 -4
- package/lib/deltaManagerProxies.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +0 -2
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +0 -8
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/legacy.d.ts +3 -1
- package/lib/messageTypes.d.ts +0 -9
- package/lib/messageTypes.d.ts.map +1 -1
- package/lib/messageTypes.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +9 -0
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +17 -5
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/duplicateBatchDetector.d.ts +32 -0
- package/lib/opLifecycle/duplicateBatchDetector.d.ts.map +1 -0
- package/lib/opLifecycle/duplicateBatchDetector.js +64 -0
- package/lib/opLifecycle/duplicateBatchDetector.js.map +1 -0
- package/lib/opLifecycle/index.d.ts +3 -2
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +2 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +0 -4
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +0 -4
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +1 -6
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +1 -4
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +37 -17
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +47 -37
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +27 -17
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +85 -56
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts +2 -4
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +6 -37
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.js +0 -2
- package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts.map +1 -1
- package/lib/summary/summaryCollection.js +5 -7
- package/lib/summary/summaryCollection.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js +1 -4
- package/lib/summary/summaryFormat.js.map +1 -1
- package/lib/tsdoc-metadata.json +1 -1
- package/package.json +49 -27
- package/src/channelCollection.ts +7 -21
- package/src/connectionTelemetry.ts +33 -3
- package/src/containerRuntime.ts +382 -233
- package/src/deltaManagerProxies.ts +11 -4
- package/src/gc/garbageCollection.ts +1 -3
- package/src/gc/gcHelpers.ts +4 -12
- package/src/index.ts +2 -0
- package/src/messageTypes.ts +0 -10
- package/src/opLifecycle/batchManager.ts +29 -7
- package/src/opLifecycle/duplicateBatchDetector.ts +78 -0
- package/src/opLifecycle/index.ts +4 -1
- package/src/opLifecycle/opCompressor.ts +2 -6
- package/src/opLifecycle/opGroupingManager.ts +2 -6
- package/src/opLifecycle/opSplitter.ts +2 -6
- package/src/opLifecycle/outbox.ts +1 -3
- package/src/opLifecycle/remoteMessageProcessor.ts +87 -59
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +114 -66
- package/src/scheduleManager.ts +8 -47
- package/src/summary/summarizerNode/summarizerNodeUtils.ts +1 -3
- package/src/summary/summaryCollection.ts +7 -9
- package/src/summary/summaryFormat.ts +1 -3
- package/src/summary/summaryFormats.md +11 -9
- package/tsconfig.json +1 -0
|
@@ -208,10 +208,17 @@ export class DeltaManagerSummarizerProxy extends BaseDeltaManagerProxy {
|
|
|
208
208
|
export class DeltaManagerPendingOpsProxy extends BaseDeltaManagerProxy {
|
|
209
209
|
public get minimumSequenceNumber(): number {
|
|
210
210
|
const minPendingSeqNum = this.pendingStateManager.minimumPendingMessageSequenceNumber;
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
211
|
+
/**
|
|
212
|
+
* The reason why the minimum pending sequence number can be less than the delta manager's minimum sequence
|
|
213
|
+
* number (DM's msn) is that when we are processing messages in the container runtime/delta manager, the delta
|
|
214
|
+
* manager's msn can be updated to continually increase. In the meantime, the pending state manager's op which
|
|
215
|
+
* hasn't been sent can still have a lower sequence number than the DM's msn (think about a disconnected
|
|
216
|
+
* scenario). To successfully resubmit that pending op it has to be rebased first by the DDS. The DDS still
|
|
217
|
+
* needs to keep the local data for that op that has a reference sequence number lower than the DM's msn. To
|
|
218
|
+
* achieve this, the msn passed to the DDS needs to be the minimum of the DM's msn and the minimum pending
|
|
219
|
+
* sequence number, so that it can keep the relevant local data to generate the right data for the new op
|
|
220
|
+
* during resubmission.
|
|
221
|
+
*/
|
|
215
222
|
if (
|
|
216
223
|
minPendingSeqNum !== undefined &&
|
|
217
224
|
minPendingSeqNum < this.deltaManager.minimumSequenceNumber
|
|
@@ -789,9 +789,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
789
789
|
if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
|
|
790
790
|
gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
|
|
791
791
|
} else {
|
|
792
|
-
|
|
793
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
794
|
-
gcDataSuperSet.gcNodes[sourceNodeId]!.push(...outboundRoutes);
|
|
792
|
+
gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
|
|
795
793
|
}
|
|
796
794
|
newOutboundRoutesSinceLastRun.push(...outboundRoutes);
|
|
797
795
|
},
|
package/src/gc/gcHelpers.ts
CHANGED
|
@@ -166,9 +166,7 @@ export function concatGarbageCollectionData(
|
|
|
166
166
|
if (combinedGCData.gcNodes[id] === undefined) {
|
|
167
167
|
combinedGCData.gcNodes[id] = Array.from(routes);
|
|
168
168
|
} else {
|
|
169
|
-
|
|
170
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
171
|
-
const combinedRoutes = [...routes, ...combinedGCData.gcNodes[id]!];
|
|
169
|
+
const combinedRoutes = [...routes, ...combinedGCData.gcNodes[id]];
|
|
172
170
|
combinedGCData.gcNodes[id] = [...new Set(combinedRoutes)];
|
|
173
171
|
}
|
|
174
172
|
}
|
|
@@ -189,17 +187,13 @@ export async function getGCDataFromSnapshot(
|
|
|
189
187
|
for (const key of Object.keys(gcSnapshotTree.blobs)) {
|
|
190
188
|
// Update deleted nodes blob.
|
|
191
189
|
if (key === gcDeletedBlobKey) {
|
|
192
|
-
|
|
193
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
194
|
-
deletedNodes = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]!);
|
|
190
|
+
deletedNodes = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]);
|
|
195
191
|
continue;
|
|
196
192
|
}
|
|
197
193
|
|
|
198
194
|
// Update tombstone blob.
|
|
199
195
|
if (key === gcTombstoneBlobKey) {
|
|
200
|
-
|
|
201
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
202
|
-
tombstones = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]!);
|
|
196
|
+
tombstones = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]);
|
|
203
197
|
continue;
|
|
204
198
|
}
|
|
205
199
|
|
|
@@ -302,9 +296,7 @@ function trimLeadingAndTrailingSlashes(str: string) {
|
|
|
302
296
|
|
|
303
297
|
/** Reformats a request URL to match expected format for a GC node path */
|
|
304
298
|
export function urlToGCNodePath(url: string): string {
|
|
305
|
-
|
|
306
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
307
|
-
return `/${trimLeadingAndTrailingSlashes(url.split("?")[0]!)}`;
|
|
299
|
+
return `/${trimLeadingAndTrailingSlashes(url.split("?")[0])}`;
|
|
308
300
|
}
|
|
309
301
|
|
|
310
302
|
/**
|
package/src/index.ts
CHANGED
package/src/messageTypes.ts
CHANGED
|
@@ -223,16 +223,6 @@ export type InboundSequencedContainerRuntimeMessage = Omit<
|
|
|
223
223
|
> &
|
|
224
224
|
InboundContainerRuntimeMessage;
|
|
225
225
|
|
|
226
|
-
/** Essentially ISequencedDocumentMessage except that `type` is not `string` to enable narrowing
|
|
227
|
-
* as `Exclude<string, InboundContainerRuntimeMessage['type']>` is not supported.
|
|
228
|
-
* There should never be a runtime value of "__not_a_...".
|
|
229
|
-
* Currently additionally replaces `contents` type until protocol-definitions update is taken with `unknown` instead of `any`.
|
|
230
|
-
*/
|
|
231
|
-
export type InboundSequencedNonContainerRuntimeMessage = Omit<
|
|
232
|
-
ISequencedDocumentMessage,
|
|
233
|
-
"type" | "contents"
|
|
234
|
-
> & { type: "__not_a_container_runtime_message_type__"; contents: unknown };
|
|
235
|
-
|
|
236
226
|
/** A [loose] InboundSequencedContainerRuntimeMessage that is recent and may contain compat details.
|
|
237
227
|
* It exists solely to to provide access to those details.
|
|
238
228
|
*/
|
|
@@ -6,9 +6,11 @@
|
|
|
6
6
|
import { assert } from "@fluidframework/core-utils/internal";
|
|
7
7
|
|
|
8
8
|
import { ICompressionRuntimeOptions } from "../containerRuntime.js";
|
|
9
|
-
import { type IBatchMetadata } from "../metadata.js";
|
|
9
|
+
import { asBatchMetadata, type IBatchMetadata } from "../metadata.js";
|
|
10
|
+
import type { IPendingMessage } from "../pendingStateManager.js";
|
|
10
11
|
|
|
11
12
|
import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.js";
|
|
13
|
+
import type { BatchStartInfo } from "./remoteMessageProcessor.js";
|
|
12
14
|
|
|
13
15
|
export interface IBatchManagerOptions {
|
|
14
16
|
readonly hardLimit: number;
|
|
@@ -33,6 +35,30 @@ export function generateBatchId(originalClientId: string, batchStartCsn: number)
|
|
|
33
35
|
return `${originalClientId}_[${batchStartCsn}]`;
|
|
34
36
|
}
|
|
35
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Get the effective batch ID for the input argument.
|
|
40
|
+
* Supports either an IPendingMessage or BatchStartInfo.
|
|
41
|
+
* If the batch ID is explicitly present, return it.
|
|
42
|
+
* Otherwise, generate a new batch ID using the client ID and batch start CSN.
|
|
43
|
+
*/
|
|
44
|
+
export function getEffectiveBatchId(
|
|
45
|
+
pendingMessageOrBatchStartInfo: IPendingMessage | BatchStartInfo,
|
|
46
|
+
): string {
|
|
47
|
+
if ("localOpMetadata" in pendingMessageOrBatchStartInfo) {
|
|
48
|
+
const pendingMessage: IPendingMessage = pendingMessageOrBatchStartInfo;
|
|
49
|
+
return (
|
|
50
|
+
asBatchMetadata(pendingMessage.opMetadata)?.batchId ??
|
|
51
|
+
generateBatchId(
|
|
52
|
+
pendingMessage.batchInfo.clientId,
|
|
53
|
+
pendingMessage.batchInfo.batchStartCsn,
|
|
54
|
+
)
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const batchStart: BatchStartInfo = pendingMessageOrBatchStartInfo;
|
|
59
|
+
return batchStart.batchId ?? generateBatchId(batchStart.clientId, batchStart.batchStartCsn);
|
|
60
|
+
}
|
|
61
|
+
|
|
36
62
|
/**
|
|
37
63
|
* Estimated size of the stringification overhead for an op accumulated
|
|
38
64
|
* from runtime to loader to the service.
|
|
@@ -64,9 +90,7 @@ export class BatchManager {
|
|
|
64
90
|
private get referenceSequenceNumber(): number | undefined {
|
|
65
91
|
return this.pendingBatch.length === 0
|
|
66
92
|
? undefined
|
|
67
|
-
:
|
|
68
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
69
|
-
this.pendingBatch[this.pendingBatch.length - 1]!.referenceSequenceNumber;
|
|
93
|
+
: this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
|
|
70
94
|
}
|
|
71
95
|
|
|
72
96
|
/**
|
|
@@ -138,9 +162,7 @@ export class BatchManager {
|
|
|
138
162
|
rollback: (process: (message: BatchMessage) => void) => {
|
|
139
163
|
for (let i = this.pendingBatch.length; i > startPoint; ) {
|
|
140
164
|
i--;
|
|
141
|
-
|
|
142
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
143
|
-
const message = this.pendingBatch[i]!;
|
|
165
|
+
const message = this.pendingBatch[i];
|
|
144
166
|
this.batchContentSize -= message.contents?.length ?? 0;
|
|
145
167
|
process(message);
|
|
146
168
|
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { assert } from "@fluidframework/core-utils/internal";
|
|
7
|
+
|
|
8
|
+
import { getEffectiveBatchId } from "./batchManager.js";
|
|
9
|
+
import { type BatchStartInfo } from "./remoteMessageProcessor.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* This class tracks recent batchIds we've seen, and checks incoming batches for duplicates.
|
|
13
|
+
*/
|
|
14
|
+
export class DuplicateBatchDetector {
|
|
15
|
+
/** All batchIds we've seen recently enough (based on MSN) that we need to watch for duplicates */
|
|
16
|
+
private readonly batchIdsAll = new Set<string>();
|
|
17
|
+
|
|
18
|
+
/** We map from sequenceNumber to batchId to find which ones we can stop tracking as MSN advances */
|
|
19
|
+
private readonly batchIdsBySeqNum = new Map<number, string>();
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Records this batch's batchId, and checks if it's a duplicate of a batch we've already seen.
|
|
23
|
+
* If it's a duplicate, also return the sequence number of the other batch for logging.
|
|
24
|
+
*
|
|
25
|
+
* @remarks - We also use the minimumSequenceNumber to clear out old batchIds that are no longer at risk for duplicates.
|
|
26
|
+
*/
|
|
27
|
+
public processInboundBatch(
|
|
28
|
+
batchStart: BatchStartInfo,
|
|
29
|
+
): { duplicate: true; otherSequenceNumber: number } | { duplicate: false } {
|
|
30
|
+
const { sequenceNumber, minimumSequenceNumber } = batchStart.keyMessage;
|
|
31
|
+
|
|
32
|
+
// Glance at this batch's MSN. Any batchIds we're tracking with a lower sequence number are now safe to forget.
|
|
33
|
+
// Why? Because any other client holding the same batch locally would have seen the earlier batch and closed before submitting its duplicate.
|
|
34
|
+
this.clearOldBatchIds(minimumSequenceNumber);
|
|
35
|
+
|
|
36
|
+
// getEffectiveBatchId is only needed in the SUPER rare/surprising case where
|
|
37
|
+
// the original batch (not resubmitted, so no batchId) arrives in parallel with a resubmitted batch.
|
|
38
|
+
// In the presence of typical network conditions, this would not be possible
|
|
39
|
+
// (the original batch should roundtrip WAY before another container could rehydrate, connect, and resubmit)
|
|
40
|
+
const batchId = getEffectiveBatchId(batchStart);
|
|
41
|
+
|
|
42
|
+
// Check this batch against the tracked batchIds to see if it's a duplicate
|
|
43
|
+
if (this.batchIdsAll.has(batchId)) {
|
|
44
|
+
for (const [otherSequenceNumber, otherBatchId] of this.batchIdsBySeqNum.entries()) {
|
|
45
|
+
if (otherBatchId === batchId) {
|
|
46
|
+
return {
|
|
47
|
+
duplicate: true,
|
|
48
|
+
otherSequenceNumber,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
assert(false, 0xa34 /* Should have found the batchId in batchIdBySeqNum map */);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Now we know it's not a duplicate, so add it to the tracked batchIds and return.
|
|
56
|
+
assert(
|
|
57
|
+
!this.batchIdsBySeqNum.has(sequenceNumber),
|
|
58
|
+
0xa35 /* batchIdsAll and batchIdsBySeqNum should be in sync */,
|
|
59
|
+
);
|
|
60
|
+
this.batchIdsBySeqNum.set(sequenceNumber, batchId);
|
|
61
|
+
this.batchIdsAll.add(batchId);
|
|
62
|
+
|
|
63
|
+
return { duplicate: false };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Batches that started before the MSN are not at risk for a sequenced duplicate to arrive,
|
|
68
|
+
* since the batch start has been processed by all clients, and local batches are deduped and the forked client would close.
|
|
69
|
+
*/
|
|
70
|
+
private clearOldBatchIds(msn: number) {
|
|
71
|
+
this.batchIdsBySeqNum.forEach((batchId, sequenceNumber) => {
|
|
72
|
+
if (sequenceNumber < msn) {
|
|
73
|
+
this.batchIdsBySeqNum.delete(sequenceNumber);
|
|
74
|
+
this.batchIdsAll.delete(batchId);
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
package/src/opLifecycle/index.ts
CHANGED
|
@@ -8,17 +8,20 @@ export {
|
|
|
8
8
|
BatchManager,
|
|
9
9
|
BatchSequenceNumbers,
|
|
10
10
|
estimateSocketSize,
|
|
11
|
+
getEffectiveBatchId,
|
|
11
12
|
generateBatchId,
|
|
12
13
|
IBatchManagerOptions,
|
|
13
14
|
} from "./batchManager.js";
|
|
14
15
|
export { BatchMessage, IBatch, IBatchCheckpoint, IChunkedOp } from "./definitions.js";
|
|
16
|
+
export { DuplicateBatchDetector } from "./duplicateBatchDetector.js";
|
|
15
17
|
export { Outbox, getLongStack } from "./outbox.js";
|
|
16
18
|
export { OpCompressor } from "./opCompressor.js";
|
|
17
19
|
export { OpDecompressor } from "./opDecompressor.js";
|
|
18
20
|
export { OpSplitter, splitOp, isChunkedMessage } from "./opSplitter.js";
|
|
19
21
|
export {
|
|
20
22
|
ensureContentsDeserialized,
|
|
21
|
-
|
|
23
|
+
InboundMessageResult,
|
|
24
|
+
BatchStartInfo,
|
|
22
25
|
RemoteMessageProcessor,
|
|
23
26
|
unpackRuntimeMessage,
|
|
24
27
|
} from "./remoteMessageProcessor.js";
|
|
@@ -47,13 +47,9 @@ export class OpCompressor {
|
|
|
47
47
|
|
|
48
48
|
const messages: BatchMessage[] = [];
|
|
49
49
|
messages.push({
|
|
50
|
-
|
|
51
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
52
|
-
...batch.messages[0]!,
|
|
50
|
+
...batch.messages[0],
|
|
53
51
|
contents: JSON.stringify({ packedContents: compressedContent }),
|
|
54
|
-
|
|
55
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
56
|
-
metadata: batch.messages[0]!.metadata,
|
|
52
|
+
metadata: batch.messages[0].metadata,
|
|
57
53
|
compression: CompressionAlgorithms.lz4,
|
|
58
54
|
});
|
|
59
55
|
|
|
@@ -99,9 +99,7 @@ export class OpGroupingManager {
|
|
|
99
99
|
length: batch.messages.length,
|
|
100
100
|
threshold: this.config.opCountThreshold,
|
|
101
101
|
reentrant: batch.hasReentrantOps,
|
|
102
|
-
|
|
103
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
104
|
-
referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
|
|
102
|
+
referenceSequenceNumber: batch.messages[0].referenceSequenceNumber,
|
|
105
103
|
});
|
|
106
104
|
}
|
|
107
105
|
// We expect this will be on the first message, if present at all.
|
|
@@ -130,9 +128,7 @@ export class OpGroupingManager {
|
|
|
130
128
|
messages: [
|
|
131
129
|
{
|
|
132
130
|
metadata: { batchId: groupedBatchId },
|
|
133
|
-
|
|
134
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
135
|
-
referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
|
|
131
|
+
referenceSequenceNumber: batch.messages[0].referenceSequenceNumber,
|
|
136
132
|
contents: serializedContent,
|
|
137
133
|
},
|
|
138
134
|
],
|
|
@@ -134,9 +134,7 @@ export class OpSplitter {
|
|
|
134
134
|
0x516 /* Chunk size needs to be smaller than the max batch size */,
|
|
135
135
|
);
|
|
136
136
|
|
|
137
|
-
//
|
|
138
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
139
|
-
const firstMessage = batch.messages[0]!; // we expect this to be the large compressed op, which needs to be split
|
|
137
|
+
const firstMessage = batch.messages[0]; // we expect this to be the large compressed op, which needs to be split
|
|
140
138
|
assert(
|
|
141
139
|
(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,
|
|
142
140
|
0x518 /* First message in the batch needs to be chunkable */,
|
|
@@ -165,9 +163,7 @@ export class OpSplitter {
|
|
|
165
163
|
// The last chunk will be part of the new batch and needs to
|
|
166
164
|
// preserve the batch metadata of the original batch
|
|
167
165
|
const lastChunk = chunkToBatchMessage(
|
|
168
|
-
|
|
169
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
170
|
-
chunks[chunks.length - 1]!,
|
|
166
|
+
chunks[chunks.length - 1],
|
|
171
167
|
batch.referenceSequenceNumber,
|
|
172
168
|
{ batch: firstMessage.metadata?.batch },
|
|
173
169
|
);
|
|
@@ -446,9 +446,7 @@ export class Outbox {
|
|
|
446
446
|
// Legacy path - supporting old loader versions. Can be removed only when LTS moves above
|
|
447
447
|
// version that has support for batches (submitBatchFn)
|
|
448
448
|
assert(
|
|
449
|
-
|
|
450
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
451
|
-
batch.messages[0]!.compression === undefined,
|
|
449
|
+
batch.messages[0].compression === undefined,
|
|
452
450
|
0x5a6 /* Compression should not have happened if the loader does not support it */,
|
|
453
451
|
);
|
|
454
452
|
|
|
@@ -21,26 +21,56 @@ import { OpDecompressor } from "./opDecompressor.js";
|
|
|
21
21
|
import { OpGroupingManager, isGroupedBatch } from "./opGroupingManager.js";
|
|
22
22
|
import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
|
|
23
23
|
|
|
24
|
-
/**
|
|
25
|
-
export interface
|
|
26
|
-
/** Messages in this batch */
|
|
27
|
-
readonly messages: InboundSequencedContainerRuntimeMessage[];
|
|
24
|
+
/** Info about the batch we learn when we process the first message */
|
|
25
|
+
export interface BatchStartInfo {
|
|
28
26
|
/** Batch ID, if present */
|
|
29
27
|
readonly batchId: string | undefined;
|
|
30
28
|
/** clientId that sent this batch. Used to compute Batch ID if needed */
|
|
31
29
|
readonly clientId: string;
|
|
32
30
|
/**
|
|
33
|
-
* Client Sequence Number of the first message in the batch.
|
|
31
|
+
* Client Sequence Number of the Grouped Batch message, or the first message in the ungrouped batch.
|
|
34
32
|
* Used to compute Batch ID if needed
|
|
35
33
|
*
|
|
36
34
|
* @remarks For chunked batches, this is the CSN of the "representative" chunk (the final chunk).
|
|
37
35
|
* For grouped batches, clientSequenceNumber on messages is overwritten, so we track this original value here.
|
|
38
36
|
*/
|
|
39
37
|
readonly batchStartCsn: number;
|
|
40
|
-
/**
|
|
41
|
-
|
|
38
|
+
/**
|
|
39
|
+
* The first message in the batch, or if the batch is empty, the empty grouped batch message.
|
|
40
|
+
* Used for accessing the sequence numbers for the (start of the) batch.
|
|
41
|
+
*
|
|
42
|
+
* @remarks Do not use clientSequenceNumber here, use batchStartCsn instead.
|
|
43
|
+
*/
|
|
44
|
+
readonly keyMessage: ISequencedDocumentMessage;
|
|
42
45
|
}
|
|
43
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Result of processing the next inbound message.
|
|
49
|
+
* Depending on the message and configuration of RemoteMessageProcessor, the result may be:
|
|
50
|
+
* - A full batch of messages (including a single-message batch)
|
|
51
|
+
* - The first message of a multi-message batch
|
|
52
|
+
* - The next message in a multi-message batch
|
|
53
|
+
*/
|
|
54
|
+
export type InboundMessageResult =
|
|
55
|
+
| {
|
|
56
|
+
type: "fullBatch";
|
|
57
|
+
messages: InboundSequencedContainerRuntimeMessage[];
|
|
58
|
+
batchStart: BatchStartInfo;
|
|
59
|
+
length: number;
|
|
60
|
+
}
|
|
61
|
+
| {
|
|
62
|
+
type: "batchStartingMessage";
|
|
63
|
+
batchStart: BatchStartInfo;
|
|
64
|
+
nextMessage: InboundSequencedContainerRuntimeMessage;
|
|
65
|
+
length?: never;
|
|
66
|
+
}
|
|
67
|
+
| {
|
|
68
|
+
type: "nextBatchMessage";
|
|
69
|
+
batchEnd?: boolean;
|
|
70
|
+
nextMessage: InboundSequencedContainerRuntimeMessage;
|
|
71
|
+
length?: never;
|
|
72
|
+
};
|
|
73
|
+
|
|
44
74
|
function assertHasClientId(
|
|
45
75
|
message: ISequencedDocumentMessage,
|
|
46
76
|
): asserts message is ISequencedDocumentMessage & { clientId: string } {
|
|
@@ -57,12 +87,7 @@ function assertHasClientId(
|
|
|
57
87
|
* @internal
|
|
58
88
|
*/
|
|
59
89
|
export class RemoteMessageProcessor {
|
|
60
|
-
|
|
61
|
-
* The current batch being received, with details needed to process it.
|
|
62
|
-
*
|
|
63
|
-
* @remarks If undefined, we are expecting the next message to start a new batch.
|
|
64
|
-
*/
|
|
65
|
-
private batchInProgress: InboundBatch | undefined;
|
|
90
|
+
private batchInProgress: boolean = false;
|
|
66
91
|
|
|
67
92
|
constructor(
|
|
68
93
|
private readonly opSplitter: OpSplitter,
|
|
@@ -100,7 +125,7 @@ export class RemoteMessageProcessor {
|
|
|
100
125
|
public process(
|
|
101
126
|
remoteMessageCopy: ISequencedDocumentMessage,
|
|
102
127
|
logLegacyCase: (codePath: string) => void,
|
|
103
|
-
):
|
|
128
|
+
): InboundMessageResult | undefined {
|
|
104
129
|
let message = remoteMessageCopy;
|
|
105
130
|
|
|
106
131
|
assertHasClientId(message);
|
|
@@ -129,80 +154,84 @@ export class RemoteMessageProcessor {
|
|
|
129
154
|
}
|
|
130
155
|
|
|
131
156
|
if (isGroupedBatch(message)) {
|
|
132
|
-
// We should be awaiting a new batch (batchInProgress
|
|
133
|
-
assert(
|
|
134
|
-
this.batchInProgress === undefined,
|
|
135
|
-
0x9d3 /* Grouped batch interrupting another batch */,
|
|
136
|
-
);
|
|
157
|
+
// We should be awaiting a new batch (batchInProgress false)
|
|
158
|
+
assert(!this.batchInProgress, 0x9d3 /* Grouped batch interrupting another batch */);
|
|
137
159
|
const batchId = asBatchMetadata(message.metadata)?.batchId;
|
|
138
160
|
const groupedMessages = this.opGroupingManager.ungroupOp(message).map(unpack);
|
|
161
|
+
|
|
139
162
|
return {
|
|
163
|
+
type: "fullBatch",
|
|
140
164
|
messages: groupedMessages, // Will be [] for an empty batch
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
165
|
+
batchStart: {
|
|
166
|
+
batchStartCsn: message.clientSequenceNumber,
|
|
167
|
+
clientId,
|
|
168
|
+
batchId,
|
|
169
|
+
keyMessage: groupedMessages[0] ?? message, // For an empty batch, this is the empty grouped batch message. Needed for sequence numbers for this batch
|
|
170
|
+
},
|
|
171
|
+
length: groupedMessages.length, // Will be 0 for an empty batch
|
|
147
172
|
};
|
|
148
173
|
}
|
|
149
174
|
|
|
150
175
|
// Do a final unpack of runtime messages in case the message was not grouped, compressed, or chunked
|
|
151
176
|
unpackRuntimeMessage(message, logLegacyCase);
|
|
152
177
|
|
|
153
|
-
|
|
178
|
+
return this.getResultBasedOnBatchMetadata(
|
|
154
179
|
message as InboundSequencedContainerRuntimeMessage & { clientId: string },
|
|
155
180
|
);
|
|
156
|
-
|
|
157
|
-
if (!batchEnded) {
|
|
158
|
-
// batch not yet complete
|
|
159
|
-
return undefined;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
const completedBatch = this.batchInProgress;
|
|
163
|
-
this.batchInProgress = undefined;
|
|
164
|
-
return completedBatch;
|
|
165
181
|
}
|
|
166
182
|
|
|
167
183
|
/**
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
* @returns batchEnded: true if the batch is now complete, batchEnded: false if more messages are expected
|
|
184
|
+
* Now that the message has been "unwrapped" as to any virtualization (grouping, compression, chunking),
|
|
185
|
+
* inspect the batch metadata flag and determine what kind of result to return.
|
|
171
186
|
*/
|
|
172
|
-
private
|
|
187
|
+
private getResultBasedOnBatchMetadata(
|
|
173
188
|
message: InboundSequencedContainerRuntimeMessage & { clientId: string },
|
|
174
|
-
):
|
|
189
|
+
): InboundMessageResult {
|
|
175
190
|
const batchMetadataFlag = asBatchMetadata(message.metadata)?.batch;
|
|
176
|
-
if (this.batchInProgress
|
|
191
|
+
if (!this.batchInProgress) {
|
|
177
192
|
// We are waiting for a new batch
|
|
178
193
|
assert(batchMetadataFlag !== false, 0x9d5 /* Unexpected batch end marker */);
|
|
179
194
|
|
|
180
195
|
// Start of a new multi-message batch
|
|
181
196
|
if (batchMetadataFlag === true) {
|
|
182
|
-
this.batchInProgress =
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
197
|
+
this.batchInProgress = true;
|
|
198
|
+
return {
|
|
199
|
+
type: "batchStartingMessage",
|
|
200
|
+
batchStart: {
|
|
201
|
+
batchId: asBatchMetadata(message.metadata)?.batchId,
|
|
202
|
+
clientId: message.clientId,
|
|
203
|
+
batchStartCsn: message.clientSequenceNumber,
|
|
204
|
+
keyMessage: message,
|
|
205
|
+
},
|
|
206
|
+
nextMessage: message,
|
|
187
207
|
};
|
|
188
|
-
|
|
189
|
-
return { batchEnded: false };
|
|
190
208
|
}
|
|
191
209
|
|
|
192
210
|
// Single-message batch (Since metadata flag is undefined)
|
|
193
|
-
|
|
211
|
+
return {
|
|
212
|
+
type: "fullBatch",
|
|
194
213
|
messages: [message],
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
214
|
+
batchStart: {
|
|
215
|
+
batchStartCsn: message.clientSequenceNumber,
|
|
216
|
+
clientId: message.clientId,
|
|
217
|
+
batchId: asBatchMetadata(message.metadata)?.batchId,
|
|
218
|
+
keyMessage: message,
|
|
219
|
+
},
|
|
220
|
+
length: 1,
|
|
198
221
|
};
|
|
199
|
-
return { batchEnded: true };
|
|
200
222
|
}
|
|
201
223
|
assert(batchMetadataFlag !== true, 0x9d6 /* Unexpected batch start marker */);
|
|
202
224
|
|
|
203
|
-
|
|
225
|
+
// Clear batchInProgress state if the batch is ending
|
|
226
|
+
if (batchMetadataFlag === false) {
|
|
227
|
+
this.batchInProgress = false;
|
|
228
|
+
}
|
|
204
229
|
|
|
205
|
-
return {
|
|
230
|
+
return {
|
|
231
|
+
type: "nextBatchMessage",
|
|
232
|
+
nextMessage: message,
|
|
233
|
+
batchEnd: batchMetadataFlag === false,
|
|
234
|
+
};
|
|
206
235
|
}
|
|
207
236
|
}
|
|
208
237
|
|
|
@@ -217,10 +246,9 @@ export function ensureContentsDeserialized(
|
|
|
217
246
|
hasModernRuntimeMessageEnvelope: boolean,
|
|
218
247
|
logLegacyCase: (codePath: string) => void,
|
|
219
248
|
): void {
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
//
|
|
223
|
-
// Only hasModernRuntimeMessageEnvelope true will be expected to have JSON contents.
|
|
249
|
+
// This should become unconditional once (Loader LTS) DeltaManager.processInboundMessage() stops parsing content (ADO #12052)
|
|
250
|
+
// Note: Until that change is made in the loader, this case will never be hit.
|
|
251
|
+
// Then there will be a long time of needing both cases, until LTS catches up to the change.
|
|
224
252
|
let didParseJsonContents: boolean;
|
|
225
253
|
if (typeof mutableMessage.contents === "string" && mutableMessage.contents !== "") {
|
|
226
254
|
mutableMessage.contents = JSON.parse(mutableMessage.contents);
|
package/src/packageVersion.ts
CHANGED