@fluidframework/container-runtime 2.4.0-299374 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +10 -4
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +6 -0
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +44 -69
- package/dist/containerRuntime.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +1 -1
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +2 -6
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +2 -0
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +6 -0
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +25 -11
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +4 -4
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +6 -18
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +8 -2
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +14 -7
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts.map +1 -1
- package/dist/summary/summaryCollection.js +3 -4
- package/dist/summary/summaryCollection.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +10 -4
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +6 -0
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +42 -68
- package/lib/containerRuntime.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +1 -1
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +3 -7
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +2 -0
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +6 -0
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +23 -10
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +4 -4
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +6 -18
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +8 -2
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +14 -7
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts.map +1 -1
- package/lib/summary/summaryCollection.js +3 -4
- package/lib/summary/summaryCollection.js.map +1 -1
- package/package.json +20 -18
- package/src/blobManager/blobManager.ts +10 -4
- package/src/containerRuntime.ts +67 -85
- package/src/gc/garbageCollection.ts +5 -12
- package/src/opLifecycle/batchManager.ts +3 -0
- package/src/opLifecycle/index.ts +1 -1
- package/src/opLifecycle/outbox.ts +32 -10
- package/src/opLifecycle/remoteMessageProcessor.ts +8 -22
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +26 -8
- package/src/summary/summaryCollection.ts +3 -4
package/src/containerRuntime.ts
CHANGED
|
@@ -102,7 +102,10 @@ import {
|
|
|
102
102
|
responseToException,
|
|
103
103
|
seqFromTree,
|
|
104
104
|
} from "@fluidframework/runtime-utils/internal";
|
|
105
|
-
import type {
|
|
105
|
+
import type {
|
|
106
|
+
IFluidErrorBase,
|
|
107
|
+
ITelemetryGenericEventExt,
|
|
108
|
+
} from "@fluidframework/telemetry-utils/internal";
|
|
106
109
|
import {
|
|
107
110
|
ITelemetryLoggerExt,
|
|
108
111
|
DataCorruptionError,
|
|
@@ -167,7 +170,7 @@ import {
|
|
|
167
170
|
type OutboundContainerRuntimeMessage,
|
|
168
171
|
type UnknownContainerRuntimeMessage,
|
|
169
172
|
} from "./messageTypes.js";
|
|
170
|
-
import {
|
|
173
|
+
import { ISavedOpMetadata } from "./metadata.js";
|
|
171
174
|
import {
|
|
172
175
|
BatchId,
|
|
173
176
|
BatchMessage,
|
|
@@ -182,6 +185,7 @@ import {
|
|
|
182
185
|
OpSplitter,
|
|
183
186
|
Outbox,
|
|
184
187
|
RemoteMessageProcessor,
|
|
188
|
+
serializeOpContents,
|
|
185
189
|
} from "./opLifecycle/index.js";
|
|
186
190
|
import { pkgVersion } from "./packageVersion.js";
|
|
187
191
|
import {
|
|
@@ -236,19 +240,32 @@ import {
|
|
|
236
240
|
import { Throttler, formExponentialFn } from "./throttler.js";
|
|
237
241
|
|
|
238
242
|
/**
|
|
239
|
-
*
|
|
243
|
+
* Creates an error object to be thrown / passed to Container's close fn in case of an unknown message type.
|
|
240
244
|
* The parameters are typed to support compile-time enforcement of handling all known types/behaviors
|
|
241
245
|
*
|
|
242
|
-
* @param
|
|
246
|
+
* @param unknownContainerRuntimeMessageType - Typed as something unexpected, to ensure all known types have been
|
|
243
247
|
* handled before calling this function (e.g. in a switch statement).
|
|
244
|
-
*
|
|
248
|
+
*
|
|
249
|
+
* @param codePath - The code path where the unexpected message type was encountered.
|
|
250
|
+
*
|
|
251
|
+
* @param sequencedMessage - The sequenced message that contained the unexpected message type.
|
|
252
|
+
*
|
|
245
253
|
*/
|
|
246
|
-
function
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
return
|
|
254
|
+
function getUnknownMessageTypeError(
|
|
255
|
+
unknownContainerRuntimeMessageType: UnknownContainerRuntimeMessage["type"],
|
|
256
|
+
codePath: string,
|
|
257
|
+
sequencedMessage?: ISequencedDocumentMessage,
|
|
258
|
+
): IFluidErrorBase {
|
|
259
|
+
return DataProcessingError.create(
|
|
260
|
+
"Runtime message of unknown type",
|
|
261
|
+
codePath,
|
|
262
|
+
sequencedMessage,
|
|
263
|
+
{
|
|
264
|
+
messageDetails: {
|
|
265
|
+
type: unknownContainerRuntimeMessageType,
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
);
|
|
252
269
|
}
|
|
253
270
|
|
|
254
271
|
/**
|
|
@@ -746,7 +763,7 @@ function lastMessageFromMetadata(metadata: IContainerRuntimeMetadata | undefined
|
|
|
746
763
|
* to understand if/when it is hit.
|
|
747
764
|
* We only want to log this once, to avoid spamming telemetry if we are wrong and these cases are hit commonly.
|
|
748
765
|
*/
|
|
749
|
-
let getSingleUseLegacyLogCallback = (logger: ITelemetryLoggerExt, type: string) => {
|
|
766
|
+
export let getSingleUseLegacyLogCallback = (logger: ITelemetryLoggerExt, type: string) => {
|
|
750
767
|
return (codePath: string) => {
|
|
751
768
|
logger.sendTelemetryEvent({
|
|
752
769
|
eventName: "LegacyMessageFormat",
|
|
@@ -2527,27 +2544,12 @@ export class ContainerRuntime
|
|
|
2527
2544
|
// GC op is only sent in summarizer which should never have stashed ops.
|
|
2528
2545
|
throw new LoggingError("GC op not expected to be stashed in summarizer");
|
|
2529
2546
|
default: {
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
"Stashed runtime message of unexpected type",
|
|
2537
|
-
"applyStashedOp",
|
|
2538
|
-
undefined /* sequencedMessage */,
|
|
2539
|
-
{
|
|
2540
|
-
messageDetails: JSON.stringify({
|
|
2541
|
-
type: opContents.type,
|
|
2542
|
-
compatBehavior,
|
|
2543
|
-
}),
|
|
2544
|
-
},
|
|
2545
|
-
);
|
|
2546
|
-
this.closeFn(error);
|
|
2547
|
-
throw error;
|
|
2548
|
-
}
|
|
2549
|
-
// Note: Even if its compat behavior allows it, we don't know how to apply this stashed op.
|
|
2550
|
-
// All we can do is ignore it (similar to on process).
|
|
2547
|
+
const error = getUnknownMessageTypeError(
|
|
2548
|
+
opContents.type,
|
|
2549
|
+
"applyStashedOp" /* codePath */,
|
|
2550
|
+
);
|
|
2551
|
+
this.closeFn(error);
|
|
2552
|
+
throw error;
|
|
2551
2553
|
}
|
|
2552
2554
|
}
|
|
2553
2555
|
}
|
|
@@ -2709,8 +2711,13 @@ export class ContainerRuntime
|
|
|
2709
2711
|
const savedOp = (messageCopy.metadata as ISavedOpMetadata)?.savedOp;
|
|
2710
2712
|
const logLegacyCase = getSingleUseLegacyLogCallback(this.logger, messageCopy.type);
|
|
2711
2713
|
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
+
let runtimeBatch: boolean =
|
|
2715
|
+
hasModernRuntimeMessageEnvelope || isUnpackedRuntimeMessage(messageCopy);
|
|
2716
|
+
if (runtimeBatch) {
|
|
2717
|
+
// We expect runtime messages to have JSON contents - deserialize it in place.
|
|
2718
|
+
ensureContentsDeserialized(messageCopy);
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2714
2721
|
if (hasModernRuntimeMessageEnvelope) {
|
|
2715
2722
|
// If the message has the modern message envelope, then process it here.
|
|
2716
2723
|
// Here we unpack the message (decompress, unchunk, and/or ungroup) into a batch of messages with ContainerMessageType
|
|
@@ -2748,7 +2755,6 @@ export class ContainerRuntime
|
|
|
2748
2755
|
}
|
|
2749
2756
|
}
|
|
2750
2757
|
|
|
2751
|
-
let runtimeBatch: boolean = true;
|
|
2752
2758
|
// Reach out to PendingStateManager, either to zip localOpMetadata into the *local* message list,
|
|
2753
2759
|
// or to check to ensure the *remote* messages don't match the batchId of a pending local batch.
|
|
2754
2760
|
// This latter case would indicate that the container has forked - two copies are trying to persist the same local changes.
|
|
@@ -2806,12 +2812,23 @@ export class ContainerRuntime
|
|
|
2806
2812
|
runtimeBatch,
|
|
2807
2813
|
);
|
|
2808
2814
|
} else {
|
|
2815
|
+
if (!runtimeBatch) {
|
|
2816
|
+
// The DeltaManager used to do this, but doesn't anymore as of Loader v2.4
|
|
2817
|
+
// Anyone listening to our "op" event would expect the contents to be parsed per this same logic
|
|
2818
|
+
if (
|
|
2819
|
+
typeof messageCopy.contents === "string" &&
|
|
2820
|
+
messageCopy.contents !== "" &&
|
|
2821
|
+
messageCopy.type !== MessageType.ClientLeave
|
|
2822
|
+
) {
|
|
2823
|
+
messageCopy.contents = JSON.parse(messageCopy.contents);
|
|
2824
|
+
}
|
|
2825
|
+
}
|
|
2809
2826
|
this.processInboundMessages(
|
|
2810
2827
|
[{ message: messageCopy, localOpMetadata: undefined }],
|
|
2811
2828
|
{ batchStart: true, batchEnd: true }, // Single message
|
|
2812
2829
|
local,
|
|
2813
2830
|
savedOp,
|
|
2814
|
-
|
|
2831
|
+
runtimeBatch,
|
|
2815
2832
|
);
|
|
2816
2833
|
}
|
|
2817
2834
|
|
|
@@ -2978,27 +2995,13 @@ export class ContainerRuntime
|
|
|
2978
2995
|
);
|
|
2979
2996
|
break;
|
|
2980
2997
|
default: {
|
|
2981
|
-
const
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
{
|
|
2989
|
-
local,
|
|
2990
|
-
messageDetails: JSON.stringify({
|
|
2991
|
-
type: message.type,
|
|
2992
|
-
contentType: typeof message.contents,
|
|
2993
|
-
compatBehavior,
|
|
2994
|
-
batch: (message.metadata as IBatchMetadata | undefined)?.batch,
|
|
2995
|
-
compression: message.compression,
|
|
2996
|
-
}),
|
|
2997
|
-
},
|
|
2998
|
-
);
|
|
2999
|
-
this.closeFn(error);
|
|
3000
|
-
throw error;
|
|
3001
|
-
}
|
|
2998
|
+
const error = getUnknownMessageTypeError(
|
|
2999
|
+
message.type,
|
|
3000
|
+
"validateAndProcessRuntimeMessage" /* codePath */,
|
|
3001
|
+
message,
|
|
3002
|
+
);
|
|
3003
|
+
this.closeFn(error);
|
|
3004
|
+
throw error;
|
|
3002
3005
|
}
|
|
3003
3006
|
}
|
|
3004
3007
|
|
|
@@ -4225,7 +4228,7 @@ export class ContainerRuntime
|
|
|
4225
4228
|
contents: idRange,
|
|
4226
4229
|
};
|
|
4227
4230
|
const idAllocationBatchMessage: BatchMessage = {
|
|
4228
|
-
contents:
|
|
4231
|
+
contents: serializeOpContents(idAllocationMessage),
|
|
4229
4232
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
4230
4233
|
};
|
|
4231
4234
|
this.outbox.submitIdAllocation(idAllocationBatchMessage);
|
|
@@ -4288,13 +4291,13 @@ export class ContainerRuntime
|
|
|
4288
4291
|
contents: schemaChangeMessage,
|
|
4289
4292
|
};
|
|
4290
4293
|
this.outbox.submit({
|
|
4291
|
-
contents:
|
|
4294
|
+
contents: serializeOpContents(msg),
|
|
4292
4295
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
4293
4296
|
});
|
|
4294
4297
|
}
|
|
4295
4298
|
|
|
4296
4299
|
const message: BatchMessage = {
|
|
4297
|
-
contents:
|
|
4300
|
+
contents: serializeOpContents(containerRuntimeMessage),
|
|
4298
4301
|
metadata,
|
|
4299
4302
|
localOpMetadata,
|
|
4300
4303
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
@@ -4462,30 +4465,9 @@ export class ContainerRuntime
|
|
|
4462
4465
|
// send any ops, as some other client already changed schema.
|
|
4463
4466
|
break;
|
|
4464
4467
|
default: {
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
|
|
4469
|
-
// We do not ultimately resubmit it, to be consistent with this version of the code.
|
|
4470
|
-
this.logger.sendTelemetryEvent({
|
|
4471
|
-
eventName: "resubmitUnrecognizedMessageTypeAllowed",
|
|
4472
|
-
messageDetails: { type: message.type, compatBehavior },
|
|
4473
|
-
});
|
|
4474
|
-
} else {
|
|
4475
|
-
const error = DataProcessingError.create(
|
|
4476
|
-
"Resubmitting runtime message of unexpected type",
|
|
4477
|
-
"reSubmitCore",
|
|
4478
|
-
undefined /* sequencedMessage */,
|
|
4479
|
-
{
|
|
4480
|
-
messageDetails: JSON.stringify({
|
|
4481
|
-
type: message.type,
|
|
4482
|
-
compatBehavior,
|
|
4483
|
-
}),
|
|
4484
|
-
},
|
|
4485
|
-
);
|
|
4486
|
-
this.closeFn(error);
|
|
4487
|
-
throw error;
|
|
4488
|
-
}
|
|
4468
|
+
const error = getUnknownMessageTypeError(message.type, "reSubmitCore" /* codePath */);
|
|
4469
|
+
this.closeFn(error);
|
|
4470
|
+
throw error;
|
|
4489
4471
|
}
|
|
4490
4472
|
}
|
|
4491
4473
|
}
|
|
@@ -51,7 +51,6 @@ import {
|
|
|
51
51
|
} from "./gcDefinitions.js";
|
|
52
52
|
import {
|
|
53
53
|
cloneGCData,
|
|
54
|
-
compatBehaviorAllowsGCMessageType,
|
|
55
54
|
concatGarbageCollectionData,
|
|
56
55
|
dataStoreNodePathOnly,
|
|
57
56
|
getGCDataFromSnapshot,
|
|
@@ -896,16 +895,10 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
896
895
|
break;
|
|
897
896
|
}
|
|
898
897
|
default: {
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
`Garbage collection message of unknown type ${gcMessageType}`,
|
|
904
|
-
"processMessage",
|
|
905
|
-
);
|
|
906
|
-
throw error;
|
|
907
|
-
}
|
|
908
|
-
break;
|
|
898
|
+
throw DataProcessingError.create(
|
|
899
|
+
`Garbage collection message of unknown type ${gcMessageType}`,
|
|
900
|
+
"processMessage",
|
|
901
|
+
);
|
|
909
902
|
}
|
|
910
903
|
}
|
|
911
904
|
}
|
|
@@ -1034,7 +1027,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1034
1027
|
*
|
|
1035
1028
|
* Submit a GC op indicating that the Tombstone with the given path has been loaded.
|
|
1036
1029
|
* Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
|
|
1037
|
-
* before
|
|
1030
|
+
* before running GC next.
|
|
1038
1031
|
*/
|
|
1039
1032
|
private triggerAutoRecovery(nodePath: string) {
|
|
1040
1033
|
// If sweep isn't enabled, auto-recovery isn't needed since its purpose is to prevent this object from being
|
|
@@ -20,6 +20,9 @@ export interface IBatchManagerOptions {
|
|
|
20
20
|
* If true, the outbox is allowed to rebase the batch during flushing.
|
|
21
21
|
*/
|
|
22
22
|
readonly canRebase: boolean;
|
|
23
|
+
|
|
24
|
+
/** If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored */
|
|
25
|
+
readonly ignoreBatchId?: boolean;
|
|
23
26
|
}
|
|
24
27
|
|
|
25
28
|
export interface BatchSequenceNumbers {
|
package/src/opLifecycle/index.ts
CHANGED
|
@@ -14,7 +14,7 @@ export {
|
|
|
14
14
|
} from "./batchManager.js";
|
|
15
15
|
export { BatchMessage, IBatch, IBatchCheckpoint, IChunkedOp } from "./definitions.js";
|
|
16
16
|
export { DuplicateBatchDetector } from "./duplicateBatchDetector.js";
|
|
17
|
-
export { Outbox, getLongStack } from "./outbox.js";
|
|
17
|
+
export { Outbox, getLongStack, serializeOpContents } from "./outbox.js";
|
|
18
18
|
export { OpCompressor } from "./opCompressor.js";
|
|
19
19
|
export { OpDecompressor } from "./opDecompressor.js";
|
|
20
20
|
export { OpSplitter, splitOp, isChunkedMessage } from "./opSplitter.js";
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
} from "@fluidframework/telemetry-utils/internal";
|
|
16
16
|
|
|
17
17
|
import { ICompressionRuntimeOptions } from "../containerRuntime.js";
|
|
18
|
+
import { OutboundContainerRuntimeMessage } from "../messageTypes.js";
|
|
18
19
|
import { PendingMessageResubmitData, PendingStateManager } from "../pendingStateManager.js";
|
|
19
20
|
|
|
20
21
|
import {
|
|
@@ -28,6 +29,8 @@ import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.js";
|
|
|
28
29
|
import { OpCompressor } from "./opCompressor.js";
|
|
29
30
|
import { OpGroupingManager } from "./opGroupingManager.js";
|
|
30
31
|
import { OpSplitter } from "./opSplitter.js";
|
|
32
|
+
// eslint-disable-next-line unused-imports/no-unused-imports -- Used by "@link" comment annotation below
|
|
33
|
+
import { ensureContentsDeserialized } from "./remoteMessageProcessor.js";
|
|
31
34
|
|
|
32
35
|
export interface IOutboxConfig {
|
|
33
36
|
readonly compressionOptions: ICompressionRuntimeOptions;
|
|
@@ -54,6 +57,14 @@ export interface IOutboxParameters {
|
|
|
54
57
|
readonly closeContainer: (error?: ICriticalContainerError) => void;
|
|
55
58
|
}
|
|
56
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Before submitting an op to the Outbox, its contents must be serialized using this function.
|
|
62
|
+
* @remarks - The deserialization on process happens via the function {@link ensureContentsDeserialized}.
|
|
63
|
+
*/
|
|
64
|
+
export function serializeOpContents(contents: OutboundContainerRuntimeMessage): string {
|
|
65
|
+
return JSON.stringify(contents);
|
|
66
|
+
}
|
|
67
|
+
|
|
57
68
|
/**
|
|
58
69
|
* Temporarily increase the stack limit while executing the provided action.
|
|
59
70
|
* If a negative value is provided for `length`, no stack frames will be collected.
|
|
@@ -123,7 +134,11 @@ export class Outbox {
|
|
|
123
134
|
|
|
124
135
|
this.mainBatch = new BatchManager({ hardLimit, canRebase: true });
|
|
125
136
|
this.blobAttachBatch = new BatchManager({ hardLimit, canRebase: true });
|
|
126
|
-
this.idAllocationBatch = new BatchManager({
|
|
137
|
+
this.idAllocationBatch = new BatchManager({
|
|
138
|
+
hardLimit,
|
|
139
|
+
canRebase: false,
|
|
140
|
+
ignoreBatchId: true,
|
|
141
|
+
});
|
|
127
142
|
}
|
|
128
143
|
|
|
129
144
|
public get messageCount(): number {
|
|
@@ -251,17 +266,20 @@ export class Outbox {
|
|
|
251
266
|
}
|
|
252
267
|
|
|
253
268
|
private flushAll(resubmittingBatchId?: BatchId) {
|
|
254
|
-
//
|
|
255
|
-
//
|
|
256
|
-
this
|
|
257
|
-
//
|
|
258
|
-
//
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
if (resubmittingBatchId &&
|
|
269
|
+
// If we're resubmitting and all batches are empty, we need to flush an empty batch.
|
|
270
|
+
// Note that we currently resubmit one batch at a time, so on resubmit, 2 of the 3 batches will *always* be empty.
|
|
271
|
+
// It's theoretically possible that we don't *need* to resubmit this empty batch, and in those cases, it'll safely be ignored
|
|
272
|
+
// by the rest of the system, including remote clients.
|
|
273
|
+
// In some cases we *must* resubmit the empty batch (to match up with a non-empty version tracked locally by a container fork), so we do it always.
|
|
274
|
+
const allBatchesEmpty =
|
|
275
|
+
this.idAllocationBatch.empty && this.blobAttachBatch.empty && this.mainBatch.empty;
|
|
276
|
+
if (resubmittingBatchId && allBatchesEmpty) {
|
|
262
277
|
this.flushEmptyBatch(resubmittingBatchId);
|
|
263
278
|
return;
|
|
264
279
|
}
|
|
280
|
+
// Don't use resubmittingBatchId for idAllocationBatch.
|
|
281
|
+
// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
|
|
282
|
+
this.flushInternal(this.idAllocationBatch);
|
|
265
283
|
this.flushInternal(
|
|
266
284
|
this.blobAttachBatch,
|
|
267
285
|
true /* disableGroupedBatching */,
|
|
@@ -332,7 +350,11 @@ export class Outbox {
|
|
|
332
350
|
);
|
|
333
351
|
}
|
|
334
352
|
|
|
335
|
-
this.params.pendingStateManager.onFlushBatch(
|
|
353
|
+
this.params.pendingStateManager.onFlushBatch(
|
|
354
|
+
rawBatch.messages,
|
|
355
|
+
clientSequenceNumber,
|
|
356
|
+
batchManager.options.ignoreBatchId,
|
|
357
|
+
);
|
|
336
358
|
}
|
|
337
359
|
|
|
338
360
|
/**
|
|
@@ -20,6 +20,8 @@ import { asBatchMetadata } from "../metadata.js";
|
|
|
20
20
|
import { OpDecompressor } from "./opDecompressor.js";
|
|
21
21
|
import { OpGroupingManager, isGroupedBatch } from "./opGroupingManager.js";
|
|
22
22
|
import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
|
|
23
|
+
// eslint-disable-next-line unused-imports/no-unused-imports -- Used by "@link" comment annotation below
|
|
24
|
+
import { serializeOpContents } from "./outbox.js";
|
|
23
25
|
|
|
24
26
|
/** Info about the batch we learn when we process the first message */
|
|
25
27
|
export interface BatchStartInfo {
|
|
@@ -236,32 +238,16 @@ export class RemoteMessageProcessor {
|
|
|
236
238
|
}
|
|
237
239
|
|
|
238
240
|
/**
|
|
239
|
-
* Takes an incoming message
|
|
241
|
+
* Takes an incoming runtime message JSON.parse's its contents in place, if needed (old Loader does this for us).
|
|
242
|
+
* Only to be used for runtine messages.
|
|
243
|
+
* @remarks - Serialization during submit happens via {@link serializeOpContents}
|
|
240
244
|
* @param mutableMessage - op message received
|
|
241
|
-
* @param hasModernRuntimeMessageEnvelope - false if the message does not contain the modern op envelop where message.type is MessageType.Operation
|
|
242
|
-
* @param logLegacyCase - callback to log when legacy op is encountered
|
|
243
245
|
*/
|
|
244
|
-
export function ensureContentsDeserialized(
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
logLegacyCase: (codePath: string) => void,
|
|
248
|
-
): void {
|
|
249
|
-
// This should become unconditional once (Loader LTS) DeltaManager.processInboundMessage() stops parsing content (ADO #12052)
|
|
250
|
-
// Note: Until that change is made in the loader, this case will never be hit.
|
|
251
|
-
// Then there will be a long time of needing both cases, until LTS catches up to the change.
|
|
252
|
-
let didParseJsonContents: boolean;
|
|
246
|
+
export function ensureContentsDeserialized(mutableMessage: ISequencedDocumentMessage): void {
|
|
247
|
+
// This should become unconditional once Loader LTS reaches 2.4 or later.
|
|
248
|
+
// There will be a long time of needing both cases, until LTS advances to that point.
|
|
253
249
|
if (typeof mutableMessage.contents === "string" && mutableMessage.contents !== "") {
|
|
254
250
|
mutableMessage.contents = JSON.parse(mutableMessage.contents);
|
|
255
|
-
didParseJsonContents = true;
|
|
256
|
-
} else {
|
|
257
|
-
didParseJsonContents = false;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
// The DeltaManager parses the contents of the message as JSON if it is a string,
|
|
261
|
-
// so we should never end up parsing it here.
|
|
262
|
-
// Let's observe if we are wrong about this to learn about these cases.
|
|
263
|
-
if (didParseJsonContents) {
|
|
264
|
-
logLegacyCase("ensureContentsDeserialized_foundJsonContents");
|
|
265
251
|
}
|
|
266
252
|
}
|
|
267
253
|
|
package/src/packageVersion.ts
CHANGED
|
@@ -41,7 +41,10 @@ export interface IPendingMessage {
|
|
|
41
41
|
localOpMetadata: unknown;
|
|
42
42
|
opMetadata: Record<string, unknown> | undefined;
|
|
43
43
|
sequenceNumber?: number;
|
|
44
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Info about the batch this pending message belongs to, for validation and for computing the batchId on reconnect
|
|
46
|
+
* We don't include batchId itself to avoid redundancy, because that's stamped on opMetadata above
|
|
47
|
+
*/
|
|
45
48
|
batchInfo: {
|
|
46
49
|
/**
|
|
47
50
|
* The Batch's original clientId, from when it was first flushed to be submitted.
|
|
@@ -55,6 +58,8 @@ export interface IPendingMessage {
|
|
|
55
58
|
batchStartCsn: number;
|
|
56
59
|
/** length of the batch (how many runtime messages here) */
|
|
57
60
|
length: number;
|
|
61
|
+
/** If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored */
|
|
62
|
+
ignoreBatchId?: boolean;
|
|
58
63
|
};
|
|
59
64
|
}
|
|
60
65
|
|
|
@@ -239,8 +244,13 @@ export class PendingStateManager implements IDisposable {
|
|
|
239
244
|
* @param batch - The batch that was flushed
|
|
240
245
|
* @param clientSequenceNumber - The CSN of the first message in the batch,
|
|
241
246
|
* or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
|
|
247
|
+
* @param ignoreBatchId - Whether to ignore the batchId in the batchStartInfo
|
|
242
248
|
*/
|
|
243
|
-
public onFlushBatch(
|
|
249
|
+
public onFlushBatch(
|
|
250
|
+
batch: BatchMessage[],
|
|
251
|
+
clientSequenceNumber: number | undefined,
|
|
252
|
+
ignoreBatchId?: boolean,
|
|
253
|
+
) {
|
|
244
254
|
// clientId and batchStartCsn are used for generating the batchId so we can detect container forks
|
|
245
255
|
// where this batch was submitted by two different clients rehydrating from the same local state.
|
|
246
256
|
// In the typical case where the batch was actually sent, use the clientId and clientSequenceNumber.
|
|
@@ -269,7 +279,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
269
279
|
localOpMetadata,
|
|
270
280
|
opMetadata,
|
|
271
281
|
// Note: We only will read this off the first message, but put it on all for simplicity
|
|
272
|
-
batchInfo: { clientId, batchStartCsn, length: batch.length },
|
|
282
|
+
batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId },
|
|
273
283
|
};
|
|
274
284
|
this.pendingMessages.push(pendingMessage);
|
|
275
285
|
}
|
|
@@ -328,15 +338,23 @@ export class PendingStateManager implements IDisposable {
|
|
|
328
338
|
* @returns whether the batch IDs match
|
|
329
339
|
*/
|
|
330
340
|
private remoteBatchMatchesPendingBatch(remoteBatchStart: BatchStartInfo): boolean {
|
|
331
|
-
//
|
|
332
|
-
|
|
333
|
-
|
|
341
|
+
// Find the first pending message that uses Batch ID, to compare to the incoming remote batch.
|
|
342
|
+
// If there is no such message, then the incoming remote batch doesn't have a match here and we can return.
|
|
343
|
+
const firstIndexUsingBatchId = Array.from({
|
|
344
|
+
length: this.pendingMessages.length,
|
|
345
|
+
}).findIndex((_, i) => this.pendingMessages.get(i)?.batchInfo.ignoreBatchId !== true);
|
|
346
|
+
const pendingMessageUsingBatchId =
|
|
347
|
+
firstIndexUsingBatchId === -1
|
|
348
|
+
? undefined
|
|
349
|
+
: this.pendingMessages.get(firstIndexUsingBatchId);
|
|
350
|
+
|
|
351
|
+
if (pendingMessageUsingBatchId === undefined) {
|
|
334
352
|
return false;
|
|
335
353
|
}
|
|
336
354
|
|
|
337
355
|
// We must compare the effective batch IDs, since one of these ops
|
|
338
356
|
// may have been the original, not resubmitted, so wouldn't have its batch ID stamped yet.
|
|
339
|
-
const pendingBatchId = getEffectiveBatchId(
|
|
357
|
+
const pendingBatchId = getEffectiveBatchId(pendingMessageUsingBatchId);
|
|
340
358
|
const inboundBatchId = getEffectiveBatchId(remoteBatchStart);
|
|
341
359
|
|
|
342
360
|
return pendingBatchId === inboundBatchId;
|
|
@@ -488,7 +506,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
488
506
|
0xa21 /* No pending message found as we start processing this remote batch */,
|
|
489
507
|
);
|
|
490
508
|
|
|
491
|
-
// If this batch became empty on resubmit, batch.messages will be empty (
|
|
509
|
+
// If this batch became empty on resubmit, batch.messages will be empty (but keyMessage is always set)
|
|
492
510
|
// and the next pending message should be an empty batch marker.
|
|
493
511
|
// More Info: We must submit empty batches and track them in case a different fork
|
|
494
512
|
// of this container also submitted the same batch (and it may not be empty for that fork).
|
|
@@ -353,11 +353,10 @@ export class SummaryCollection extends TypedEventEmitter<ISummaryCollectionOpEve
|
|
|
353
353
|
}
|
|
354
354
|
|
|
355
355
|
private parseContent(op: ISequencedDocumentMessage) {
|
|
356
|
-
// This should become unconditional once (Loader LTS)
|
|
357
|
-
//
|
|
358
|
-
// Then there will be a long time of needing both cases, until LTS catches up to the change.
|
|
356
|
+
// This should become unconditional once (Loader LTS) reaches 2.4 or later
|
|
357
|
+
// There will be a long time of needing both cases, until LTS catches up to the change.
|
|
359
358
|
// That said, we may instead move to listen for "op" events from ContainerRuntime,
|
|
360
|
-
// and parsing may not be required at all if ContainerRuntime.process()
|
|
359
|
+
// and parsing may not be required at all if ContainerRuntime.process() continues to parse it for all types of ops.
|
|
361
360
|
if (typeof op.contents === "string") {
|
|
362
361
|
op.contents = JSON.parse(op.contents);
|
|
363
362
|
}
|