@fluidframework/container-runtime 2.33.0-333010 → 2.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/api-report/container-runtime.legacy.alpha.api.md +71 -67
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts +7 -4
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +38 -12
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/channelCollection.d.ts +4 -0
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +24 -0
- package/dist/channelCollection.js.map +1 -1
- package/dist/compatUtils.d.ts +74 -0
- package/dist/compatUtils.d.ts.map +1 -0
- package/dist/compatUtils.js +151 -0
- package/dist/compatUtils.js.map +1 -0
- package/dist/compressionDefinitions.d.ts +39 -0
- package/dist/compressionDefinitions.d.ts.map +1 -0
- package/dist/compressionDefinitions.js +30 -0
- package/dist/compressionDefinitions.js.map +1 -0
- package/dist/containerRuntime.d.ts +78 -52
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +141 -54
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +3 -0
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +122 -66
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/deltaManagerProxies.d.ts +55 -12
- package/dist/deltaManagerProxies.d.ts.map +1 -1
- package/dist/deltaManagerProxies.js +63 -55
- package/dist/deltaManagerProxies.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +2 -0
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/legacy.d.ts +1 -0
- package/dist/opLifecycle/batchManager.d.ts +1 -1
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +4 -1
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +35 -4
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.js +2 -2
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +3 -3
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts +2 -2
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +1 -2
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/opSerialization.d.ts +3 -1
- package/dist/opLifecycle/opSerialization.d.ts.map +1 -1
- package/dist/opLifecycle/opSerialization.js +4 -2
- package/dist/opLifecycle/opSerialization.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +6 -3
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +46 -20
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +36 -7
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +83 -16
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/runtimeLayerCompatState.d.ts.map +1 -1
- package/dist/runtimeLayerCompatState.js +1 -1
- package/dist/runtimeLayerCompatState.js.map +1 -1
- package/dist/summary/documentSchema.d.ts +1 -0
- package/dist/summary/documentSchema.d.ts.map +1 -1
- package/dist/summary/documentSchema.js +2 -0
- package/dist/summary/documentSchema.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts +7 -4
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +38 -12
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/channelCollection.d.ts +4 -0
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +24 -0
- package/lib/channelCollection.js.map +1 -1
- package/lib/compatUtils.d.ts +74 -0
- package/lib/compatUtils.d.ts.map +1 -0
- package/lib/compatUtils.js +142 -0
- package/lib/compatUtils.js.map +1 -0
- package/lib/compressionDefinitions.d.ts +39 -0
- package/lib/compressionDefinitions.d.ts.map +1 -0
- package/lib/compressionDefinitions.js +27 -0
- package/lib/compressionDefinitions.js.map +1 -0
- package/lib/containerRuntime.d.ts +78 -52
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +143 -56
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +3 -0
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +57 -1
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/deltaManagerProxies.d.ts +55 -12
- package/lib/deltaManagerProxies.d.ts.map +1 -1
- package/lib/deltaManagerProxies.js +63 -55
- package/lib/deltaManagerProxies.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +2 -0
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/index.d.ts +4 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/legacy.d.ts +1 -0
- package/lib/opLifecycle/batchManager.d.ts +1 -1
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +4 -1
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +35 -4
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.js +1 -1
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +1 -1
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts +2 -2
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +1 -2
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/opSerialization.d.ts +3 -1
- package/lib/opLifecycle/opSerialization.d.ts.map +1 -1
- package/lib/opLifecycle/opSerialization.js +4 -2
- package/lib/opLifecycle/opSerialization.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +6 -3
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +46 -20
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +36 -7
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +84 -17
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/runtimeLayerCompatState.d.ts.map +1 -1
- package/lib/runtimeLayerCompatState.js +2 -2
- package/lib/runtimeLayerCompatState.js.map +1 -1
- package/lib/summary/documentSchema.d.ts +1 -0
- package/lib/summary/documentSchema.d.ts.map +1 -1
- package/lib/summary/documentSchema.js +2 -0
- package/lib/summary/documentSchema.js.map +1 -1
- package/lib/tsdoc-metadata.json +1 -1
- package/package.json +21 -20
- package/src/blobManager/blobManager.ts +48 -15
- package/src/channelCollection.ts +27 -0
- package/src/compatUtils.ts +211 -0
- package/src/compressionDefinitions.ts +47 -0
- package/src/containerRuntime.ts +259 -108
- package/src/dataStoreContext.ts +82 -2
- package/src/deltaManagerProxies.ts +132 -70
- package/src/gc/gcDefinitions.ts +2 -0
- package/src/index.ts +5 -3
- package/src/opLifecycle/batchManager.ts +5 -4
- package/src/opLifecycle/definitions.ts +34 -4
- package/src/opLifecycle/index.ts +1 -0
- package/src/opLifecycle/opCompressor.ts +1 -1
- package/src/opLifecycle/opDecompressor.ts +1 -1
- package/src/opLifecycle/opGroupingManager.ts +7 -5
- package/src/opLifecycle/opSerialization.ts +6 -2
- package/src/opLifecycle/outbox.ts +65 -30
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +135 -21
- package/src/runtimeLayerCompatState.ts +5 -2
- package/src/summary/documentSchema.ts +3 -0
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
type ITelemetryLoggerExt,
|
|
18
18
|
} from "@fluidframework/telemetry-utils/internal";
|
|
19
19
|
|
|
20
|
-
import { ICompressionRuntimeOptions } from "../
|
|
20
|
+
import { ICompressionRuntimeOptions } from "../compressionDefinitions.js";
|
|
21
21
|
import { PendingMessageResubmitData, PendingStateManager } from "../pendingStateManager.js";
|
|
22
22
|
|
|
23
23
|
import {
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
} from "./definitions.js";
|
|
37
37
|
import { OpCompressor } from "./opCompressor.js";
|
|
38
38
|
import { OpGroupingManager } from "./opGroupingManager.js";
|
|
39
|
+
import { serializeOp } from "./opSerialization.js";
|
|
39
40
|
import { OpSplitter } from "./opSplitter.js";
|
|
40
41
|
|
|
41
42
|
export interface IOutboxConfig {
|
|
@@ -111,11 +112,17 @@ export function getLongStack<T>(action: () => T, length: number = 50): T {
|
|
|
111
112
|
/**
|
|
112
113
|
* Convert from local batch to outbound batch, including computing contentSizeInBytes.
|
|
113
114
|
*/
|
|
114
|
-
export function localBatchToOutboundBatch(
|
|
115
|
+
export function localBatchToOutboundBatch({
|
|
116
|
+
staged: _staged, // Peel this off the incoming batch, it's irrelevant (see Note below)
|
|
117
|
+
...localBatch
|
|
118
|
+
}: LocalBatch): OutboundBatch {
|
|
119
|
+
// Note: the staged property might be misleading here, in case a pre-staging batch is resubmitted during staging mode.
|
|
120
|
+
// It will be set to true, but the batch was not actually staged.
|
|
121
|
+
|
|
115
122
|
// Shallow copy each message as we switch types
|
|
116
123
|
const outboundMessages = localBatch.messages.map<OutboundBatchMessage>(
|
|
117
|
-
({
|
|
118
|
-
contents:
|
|
124
|
+
({ runtimeOp, ...message }) => ({
|
|
125
|
+
contents: serializeOp(runtimeOp),
|
|
119
126
|
...message,
|
|
120
127
|
}),
|
|
121
128
|
);
|
|
@@ -266,7 +273,7 @@ export class Outbox {
|
|
|
266
273
|
? "generic"
|
|
267
274
|
: "error",
|
|
268
275
|
eventName: "ReferenceSequenceNumberMismatch",
|
|
269
|
-
|
|
276
|
+
details: {
|
|
270
277
|
expectedDueToReentrancy,
|
|
271
278
|
mainReferenceSequenceNumber: mainBatchSeqNums.referenceSequenceNumber,
|
|
272
279
|
mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
|
|
@@ -330,17 +337,19 @@ export class Outbox {
|
|
|
330
337
|
* @throws If called from a reentrant context, or if the batch being flushed is too large.
|
|
331
338
|
* @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
|
|
332
339
|
* with the given Batch ID, which must be preserved
|
|
340
|
+
* @param resubmittingStagedBatch - If defined, indicates this is a resubmission of a batch that is staged,
|
|
341
|
+
* meaning it should not be sent to the ordering service yet.
|
|
333
342
|
*/
|
|
334
|
-
public flush(resubmittingBatchId?: BatchId): void {
|
|
343
|
+
public flush(resubmittingBatchId?: BatchId, resubmittingStagedBatch?: boolean): void {
|
|
335
344
|
assert(
|
|
336
345
|
!this.isContextReentrant(),
|
|
337
346
|
0xb7b /* Flushing must not happen while incoming changes are being processed */,
|
|
338
347
|
);
|
|
339
348
|
|
|
340
|
-
this.flushAll(resubmittingBatchId);
|
|
349
|
+
this.flushAll(resubmittingBatchId, resubmittingStagedBatch);
|
|
341
350
|
}
|
|
342
351
|
|
|
343
|
-
private flushAll(resubmittingBatchId?: BatchId): void {
|
|
352
|
+
private flushAll(resubmittingBatchId?: BatchId, resubmittingStagedBatch?: boolean): void {
|
|
344
353
|
const allBatchesEmpty =
|
|
345
354
|
this.idAllocationBatch.empty && this.blobAttachBatch.empty && this.mainBatch.empty;
|
|
346
355
|
if (allBatchesEmpty) {
|
|
@@ -350,27 +359,35 @@ export class Outbox {
|
|
|
350
359
|
// by the rest of the system, including remote clients.
|
|
351
360
|
// In some cases we *must* resubmit the empty batch (to match up with a non-empty version tracked locally by a container fork), so we do it always.
|
|
352
361
|
if (resubmittingBatchId) {
|
|
353
|
-
this.flushEmptyBatch(resubmittingBatchId);
|
|
362
|
+
this.flushEmptyBatch(resubmittingBatchId, resubmittingStagedBatch === true);
|
|
354
363
|
}
|
|
355
364
|
return;
|
|
356
365
|
}
|
|
357
366
|
|
|
358
367
|
// Don't use resubmittingBatchId for idAllocationBatch.
|
|
359
368
|
// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
|
|
360
|
-
this.flushInternal(
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
369
|
+
this.flushInternal({
|
|
370
|
+
batchManager: this.idAllocationBatch,
|
|
371
|
+
// Note: For now, we will never stage ID Allocation messages.
|
|
372
|
+
// They won't contain personal info and no harm in extra allocations in case of discarding the staged changes
|
|
373
|
+
});
|
|
374
|
+
this.flushInternal({
|
|
375
|
+
batchManager: this.blobAttachBatch,
|
|
376
|
+
disableGroupedBatching: true,
|
|
364
377
|
resubmittingBatchId,
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
378
|
+
resubmittingStagedBatch,
|
|
379
|
+
});
|
|
380
|
+
this.flushInternal({
|
|
381
|
+
batchManager: this.mainBatch,
|
|
369
382
|
resubmittingBatchId,
|
|
370
|
-
|
|
383
|
+
resubmittingStagedBatch,
|
|
384
|
+
});
|
|
371
385
|
}
|
|
372
386
|
|
|
373
|
-
private flushEmptyBatch(
|
|
387
|
+
private flushEmptyBatch(
|
|
388
|
+
resubmittingBatchId: BatchId,
|
|
389
|
+
resubmittingStagedBatch: boolean,
|
|
390
|
+
): void {
|
|
374
391
|
const referenceSequenceNumber =
|
|
375
392
|
this.params.getCurrentSequenceNumbers().referenceSequenceNumber;
|
|
376
393
|
assert(
|
|
@@ -383,28 +400,42 @@ export class Outbox {
|
|
|
383
400
|
referenceSequenceNumber,
|
|
384
401
|
);
|
|
385
402
|
let clientSequenceNumber: number | undefined;
|
|
386
|
-
if (this.params.shouldSend()) {
|
|
403
|
+
if (this.params.shouldSend() && !resubmittingStagedBatch) {
|
|
387
404
|
clientSequenceNumber = this.sendBatch(outboundBatch);
|
|
388
405
|
}
|
|
389
406
|
|
|
390
407
|
// Push the empty batch placeholder to the PendingStateManager
|
|
391
|
-
this.params.pendingStateManager.
|
|
392
|
-
|
|
408
|
+
this.params.pendingStateManager.onFlushEmptyBatch(
|
|
409
|
+
placeholderMessage,
|
|
393
410
|
clientSequenceNumber,
|
|
411
|
+
resubmittingStagedBatch,
|
|
394
412
|
);
|
|
395
413
|
return;
|
|
396
414
|
}
|
|
397
415
|
|
|
398
|
-
private flushInternal(
|
|
399
|
-
batchManager: BatchManager
|
|
400
|
-
disableGroupedBatching
|
|
401
|
-
resubmittingBatchId?: BatchId
|
|
402
|
-
|
|
416
|
+
private flushInternal(params: {
|
|
417
|
+
batchManager: BatchManager;
|
|
418
|
+
disableGroupedBatching?: boolean;
|
|
419
|
+
resubmittingBatchId?: BatchId; // undefined if not resubmitting
|
|
420
|
+
resubmittingStagedBatch?: boolean; // undefined if not resubmitting
|
|
421
|
+
}): void {
|
|
422
|
+
const {
|
|
423
|
+
batchManager,
|
|
424
|
+
disableGroupedBatching = false,
|
|
425
|
+
resubmittingBatchId,
|
|
426
|
+
resubmittingStagedBatch,
|
|
427
|
+
} = params;
|
|
403
428
|
if (batchManager.empty) {
|
|
404
429
|
return;
|
|
405
430
|
}
|
|
406
431
|
|
|
407
432
|
const rawBatch = batchManager.popBatch(resubmittingBatchId);
|
|
433
|
+
|
|
434
|
+
// When resubmitting, we respect the staged state of the original batch.
|
|
435
|
+
// In this case rawBatch.staged will match the state of inStagingMode when
|
|
436
|
+
// the resubmit occurred, which is not relevant.
|
|
437
|
+
const staged = resubmittingStagedBatch ?? rawBatch.staged === true;
|
|
438
|
+
|
|
408
439
|
const groupingEnabled =
|
|
409
440
|
!disableGroupedBatching && this.params.groupingManager.groupedBatchingEnabled();
|
|
410
441
|
if (
|
|
@@ -419,6 +450,9 @@ export class Outbox {
|
|
|
419
450
|
// If a batch contains reentrant ops (ops created as a result from processing another op)
|
|
420
451
|
// it needs to be rebased so that we can ensure consistent reference sequence numbers
|
|
421
452
|
// and eventual consistency at the DDS level.
|
|
453
|
+
// Note: Since this is happening in the same turn the ops were originally created with,
|
|
454
|
+
// and they haven't gone to PendingStateManager yet, we can just let them respect
|
|
455
|
+
// ContainerRuntime.inStagingMode
|
|
422
456
|
this.rebase(rawBatch, batchManager);
|
|
423
457
|
return;
|
|
424
458
|
}
|
|
@@ -427,7 +461,7 @@ export class Outbox {
|
|
|
427
461
|
// Did we disconnect? (i.e. is shouldSend false?)
|
|
428
462
|
// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
|
|
429
463
|
// Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
|
|
430
|
-
if (this.params.shouldSend()) {
|
|
464
|
+
if (this.params.shouldSend() && !staged) {
|
|
431
465
|
const virtualizedBatch = this.virtualizeBatch(rawBatch, groupingEnabled);
|
|
432
466
|
|
|
433
467
|
clientSequenceNumber = this.sendBatch(virtualizedBatch);
|
|
@@ -440,6 +474,7 @@ export class Outbox {
|
|
|
440
474
|
this.params.pendingStateManager.onFlushBatch(
|
|
441
475
|
rawBatch.messages,
|
|
442
476
|
clientSequenceNumber,
|
|
477
|
+
staged,
|
|
443
478
|
batchManager.options.ignoreBatchId,
|
|
444
479
|
);
|
|
445
480
|
}
|
|
@@ -457,7 +492,7 @@ export class Outbox {
|
|
|
457
492
|
this.rebasing = true;
|
|
458
493
|
for (const message of rawBatch.messages) {
|
|
459
494
|
this.params.reSubmit({
|
|
460
|
-
|
|
495
|
+
runtimeOp: message.runtimeOp,
|
|
461
496
|
localOpMetadata: message.localOpMetadata,
|
|
462
497
|
opMetadata: message.metadata,
|
|
463
498
|
});
|
|
@@ -475,7 +510,7 @@ export class Outbox {
|
|
|
475
510
|
this.batchRebasesToReport--;
|
|
476
511
|
}
|
|
477
512
|
|
|
478
|
-
this.flushInternal(batchManager);
|
|
513
|
+
this.flushInternal({ batchManager });
|
|
479
514
|
this.rebasing = false;
|
|
480
515
|
}
|
|
481
516
|
|
package/src/packageVersion.ts
CHANGED
|
@@ -27,6 +27,8 @@ import {
|
|
|
27
27
|
getEffectiveBatchId,
|
|
28
28
|
BatchStartInfo,
|
|
29
29
|
InboundMessageResult,
|
|
30
|
+
serializeOp,
|
|
31
|
+
type LocalEmptyBatchPlaceholder,
|
|
30
32
|
} from "./opLifecycle/index.js";
|
|
31
33
|
|
|
32
34
|
/**
|
|
@@ -38,7 +40,15 @@ import {
|
|
|
38
40
|
export interface IPendingMessage {
|
|
39
41
|
type: "message";
|
|
40
42
|
referenceSequenceNumber: number;
|
|
43
|
+
/**
|
|
44
|
+
* Serialized copy of runtimeOp
|
|
45
|
+
*/
|
|
41
46
|
content: string;
|
|
47
|
+
/**
|
|
48
|
+
* The original runtime op that was submitted to the ContainerRuntime
|
|
49
|
+
* Unless this pending message came from stashed content, in which case this was roundtripped through string
|
|
50
|
+
*/
|
|
51
|
+
runtimeOp?: LocalContainerRuntimeMessage | undefined; // Undefined for empty batches and initial messages before parsing
|
|
42
52
|
localOpMetadata: unknown;
|
|
43
53
|
opMetadata: Record<string, unknown> | undefined;
|
|
44
54
|
sequenceNumber?: number;
|
|
@@ -65,6 +75,10 @@ export interface IPendingMessage {
|
|
|
65
75
|
* If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored
|
|
66
76
|
*/
|
|
67
77
|
ignoreBatchId?: boolean;
|
|
78
|
+
/**
|
|
79
|
+
* If true, this batch is staged and should not actually be submitted on replayPendingStates.
|
|
80
|
+
*/
|
|
81
|
+
staged: boolean;
|
|
68
82
|
};
|
|
69
83
|
}
|
|
70
84
|
|
|
@@ -94,14 +108,17 @@ export interface IPendingLocalState {
|
|
|
94
108
|
*/
|
|
95
109
|
export type PendingMessageResubmitData = Pick<
|
|
96
110
|
IPendingMessage,
|
|
97
|
-
"
|
|
98
|
-
|
|
111
|
+
"runtimeOp" | "localOpMetadata" | "opMetadata"
|
|
112
|
+
> & {
|
|
113
|
+
// Required (it's only missing on IPendingMessage for empty batch, which will be resubmitted as an empty array)
|
|
114
|
+
runtimeOp: LocalContainerRuntimeMessage;
|
|
115
|
+
};
|
|
99
116
|
|
|
100
117
|
export interface IRuntimeStateHandler {
|
|
101
118
|
connected(): boolean;
|
|
102
119
|
clientId(): string | undefined;
|
|
103
|
-
applyStashedOp(
|
|
104
|
-
reSubmitBatch(batch: PendingMessageResubmitData[], batchId: BatchId): void;
|
|
120
|
+
applyStashedOp(serializedOp: string): Promise<unknown>;
|
|
121
|
+
reSubmitBatch(batch: PendingMessageResubmitData[], batchId: BatchId, staged: boolean): void;
|
|
105
122
|
isActiveConnection: () => boolean;
|
|
106
123
|
isAttached: () => boolean;
|
|
107
124
|
}
|
|
@@ -180,10 +197,17 @@ export function findFirstCharacterMismatched(
|
|
|
180
197
|
return [index, charA, charB];
|
|
181
198
|
}
|
|
182
199
|
|
|
183
|
-
|
|
200
|
+
/**
|
|
201
|
+
* Returns a shallow copy of the given message with the non-serializable properties removed.
|
|
202
|
+
* Note that the runtimeOp's data has already been serialized in the content property.
|
|
203
|
+
*/
|
|
204
|
+
function toSerializableForm(
|
|
205
|
+
message: IPendingMessage,
|
|
206
|
+
): IPendingMessage & { runtimeOp: undefined; localOpMetadata: undefined } {
|
|
184
207
|
return {
|
|
185
208
|
...message,
|
|
186
209
|
localOpMetadata: undefined,
|
|
210
|
+
runtimeOp: undefined,
|
|
187
211
|
};
|
|
188
212
|
}
|
|
189
213
|
|
|
@@ -273,7 +297,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
273
297
|
return {
|
|
274
298
|
pendingStates: [
|
|
275
299
|
...newSavedOps,
|
|
276
|
-
...this.pendingMessages.toArray().map((message) =>
|
|
300
|
+
...this.pendingMessages.toArray().map((message) => toSerializableForm(message)),
|
|
277
301
|
],
|
|
278
302
|
};
|
|
279
303
|
}
|
|
@@ -296,17 +320,36 @@ export class PendingStateManager implements IDisposable {
|
|
|
296
320
|
}
|
|
297
321
|
public readonly dispose = (): void => this.disposeOnce.value;
|
|
298
322
|
|
|
323
|
+
/**
|
|
324
|
+
* We've flushed an empty batch, and need to track it locally until the corresponding
|
|
325
|
+
* ack is processed, to properly track batch IDs
|
|
326
|
+
*/
|
|
327
|
+
public onFlushEmptyBatch(
|
|
328
|
+
placeholder: LocalEmptyBatchPlaceholder,
|
|
329
|
+
clientSequenceNumber: number | undefined,
|
|
330
|
+
staged: boolean,
|
|
331
|
+
): void {
|
|
332
|
+
// We have to cast because runtimeOp doesn't apply for empty batches and is missing on LocalEmptyBatchPlaceholder
|
|
333
|
+
this.onFlushBatch(
|
|
334
|
+
[placeholder satisfies Omit<LocalBatchMessage, "runtimeOp"> as LocalBatchMessage],
|
|
335
|
+
clientSequenceNumber,
|
|
336
|
+
staged,
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
|
|
299
340
|
/**
|
|
300
341
|
* The given batch has been flushed, and needs to be tracked locally until the corresponding
|
|
301
342
|
* acks are processed, to ensure it is successfully sent.
|
|
302
343
|
* @param batch - The batch that was flushed
|
|
303
344
|
* @param clientSequenceNumber - The CSN of the first message in the batch,
|
|
304
345
|
* or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
|
|
346
|
+
* @param staged - Indicates whether batch is staged (not to be submitted while runtime is in Staging Mode)
|
|
305
347
|
* @param ignoreBatchId - Whether to ignore the batchId in the batchStartInfo
|
|
306
348
|
*/
|
|
307
349
|
public onFlushBatch(
|
|
308
350
|
batch: LocalBatchMessage[],
|
|
309
351
|
clientSequenceNumber: number | undefined,
|
|
352
|
+
staged: boolean,
|
|
310
353
|
ignoreBatchId?: boolean,
|
|
311
354
|
): void {
|
|
312
355
|
// clientId and batchStartCsn are used for generating the batchId so we can detect container forks
|
|
@@ -315,6 +358,9 @@ export class PendingStateManager implements IDisposable {
|
|
|
315
358
|
// In the case where the batch was not sent, use a random uuid for clientId, and -1 for clientSequenceNumber to indicate this case.
|
|
316
359
|
// This will guarantee uniqueness of the batchId, and is a suitable fallback since clientId/CSN is only needed if the batch was actually sent/sequenced.
|
|
317
360
|
const batchWasSent = clientSequenceNumber !== undefined;
|
|
361
|
+
if (batchWasSent) {
|
|
362
|
+
assert(!staged, 0xb84 /* Staged batches should not have been submitted */);
|
|
363
|
+
}
|
|
318
364
|
const [clientId, batchStartCsn] = batchWasSent
|
|
319
365
|
? [this.stateHandler.clientId(), clientSequenceNumber]
|
|
320
366
|
: [uuid(), -1]; // -1 will indicate not a real clientId/CSN pair
|
|
@@ -325,7 +371,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
325
371
|
|
|
326
372
|
for (const message of batch) {
|
|
327
373
|
const {
|
|
328
|
-
|
|
374
|
+
runtimeOp,
|
|
329
375
|
referenceSequenceNumber,
|
|
330
376
|
localOpMetadata,
|
|
331
377
|
metadata: opMetadata,
|
|
@@ -333,11 +379,12 @@ export class PendingStateManager implements IDisposable {
|
|
|
333
379
|
const pendingMessage: IPendingMessage = {
|
|
334
380
|
type: "message",
|
|
335
381
|
referenceSequenceNumber,
|
|
336
|
-
content,
|
|
382
|
+
content: serializeOp(runtimeOp),
|
|
383
|
+
runtimeOp,
|
|
337
384
|
localOpMetadata,
|
|
338
385
|
opMetadata,
|
|
339
386
|
// Note: We only will read this off the first message, but put it on all for simplicity
|
|
340
|
-
batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId },
|
|
387
|
+
batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId, staged },
|
|
341
388
|
};
|
|
342
389
|
this.pendingMessages.push(pendingMessage);
|
|
343
390
|
}
|
|
@@ -375,6 +422,11 @@ export class PendingStateManager implements IDisposable {
|
|
|
375
422
|
const localOpMetadata = await this.stateHandler.applyStashedOp(nextMessage.content);
|
|
376
423
|
if (this.stateHandler.isAttached()) {
|
|
377
424
|
nextMessage.localOpMetadata = localOpMetadata;
|
|
425
|
+
// NOTE: This runtimeOp has been roundtripped through string, which is technically lossy.
|
|
426
|
+
// e.g. At this point, handles are in their encoded form.
|
|
427
|
+
nextMessage.runtimeOp = JSON.parse(
|
|
428
|
+
nextMessage.content,
|
|
429
|
+
) as LocalContainerRuntimeMessage;
|
|
378
430
|
// then we push onto pendingMessages which will cause PendingStateManager to resubmit when we connect
|
|
379
431
|
patchbatchInfo(nextMessage); // Back compat
|
|
380
432
|
this.pendingMessages.push(nextMessage);
|
|
@@ -508,7 +560,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
508
560
|
);
|
|
509
561
|
|
|
510
562
|
pendingMessage.sequenceNumber = sequenceNumber;
|
|
511
|
-
this.savedOps.push(
|
|
563
|
+
this.savedOps.push(toSerializableForm(pendingMessage));
|
|
512
564
|
|
|
513
565
|
this.pendingMessages.shift();
|
|
514
566
|
|
|
@@ -579,6 +631,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
579
631
|
pendingMessage !== undefined,
|
|
580
632
|
0xa21 /* No pending message found as we start processing this remote batch */,
|
|
581
633
|
);
|
|
634
|
+
assert(!pendingMessage.batchInfo.staged, 0xb85 /* Can't get an ack from a staged batch */);
|
|
582
635
|
|
|
583
636
|
// If this batch became empty on resubmit, batch.messages will be empty (but keyMessage is always set)
|
|
584
637
|
// and the next pending message should be an empty batch marker.
|
|
@@ -630,18 +683,22 @@ export class PendingStateManager implements IDisposable {
|
|
|
630
683
|
* Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
|
|
631
684
|
* states in its queue. This includes triggering resubmission of unacked ops.
|
|
632
685
|
* ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
|
|
686
|
+
* @param onlyStagedBatches - If true, only replay staged batches. This is used when we are exiting staging mode and want to rebase and submit the staged batches.
|
|
633
687
|
*/
|
|
634
|
-
public replayPendingStates(): void {
|
|
688
|
+
public replayPendingStates(onlyStagedBatches?: boolean): void {
|
|
635
689
|
assert(
|
|
636
690
|
this.stateHandler.connected(),
|
|
637
691
|
0x172 /* "The connection state is not consistent with the runtime" */,
|
|
638
692
|
);
|
|
639
693
|
|
|
640
|
-
//
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
694
|
+
// Staged batches have not yet been submitted so check doesn't apply
|
|
695
|
+
if (!onlyStagedBatches) {
|
|
696
|
+
// This assert suggests we are about to send same ops twice, which will result in data loss.
|
|
697
|
+
assert(
|
|
698
|
+
this.clientIdFromLastReplay !== this.stateHandler.clientId(),
|
|
699
|
+
0x173 /* "replayPendingStates called twice for same clientId!" */,
|
|
700
|
+
);
|
|
701
|
+
}
|
|
645
702
|
this.clientIdFromLastReplay = this.stateHandler.clientId();
|
|
646
703
|
|
|
647
704
|
assert(
|
|
@@ -652,6 +709,8 @@ export class PendingStateManager implements IDisposable {
|
|
|
652
709
|
const initialPendingMessagesCount = this.pendingMessages.length;
|
|
653
710
|
let remainingPendingMessagesCount = this.pendingMessages.length;
|
|
654
711
|
|
|
712
|
+
let seenStagedBatch = false;
|
|
713
|
+
|
|
655
714
|
// Process exactly `pendingMessagesCount` items in the queue as it represents the number of messages that were
|
|
656
715
|
// pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
|
|
657
716
|
// which must not be replayed.
|
|
@@ -660,18 +719,37 @@ export class PendingStateManager implements IDisposable {
|
|
|
660
719
|
let pendingMessage = this.pendingMessages.shift()!;
|
|
661
720
|
remainingPendingMessagesCount--;
|
|
662
721
|
|
|
722
|
+
// Re-queue pre-staging messages if we are only processing staged batches
|
|
723
|
+
if (onlyStagedBatches) {
|
|
724
|
+
if (!pendingMessage.batchInfo.staged) {
|
|
725
|
+
assert(!seenStagedBatch, 0xb86 /* Staged batch was followed by non-staged batch */);
|
|
726
|
+
this.pendingMessages.push(pendingMessage);
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
seenStagedBatch = true;
|
|
731
|
+
pendingMessage.batchInfo.staged = false; // Clear staged flag so we can submit
|
|
732
|
+
}
|
|
733
|
+
|
|
663
734
|
const batchMetadataFlag = asBatchMetadata(pendingMessage.opMetadata)?.batch;
|
|
664
735
|
assert(batchMetadataFlag !== false, 0x41b /* We cannot process batches in chunks */);
|
|
665
736
|
|
|
666
737
|
// The next message starts a batch (possibly single-message), and we'll need its batchId.
|
|
667
738
|
const batchId = getEffectiveBatchId(pendingMessage);
|
|
668
739
|
|
|
740
|
+
const staged = pendingMessage.batchInfo.staged;
|
|
741
|
+
|
|
669
742
|
if (asEmptyBatchLocalOpMetadata(pendingMessage.localOpMetadata)?.emptyBatch === true) {
|
|
670
743
|
// Resubmit no messages, with the batchId. Will result in another empty batch marker.
|
|
671
|
-
this.stateHandler.reSubmitBatch([], batchId);
|
|
744
|
+
this.stateHandler.reSubmitBatch([], batchId, staged);
|
|
672
745
|
continue;
|
|
673
746
|
}
|
|
674
747
|
|
|
748
|
+
assert(
|
|
749
|
+
pendingMessage.runtimeOp !== undefined,
|
|
750
|
+
0xb87 /* viableOp is only undefined for empty batches */,
|
|
751
|
+
);
|
|
752
|
+
|
|
675
753
|
/**
|
|
676
754
|
* We must preserve the distinct batches on resubmit.
|
|
677
755
|
* Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will
|
|
@@ -683,12 +761,13 @@ export class PendingStateManager implements IDisposable {
|
|
|
683
761
|
this.stateHandler.reSubmitBatch(
|
|
684
762
|
[
|
|
685
763
|
{
|
|
686
|
-
|
|
764
|
+
runtimeOp: pendingMessage.runtimeOp,
|
|
687
765
|
localOpMetadata: pendingMessage.localOpMetadata,
|
|
688
766
|
opMetadata: pendingMessage.opMetadata,
|
|
689
767
|
},
|
|
690
768
|
],
|
|
691
769
|
batchId,
|
|
770
|
+
staged,
|
|
692
771
|
);
|
|
693
772
|
continue;
|
|
694
773
|
}
|
|
@@ -703,12 +782,17 @@ export class PendingStateManager implements IDisposable {
|
|
|
703
782
|
|
|
704
783
|
// check is >= because batch end may be last pending message
|
|
705
784
|
while (remainingPendingMessagesCount >= 0) {
|
|
785
|
+
assert(
|
|
786
|
+
pendingMessage.runtimeOp !== undefined,
|
|
787
|
+
0xb88 /* viableOp is only undefined for empty batches */,
|
|
788
|
+
);
|
|
706
789
|
batch.push({
|
|
707
|
-
|
|
790
|
+
runtimeOp: pendingMessage.runtimeOp,
|
|
708
791
|
localOpMetadata: pendingMessage.localOpMetadata,
|
|
709
792
|
opMetadata: pendingMessage.opMetadata,
|
|
710
793
|
});
|
|
711
794
|
|
|
795
|
+
// End of the batch
|
|
712
796
|
if (pendingMessage.opMetadata?.batch === false) {
|
|
713
797
|
break;
|
|
714
798
|
}
|
|
@@ -723,7 +807,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
723
807
|
);
|
|
724
808
|
}
|
|
725
809
|
|
|
726
|
-
this.stateHandler.reSubmitBatch(batch, batchId);
|
|
810
|
+
this.stateHandler.reSubmitBatch(batch, batchId, staged);
|
|
727
811
|
}
|
|
728
812
|
|
|
729
813
|
// pending ops should no longer depend on previous sequenced local ops after resubmit
|
|
@@ -740,6 +824,36 @@ export class PendingStateManager implements IDisposable {
|
|
|
740
824
|
});
|
|
741
825
|
}
|
|
742
826
|
}
|
|
827
|
+
|
|
828
|
+
/**
|
|
829
|
+
* Clears the 'staged' flag off all pending messages.
|
|
830
|
+
*/
|
|
831
|
+
public clearStagingFlags(): void {
|
|
832
|
+
for (const message of this.pendingMessages.toArray()) {
|
|
833
|
+
if (message.batchInfo.staged) {
|
|
834
|
+
message.batchInfo.staged = false;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/**
|
|
840
|
+
* Pops all staged batches, invoking the callback on each one in order (LIFO)
|
|
841
|
+
*/
|
|
842
|
+
public popStagedBatches(callback: (stagedMessage: IPendingMessage) => void): void {
|
|
843
|
+
while (!this.pendingMessages.isEmpty()) {
|
|
844
|
+
const stagedMessage = this.pendingMessages.peekBack();
|
|
845
|
+
if (stagedMessage?.batchInfo.staged === true) {
|
|
846
|
+
callback(stagedMessage);
|
|
847
|
+
this.pendingMessages.pop();
|
|
848
|
+
} else {
|
|
849
|
+
break; // no more staged messages
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
assert(
|
|
853
|
+
this.pendingMessages.toArray().every((m) => m.batchInfo.staged !== true),
|
|
854
|
+
0xb89 /* Shouldn't be any more staged messages */,
|
|
855
|
+
);
|
|
856
|
+
}
|
|
743
857
|
}
|
|
744
858
|
|
|
745
859
|
/**
|
|
@@ -751,6 +865,6 @@ function patchbatchInfo(
|
|
|
751
865
|
const batchInfo: IPendingMessageFromStash["batchInfo"] = message.batchInfo;
|
|
752
866
|
if (batchInfo === undefined) {
|
|
753
867
|
// Using uuid guarantees uniqueness, retaining existing behavior
|
|
754
|
-
message.batchInfo = { clientId: uuid(), batchStartCsn: -1, length: -1 };
|
|
868
|
+
message.batchInfo = { clientId: uuid(), batchStartCsn: -1, length: -1, staged: false };
|
|
755
869
|
}
|
|
756
870
|
}
|
|
@@ -9,7 +9,10 @@ import {
|
|
|
9
9
|
type ILayerCompatSupportRequirements,
|
|
10
10
|
} from "@fluid-internal/client-utils";
|
|
11
11
|
import type { ICriticalContainerError } from "@fluidframework/container-definitions";
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
encodeHandlesInContainerRuntime,
|
|
14
|
+
notifiesReadOnlyState,
|
|
15
|
+
} from "@fluidframework/runtime-definitions/internal";
|
|
13
16
|
import { UsageError } from "@fluidframework/telemetry-utils/internal";
|
|
14
17
|
|
|
15
18
|
import { pkgVersion } from "./packageVersion.js";
|
|
@@ -66,7 +69,7 @@ export const runtimeCompatDetailsForDataStore: ILayerCompatDetails = {
|
|
|
66
69
|
/**
|
|
67
70
|
* The features supported by the Runtime layer across the Runtime / DataStore boundary.
|
|
68
71
|
*/
|
|
69
|
-
supportedFeatures: new Set<string>([encodeHandlesInContainerRuntime]),
|
|
72
|
+
supportedFeatures: new Set<string>([encodeHandlesInContainerRuntime, notifiesReadOnlyState]),
|
|
70
73
|
};
|
|
71
74
|
|
|
72
75
|
/**
|
|
@@ -96,6 +96,7 @@ export interface IDocumentSchemaFeatures {
|
|
|
96
96
|
compressionLz4: boolean;
|
|
97
97
|
idCompressorMode: IdCompressorMode;
|
|
98
98
|
opGroupingEnabled: boolean;
|
|
99
|
+
createBlobPayloadPending: true | undefined;
|
|
99
100
|
|
|
100
101
|
/**
|
|
101
102
|
* List of disallowed versions of the runtime.
|
|
@@ -227,6 +228,7 @@ const documentSchemaSupportedConfigs = {
|
|
|
227
228
|
idCompressorMode: new IdCompressorProperty(["delayed", "on"]),
|
|
228
229
|
opGroupingEnabled: new TrueOrUndefined(),
|
|
229
230
|
compressionLz4: new TrueOrUndefined(),
|
|
231
|
+
createBlobPayloadPending: new TrueOrUndefined(),
|
|
230
232
|
disallowedVersions: new CheckVersions(),
|
|
231
233
|
};
|
|
232
234
|
|
|
@@ -482,6 +484,7 @@ export class DocumentsSchemaController {
|
|
|
482
484
|
compressionLz4: boolToProp(features.compressionLz4),
|
|
483
485
|
idCompressorMode: features.idCompressorMode,
|
|
484
486
|
opGroupingEnabled: boolToProp(features.opGroupingEnabled),
|
|
487
|
+
createBlobPayloadPending: features.createBlobPayloadPending,
|
|
485
488
|
disallowedVersions: arrayToProp(features.disallowedVersions),
|
|
486
489
|
},
|
|
487
490
|
};
|