@fluidframework/container-runtime 2.10.0-307399 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/api-report/container-runtime.legacy.alpha.api.md +1 -1
- package/container-runtime.test-files.tar +0 -0
- package/dist/containerRuntime.d.ts +1 -1
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +23 -25
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +15 -2
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +39 -1
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreRegistry.d.ts +1 -0
- package/dist/dataStoreRegistry.d.ts.map +1 -1
- package/dist/dataStoreRegistry.js +10 -2
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +2 -0
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +40 -11
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +0 -7
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +0 -12
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/messageTypes.d.ts +3 -39
- package/dist/messageTypes.d.ts.map +1 -1
- package/dist/messageTypes.js.map +1 -1
- package/dist/opLifecycle/duplicateBatchDetector.d.ts +10 -0
- package/dist/opLifecycle/duplicateBatchDetector.d.ts.map +1 -1
- package/dist/opLifecycle/duplicateBatchDetector.js +21 -1
- package/dist/opLifecycle/duplicateBatchDetector.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +1 -5
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +2 -3
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summary/index.d.ts +1 -1
- package/dist/summary/index.d.ts.map +1 -1
- package/dist/summary/index.js +2 -1
- package/dist/summary/index.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +1 -0
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js +2 -1
- package/dist/summary/summaryFormat.js.map +1 -1
- package/lib/containerRuntime.d.ts +1 -1
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +24 -26
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +15 -2
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +40 -2
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreRegistry.d.ts +1 -0
- package/lib/dataStoreRegistry.d.ts.map +1 -1
- package/lib/dataStoreRegistry.js +8 -0
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +2 -0
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +40 -11
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +0 -7
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +0 -12
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/messageTypes.d.ts +3 -39
- package/lib/messageTypes.d.ts.map +1 -1
- package/lib/messageTypes.js.map +1 -1
- package/lib/opLifecycle/duplicateBatchDetector.d.ts +10 -0
- package/lib/opLifecycle/duplicateBatchDetector.d.ts.map +1 -1
- package/lib/opLifecycle/duplicateBatchDetector.js +21 -1
- package/lib/opLifecycle/duplicateBatchDetector.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +1 -5
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +2 -3
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summary/index.d.ts +1 -1
- package/lib/summary/index.d.ts.map +1 -1
- package/lib/summary/index.js +1 -1
- package/lib/summary/index.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +1 -0
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js +1 -0
- package/lib/summary/summaryFormat.js.map +1 -1
- package/package.json +20 -72
- package/src/containerRuntime.ts +40 -31
- package/src/dataStoreContext.ts +57 -1
- package/src/dataStoreRegistry.ts +10 -0
- package/src/gc/garbageCollection.ts +41 -11
- package/src/gc/gcSummaryStateTracker.ts +0 -13
- package/src/index.ts +0 -3
- package/src/messageTypes.ts +3 -50
- package/src/opLifecycle/duplicateBatchDetector.ts +33 -0
- package/src/opLifecycle/remoteMessageProcessor.ts +1 -6
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +2 -3
- package/src/summary/index.ts +1 -0
- package/src/summary/summaryFormat.ts +1 -0
package/src/containerRuntime.ts
CHANGED
|
@@ -234,6 +234,7 @@ import {
|
|
|
234
234
|
SummaryManager,
|
|
235
235
|
aliasBlobName,
|
|
236
236
|
chunksBlobName,
|
|
237
|
+
recentBatchInfoBlobName,
|
|
237
238
|
createRootSummarizerNodeWithGC,
|
|
238
239
|
electedSummarizerBlobName,
|
|
239
240
|
extractSummaryMetadataMessage,
|
|
@@ -930,14 +931,23 @@ export class ContainerRuntime
|
|
|
930
931
|
}
|
|
931
932
|
};
|
|
932
933
|
|
|
933
|
-
const [
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
934
|
+
const [
|
|
935
|
+
chunks,
|
|
936
|
+
recentBatchInfo,
|
|
937
|
+
metadata,
|
|
938
|
+
electedSummarizerData,
|
|
939
|
+
aliases,
|
|
940
|
+
serializedIdCompressor,
|
|
941
|
+
] = await Promise.all([
|
|
942
|
+
tryFetchBlob<[string, string[]][]>(chunksBlobName),
|
|
943
|
+
tryFetchBlob<ReturnType<DuplicateBatchDetector["getRecentBatchInfoForSummary"]>>(
|
|
944
|
+
recentBatchInfoBlobName,
|
|
945
|
+
),
|
|
946
|
+
tryFetchBlob<IContainerRuntimeMetadata>(metadataBlobName),
|
|
947
|
+
tryFetchBlob<ISerializedElection>(electedSummarizerBlobName),
|
|
948
|
+
tryFetchBlob<[string, string][]>(aliasBlobName),
|
|
949
|
+
tryFetchBlob<SerializedIdCompressorWithNoSession>(idCompressorBlobName),
|
|
950
|
+
]);
|
|
941
951
|
|
|
942
952
|
// read snapshot blobs needed for BlobManager to load
|
|
943
953
|
const blobManagerSnapshot = await loadBlobManagerLoadInfo(context);
|
|
@@ -1114,6 +1124,7 @@ export class ContainerRuntime
|
|
|
1114
1124
|
provideEntryPoint,
|
|
1115
1125
|
requestHandler,
|
|
1116
1126
|
undefined, // summaryConfiguration
|
|
1127
|
+
recentBatchInfo,
|
|
1117
1128
|
);
|
|
1118
1129
|
|
|
1119
1130
|
runtime.blobManager.stashedBlobsUploadP.then(
|
|
@@ -1492,6 +1503,7 @@ export class ContainerRuntime
|
|
|
1492
1503
|
// the runtime configuration overrides
|
|
1493
1504
|
...runtimeOptions.summaryOptions?.summaryConfigOverrides,
|
|
1494
1505
|
},
|
|
1506
|
+
recentBatchInfo?: [number, string][],
|
|
1495
1507
|
) {
|
|
1496
1508
|
super();
|
|
1497
1509
|
|
|
@@ -1532,7 +1544,7 @@ export class ContainerRuntime
|
|
|
1532
1544
|
compressionAlgorithm: CompressionAlgorithms.lz4,
|
|
1533
1545
|
};
|
|
1534
1546
|
|
|
1535
|
-
assert(isIDeltaManagerFull(deltaManager),
|
|
1547
|
+
assert(isIDeltaManagerFull(deltaManager), 0xa80 /* Invalid delta manager */);
|
|
1536
1548
|
this.innerDeltaManager = deltaManager;
|
|
1537
1549
|
|
|
1538
1550
|
// Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
|
|
@@ -1707,7 +1719,7 @@ export class ContainerRuntime
|
|
|
1707
1719
|
// It maintains a cache of all batchIds/sequenceNumbers within the collab window.
|
|
1708
1720
|
// Don't waste resources doing so if not needed.
|
|
1709
1721
|
if (this.offlineEnabled) {
|
|
1710
|
-
this.duplicateBatchDetector = new DuplicateBatchDetector();
|
|
1722
|
+
this.duplicateBatchDetector = new DuplicateBatchDetector(recentBatchInfo);
|
|
1711
1723
|
}
|
|
1712
1724
|
|
|
1713
1725
|
if (context.attachState === AttachState.Attached) {
|
|
@@ -2414,6 +2426,12 @@ export class ContainerRuntime
|
|
|
2414
2426
|
addBlobToSummary(summaryTree, chunksBlobName, content);
|
|
2415
2427
|
}
|
|
2416
2428
|
|
|
2429
|
+
const recentBatchInfo =
|
|
2430
|
+
this.duplicateBatchDetector?.getRecentBatchInfoForSummary(telemetryContext);
|
|
2431
|
+
if (recentBatchInfo !== undefined) {
|
|
2432
|
+
addBlobToSummary(summaryTree, recentBatchInfoBlobName, JSON.stringify(recentBatchInfo));
|
|
2433
|
+
}
|
|
2434
|
+
|
|
2417
2435
|
const dataStoreAliases = this.channelCollection.aliases;
|
|
2418
2436
|
if (dataStoreAliases.size > 0) {
|
|
2419
2437
|
addBlobToSummary(summaryTree, aliasBlobName, JSON.stringify([...dataStoreAliases]));
|
|
@@ -2651,20 +2669,21 @@ export class ContainerRuntime
|
|
|
2651
2669
|
|
|
2652
2670
|
this._connected = connected;
|
|
2653
2671
|
|
|
2654
|
-
if (
|
|
2655
|
-
this._signalTracking.signalsLost = 0;
|
|
2656
|
-
this._signalTracking.signalsOutOfOrder = 0;
|
|
2657
|
-
this._signalTracking.signalTimestamp = 0;
|
|
2658
|
-
this._signalTracking.signalsSentSinceLastLatencyMeasurement = 0;
|
|
2659
|
-
this._signalTracking.totalSignalsSentInLatencyWindow = 0;
|
|
2660
|
-
this._signalTracking.roundTripSignalSequenceNumber = undefined;
|
|
2661
|
-
this._signalTracking.trackingSignalSequenceNumber = undefined;
|
|
2662
|
-
this._signalTracking.minimumTrackingSignalSequenceNumber = undefined;
|
|
2663
|
-
} else {
|
|
2672
|
+
if (connected) {
|
|
2664
2673
|
assert(
|
|
2665
2674
|
this.attachState === AttachState.Attached,
|
|
2666
2675
|
0x3cd /* Connection is possible only if container exists in storage */,
|
|
2667
2676
|
);
|
|
2677
|
+
if (changeOfState) {
|
|
2678
|
+
this._signalTracking.signalsLost = 0;
|
|
2679
|
+
this._signalTracking.signalsOutOfOrder = 0;
|
|
2680
|
+
this._signalTracking.signalTimestamp = 0;
|
|
2681
|
+
this._signalTracking.signalsSentSinceLastLatencyMeasurement = 0;
|
|
2682
|
+
this._signalTracking.totalSignalsSentInLatencyWindow = 0;
|
|
2683
|
+
this._signalTracking.roundTripSignalSequenceNumber = undefined;
|
|
2684
|
+
this._signalTracking.trackingSignalSequenceNumber = undefined;
|
|
2685
|
+
this._signalTracking.minimumTrackingSignalSequenceNumber = undefined;
|
|
2686
|
+
}
|
|
2668
2687
|
}
|
|
2669
2688
|
|
|
2670
2689
|
// Fail while disconnected
|
|
@@ -3209,16 +3228,7 @@ export class ContainerRuntime
|
|
|
3209
3228
|
};
|
|
3210
3229
|
|
|
3211
3230
|
// Only collect signal telemetry for broadcast messages sent by the current client.
|
|
3212
|
-
if (
|
|
3213
|
-
message.clientId === this.clientId &&
|
|
3214
|
-
// jason-ha: This `connected` check seems incorrect. Signals that come through
|
|
3215
|
-
// here must have been received while connected to service and there is no need
|
|
3216
|
-
// to avoid processing when connection has dropped. Because container runtime's
|
|
3217
|
-
// `connected` (and `_connected`) state also reflects some ops state, it may
|
|
3218
|
-
// easily be false when newly connected and signal tracking may very well
|
|
3219
|
-
// complain lost signals (that were simply skipped per this check).
|
|
3220
|
-
this.connected
|
|
3221
|
-
) {
|
|
3231
|
+
if (message.clientId === this.clientId) {
|
|
3222
3232
|
this.processSignalForTelemetry(envelope);
|
|
3223
3233
|
}
|
|
3224
3234
|
|
|
@@ -4603,7 +4613,6 @@ export class ContainerRuntime
|
|
|
4603
4613
|
this.channelCollection.rollback(type, contents, localOpMetadata);
|
|
4604
4614
|
break;
|
|
4605
4615
|
default:
|
|
4606
|
-
// Don't check message.compatDetails because this is for rolling back a local op so the type will be known
|
|
4607
4616
|
throw new Error(`Can't rollback ${type}`);
|
|
4608
4617
|
}
|
|
4609
4618
|
}
|
package/src/dataStoreContext.ts
CHANGED
|
@@ -55,6 +55,7 @@ import {
|
|
|
55
55
|
IInboundSignalMessage,
|
|
56
56
|
type IPendingMessagesState,
|
|
57
57
|
type IRuntimeMessageCollection,
|
|
58
|
+
type IFluidDataStoreFactory,
|
|
58
59
|
} from "@fluidframework/runtime-definitions/internal";
|
|
59
60
|
import {
|
|
60
61
|
addBlobToSummary,
|
|
@@ -65,6 +66,7 @@ import {
|
|
|
65
66
|
LoggingError,
|
|
66
67
|
MonitoringContext,
|
|
67
68
|
ThresholdCounter,
|
|
69
|
+
UsageError,
|
|
68
70
|
createChildMonitoringContext,
|
|
69
71
|
extractSafePropertiesFromMessage,
|
|
70
72
|
generateStack,
|
|
@@ -496,7 +498,7 @@ export abstract class FluidDataStoreContext
|
|
|
496
498
|
this.rejectDeferredRealize("No registry for package", lastPkg, packages);
|
|
497
499
|
}
|
|
498
500
|
lastPkg = pkg;
|
|
499
|
-
entry = await registry.get(pkg);
|
|
501
|
+
entry = registry.getSync?.(pkg) ?? (await registry.get(pkg));
|
|
500
502
|
if (!entry) {
|
|
501
503
|
this.rejectDeferredRealize(
|
|
502
504
|
"Registry does not contain entry for the package",
|
|
@@ -517,6 +519,42 @@ export abstract class FluidDataStoreContext
|
|
|
517
519
|
return factory;
|
|
518
520
|
}
|
|
519
521
|
|
|
522
|
+
createChildDataStore<T extends IFluidDataStoreFactory>(
|
|
523
|
+
childFactory: T,
|
|
524
|
+
): ReturnType<Exclude<T["createDataStore"], undefined>> {
|
|
525
|
+
const maybe = this.registry?.getSync?.(childFactory.type);
|
|
526
|
+
|
|
527
|
+
const isUndefined = maybe === undefined;
|
|
528
|
+
const diffInstance = maybe?.IFluidDataStoreFactory !== childFactory;
|
|
529
|
+
|
|
530
|
+
if (isUndefined || diffInstance) {
|
|
531
|
+
throw new UsageError(
|
|
532
|
+
"The provided factory instance must be synchronously available as a child of this datastore",
|
|
533
|
+
{ isUndefined, diffInstance },
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
if (childFactory?.createDataStore === undefined) {
|
|
537
|
+
throw new UsageError("createDataStore must exist on the provided factory", {
|
|
538
|
+
noCreateDataStore: true,
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
const context = this._containerRuntime.createDetachedDataStore([
|
|
543
|
+
...this.packagePath,
|
|
544
|
+
childFactory.type,
|
|
545
|
+
]);
|
|
546
|
+
assert(
|
|
547
|
+
context instanceof LocalDetachedFluidDataStoreContext,
|
|
548
|
+
0xa89 /* must be a LocalDetachedFluidDataStoreContext */,
|
|
549
|
+
);
|
|
550
|
+
|
|
551
|
+
const created = childFactory.createDataStore(context) as ReturnType<
|
|
552
|
+
Exclude<T["createDataStore"], undefined>
|
|
553
|
+
>;
|
|
554
|
+
context.unsafe_AttachRuntimeSync(created.runtime);
|
|
555
|
+
return created;
|
|
556
|
+
}
|
|
557
|
+
|
|
520
558
|
private async realizeCore(existing: boolean) {
|
|
521
559
|
const details = await this.getInitialSnapshotDetails();
|
|
522
560
|
// Base snapshot is the baseline where pending ops are applied to.
|
|
@@ -1428,6 +1466,24 @@ export class LocalDetachedFluidDataStoreContext
|
|
|
1428
1466
|
return this.channelToDataStoreFn(await this.channelP);
|
|
1429
1467
|
}
|
|
1430
1468
|
|
|
1469
|
+
/**
|
|
1470
|
+
* This method provides a synchronous path for binding a runtime to the context.
|
|
1471
|
+
*
|
|
1472
|
+
* Due to its synchronous nature, it is unable to validate that the runtime
|
|
1473
|
+
* represents a datastore which is instantiable by remote clients. This could
|
|
1474
|
+
* happen if the runtime's package path does not return a factory when looked up
|
|
1475
|
+
* in the container runtime's registry, or if the runtime's entrypoint is not
|
|
1476
|
+
* properly initialized. As both of these validation's are asynchronous to preform.
|
|
1477
|
+
*
|
|
1478
|
+
* If used incorrectly, this function can result in permanent data corruption.
|
|
1479
|
+
*/
|
|
1480
|
+
public unsafe_AttachRuntimeSync(channel: IFluidDataStoreChannel) {
|
|
1481
|
+
this.channelP = Promise.resolve(channel);
|
|
1482
|
+
this.processPendingOps(channel);
|
|
1483
|
+
this.completeBindingRuntime(channel);
|
|
1484
|
+
return this.channelToDataStoreFn(channel);
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1431
1487
|
public async getInitialSnapshotDetails(): Promise<ISnapshotDetails> {
|
|
1432
1488
|
if (this.detachedRuntimeCreation) {
|
|
1433
1489
|
throw new Error(
|
package/src/dataStoreRegistry.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
+
import { isPromiseLike } from "@fluidframework/core-utils/internal";
|
|
6
7
|
import {
|
|
7
8
|
FluidDataStoreRegistryEntry,
|
|
8
9
|
IFluidDataStoreRegistry,
|
|
@@ -40,4 +41,13 @@ export class FluidDataStoreRegistry implements IFluidDataStoreRegistry {
|
|
|
40
41
|
|
|
41
42
|
return undefined;
|
|
42
43
|
}
|
|
44
|
+
|
|
45
|
+
public getSync(name: string): FluidDataStoreRegistryEntry | undefined {
|
|
46
|
+
const entry = this.map.get(name);
|
|
47
|
+
if (!isPromiseLike(entry)) {
|
|
48
|
+
return entry;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
43
53
|
}
|
|
@@ -338,6 +338,43 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
338
338
|
});
|
|
339
339
|
}
|
|
340
340
|
|
|
341
|
+
/** API for ensuring the correct auto-recovery mitigations */
|
|
342
|
+
private readonly autoRecovery = (() => {
|
|
343
|
+
// This uses a hidden state machine for forcing fullGC as part of autorecovery,
|
|
344
|
+
// to regenerate the GC data for each node.
|
|
345
|
+
//
|
|
346
|
+
// Once fullGC has been requested, we need to wait until GC has run and the summary has been acked before clearing the state.
|
|
347
|
+
//
|
|
348
|
+
// States:
|
|
349
|
+
// - undefined: No need to run fullGC now.
|
|
350
|
+
// - "requested": FullGC requested, but GC has not yet run. Keep using fullGC until back to undefined.
|
|
351
|
+
// - "ran": FullGC ran, but the following summary has not yet been acked. Keep using fullGC until back to undefined.
|
|
352
|
+
//
|
|
353
|
+
// Transitions:
|
|
354
|
+
// - autoRecovery.requestFullGCOnNextRun :: [anything] --> "requested"
|
|
355
|
+
// - autoRecovery.onCompletedGCRun :: "requested" --> "ran"
|
|
356
|
+
// - autoRecovery.onSummaryAck :: "ran" --> undefined
|
|
357
|
+
let state: "requested" | "ran" | undefined;
|
|
358
|
+
return {
|
|
359
|
+
requestFullGCOnNextRun: () => {
|
|
360
|
+
state = "requested";
|
|
361
|
+
},
|
|
362
|
+
onCompletedGCRun: () => {
|
|
363
|
+
if (state === "requested") {
|
|
364
|
+
state = "ran";
|
|
365
|
+
}
|
|
366
|
+
},
|
|
367
|
+
onSummaryAck: () => {
|
|
368
|
+
if (state === "ran") {
|
|
369
|
+
state = undefined;
|
|
370
|
+
}
|
|
371
|
+
},
|
|
372
|
+
useFullGC: () => {
|
|
373
|
+
return state !== undefined;
|
|
374
|
+
},
|
|
375
|
+
};
|
|
376
|
+
})();
|
|
377
|
+
|
|
341
378
|
/**
|
|
342
379
|
* Called during container initialization. Initializes the tombstone and deleted nodes state from the base snapshot.
|
|
343
380
|
* Also, initializes the GC state including unreferenced nodes tracking if a current reference timestamp exists.
|
|
@@ -460,9 +497,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
460
497
|
telemetryContext?: ITelemetryContext,
|
|
461
498
|
): Promise<IGCStats | undefined> {
|
|
462
499
|
const fullGC =
|
|
463
|
-
options.fullGC ??
|
|
464
|
-
(this.configs.runFullGC === true ||
|
|
465
|
-
this.summaryStateTracker.autoRecovery.fullGCRequested());
|
|
500
|
+
options.fullGC ?? (this.configs.runFullGC === true || this.autoRecovery.useFullGC());
|
|
466
501
|
|
|
467
502
|
// Add the options that are used to run GC to the telemetry context.
|
|
468
503
|
telemetryContext?.setMultiple("fluid_GC", "Options", {
|
|
@@ -521,6 +556,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
521
556
|
await this.telemetryTracker.logPendingEvents(logger);
|
|
522
557
|
// Update the state of summary state tracker from this run's stats.
|
|
523
558
|
this.summaryStateTracker.updateStateFromGCRunStats(gcStats);
|
|
559
|
+
this.autoRecovery.onCompletedGCRun();
|
|
524
560
|
this.newReferencesSinceLastRun.clear();
|
|
525
561
|
this.completedRuns++;
|
|
526
562
|
|
|
@@ -709,13 +745,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
709
745
|
deletedNodeIds: sweepReadyDSAndBlobs,
|
|
710
746
|
};
|
|
711
747
|
|
|
712
|
-
// Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
|
|
713
|
-
// is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
|
|
714
|
-
// accidental usage. The clients will sync without the delete op too but it may take longer.
|
|
715
748
|
const containerGCMessage: ContainerRuntimeGCMessage = {
|
|
716
749
|
type: ContainerMessageType.GC,
|
|
717
750
|
contents,
|
|
718
|
-
compatDetails: { behavior: "Ignore" }, // DEPRECATED: For temporary back compat only
|
|
719
751
|
};
|
|
720
752
|
this.submitMessage(containerGCMessage);
|
|
721
753
|
return;
|
|
@@ -857,6 +889,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
857
889
|
* Called to refresh the latest summary state. This happens when either a pending summary is acked.
|
|
858
890
|
*/
|
|
859
891
|
public async refreshLatestSummary(result: IRefreshSummaryResult): Promise<void> {
|
|
892
|
+
this.autoRecovery.onSummaryAck();
|
|
860
893
|
return this.summaryStateTracker.refreshLatestSummary(result);
|
|
861
894
|
}
|
|
862
895
|
|
|
@@ -891,7 +924,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
891
924
|
|
|
892
925
|
// In case the cause of the TombstoneLoaded event is incorrect GC Data (i.e. the object is actually reachable),
|
|
893
926
|
// do fullGC on the next run to get a chance to repair (in the likely case the bug is not deterministic)
|
|
894
|
-
this.
|
|
927
|
+
this.autoRecovery.requestFullGCOnNextRun();
|
|
895
928
|
break;
|
|
896
929
|
}
|
|
897
930
|
default:
|
|
@@ -1036,15 +1069,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1036
1069
|
return;
|
|
1037
1070
|
}
|
|
1038
1071
|
|
|
1039
|
-
// Use compat behavior "Ignore" since this is an optimization to opportunistically protect
|
|
1040
|
-
// objects from deletion, so it's fine for older clients to ignore this op.
|
|
1041
1072
|
const containerGCMessage: ContainerRuntimeGCMessage = {
|
|
1042
1073
|
type: ContainerMessageType.GC,
|
|
1043
1074
|
contents: {
|
|
1044
1075
|
type: GarbageCollectionMessageType.TombstoneLoaded,
|
|
1045
1076
|
nodePath,
|
|
1046
1077
|
},
|
|
1047
|
-
compatDetails: { behavior: "Ignore" }, // DEPRECATED: For temporary back compat only
|
|
1048
1078
|
};
|
|
1049
1079
|
this.submitMessage(containerGCMessage);
|
|
1050
1080
|
}
|
|
@@ -50,18 +50,6 @@ export class GCSummaryStateTracker {
|
|
|
50
50
|
// to unreferenced or vice-versa.
|
|
51
51
|
public updatedDSCountSinceLastSummary: number = 0;
|
|
52
52
|
|
|
53
|
-
/** API for ensuring the correct auto-recovery mitigations */
|
|
54
|
-
public autoRecovery = {
|
|
55
|
-
requestFullGCOnNextRun: () => {
|
|
56
|
-
this.fullGCModeForAutoRecovery = true;
|
|
57
|
-
},
|
|
58
|
-
fullGCRequested: () => {
|
|
59
|
-
return this.fullGCModeForAutoRecovery;
|
|
60
|
-
},
|
|
61
|
-
};
|
|
62
|
-
/** If true, the next GC run will do fullGC mode to regenerate the GC data for each node */
|
|
63
|
-
private fullGCModeForAutoRecovery: boolean = false;
|
|
64
|
-
|
|
65
53
|
constructor(
|
|
66
54
|
// Tells whether GC should run or not.
|
|
67
55
|
private readonly configs: Pick<
|
|
@@ -235,7 +223,6 @@ export class GCSummaryStateTracker {
|
|
|
235
223
|
this.latestSummaryData = this.pendingSummaryData;
|
|
236
224
|
this.pendingSummaryData = undefined;
|
|
237
225
|
this.updatedDSCountSinceLastSummary = 0;
|
|
238
|
-
this.fullGCModeForAutoRecovery = false;
|
|
239
226
|
}
|
|
240
227
|
|
|
241
228
|
/**
|
package/src/index.ts
CHANGED
|
@@ -26,9 +26,6 @@ export {
|
|
|
26
26
|
} from "./containerRuntime.js";
|
|
27
27
|
export {
|
|
28
28
|
ContainerMessageType,
|
|
29
|
-
IContainerRuntimeMessageCompatDetails,
|
|
30
|
-
CompatModeBehavior,
|
|
31
|
-
RecentlyAddedContainerRuntimeMessageDetails,
|
|
32
29
|
UnknownContainerRuntimeMessage,
|
|
33
30
|
} from "./messageTypes.js";
|
|
34
31
|
export { IBlobManagerLoadInfo } from "./blobManager/index.js";
|
package/src/messageTypes.ts
CHANGED
|
@@ -58,29 +58,6 @@ export enum ContainerMessageType {
|
|
|
58
58
|
GC = "GC",
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
/**
|
|
62
|
-
* How should an older client handle an unrecognized remote op type?
|
|
63
|
-
*
|
|
64
|
-
* @deprecated The utility of a mechanism to handle unknown messages is outweighed by the nuance required to get it right.
|
|
65
|
-
* @internal
|
|
66
|
-
*/
|
|
67
|
-
export type CompatModeBehavior =
|
|
68
|
-
/** Ignore the op. It won't be persisted if this client summarizes */
|
|
69
|
-
| "Ignore"
|
|
70
|
-
/** Fail processing immediately. (The container will close) */
|
|
71
|
-
| "FailToProcess";
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* All the info an older client would need to know how to handle an unrecognized remote op type
|
|
75
|
-
*
|
|
76
|
-
* @deprecated The utility of a mechanism to handle unknown messages is outweighed by the nuance required to get it right.
|
|
77
|
-
* @internal
|
|
78
|
-
*/
|
|
79
|
-
export interface IContainerRuntimeMessageCompatDetails {
|
|
80
|
-
/** How should an older client handle an unrecognized remote op type? */
|
|
81
|
-
behavior: CompatModeBehavior;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
61
|
/**
|
|
85
62
|
* The unpacked runtime message / details to be handled or dispatched by the ContainerRuntime.
|
|
86
63
|
* Message type are differentiated via a `type` string and contain different contents depending on their type.
|
|
@@ -88,27 +65,11 @@ export interface IContainerRuntimeMessageCompatDetails {
|
|
|
88
65
|
* IMPORTANT: when creating one to be serialized, set the properties in the order they appear here.
|
|
89
66
|
* This way stringified values can be compared.
|
|
90
67
|
*/
|
|
91
|
-
|
|
92
|
-
TType extends ContainerMessageType,
|
|
93
|
-
TContents,
|
|
94
|
-
TUSedCompatDetails extends boolean = false,
|
|
95
|
-
> = {
|
|
68
|
+
interface TypedContainerRuntimeMessage<TType extends ContainerMessageType, TContents> {
|
|
96
69
|
/** Type of the op, within the ContainerRuntime's domain */
|
|
97
70
|
type: TType;
|
|
98
71
|
/** Domain-specific contents, interpreted according to the type */
|
|
99
72
|
contents: TContents;
|
|
100
|
-
} & (TUSedCompatDetails extends true
|
|
101
|
-
? Partial<RecentlyAddedContainerRuntimeMessageDetails>
|
|
102
|
-
: { compatDetails?: undefined });
|
|
103
|
-
|
|
104
|
-
/**
|
|
105
|
-
* Additional details expected for any recently added message.
|
|
106
|
-
* @deprecated The utility of a mechanism to handle unknown messages is outweighed by the nuance required to get it right.
|
|
107
|
-
* @internal
|
|
108
|
-
*/
|
|
109
|
-
export interface RecentlyAddedContainerRuntimeMessageDetails {
|
|
110
|
-
/** Info describing how to handle this op in case the type is unrecognized (default: fail to process) */
|
|
111
|
-
compatDetails: IContainerRuntimeMessageCompatDetails;
|
|
112
73
|
}
|
|
113
74
|
|
|
114
75
|
export type ContainerRuntimeDataStoreOpMessage = TypedContainerRuntimeMessage<
|
|
@@ -145,8 +106,7 @@ export type ContainerRuntimeIdAllocationMessage = TypedContainerRuntimeMessage<
|
|
|
145
106
|
>;
|
|
146
107
|
export type ContainerRuntimeGCMessage = TypedContainerRuntimeMessage<
|
|
147
108
|
ContainerMessageType.GC,
|
|
148
|
-
GarbageCollectionMessage
|
|
149
|
-
true // TUsedCompatDetails
|
|
109
|
+
GarbageCollectionMessage
|
|
150
110
|
>;
|
|
151
111
|
export type ContainerRuntimeDocumentSchemaMessage = TypedContainerRuntimeMessage<
|
|
152
112
|
ContainerMessageType.DocumentSchemaChange,
|
|
@@ -157,8 +117,7 @@ export type ContainerRuntimeDocumentSchemaMessage = TypedContainerRuntimeMessage
|
|
|
157
117
|
* Represents an unrecognized TypedContainerRuntimeMessage, e.g. a message from a future version of the container runtime.
|
|
158
118
|
* @internal
|
|
159
119
|
*/
|
|
160
|
-
export interface UnknownContainerRuntimeMessage
|
|
161
|
-
extends Partial<RecentlyAddedContainerRuntimeMessageDetails> {
|
|
120
|
+
export interface UnknownContainerRuntimeMessage {
|
|
162
121
|
/** Invalid type of the op, within the ContainerRuntime's domain. This value should never exist at runtime.
|
|
163
122
|
* This is useful for type narrowing but should never be used as an actual message type at runtime.
|
|
164
123
|
* Actual value will not be "__unknown...", but the type `Exclude<string, ContainerMessageType>` is not supported.
|
|
@@ -222,9 +181,3 @@ export type InboundSequencedContainerRuntimeMessage = Omit<
|
|
|
222
181
|
"type" | "contents"
|
|
223
182
|
> &
|
|
224
183
|
InboundContainerRuntimeMessage;
|
|
225
|
-
|
|
226
|
-
/** A [loose] InboundSequencedContainerRuntimeMessage that is recent and may contain compat details.
|
|
227
|
-
* It exists solely to to provide access to those details.
|
|
228
|
-
*/
|
|
229
|
-
export type InboundSequencedRecentlyAddedContainerRuntimeMessage = ISequencedDocumentMessage &
|
|
230
|
-
Partial<RecentlyAddedContainerRuntimeMessageDetails>;
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { assert } from "@fluidframework/core-utils/internal";
|
|
7
|
+
import type { ITelemetryContext } from "@fluidframework/runtime-definitions/internal";
|
|
7
8
|
|
|
8
9
|
import { getEffectiveBatchId } from "./batchManager.js";
|
|
9
10
|
import { type BatchStartInfo } from "./remoteMessageProcessor.js";
|
|
@@ -18,6 +19,16 @@ export class DuplicateBatchDetector {
|
|
|
18
19
|
/** We map from sequenceNumber to batchId to find which ones we can stop tracking as MSN advances */
|
|
19
20
|
private readonly batchIdsBySeqNum = new Map<number, string>();
|
|
20
21
|
|
|
22
|
+
/** Initialize from snapshot data if provided - otherwise initialize empty */
|
|
23
|
+
constructor(batchIdsFromSnapshot: [number, string][] | undefined) {
|
|
24
|
+
if (batchIdsFromSnapshot) {
|
|
25
|
+
this.batchIdsBySeqNum = new Map(batchIdsFromSnapshot);
|
|
26
|
+
for (const batchId of this.batchIdsBySeqNum.values()) {
|
|
27
|
+
this.batchIdsAll.add(batchId);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
21
32
|
/**
|
|
22
33
|
* Records this batch's batchId, and checks if it's a duplicate of a batch we've already seen.
|
|
23
34
|
* If it's a duplicate, also return the sequence number of the other batch for logging.
|
|
@@ -75,4 +86,26 @@ export class DuplicateBatchDetector {
|
|
|
75
86
|
}
|
|
76
87
|
});
|
|
77
88
|
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Returns a snapshot of the state of the detector which can be included in a summary
|
|
92
|
+
* and used to "rehydrate" this class when loading from a snapshot.
|
|
93
|
+
*
|
|
94
|
+
* @returns - A serializable object representing the state of the detector, or undefined if there is nothing to save.
|
|
95
|
+
*/
|
|
96
|
+
public getRecentBatchInfoForSummary(
|
|
97
|
+
telemetryContext?: ITelemetryContext,
|
|
98
|
+
): [number, string][] | undefined {
|
|
99
|
+
if (this.batchIdsBySeqNum.size === 0) {
|
|
100
|
+
return undefined;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
telemetryContext?.set(
|
|
104
|
+
"fluid_DuplicateBatchDetector_",
|
|
105
|
+
"recentBatchCount",
|
|
106
|
+
this.batchIdsBySeqNum.size,
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
return [...this.batchIdsBySeqNum.entries()];
|
|
110
|
+
}
|
|
78
111
|
}
|
|
@@ -13,7 +13,6 @@ import {
|
|
|
13
13
|
ContainerMessageType,
|
|
14
14
|
type InboundContainerRuntimeMessage,
|
|
15
15
|
type InboundSequencedContainerRuntimeMessage,
|
|
16
|
-
type InboundSequencedRecentlyAddedContainerRuntimeMessage,
|
|
17
16
|
} from "../messageTypes.js";
|
|
18
17
|
import { asBatchMetadata } from "../metadata.js";
|
|
19
18
|
|
|
@@ -259,7 +258,7 @@ export function ensureContentsDeserialized(mutableMessage: ISequencedDocumentMes
|
|
|
259
258
|
*
|
|
260
259
|
* The return type illustrates the assumption that the message param
|
|
261
260
|
* becomes a InboundSequencedContainerRuntimeMessage by the time the function returns
|
|
262
|
-
* (but there is no runtime validation of the 'type'
|
|
261
|
+
* (but there is no runtime validation of the 'type').
|
|
263
262
|
*/
|
|
264
263
|
function unpack(message: ISequencedDocumentMessage): InboundSequencedContainerRuntimeMessage {
|
|
265
264
|
// We assume the contents is an InboundContainerRuntimeMessage (the message is "packed")
|
|
@@ -270,10 +269,6 @@ function unpack(message: ISequencedDocumentMessage): InboundSequencedContainerRu
|
|
|
270
269
|
|
|
271
270
|
messageUnpacked.type = contents.type;
|
|
272
271
|
messageUnpacked.contents = contents.contents;
|
|
273
|
-
if ("compatDetails" in contents) {
|
|
274
|
-
(messageUnpacked as InboundSequencedRecentlyAddedContainerRuntimeMessage).compatDetails =
|
|
275
|
-
contents.compatDetails;
|
|
276
|
-
}
|
|
277
272
|
return messageUnpacked;
|
|
278
273
|
}
|
|
279
274
|
|
package/src/packageVersion.ts
CHANGED
|
@@ -105,9 +105,9 @@ function isEmptyBatchPendingMessage(message: IPendingMessageFromStash): boolean
|
|
|
105
105
|
function buildPendingMessageContent(message: InboundSequencedContainerRuntimeMessage): string {
|
|
106
106
|
// IMPORTANT: Order matters here, this must match the order of the properties used
|
|
107
107
|
// when submitting the message.
|
|
108
|
-
const { type, contents
|
|
108
|
+
const { type, contents }: InboundContainerRuntimeMessage = message;
|
|
109
109
|
// Any properties that are not defined, won't be emitted by stringify.
|
|
110
|
-
return JSON.stringify({ type, contents
|
|
110
|
+
return JSON.stringify({ type, contents });
|
|
111
111
|
}
|
|
112
112
|
|
|
113
113
|
function typesOfKeys<T extends object>(obj: T): Record<keyof T, string> {
|
|
@@ -126,7 +126,6 @@ function scrubAndStringify(
|
|
|
126
126
|
// For these known/expected keys, we can either drill in (for contents)
|
|
127
127
|
// or just use the value as-is (since it's not personal info)
|
|
128
128
|
scrubbed.contents = message.contents && typesOfKeys(message.contents);
|
|
129
|
-
scrubbed.compatDetails = message.compatDetails;
|
|
130
129
|
scrubbed.type = message.type;
|
|
131
130
|
|
|
132
131
|
return JSON.stringify(scrubbed);
|
package/src/summary/index.ts
CHANGED
|
@@ -208,6 +208,7 @@ export function getMetadataFormatVersion(metadata?: IContainerRuntimeMetadata):
|
|
|
208
208
|
export const aliasBlobName = ".aliases";
|
|
209
209
|
export const metadataBlobName = ".metadata";
|
|
210
210
|
export const chunksBlobName = ".chunks";
|
|
211
|
+
export const recentBatchInfoBlobName = ".recentBatchInfo";
|
|
211
212
|
export const electedSummarizerBlobName = ".electedSummarizer";
|
|
212
213
|
export const idCompressorBlobName = ".idCompressor";
|
|
213
214
|
export const blobHeadersBlobName = blobNameForBlobHeaders;
|