@fluidframework/container-runtime 0.58.2001 → 0.59.1000-61898
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts +15 -2
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +65 -9
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +63 -23
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +39 -7
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +161 -29
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.js +8 -1
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +9 -3
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +22 -6
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +13 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +39 -18
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaScheduler.d.ts +4 -5
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +54 -35
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/garbageCollection.d.ts +31 -27
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +76 -75
- package/dist/garbageCollection.js.map +1 -1
- package/dist/opTelemetry.d.ts +22 -0
- package/dist/opTelemetry.d.ts.map +1 -0
- package/dist/opTelemetry.js +59 -0
- package/dist/opTelemetry.js.map +1 -0
- package/dist/orderedClientElection.d.ts +57 -6
- package/dist/orderedClientElection.d.ts.map +1 -1
- package/dist/orderedClientElection.js +140 -25
- package/dist/orderedClientElection.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/summarizerClientElection.d.ts +2 -0
- package/dist/summarizerClientElection.d.ts.map +1 -1
- package/dist/summarizerClientElection.js +7 -2
- package/dist/summarizerClientElection.js.map +1 -1
- package/dist/summarizerTypes.d.ts +9 -0
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +1 -1
- package/dist/summaryGenerator.js.map +1 -1
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +14 -3
- package/dist/summaryManager.js.map +1 -1
- package/lib/blobManager.d.ts +15 -2
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +66 -10
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +63 -23
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +39 -7
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +163 -31
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.js +8 -1
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +9 -3
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +22 -6
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +13 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +39 -18
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaScheduler.d.ts +4 -5
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +54 -35
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/garbageCollection.d.ts +31 -27
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +75 -74
- package/lib/garbageCollection.js.map +1 -1
- package/lib/opTelemetry.d.ts +22 -0
- package/lib/opTelemetry.d.ts.map +1 -0
- package/lib/opTelemetry.js +55 -0
- package/lib/opTelemetry.js.map +1 -0
- package/lib/orderedClientElection.d.ts +57 -6
- package/lib/orderedClientElection.d.ts.map +1 -1
- package/lib/orderedClientElection.js +140 -25
- package/lib/orderedClientElection.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/summarizerClientElection.d.ts +2 -0
- package/lib/summarizerClientElection.d.ts.map +1 -1
- package/lib/summarizerClientElection.js +7 -2
- package/lib/summarizerClientElection.js.map +1 -1
- package/lib/summarizerTypes.d.ts +9 -0
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +1 -1
- package/lib/summaryGenerator.js.map +1 -1
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +14 -3
- package/lib/summaryManager.js.map +1 -1
- package/package.json +63 -19
- package/src/blobManager.ts +78 -11
- package/src/connectionTelemetry.ts +110 -19
- package/src/containerRuntime.ts +191 -36
- package/src/dataStore.ts +7 -1
- package/src/dataStoreContext.ts +22 -7
- package/src/dataStores.ts +40 -19
- package/src/deltaScheduler.ts +65 -39
- package/src/garbageCollection.ts +92 -78
- package/src/opTelemetry.ts +71 -0
- package/src/orderedClientElection.ts +154 -25
- package/src/packageVersion.ts +1 -1
- package/src/summarizerClientElection.ts +7 -2
- package/src/summarizerTypes.ts +9 -0
- package/src/summaryGenerator.ts +9 -1
- package/src/summaryManager.ts +15 -4
package/src/containerRuntime.ts
CHANGED
|
@@ -10,7 +10,6 @@ import {
|
|
|
10
10
|
FluidObject,
|
|
11
11
|
IFluidHandle,
|
|
12
12
|
IFluidHandleContext,
|
|
13
|
-
IFluidObject,
|
|
14
13
|
IFluidRouter,
|
|
15
14
|
IRequest,
|
|
16
15
|
IResponse,
|
|
@@ -91,7 +90,6 @@ import {
|
|
|
91
90
|
import {
|
|
92
91
|
addBlobToSummary,
|
|
93
92
|
addTreeToSummary,
|
|
94
|
-
convertToSummaryTree,
|
|
95
93
|
createRootSummarizerNodeWithGC,
|
|
96
94
|
IRootSummarizerNodeWithGC,
|
|
97
95
|
RequestParser,
|
|
@@ -100,7 +98,9 @@ import {
|
|
|
100
98
|
requestFluidObject,
|
|
101
99
|
responseToException,
|
|
102
100
|
seqFromTree,
|
|
101
|
+
calculateStats,
|
|
103
102
|
} from "@fluidframework/runtime-utils";
|
|
103
|
+
import { GCDataBuilder } from "@fluidframework/garbage-collector";
|
|
104
104
|
import { v4 as uuid } from "uuid";
|
|
105
105
|
import { ContainerFluidHandleContext } from "./containerHandleContext";
|
|
106
106
|
import { FluidDataStoreRegistry } from "./dataStoreRegistry";
|
|
@@ -141,6 +141,7 @@ import { formExponentialFn, Throttler } from "./throttler";
|
|
|
141
141
|
import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
|
|
142
142
|
import {
|
|
143
143
|
GarbageCollector,
|
|
144
|
+
GCNodeType,
|
|
144
145
|
gcTreeKey,
|
|
145
146
|
IGarbageCollectionRuntime,
|
|
146
147
|
IGarbageCollector,
|
|
@@ -152,6 +153,7 @@ import {
|
|
|
152
153
|
isDataStoreAliasMessage,
|
|
153
154
|
} from "./dataStore";
|
|
154
155
|
import { BindBatchTracker } from "./batchTracker";
|
|
156
|
+
import { OpTracker } from "./opTelemetry";
|
|
155
157
|
|
|
156
158
|
export enum ContainerMessageType {
|
|
157
159
|
// An op to be delivered to store
|
|
@@ -277,8 +279,8 @@ export interface ISummaryRuntimeOptions {
|
|
|
277
279
|
* Options for container runtime.
|
|
278
280
|
*/
|
|
279
281
|
export interface IContainerRuntimeOptions {
|
|
280
|
-
summaryOptions?: ISummaryRuntimeOptions;
|
|
281
|
-
gcOptions?: IGCRuntimeOptions;
|
|
282
|
+
readonly summaryOptions?: ISummaryRuntimeOptions;
|
|
283
|
+
readonly gcOptions?: IGCRuntimeOptions;
|
|
282
284
|
/**
|
|
283
285
|
* Affects the behavior while loading the runtime when the data verification check which
|
|
284
286
|
* compares the DeltaManager sequence number (obtained from protocol in summary) to the
|
|
@@ -287,13 +289,20 @@ export interface IContainerRuntimeOptions {
|
|
|
287
289
|
* 2. "log" will log an error event to telemetry, but still continue to load.
|
|
288
290
|
* 3. "bypass" will skip the check entirely. This is not recommended.
|
|
289
291
|
*/
|
|
290
|
-
loadSequenceNumberVerification?: "close" | "log" | "bypass";
|
|
292
|
+
readonly loadSequenceNumberVerification?: "close" | "log" | "bypass";
|
|
291
293
|
/**
|
|
292
294
|
* Should the runtime use data store aliasing for creating root datastores.
|
|
293
295
|
* In case of aliasing conflicts, the runtime will raise an exception which does
|
|
294
296
|
* not effect the status of the container.
|
|
295
297
|
*/
|
|
296
|
-
useDataStoreAliasing?: boolean;
|
|
298
|
+
readonly useDataStoreAliasing?: boolean;
|
|
299
|
+
/**
|
|
300
|
+
* Sets the flush mode for the runtime. In Immediate flush mode the runtime will immediately
|
|
301
|
+
* send all operations to the driver layer, while in TurnBased the operations will be buffered
|
|
302
|
+
* and then sent them as a single batch at the end of the turn.
|
|
303
|
+
* By default, flush mode is TurnBased.
|
|
304
|
+
*/
|
|
305
|
+
readonly flushMode?: FlushMode;
|
|
297
306
|
}
|
|
298
307
|
|
|
299
308
|
type IRuntimeMessageMetadata = undefined | {
|
|
@@ -347,6 +356,13 @@ const maxOpSizeInBytesKey = "Fluid.ContainerRuntime.MaxOpSizeInBytes";
|
|
|
347
356
|
// to not reach the 1MB limits in socket.io and Kafka.
|
|
348
357
|
const defaultMaxOpSizeInBytes = 768000;
|
|
349
358
|
|
|
359
|
+
// By default, the size of the contents for the incoming ops is tracked.
|
|
360
|
+
// However, in certain situations, this may incur a performance hit.
|
|
361
|
+
// The feature-gate below can be used to disable this feature.
|
|
362
|
+
const disableOpTrackingKey = "Fluid.ContainerRuntime.DisableOpTracking";
|
|
363
|
+
|
|
364
|
+
const defaultFlushMode = FlushMode.TurnBased;
|
|
365
|
+
|
|
350
366
|
export enum RuntimeMessage {
|
|
351
367
|
FluidDataStoreOp = "component",
|
|
352
368
|
Attach = "attach",
|
|
@@ -394,6 +410,7 @@ class ScheduleManagerCore {
|
|
|
394
410
|
private currentBatchClientId: string | undefined;
|
|
395
411
|
private localPaused = false;
|
|
396
412
|
private timePaused = 0;
|
|
413
|
+
private batchCount = 0;
|
|
397
414
|
|
|
398
415
|
constructor(
|
|
399
416
|
private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
|
|
@@ -482,14 +499,30 @@ class ScheduleManagerCore {
|
|
|
482
499
|
this.deltaManager.inbound.pause();
|
|
483
500
|
}
|
|
484
501
|
|
|
485
|
-
private resumeQueue(startBatch: number,
|
|
502
|
+
private resumeQueue(startBatch: number, messageEndBatch: ISequencedDocumentMessage) {
|
|
503
|
+
const endBatch = messageEndBatch.sequenceNumber;
|
|
504
|
+
const duration = performance.now() - this.timePaused;
|
|
505
|
+
|
|
506
|
+
this.batchCount++;
|
|
507
|
+
if (this.batchCount % 1000 === 1) {
|
|
508
|
+
this.logger.sendTelemetryEvent({
|
|
509
|
+
eventName: "BatchStats",
|
|
510
|
+
sequenceNumber: endBatch,
|
|
511
|
+
length: endBatch - startBatch + 1,
|
|
512
|
+
msnDistance: endBatch - messageEndBatch.minimumSequenceNumber,
|
|
513
|
+
duration,
|
|
514
|
+
batchCount: this.batchCount,
|
|
515
|
+
interrupted: this.localPaused,
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
|
|
486
519
|
// Return early if no change in value
|
|
487
520
|
if (!this.localPaused) {
|
|
488
521
|
return;
|
|
489
522
|
}
|
|
490
523
|
|
|
491
524
|
this.localPaused = false;
|
|
492
|
-
|
|
525
|
+
|
|
493
526
|
// Random round number - we want to know when batch waiting paused op processing.
|
|
494
527
|
if (duration > latencyThreshold) {
|
|
495
528
|
this.logger.sendErrorEvent({
|
|
@@ -564,7 +597,7 @@ class ScheduleManagerCore {
|
|
|
564
597
|
} else if (batchMetadata === false) {
|
|
565
598
|
assert(this.pauseSequenceNumber !== undefined, 0x2a0 /* "batch presence was validated above" */);
|
|
566
599
|
// Batch is complete, we can process it!
|
|
567
|
-
this.resumeQueue(this.pauseSequenceNumber, message
|
|
600
|
+
this.resumeQueue(this.pauseSequenceNumber, message);
|
|
568
601
|
this.pauseSequenceNumber = undefined;
|
|
569
602
|
this.currentBatchClientId = undefined;
|
|
570
603
|
} else {
|
|
@@ -605,7 +638,7 @@ export class ScheduleManager {
|
|
|
605
638
|
|
|
606
639
|
// This could be the beginning of a new batch or an individual message.
|
|
607
640
|
this.emitter.emit("batchBegin", message);
|
|
608
|
-
this.deltaScheduler.batchBegin();
|
|
641
|
+
this.deltaScheduler.batchBegin(message);
|
|
609
642
|
|
|
610
643
|
const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
|
|
611
644
|
if (batch) {
|
|
@@ -626,7 +659,7 @@ export class ScheduleManager {
|
|
|
626
659
|
this.hitError = true;
|
|
627
660
|
this.batchClientId = undefined;
|
|
628
661
|
this.emitter.emit("batchEnd", error, message);
|
|
629
|
-
this.deltaScheduler.batchEnd();
|
|
662
|
+
this.deltaScheduler.batchEnd(message);
|
|
630
663
|
return;
|
|
631
664
|
}
|
|
632
665
|
|
|
@@ -636,7 +669,7 @@ export class ScheduleManager {
|
|
|
636
669
|
if (this.batchClientId === undefined || batch === false) {
|
|
637
670
|
this.batchClientId = undefined;
|
|
638
671
|
this.emitter.emit("batchEnd", undefined, message);
|
|
639
|
-
this.deltaScheduler.batchEnd();
|
|
672
|
+
this.deltaScheduler.batchEnd(message);
|
|
640
673
|
return;
|
|
641
674
|
}
|
|
642
675
|
}
|
|
@@ -710,6 +743,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
710
743
|
gcOptions = {},
|
|
711
744
|
loadSequenceNumberVerification = "close",
|
|
712
745
|
useDataStoreAliasing = false,
|
|
746
|
+
flushMode = defaultFlushMode,
|
|
713
747
|
} = runtimeOptions;
|
|
714
748
|
|
|
715
749
|
// We pack at data store level only. If isolated channels are disabled,
|
|
@@ -805,6 +839,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
805
839
|
gcOptions,
|
|
806
840
|
loadSequenceNumberVerification,
|
|
807
841
|
useDataStoreAliasing,
|
|
842
|
+
flushMode,
|
|
808
843
|
},
|
|
809
844
|
containerScope,
|
|
810
845
|
logger,
|
|
@@ -873,7 +908,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
873
908
|
return this._flushMode;
|
|
874
909
|
}
|
|
875
910
|
|
|
876
|
-
public get scope():
|
|
911
|
+
public get scope(): FluidObject {
|
|
877
912
|
return this.containerScope;
|
|
878
913
|
}
|
|
879
914
|
|
|
@@ -909,7 +944,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
909
944
|
private readonly defaultMaxConsecutiveReconnects = 15;
|
|
910
945
|
|
|
911
946
|
private _orderSequentiallyCalls: number = 0;
|
|
912
|
-
private _flushMode: FlushMode
|
|
947
|
+
private _flushMode: FlushMode;
|
|
913
948
|
private needsFlush = false;
|
|
914
949
|
private flushTrigger = false;
|
|
915
950
|
|
|
@@ -983,6 +1018,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
983
1018
|
|
|
984
1019
|
private readonly createContainerMetadata: ICreateContainerMetadata;
|
|
985
1020
|
private summaryCount: number | undefined;
|
|
1021
|
+
private readonly opTracker: OpTracker;
|
|
986
1022
|
|
|
987
1023
|
private constructor(
|
|
988
1024
|
private readonly context: IContainerContext,
|
|
@@ -1037,16 +1073,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1037
1073
|
this.maxConsecutiveReconnects =
|
|
1038
1074
|
this.mc.config.getNumber(maxConsecutiveReconnectsKey) ?? this.defaultMaxConsecutiveReconnects;
|
|
1039
1075
|
|
|
1076
|
+
this._flushMode = runtimeOptions.flushMode;
|
|
1040
1077
|
this.garbageCollector = GarbageCollector.create(
|
|
1041
1078
|
this,
|
|
1042
1079
|
this.runtimeOptions.gcOptions,
|
|
1043
|
-
(
|
|
1044
|
-
(nodePath: string) => this.dataStores.getNodePackagePath(nodePath),
|
|
1045
|
-
/**
|
|
1046
|
-
* Returns the timestamp of the last message seen by this client. This is used by garbage collector as
|
|
1047
|
-
* the current reference timestamp for tracking unreferenced objects.
|
|
1048
|
-
*/
|
|
1049
|
-
() => this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp,
|
|
1080
|
+
(nodePath: string) => this.getGCNodePackagePath(nodePath),
|
|
1050
1081
|
() => this.messageAtLastSummary?.timestamp,
|
|
1051
1082
|
context.baseSnapshot,
|
|
1052
1083
|
async <T>(id: string) => readAndParse<T>(this.storage, id),
|
|
@@ -1098,7 +1129,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1098
1129
|
),
|
|
1099
1130
|
(id: string) => this.summarizerNode.deleteChild(id),
|
|
1100
1131
|
this.mc.logger,
|
|
1101
|
-
async () => this.garbageCollector.
|
|
1132
|
+
async () => this.garbageCollector.getBaseGCDetails(),
|
|
1102
1133
|
(path: string, timestampMs: number, packagePath?: readonly string[]) => this.garbageCollector.nodeUpdated(
|
|
1103
1134
|
path,
|
|
1104
1135
|
"Changed",
|
|
@@ -1277,6 +1308,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1277
1308
|
|
|
1278
1309
|
ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
|
|
1279
1310
|
BindBatchTracker(this, this.logger);
|
|
1311
|
+
this.opTracker = new OpTracker(this.deltaManager, this.mc.config.getBoolean(disableOpTrackingKey) === true);
|
|
1280
1312
|
}
|
|
1281
1313
|
|
|
1282
1314
|
public dispose(error?: Error): void {
|
|
@@ -1446,13 +1478,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1446
1478
|
const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
|
|
1447
1479
|
addBlobToSummary(summaryTree, electedSummarizerBlobName, electedSummarizerContent);
|
|
1448
1480
|
}
|
|
1449
|
-
const snapshot = this.blobManager.snapshot();
|
|
1450
1481
|
|
|
1482
|
+
const summary = this.blobManager.summarize();
|
|
1451
1483
|
// Some storage (like git) doesn't allow empty tree, so we can omit it.
|
|
1452
1484
|
// and the blob manager can handle the tree not existing when loading
|
|
1453
|
-
if (
|
|
1454
|
-
|
|
1455
|
-
addTreeToSummary(summaryTree, blobsTreeName, blobsTree);
|
|
1485
|
+
if (Object.keys(summary.summary.tree).length > 0) {
|
|
1486
|
+
addTreeToSummary(summaryTree, blobsTreeName, summary);
|
|
1456
1487
|
}
|
|
1457
1488
|
|
|
1458
1489
|
if (this.garbageCollector.writeDataAtRoot) {
|
|
@@ -1700,6 +1731,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1700
1731
|
return;
|
|
1701
1732
|
}
|
|
1702
1733
|
|
|
1734
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1735
|
+
eventName: "FlushMode Updated",
|
|
1736
|
+
old: this._flushMode,
|
|
1737
|
+
new: mode,
|
|
1738
|
+
});
|
|
1739
|
+
|
|
1703
1740
|
// Flush any pending batches if switching to immediate
|
|
1704
1741
|
if (mode === FlushMode.Immediate) {
|
|
1705
1742
|
this.flush();
|
|
@@ -1791,7 +1828,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1791
1828
|
*/
|
|
1792
1829
|
private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1793
1830
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
1794
|
-
|
|
1831
|
+
// back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
|
|
1832
|
+
// older versions, we still have to call bindToContext.
|
|
1833
|
+
if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
|
|
1834
|
+
fluidDataStore.makeVisibleAndAttachGraph();
|
|
1835
|
+
} else {
|
|
1836
|
+
fluidDataStore.bindToContext();
|
|
1837
|
+
}
|
|
1795
1838
|
return fluidDataStore;
|
|
1796
1839
|
}
|
|
1797
1840
|
|
|
@@ -1863,7 +1906,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1863
1906
|
const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
|
|
1864
1907
|
Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
|
|
1865
1908
|
if (isRoot) {
|
|
1866
|
-
|
|
1909
|
+
// back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
|
|
1910
|
+
// For older versions, we still have to call bindToContext.
|
|
1911
|
+
if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
|
|
1912
|
+
fluidDataStore.makeVisibleAndAttachGraph();
|
|
1913
|
+
} else {
|
|
1914
|
+
fluidDataStore.bindToContext();
|
|
1915
|
+
}
|
|
1916
|
+
this.logger.sendTelemetryEvent({
|
|
1917
|
+
eventName: "Root datastore with props",
|
|
1918
|
+
hasProps: props !== undefined,
|
|
1919
|
+
});
|
|
1867
1920
|
}
|
|
1868
1921
|
return channelToDataStore(fluidDataStore, id, this, this.dataStores, this.mc.logger);
|
|
1869
1922
|
}
|
|
@@ -2041,11 +2094,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2041
2094
|
gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
2042
2095
|
}
|
|
2043
2096
|
|
|
2044
|
-
const
|
|
2045
|
-
|
|
2097
|
+
const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState);
|
|
2098
|
+
|
|
2099
|
+
assert(summary.type === SummaryType.Tree,
|
|
2046
2100
|
0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
2047
2101
|
|
|
2048
|
-
return {
|
|
2102
|
+
return { stats, summary, gcStats };
|
|
2049
2103
|
}
|
|
2050
2104
|
|
|
2051
2105
|
/**
|
|
@@ -2064,7 +2118,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2064
2118
|
* @param fullGC - true to bypass optimizations and force full generation of GC data.
|
|
2065
2119
|
*/
|
|
2066
2120
|
public async getGCData(fullGC?: boolean): Promise<IGarbageCollectionData> {
|
|
2067
|
-
|
|
2121
|
+
const builder = new GCDataBuilder();
|
|
2122
|
+
const dsGCData = await this.dataStores.getGCData(fullGC);
|
|
2123
|
+
builder.addNodes(dsGCData.gcNodes);
|
|
2124
|
+
|
|
2125
|
+
const blobsGCData = this.blobManager.getGCData(fullGC);
|
|
2126
|
+
builder.addNodes(blobsGCData.gcNodes);
|
|
2127
|
+
return builder.getGCData();
|
|
2068
2128
|
}
|
|
2069
2129
|
|
|
2070
2130
|
/**
|
|
@@ -2080,7 +2140,83 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2080
2140
|
// always referenced, so the used routes is only self-route (empty string).
|
|
2081
2141
|
this.summarizerNode.updateUsedRoutes([""]);
|
|
2082
2142
|
|
|
2083
|
-
|
|
2143
|
+
const dataStoreUsedRoutes: string[] = [];
|
|
2144
|
+
for (const route of usedRoutes) {
|
|
2145
|
+
if (route.split("/")[1] !== BlobManager.basePath) {
|
|
2146
|
+
dataStoreUsedRoutes.push(route);
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
|
|
2150
|
+
return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes, gcTimestamp);
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
/**
|
|
2154
|
+
* When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
|
|
2155
|
+
* scenarios with accessing deleted content.
|
|
2156
|
+
* @param unusedRoutes - The routes that are unused in all data stores in this Container.
|
|
2157
|
+
*/
|
|
2158
|
+
public deleteUnusedRoutes(unusedRoutes: string[]) {
|
|
2159
|
+
const blobManagerUnusedRoutes: string[] = [];
|
|
2160
|
+
const dataStoreUnusedRoutes: string[] = [];
|
|
2161
|
+
for (const route of unusedRoutes) {
|
|
2162
|
+
if (this.isBlobPath(route)) {
|
|
2163
|
+
blobManagerUnusedRoutes.push(route);
|
|
2164
|
+
} else {
|
|
2165
|
+
dataStoreUnusedRoutes.push(route);
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
this.blobManager.deleteUnusedRoutes(blobManagerUnusedRoutes);
|
|
2170
|
+
this.dataStores.deleteUnusedRoutes(dataStoreUnusedRoutes);
|
|
2171
|
+
}
|
|
2172
|
+
|
|
2173
|
+
/**
|
|
2174
|
+
* Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
|
|
2175
|
+
*/
|
|
2176
|
+
public getCurrentReferenceTimestampMs(): number | undefined {
|
|
2177
|
+
// Use the timestamp of the last message seen by this client as that is server generated. If no messages have
|
|
2178
|
+
// been processed, use the timestamp of the message from the last summary.
|
|
2179
|
+
return this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp;
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
/**
|
|
2183
|
+
* Returns the type of the GC node. Currently, there are nodes that belong to data store and nodes that belong
|
|
2184
|
+
* to the blob manager.
|
|
2185
|
+
*/
|
|
2186
|
+
public getNodeType(nodePath: string): GCNodeType {
|
|
2187
|
+
if (this.isBlobPath(nodePath)) {
|
|
2188
|
+
return GCNodeType.Blob;
|
|
2189
|
+
}
|
|
2190
|
+
if (this.dataStores.isDataStoreNode(nodePath)) {
|
|
2191
|
+
return GCNodeType.DataStore;
|
|
2192
|
+
}
|
|
2193
|
+
// Root node ("/") and DDS nodes belong to "Other" node types.
|
|
2194
|
+
return GCNodeType.Other;
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
/**
|
|
2198
|
+
* Called by GC to retrieve the package path of the node with the given path. The node should belong to a
|
|
2199
|
+
* data store or an attachment blob.
|
|
2200
|
+
*/
|
|
2201
|
+
public getGCNodePackagePath(nodePath: string): readonly string[] | undefined {
|
|
2202
|
+
// If the node is a blob, return "_blobs" as the package path.
|
|
2203
|
+
if (this.isBlobPath(nodePath)) {
|
|
2204
|
+
return ["_blobs"];
|
|
2205
|
+
}
|
|
2206
|
+
const dataStorePkgPath = this.dataStores.getDataStorePackagePath(nodePath);
|
|
2207
|
+
assert(dataStorePkgPath !== undefined, 0x2d6 /* "Package path requested for unknown node type." */);
|
|
2208
|
+
return dataStorePkgPath;
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
/**
|
|
2212
|
+
* Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
|
|
2213
|
+
*/
|
|
2214
|
+
private isBlobPath(path: string): boolean {
|
|
2215
|
+
const pathParts = path.split("/");
|
|
2216
|
+
if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
|
|
2217
|
+
return false;
|
|
2218
|
+
}
|
|
2219
|
+
return true;
|
|
2084
2220
|
}
|
|
2085
2221
|
|
|
2086
2222
|
/**
|
|
@@ -2144,6 +2280,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2144
2280
|
await this.deltaManager.inbound.pause();
|
|
2145
2281
|
|
|
2146
2282
|
const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
|
|
2283
|
+
const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
|
|
2147
2284
|
const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
|
|
2148
2285
|
|
|
2149
2286
|
// We should be here is we haven't processed be here. If we are of if the last message's sequence number
|
|
@@ -2188,7 +2325,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2188
2325
|
|
|
2189
2326
|
let continueResult = checkContinue();
|
|
2190
2327
|
if (!continueResult.continue) {
|
|
2191
|
-
return {
|
|
2328
|
+
return {
|
|
2329
|
+
stage: "base",
|
|
2330
|
+
referenceSequenceNumber: summaryRefSeqNum,
|
|
2331
|
+
minimumSequenceNumber,
|
|
2332
|
+
error: continueResult.error,
|
|
2333
|
+
};
|
|
2192
2334
|
}
|
|
2193
2335
|
|
|
2194
2336
|
// increment summary count
|
|
@@ -2211,7 +2353,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2211
2353
|
runGC: this.garbageCollector.shouldRunGC,
|
|
2212
2354
|
});
|
|
2213
2355
|
} catch (error) {
|
|
2214
|
-
return {
|
|
2356
|
+
return {
|
|
2357
|
+
stage: "base",
|
|
2358
|
+
referenceSequenceNumber: summaryRefSeqNum,
|
|
2359
|
+
minimumSequenceNumber,
|
|
2360
|
+
error,
|
|
2361
|
+
};
|
|
2215
2362
|
}
|
|
2216
2363
|
const { summary: summaryTree, stats: partialStats } = summarizeResult;
|
|
2217
2364
|
|
|
@@ -2226,15 +2373,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2226
2373
|
assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
|
|
2227
2374
|
const handleCount = Object.values(dataStoreTree.tree).filter(
|
|
2228
2375
|
(value) => value.type === SummaryType.Handle).length;
|
|
2376
|
+
const gcSummaryTreeStats = summaryTree.tree[gcTreeKey]
|
|
2377
|
+
? calculateStats((summaryTree.tree[gcTreeKey] as ISummaryTree))
|
|
2378
|
+
: undefined;
|
|
2229
2379
|
|
|
2230
2380
|
const summaryStats: IGeneratedSummaryStats = {
|
|
2231
2381
|
dataStoreCount: this.dataStores.size,
|
|
2232
2382
|
summarizedDataStoreCount: this.dataStores.size - handleCount,
|
|
2233
2383
|
gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
|
|
2384
|
+
gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
|
|
2385
|
+
gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
|
|
2386
|
+
opsSizesSinceLastSummary: this.opTracker.opsSizeAccumulator,
|
|
2387
|
+
nonSystemOpsSinceLastSummary: this.opTracker.nonSystemOpCount,
|
|
2234
2388
|
...partialStats,
|
|
2235
2389
|
};
|
|
2236
2390
|
const generateSummaryData = {
|
|
2237
2391
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
2392
|
+
minimumSequenceNumber,
|
|
2238
2393
|
summaryTree,
|
|
2239
2394
|
summaryStats,
|
|
2240
2395
|
generateDuration: trace.trace().duration,
|
|
@@ -2301,7 +2456,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2301
2456
|
} as const;
|
|
2302
2457
|
|
|
2303
2458
|
this.summarizerNode.completeSummary(handle);
|
|
2304
|
-
|
|
2459
|
+
this.opTracker.reset();
|
|
2305
2460
|
return submitData;
|
|
2306
2461
|
} finally {
|
|
2307
2462
|
// Cleanup wip summary in case of failure
|
package/src/dataStore.ts
CHANGED
|
@@ -75,7 +75,13 @@ class DataStore implements IDataStore {
|
|
|
75
75
|
alias,
|
|
76
76
|
};
|
|
77
77
|
|
|
78
|
-
this.
|
|
78
|
+
// back-compat 0.58.2000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
|
|
79
|
+
// older versions, we still have to call bindToContext.
|
|
80
|
+
if (this.fluidDataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
|
|
81
|
+
this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
|
|
82
|
+
} else {
|
|
83
|
+
this.fluidDataStoreChannel.bindToContext();
|
|
84
|
+
}
|
|
79
85
|
|
|
80
86
|
if (this.runtime.attachState === AttachState.Detached) {
|
|
81
87
|
const localResult = this.datastores.processAliasMessageCore(message);
|
package/src/dataStoreContext.ts
CHANGED
|
@@ -132,7 +132,7 @@ export interface ILocalFluidDataStoreContextProps extends IFluidDataStoreContext
|
|
|
132
132
|
readonly pkg: Readonly<string[]> | undefined;
|
|
133
133
|
readonly snapshotTree: ISnapshotTree | undefined;
|
|
134
134
|
readonly isRootDataStore: boolean | undefined;
|
|
135
|
-
readonly
|
|
135
|
+
readonly makeLocallyVisibleFn: () => void;
|
|
136
136
|
/**
|
|
137
137
|
* @deprecated 0.16 Issue #1635, #3631
|
|
138
138
|
*/
|
|
@@ -261,7 +261,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
261
261
|
private readonly existing: boolean,
|
|
262
262
|
private bindState: BindState,
|
|
263
263
|
public readonly isLocalDataStore: boolean,
|
|
264
|
-
|
|
264
|
+
private readonly makeLocallyVisibleFn: () => void,
|
|
265
265
|
) {
|
|
266
266
|
super();
|
|
267
267
|
|
|
@@ -284,7 +284,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
284
284
|
assert(this.bindState === BindState.NotBound, 0x13b /* "datastore context is already in bound state" */);
|
|
285
285
|
this.bindState = BindState.Binding;
|
|
286
286
|
assert(this.channel !== undefined, 0x13c /* "undefined channel on datastore context" */);
|
|
287
|
-
|
|
287
|
+
this.makeLocallyVisible();
|
|
288
288
|
this.bindState = BindState.Bound;
|
|
289
289
|
};
|
|
290
290
|
|
|
@@ -627,6 +627,15 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
627
627
|
return this._containerRuntime.submitDataStoreSignal(this.id, type, content);
|
|
628
628
|
}
|
|
629
629
|
|
|
630
|
+
/**
|
|
631
|
+
* This is called by the data store channel when it becomes locally visible indicating that it is ready to become
|
|
632
|
+
* globally visible now.
|
|
633
|
+
*/
|
|
634
|
+
public makeLocallyVisible() {
|
|
635
|
+
assert(this.channel !== undefined, 0x2cf /* "undefined channel on datastore context" */);
|
|
636
|
+
this.makeLocallyVisibleFn();
|
|
637
|
+
}
|
|
638
|
+
|
|
630
639
|
protected bindRuntime(channel: IFluidDataStoreChannel) {
|
|
631
640
|
if (this.channel) {
|
|
632
641
|
throw new Error("Runtime already bound");
|
|
@@ -864,7 +873,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
|
|
|
864
873
|
props.snapshotTree !== undefined ? true : false /* existing */,
|
|
865
874
|
props.snapshotTree ? BindState.Bound : BindState.NotBound,
|
|
866
875
|
true /* isLocalDataStore */,
|
|
867
|
-
props.
|
|
876
|
+
props.makeLocallyVisibleFn,
|
|
868
877
|
);
|
|
869
878
|
|
|
870
879
|
this.snapshotTree = props.snapshotTree;
|
|
@@ -993,7 +1002,7 @@ export class LocalDetachedFluidDataStoreContext
|
|
|
993
1002
|
|
|
994
1003
|
public async attachRuntime(
|
|
995
1004
|
registry: IProvideFluidDataStoreFactory,
|
|
996
|
-
|
|
1005
|
+
dataStoreChannel: IFluidDataStoreChannel)
|
|
997
1006
|
{
|
|
998
1007
|
assert(this.detachedRuntimeCreation, 0x154 /* "runtime creation is already attached" */);
|
|
999
1008
|
assert(this.channelDeferred === undefined, 0x155 /* "channel deferral is already set" */);
|
|
@@ -1009,10 +1018,16 @@ export class LocalDetachedFluidDataStoreContext
|
|
|
1009
1018
|
this.detachedRuntimeCreation = false;
|
|
1010
1019
|
this.channelDeferred = new Deferred<IFluidDataStoreChannel>();
|
|
1011
1020
|
|
|
1012
|
-
super.bindRuntime(
|
|
1021
|
+
super.bindRuntime(dataStoreChannel);
|
|
1013
1022
|
|
|
1014
1023
|
if (await this.isRoot()) {
|
|
1015
|
-
|
|
1024
|
+
// back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
|
|
1025
|
+
// For older versions, we still have to call bindToContext.
|
|
1026
|
+
if (dataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
|
|
1027
|
+
dataStoreChannel.makeVisibleAndAttachGraph();
|
|
1028
|
+
} else {
|
|
1029
|
+
dataStoreChannel.bindToContext();
|
|
1030
|
+
}
|
|
1016
1031
|
}
|
|
1017
1032
|
}
|
|
1018
1033
|
|