@fluidframework/container-runtime 0.51.0-43124 → 0.51.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerRuntime.d.ts +31 -31
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +61 -144
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.js +1 -1
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +3 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +3 -4
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +116 -0
- package/dist/garbageCollection.d.ts.map +1 -0
- package/dist/garbageCollection.js +148 -0
- package/dist/garbageCollection.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +0 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +0 -36
- package/dist/pendingStateManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +31 -31
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +62 -145
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.js +1 -1
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +3 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +3 -4
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +116 -0
- package/lib/garbageCollection.d.ts.map +1 -0
- package/lib/garbageCollection.js +144 -0
- package/lib/garbageCollection.js.map +1 -0
- package/lib/index.d.ts +1 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +0 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +0 -36
- package/lib/pendingStateManager.js.map +1 -1
- package/package.json +11 -11
- package/src/containerRuntime.ts +89 -188
- package/src/dataStoreContext.ts +1 -1
- package/src/dataStores.ts +5 -5
- package/src/garbageCollection.ts +269 -0
- package/src/index.ts +1 -0
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +0 -43
package/lib/containerRuntime.js
CHANGED
|
@@ -7,7 +7,6 @@ import { assert, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframew
|
|
|
7
7
|
import { ChildLogger, raiseConnectedEvent, PerformanceEvent, normalizeError, TaggedLoggerAdapter, } from "@fluidframework/telemetry-utils";
|
|
8
8
|
import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
|
|
9
9
|
import { DataCorruptionError, GenericError } from "@fluidframework/container-utils";
|
|
10
|
-
import { runGarbageCollection } from "@fluidframework/garbage-collector";
|
|
11
10
|
import { BlobTreeEntry, TreeTreeEntry, } from "@fluidframework/protocol-base";
|
|
12
11
|
import { MessageType, } from "@fluidframework/protocol-definitions";
|
|
13
12
|
import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
|
|
@@ -23,13 +22,14 @@ import { PendingStateManager } from "./pendingStateManager";
|
|
|
23
22
|
import { pkgVersion } from "./packageVersion";
|
|
24
23
|
import { BlobManager } from "./blobManager";
|
|
25
24
|
import { DataStores, getSummaryForDatastores } from "./dataStores";
|
|
26
|
-
import { blobsTreeName, chunksBlobName, electedSummarizerBlobName, extractSummaryMetadataMessage,
|
|
25
|
+
import { blobsTreeName, chunksBlobName, electedSummarizerBlobName, extractSummaryMetadataMessage, metadataBlobName, wrapSummaryInChannelsTree, } from "./summaryFormat";
|
|
27
26
|
import { SummaryCollection } from "./summaryCollection";
|
|
28
27
|
import { getLocalStorageFeatureGate } from "./localStorageFeatureGates";
|
|
29
28
|
import { OrderedClientCollection, OrderedClientElection } from "./orderedClientElection";
|
|
30
29
|
import { SummarizerClientElection, summarizerClientType } from "./summarizerClientElection";
|
|
31
30
|
import { formExponentialFn, Throttler } from "./throttler";
|
|
32
31
|
import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
|
|
32
|
+
import { GarbageCollector, } from "./garbageCollection";
|
|
33
33
|
export var ContainerMessageType;
|
|
34
34
|
(function (ContainerMessageType) {
|
|
35
35
|
// An op to be delivered to store
|
|
@@ -55,14 +55,6 @@ const DefaultSummaryConfiguration = {
|
|
|
55
55
|
// the min of the two will be chosen
|
|
56
56
|
maxAckWaitTime: 120000,
|
|
57
57
|
};
|
|
58
|
-
/** This is the current version of garbage collection */
|
|
59
|
-
const GCVersion = 1;
|
|
60
|
-
// Local storage key to turn GC on / off.
|
|
61
|
-
const runGCKey = "FluidRunGC";
|
|
62
|
-
// Local storage key to turn GC test mode on / off.
|
|
63
|
-
const gcTestModeKey = "FluidGCTestMode";
|
|
64
|
-
// Local storage key to turn GC sweep on / off.
|
|
65
|
-
const runSweepKey = "FluidRunSweep";
|
|
66
58
|
// Local storage key to set the default flush mode to TurnBased
|
|
67
59
|
const turnBasedFlushModeKey = "FluidFlushModeTurnBased";
|
|
68
60
|
export function isRuntimeMessage(message) {
|
|
@@ -250,7 +242,7 @@ export const agentSchedulerId = "_scheduler";
|
|
|
250
242
|
*/
|
|
251
243
|
export class ContainerRuntime extends TypedEventEmitter {
|
|
252
244
|
constructor(context, registry, metadata, electedSummarizerData, chunks, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, requestHandler, _storage) {
|
|
253
|
-
var _a, _b, _c, _d
|
|
245
|
+
var _a, _b, _c, _d;
|
|
254
246
|
super();
|
|
255
247
|
this.context = context;
|
|
256
248
|
this.registry = registry;
|
|
@@ -273,8 +265,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
273
265
|
this._disposed = false;
|
|
274
266
|
this.dirtyContainer = false;
|
|
275
267
|
this.emitDirtyDocumentEvent = true;
|
|
276
|
-
// The current GC version that this container is running.
|
|
277
|
-
this.currentGCVersion = GCVersion;
|
|
278
268
|
/**
|
|
279
269
|
* Used to apply stashed ops at their reference sequence number.
|
|
280
270
|
* Normal op processing is synchronous, but applying stashed ops is async since the
|
|
@@ -329,31 +319,14 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
329
319
|
}
|
|
330
320
|
};
|
|
331
321
|
this.baseSummaryMessage = metadata === null || metadata === void 0 ? void 0 : metadata.message;
|
|
332
|
-
/**
|
|
333
|
-
* gcFeature in metadata is introduced with v1 in the metadata blob. Forced to 0/disallowed before that.
|
|
334
|
-
* For existing documents, we get this value from the metadata blob.
|
|
335
|
-
* For new documents, we get this value based on the gcAllowed flag in runtimeOptions.
|
|
336
|
-
*/
|
|
337
|
-
const prevSummaryGCVersion = existing ? getGCVersion(metadata) : undefined;
|
|
338
|
-
// Default to false for now.
|
|
339
|
-
this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : (this.runtimeOptions.gcOptions.gcAllowed === true ? this.currentGCVersion : 0);
|
|
340
|
-
// Whether GC should run or not. Can override with localStorage flag.
|
|
341
|
-
this.shouldRunGC = (_a = getLocalStorageFeatureGate(runGCKey)) !== null && _a !== void 0 ? _a : (
|
|
342
|
-
// GC must be enabled for the document.
|
|
343
|
-
this.gcEnabled
|
|
344
|
-
// Must not be disabled by runtime option.
|
|
345
|
-
&& !this.runtimeOptions.gcOptions.disableGC);
|
|
346
|
-
// Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
|
|
347
|
-
// localStorage flag.
|
|
348
|
-
this.shouldRunSweep = this.shouldRunGC &&
|
|
349
|
-
((_b = getLocalStorageFeatureGate(runSweepKey)) !== null && _b !== void 0 ? _b : this.runtimeOptions.gcOptions.runSweep === true);
|
|
350
322
|
// Default to false (enabled).
|
|
351
|
-
this.disableIsolatedChannels = (
|
|
323
|
+
this.disableIsolatedChannels = (_a = this.runtimeOptions.summaryOptions.disableIsolatedChannels) !== null && _a !== void 0 ? _a : false;
|
|
352
324
|
this._connected = this.context.connected;
|
|
353
325
|
this.chunkMap = new Map(chunks);
|
|
354
326
|
this.IFluidHandleContext = new ContainerFluidHandleContext("", this);
|
|
355
327
|
this.IFluidSerializer = new FluidSerializer(this.IFluidHandleContext);
|
|
356
328
|
this._logger = ChildLogger.create(this.logger, "ContainerRuntime");
|
|
329
|
+
this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), this._logger, existing, metadata);
|
|
357
330
|
const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
|
|
358
331
|
this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"),
|
|
359
332
|
// Summarize function to call when summarize is called. Summarizer node always tracks summary state.
|
|
@@ -368,8 +341,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
368
341
|
// Must set to true to throw on any data stores failure that was too severe to be handled.
|
|
369
342
|
// We also are not decoding the base summaries at the root.
|
|
370
343
|
throwOnFailure: true,
|
|
371
|
-
// If GC
|
|
372
|
-
gcDisabled: !this.shouldRunGC,
|
|
344
|
+
// If GC should not run, let the summarizer node know so that it does not track GC state.
|
|
345
|
+
gcDisabled: !this.garbageCollector.shouldRunGC,
|
|
373
346
|
// The max duration for which objects can be unreferenced before they are eligible for deletion.
|
|
374
347
|
maxUnreferencedDurationMs: this.runtimeOptions.gcOptions.maxUnreferencedDurationMs,
|
|
375
348
|
});
|
|
@@ -395,7 +368,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
395
368
|
this._logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
|
|
396
369
|
}
|
|
397
370
|
else {
|
|
398
|
-
const maxOpsSinceLastSummary = (
|
|
371
|
+
const maxOpsSinceLastSummary = (_b = this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary) !== null && _b !== void 0 ? _b : 7000;
|
|
399
372
|
const defaultAction = () => {
|
|
400
373
|
if (this.summaryCollection.opsSinceLastAck > maxOpsSinceLastSummary) {
|
|
401
374
|
this.logger.sendErrorEvent({ eventName: "SummaryStatus:Behind" });
|
|
@@ -415,7 +388,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
415
388
|
const orderedClientLogger = ChildLogger.create(this.logger, "OrderedClientElection");
|
|
416
389
|
const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
|
|
417
390
|
const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
|
|
418
|
-
const summarizerClientElectionEnabled = (
|
|
391
|
+
const summarizerClientElectionEnabled = (_c = getLocalStorageFeatureGate("summarizerClientElection")) !== null && _c !== void 0 ? _c : ((_d = this.runtimeOptions.summaryOptions) === null || _d === void 0 ? void 0 : _d.summarizerClientElection) === true;
|
|
419
392
|
this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, maxOpsSinceLastSummary, summarizerClientElectionEnabled);
|
|
420
393
|
if (this.context.clientDetails.type === summarizerClientType) {
|
|
421
394
|
this._summarizer = new Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.IFluidHandleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime));
|
|
@@ -614,16 +587,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
614
587
|
static get defaultFlushMode() {
|
|
615
588
|
return getLocalStorageFeatureGate(turnBasedFlushModeKey) ? FlushMode.TurnBased : FlushMode.Immediate;
|
|
616
589
|
}
|
|
617
|
-
// Tells whether GC is enabled for this document or not. If the summaryGCVersion is > 0, GC is enabled.
|
|
618
|
-
get gcEnabled() {
|
|
619
|
-
return this.latestSummaryGCVersion > 0;
|
|
620
|
-
}
|
|
621
|
-
// Tells whether this container is running in GC test mode. If so, unreferenced data stores are immediately
|
|
622
|
-
// deleted as soon as GC runs.
|
|
623
|
-
get gcTestMode() {
|
|
624
|
-
var _a, _b;
|
|
625
|
-
return (_a = getLocalStorageFeatureGate(gcTestModeKey)) !== null && _a !== void 0 ? _a : ((_b = this.runtimeOptions.gcOptions) === null || _b === void 0 ? void 0 : _b.runGCInTestMode) === true;
|
|
626
|
-
}
|
|
627
590
|
get summarizer() {
|
|
628
591
|
assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
|
|
629
592
|
return this._summarizer;
|
|
@@ -716,7 +679,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
716
679
|
}
|
|
717
680
|
else if (requestParser.pathParts.length > 0) {
|
|
718
681
|
/**
|
|
719
|
-
* If GC
|
|
682
|
+
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
720
683
|
* an error if the data store being requested is marked as unreferenced as per the data store's initial
|
|
721
684
|
* summary.
|
|
722
685
|
*
|
|
@@ -724,7 +687,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
724
687
|
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
725
688
|
*/
|
|
726
689
|
const wait = typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a.wait) === "boolean" ? request.headers.wait : undefined;
|
|
727
|
-
const dataStore = ((_b = request.headers) === null || _b === void 0 ? void 0 : _b.externalRequest) && this.shouldRunGC
|
|
690
|
+
const dataStore = ((_b = request.headers) === null || _b === void 0 ? void 0 : _b.externalRequest) && this.garbageCollector.shouldRunGC
|
|
728
691
|
? await this.getDataStoreIfInitiallyReferenced(id, wait)
|
|
729
692
|
: await this.getDataStore(id, wait);
|
|
730
693
|
const subRequest = requestParser.createSubRequest(1);
|
|
@@ -739,19 +702,12 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
739
702
|
return exceptionToResponse(error);
|
|
740
703
|
}
|
|
741
704
|
}
|
|
742
|
-
get shouldWriteMetadata() {
|
|
743
|
-
// We need the metadata blob if either isolated channels are enabled
|
|
744
|
-
// or GC is enabled at the document level.
|
|
745
|
-
return !this.disableIsolatedChannels || this.gcEnabled;
|
|
746
|
-
}
|
|
747
705
|
formMetadata() {
|
|
748
706
|
var _a;
|
|
749
707
|
return {
|
|
750
708
|
summaryFormatVersion: 1,
|
|
751
709
|
disableIsolatedChannels: this.disableIsolatedChannels || undefined,
|
|
752
|
-
|
|
753
|
-
// we always write the current GC version as that is what is used to generate the GC data.
|
|
754
|
-
gcFeature: this.gcEnabled ? this.currentGCVersion : this.latestSummaryGCVersion,
|
|
710
|
+
gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
|
|
755
711
|
// The last message processed at the time of summary. If there are no messages, nothing has changed from
|
|
756
712
|
// the base summary we loaded from. So, use the message from its metadata blob.
|
|
757
713
|
message: (_a = extractSummaryMetadataMessage(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.baseSummaryMessage,
|
|
@@ -782,8 +738,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
782
738
|
* @deprecated - Use summarize to get summary of the container runtime.
|
|
783
739
|
*/
|
|
784
740
|
async snapshot() {
|
|
785
|
-
if (this.shouldRunGC) {
|
|
786
|
-
await this.collectGarbage(this.logger, true /* fullGC */);
|
|
741
|
+
if (this.garbageCollector.shouldRunGC) {
|
|
742
|
+
await this.collectGarbage({ logger: this.logger, fullGC: true /* fullGC */ });
|
|
787
743
|
}
|
|
788
744
|
const root = { entries: [] };
|
|
789
745
|
const entries = await this.dataStores.snapshot();
|
|
@@ -793,9 +749,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
793
749
|
else {
|
|
794
750
|
root.entries.push(new TreeTreeEntry(channelsTreeName, { entries }));
|
|
795
751
|
}
|
|
796
|
-
|
|
797
|
-
root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata())));
|
|
798
|
-
}
|
|
752
|
+
root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata())));
|
|
799
753
|
if (this.chunkMap.size > 0) {
|
|
800
754
|
root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap])));
|
|
801
755
|
}
|
|
@@ -803,9 +757,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
803
757
|
}
|
|
804
758
|
addContainerBlobsToSummary(summaryTree) {
|
|
805
759
|
var _a;
|
|
806
|
-
|
|
807
|
-
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
808
|
-
}
|
|
760
|
+
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
809
761
|
if (this.chunkMap.size > 0) {
|
|
810
762
|
const content = JSON.stringify([...this.chunkMap]);
|
|
811
763
|
addBlobToSummary(summaryTree, chunksBlobName, content);
|
|
@@ -1016,7 +968,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1016
968
|
}
|
|
1017
969
|
async createRootDataStore(pkg, rootDataStoreId) {
|
|
1018
970
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
1019
|
-
fluidDataStore.
|
|
971
|
+
fluidDataStore.bindToContext();
|
|
1020
972
|
return fluidDataStore;
|
|
1021
973
|
}
|
|
1022
974
|
createDetachedRootDataStore(pkg, rootDataStoreId) {
|
|
@@ -1028,7 +980,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1028
980
|
async _createDataStoreWithProps(pkg, props, id = uuid(), isRoot = false) {
|
|
1029
981
|
const fluidDataStore = await this.dataStores._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
|
|
1030
982
|
if (isRoot) {
|
|
1031
|
-
fluidDataStore.
|
|
983
|
+
fluidDataStore.bindToContext();
|
|
1032
984
|
}
|
|
1033
985
|
return fluidDataStore;
|
|
1034
986
|
}
|
|
@@ -1121,42 +1073,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1121
1073
|
}
|
|
1122
1074
|
return this.context.getAbsoluteUrl(relativeUrl);
|
|
1123
1075
|
}
|
|
1124
|
-
/**
|
|
1125
|
-
* Runs garbage collection and udpates the reference / used state of the nodes in the container.
|
|
1126
|
-
* @returns the number of data stores that have been marked as unreferenced.
|
|
1127
|
-
*/
|
|
1128
|
-
async collectGarbage(logger, fullGC = false) {
|
|
1129
|
-
return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
1130
|
-
var _a;
|
|
1131
|
-
const gcStats = {};
|
|
1132
|
-
// Get the container's GC data and run GC on the reference graph in it.
|
|
1133
|
-
const gcData = await this.dataStores.getGCData(fullGC);
|
|
1134
|
-
const { referencedNodeIds, deletedNodeIds } = runGarbageCollection(gcData.gcNodes, ["/"], this.logger);
|
|
1135
|
-
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1136
|
-
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1137
|
-
// always referenced, so the used routes is only self-route (empty string).
|
|
1138
|
-
this.summarizerNode.updateUsedRoutes([""]);
|
|
1139
|
-
// Remove this node's route ("/") and notify data stores of routes that are used in it.
|
|
1140
|
-
const usedRoutes = referencedNodeIds.filter((id) => { return id !== "/"; });
|
|
1141
|
-
const { dataStoreCount, unusedDataStoreCount } = this.dataStores.updateUsedRoutes(usedRoutes, (_a =
|
|
1142
|
-
// For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
|
|
1143
|
-
// we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
1144
|
-
// of this client's connection - https://github.com/microsoft/FluidFramework/issues/7152.
|
|
1145
|
-
this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp);
|
|
1146
|
-
// Update stats to be reported in the peformance event.
|
|
1147
|
-
gcStats.deletedNodes = deletedNodeIds.length;
|
|
1148
|
-
gcStats.totalNodes = referencedNodeIds.length + deletedNodeIds.length;
|
|
1149
|
-
gcStats.deletedDataStores = unusedDataStoreCount;
|
|
1150
|
-
gcStats.totalDataStores = dataStoreCount;
|
|
1151
|
-
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
1152
|
-
// involving access to deleted data.
|
|
1153
|
-
if (this.gcTestMode) {
|
|
1154
|
-
this.dataStores.deleteUnusedRoutes(deletedNodeIds);
|
|
1155
|
-
}
|
|
1156
|
-
event.end(gcStats);
|
|
1157
|
-
return gcStats;
|
|
1158
|
-
}, { end: true, cancel: "error" });
|
|
1159
|
-
}
|
|
1160
1076
|
async summarizeInternal(fullTree, trackState) {
|
|
1161
1077
|
const summarizeResult = await this.dataStores.summarize(fullTree, trackState);
|
|
1162
1078
|
let pathPartsForChildren;
|
|
@@ -1172,14 +1088,47 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1172
1088
|
* Returns a summary of the runtime at the current sequence number.
|
|
1173
1089
|
*/
|
|
1174
1090
|
async summarize(options) {
|
|
1175
|
-
const { summaryLogger, fullTree = false, trackState = true, runGC = true, fullGC
|
|
1091
|
+
const { summaryLogger, fullTree = false, trackState = true, runGC = true, runSweep, fullGC } = options;
|
|
1176
1092
|
if (runGC) {
|
|
1177
|
-
await this.collectGarbage(summaryLogger, fullGC);
|
|
1093
|
+
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1178
1094
|
}
|
|
1179
1095
|
const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
|
|
1180
1096
|
assert(summarizeResult.summary.type === 1 /* Tree */, 0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1181
1097
|
return summarizeResult;
|
|
1182
1098
|
}
|
|
1099
|
+
/**
|
|
1100
|
+
* Implementation of IGarbageCollectionRuntime::getGCData.
|
|
1101
|
+
* Generates and returns the GC data for this container.
|
|
1102
|
+
* @param fullGC - true to bypass optimizations and force full generation of GC data.
|
|
1103
|
+
*/
|
|
1104
|
+
async getGCData(fullGC) {
|
|
1105
|
+
return this.dataStores.getGCData(fullGC);
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
|
|
1109
|
+
* After GC has run, called to notify this container's nodes of routes that are used in it.
|
|
1110
|
+
* @param usedRoutes - The routes that are used in all nodes in this Container.
|
|
1111
|
+
* @returns the statistics of the used state of the data stores.
|
|
1112
|
+
*/
|
|
1113
|
+
updateUsedRoutes(usedRoutes) {
|
|
1114
|
+
var _a;
|
|
1115
|
+
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1116
|
+
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1117
|
+
// always referenced, so the used routes is only self-route (empty string).
|
|
1118
|
+
this.summarizerNode.updateUsedRoutes([""]);
|
|
1119
|
+
return this.dataStores.updateUsedRoutes(usedRoutes, (_a =
|
|
1120
|
+
// For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
|
|
1121
|
+
// we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
1122
|
+
// of this client's connection - https://github.com/microsoft/FluidFramework/issues/7152.
|
|
1123
|
+
this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp);
|
|
1124
|
+
}
|
|
1125
|
+
/**
|
|
1126
|
+
* Runs garbage collection and udpates the reference / used state of the nodes in the container.
|
|
1127
|
+
* @returns the statistics of the garbage collection run.
|
|
1128
|
+
*/
|
|
1129
|
+
async collectGarbage(options) {
|
|
1130
|
+
return this.garbageCollector.collectGarbage(options);
|
|
1131
|
+
}
|
|
1183
1132
|
/**
|
|
1184
1133
|
* Generates the summary tree, uploads it to storage, and then submits the summarize op.
|
|
1185
1134
|
* This is intended to be called by the summarizer, since it is the implementation of
|
|
@@ -1239,23 +1188,17 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1239
1188
|
if (!continueResult.continue) {
|
|
1240
1189
|
return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error: continueResult.error };
|
|
1241
1190
|
}
|
|
1242
|
-
// If the GC version that this container is loaded from differs from the current GC version that this
|
|
1243
|
-
// container is running, we need to regenerate the GC data and run full summary. This is used to handle
|
|
1244
|
-
// scenarios where we upgrade the GC version because we cannot trust the data from the previous GC version.
|
|
1245
|
-
let forceRegenerateData = false;
|
|
1246
|
-
if (this.gcEnabled && this.latestSummaryGCVersion !== this.currentGCVersion) {
|
|
1247
|
-
forceRegenerateData = true;
|
|
1248
|
-
}
|
|
1249
1191
|
const trace = Trace.start();
|
|
1250
1192
|
let summarizeResult;
|
|
1251
1193
|
try {
|
|
1252
1194
|
summarizeResult = await this.summarize({
|
|
1253
1195
|
summaryLogger,
|
|
1254
|
-
|
|
1196
|
+
// If the GC version changed since the last summary was submitted, we need to regenerate summary by
|
|
1197
|
+
// running full summary. This is used to handle scenarios where we upgrade the GC version because we
|
|
1198
|
+
// cannot trust the data from the previous GC version anymore.
|
|
1199
|
+
fullTree: fullTree || this.garbageCollector.hasGCVersionChanged,
|
|
1255
1200
|
trackState: true,
|
|
1256
|
-
runGC: this.shouldRunGC,
|
|
1257
|
-
fullGC: this.runtimeOptions.gcOptions.runFullGC || forceRegenerateData,
|
|
1258
|
-
runSweep: this.shouldRunSweep,
|
|
1201
|
+
runGC: this.garbageCollector.shouldRunGC,
|
|
1259
1202
|
});
|
|
1260
1203
|
}
|
|
1261
1204
|
catch (error) {
|
|
@@ -1506,18 +1449,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1506
1449
|
eventName: "RefreshLatestSummaryGetSnapshot",
|
|
1507
1450
|
fetchLatest: false,
|
|
1508
1451
|
}), readAndParseBlob, summaryLogger);
|
|
1509
|
-
//
|
|
1510
|
-
|
|
1511
|
-
// If the summary was tracked by this client, it was the one that generated the summary in the first place.
|
|
1512
|
-
// Update the summaryGCVersion to the currentGCVersion of this client.
|
|
1513
|
-
if (result.wasSummaryTracked) {
|
|
1514
|
-
this.latestSummaryGCVersion = this.currentGCVersion;
|
|
1515
|
-
return;
|
|
1516
|
-
}
|
|
1517
|
-
// If the summary was not tracked by this client, update summaryGCVersion from the snapshot that was used
|
|
1518
|
-
// to update the latest summary.
|
|
1519
|
-
await this.updateSummaryGCVersionFromSnapshot(result.snapshot);
|
|
1520
|
-
}
|
|
1452
|
+
// Notify the garbage collector so it can update its latest summary state.
|
|
1453
|
+
await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
|
|
1521
1454
|
}
|
|
1522
1455
|
/**
|
|
1523
1456
|
* Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
|
|
@@ -1533,26 +1466,10 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1533
1466
|
const readAndParseBlob = async (id) => readAndParse(this.storage, id);
|
|
1534
1467
|
const snapshotRefSeq = await seqFromTree(snapshot, readAndParseBlob);
|
|
1535
1468
|
const result = await this.summarizerNode.refreshLatestSummary(undefined, snapshotRefSeq, async () => snapshot, readAndParseBlob, summaryLogger);
|
|
1536
|
-
//
|
|
1537
|
-
|
|
1538
|
-
// Since there is not proposal handle for this summary, it should not have been tracked.
|
|
1539
|
-
assert(!result.wasSummaryTracked, 0x1fd /* "Summary without proposal handle should not have been tracked" */);
|
|
1540
|
-
// Update summaryGCVersion from the snapshot that was used to update the latest summary.
|
|
1541
|
-
await this.updateSummaryGCVersionFromSnapshot(result.snapshot);
|
|
1542
|
-
}
|
|
1469
|
+
// Notify the garbage collector so it can update its latest summary state.
|
|
1470
|
+
await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
|
|
1543
1471
|
return snapshotRefSeq;
|
|
1544
1472
|
}
|
|
1545
|
-
/**
|
|
1546
|
-
* Updates the summary GC version as per the metadata blob in given snapshot.
|
|
1547
|
-
*/
|
|
1548
|
-
async updateSummaryGCVersionFromSnapshot(snapshot) {
|
|
1549
|
-
assert(this.gcEnabled, 0x25a /* "GC version should not be updated when GC is disabled" */);
|
|
1550
|
-
const metadataBlobId = snapshot.blobs[metadataBlobName];
|
|
1551
|
-
if (metadataBlobId) {
|
|
1552
|
-
const metadata = await readAndParse(this.storage, metadataBlobId);
|
|
1553
|
-
this.latestSummaryGCVersion = getGCVersion(metadata);
|
|
1554
|
-
}
|
|
1555
|
-
}
|
|
1556
1473
|
async fetchSnapshotFromStorage(versionId, logger, event) {
|
|
1557
1474
|
return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
1558
1475
|
const stats = {};
|