@fluidframework/container-runtime 0.51.0-43124 → 0.51.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerRuntime.d.ts +31 -31
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +61 -144
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.js +1 -1
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +3 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +3 -4
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +116 -0
- package/dist/garbageCollection.d.ts.map +1 -0
- package/dist/garbageCollection.js +148 -0
- package/dist/garbageCollection.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +0 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +0 -36
- package/dist/pendingStateManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +31 -31
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +62 -145
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.js +1 -1
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +3 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +3 -4
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +116 -0
- package/lib/garbageCollection.d.ts.map +1 -0
- package/lib/garbageCollection.js +144 -0
- package/lib/garbageCollection.js.map +1 -0
- package/lib/index.d.ts +1 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +0 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +0 -36
- package/lib/pendingStateManager.js.map +1 -1
- package/package.json +11 -11
- package/src/containerRuntime.ts +89 -188
- package/src/dataStoreContext.ts +1 -1
- package/src/dataStores.ts +5 -5
- package/src/garbageCollection.ts +269 -0
- package/src/index.ts +1 -0
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +0 -43
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fluidframework/container-runtime",
|
|
3
|
-
"version": "0.51.
|
|
3
|
+
"version": "0.51.3",
|
|
4
4
|
"description": "Fluid container runtime",
|
|
5
5
|
"homepage": "https://fluidframework.com",
|
|
6
6
|
"repository": "https://github.com/microsoft/FluidFramework",
|
|
@@ -59,26 +59,26 @@
|
|
|
59
59
|
"@fluidframework/common-definitions": "^0.20.1",
|
|
60
60
|
"@fluidframework/common-utils": "^0.32.1",
|
|
61
61
|
"@fluidframework/container-definitions": "^0.41.0",
|
|
62
|
-
"@fluidframework/container-runtime-definitions": "0.51.
|
|
63
|
-
"@fluidframework/container-utils": "0.51.
|
|
62
|
+
"@fluidframework/container-runtime-definitions": "^0.51.3",
|
|
63
|
+
"@fluidframework/container-utils": "^0.51.3",
|
|
64
64
|
"@fluidframework/core-interfaces": "^0.40.0",
|
|
65
|
-
"@fluidframework/datastore": "0.51.
|
|
65
|
+
"@fluidframework/datastore": "^0.51.3",
|
|
66
66
|
"@fluidframework/driver-definitions": "^0.41.0",
|
|
67
|
-
"@fluidframework/driver-utils": "0.51.
|
|
68
|
-
"@fluidframework/garbage-collector": "0.51.
|
|
67
|
+
"@fluidframework/driver-utils": "^0.51.3",
|
|
68
|
+
"@fluidframework/garbage-collector": "^0.51.3",
|
|
69
69
|
"@fluidframework/protocol-base": "^0.1033.0",
|
|
70
70
|
"@fluidframework/protocol-definitions": "^0.1025.0",
|
|
71
|
-
"@fluidframework/runtime-definitions": "0.51.
|
|
72
|
-
"@fluidframework/runtime-utils": "0.51.
|
|
73
|
-
"@fluidframework/telemetry-utils": "0.51.
|
|
71
|
+
"@fluidframework/runtime-definitions": "^0.51.3",
|
|
72
|
+
"@fluidframework/runtime-utils": "^0.51.3",
|
|
73
|
+
"@fluidframework/telemetry-utils": "^0.51.3",
|
|
74
74
|
"double-ended-queue": "^2.1.0-0",
|
|
75
75
|
"uuid": "^8.3.1"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@fluidframework/build-common": "^0.23.0",
|
|
79
79
|
"@fluidframework/eslint-config-fluid": "^0.24.0",
|
|
80
|
-
"@fluidframework/mocha-test-setup": "0.51.
|
|
81
|
-
"@fluidframework/test-runtime-utils": "0.51.
|
|
80
|
+
"@fluidframework/mocha-test-setup": "^0.51.3",
|
|
81
|
+
"@fluidframework/test-runtime-utils": "^0.51.3",
|
|
82
82
|
"@microsoft/api-extractor": "^7.16.1",
|
|
83
83
|
"@types/double-ended-queue": "^2.1.0",
|
|
84
84
|
"@types/mocha": "^8.2.2",
|
package/src/containerRuntime.ts
CHANGED
|
@@ -47,7 +47,6 @@ import {
|
|
|
47
47
|
import { IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
48
48
|
import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
|
|
49
49
|
import { DataCorruptionError, GenericError } from "@fluidframework/container-utils";
|
|
50
|
-
import { runGarbageCollection } from "@fluidframework/garbage-collector";
|
|
51
50
|
import {
|
|
52
51
|
BlobTreeEntry,
|
|
53
52
|
TreeTreeEntry,
|
|
@@ -58,7 +57,6 @@ import {
|
|
|
58
57
|
IQuorum,
|
|
59
58
|
ISequencedDocumentMessage,
|
|
60
59
|
ISignalMessage,
|
|
61
|
-
ISnapshotTree,
|
|
62
60
|
ISummaryConfiguration,
|
|
63
61
|
ISummaryContent,
|
|
64
62
|
ISummaryTree,
|
|
@@ -114,8 +112,6 @@ import {
|
|
|
114
112
|
chunksBlobName,
|
|
115
113
|
electedSummarizerBlobName,
|
|
116
114
|
extractSummaryMetadataMessage,
|
|
117
|
-
getGCVersion,
|
|
118
|
-
GCVersion,
|
|
119
115
|
IContainerRuntimeMetadata,
|
|
120
116
|
ISummaryMetadataMessage,
|
|
121
117
|
metadataBlobName,
|
|
@@ -137,6 +133,13 @@ import {
|
|
|
137
133
|
} from "./summarizerTypes";
|
|
138
134
|
import { formExponentialFn, Throttler } from "./throttler";
|
|
139
135
|
import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
|
|
136
|
+
import {
|
|
137
|
+
GarbageCollector,
|
|
138
|
+
IGarbageCollectionRuntime,
|
|
139
|
+
IGarbageCollector,
|
|
140
|
+
IGCStats,
|
|
141
|
+
IUsedStateStats,
|
|
142
|
+
} from "./garbageCollection";
|
|
140
143
|
|
|
141
144
|
export enum ContainerMessageType {
|
|
142
145
|
// An op to be delivered to store
|
|
@@ -187,21 +190,6 @@ const DefaultSummaryConfiguration: ISummaryConfiguration = {
|
|
|
187
190
|
maxAckWaitTime: 120000,
|
|
188
191
|
};
|
|
189
192
|
|
|
190
|
-
/** This is the current version of garbage collection */
|
|
191
|
-
const GCVersion = 1;
|
|
192
|
-
|
|
193
|
-
/** The statistics of a garbage collection run */
|
|
194
|
-
export interface IGCStats {
|
|
195
|
-
/** Total number of nodes in the GC graph */
|
|
196
|
-
totalNodes: number;
|
|
197
|
-
/** Number of nodes that have been marked as deleted */
|
|
198
|
-
deletedNodes: number;
|
|
199
|
-
/** Total number of data stores in the GC graph */
|
|
200
|
-
totalDataStores: number;
|
|
201
|
-
/** Number of data stores that have been marked as deleted */
|
|
202
|
-
deletedDataStores: number;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
193
|
export interface IGCRuntimeOptions {
|
|
206
194
|
/* Flag that will disable garbage collection if set to true. */
|
|
207
195
|
disableGC?: boolean;
|
|
@@ -285,12 +273,6 @@ interface IRuntimeMessageMetadata {
|
|
|
285
273
|
batch?: boolean;
|
|
286
274
|
}
|
|
287
275
|
|
|
288
|
-
// Local storage key to turn GC on / off.
|
|
289
|
-
const runGCKey = "FluidRunGC";
|
|
290
|
-
// Local storage key to turn GC test mode on / off.
|
|
291
|
-
const gcTestModeKey = "FluidGCTestMode";
|
|
292
|
-
// Local storage key to turn GC sweep on / off.
|
|
293
|
-
const runSweepKey = "FluidRunSweep";
|
|
294
276
|
// Local storage key to set the default flush mode to TurnBased
|
|
295
277
|
const turnBasedFlushModeKey = "FluidFlushModeTurnBased";
|
|
296
278
|
|
|
@@ -509,6 +491,7 @@ export const agentSchedulerId = "_scheduler";
|
|
|
509
491
|
export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
510
492
|
implements
|
|
511
493
|
IContainerRuntime,
|
|
494
|
+
IGarbageCollectionRuntime,
|
|
512
495
|
IRuntime,
|
|
513
496
|
ISummarizerRuntime,
|
|
514
497
|
ISummarizerInternalsProvider
|
|
@@ -783,20 +766,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
783
766
|
private readonly scheduleManager: ScheduleManager;
|
|
784
767
|
private readonly blobManager: BlobManager;
|
|
785
768
|
private readonly pendingStateManager: PendingStateManager;
|
|
769
|
+
private readonly garbageCollector: IGarbageCollector;
|
|
786
770
|
|
|
787
771
|
// Local copy of incomplete received chunks.
|
|
788
772
|
private readonly chunkMap: Map<string, string[]>;
|
|
789
773
|
|
|
790
774
|
private readonly dataStores: DataStores;
|
|
791
775
|
|
|
792
|
-
// The current GC version that this container is running.
|
|
793
|
-
private readonly currentGCVersion = GCVersion;
|
|
794
|
-
// This is the version of GC data in the latest summary this client has seen.
|
|
795
|
-
private latestSummaryGCVersion: GCVersion;
|
|
796
|
-
// This is the source of truth for whether GC is enabled or not.
|
|
797
|
-
private readonly shouldRunGC: boolean;
|
|
798
|
-
// This is the source of truth for whether GC sweep phase should run or not.
|
|
799
|
-
private readonly shouldRunSweep: boolean;
|
|
800
776
|
/**
|
|
801
777
|
* True if generating summaries with isolated channels is
|
|
802
778
|
* explicitly disabled. This only affects how summaries are written,
|
|
@@ -810,17 +786,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
810
786
|
return getLocalStorageFeatureGate(turnBasedFlushModeKey) ? FlushMode.TurnBased : FlushMode.Immediate;
|
|
811
787
|
}
|
|
812
788
|
|
|
813
|
-
// Tells whether GC is enabled for this document or not. If the summaryGCVersion is > 0, GC is enabled.
|
|
814
|
-
private get gcEnabled(): boolean {
|
|
815
|
-
return this.latestSummaryGCVersion > 0;
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
// Tells whether this container is running in GC test mode. If so, unreferenced data stores are immediately
|
|
819
|
-
// deleted as soon as GC runs.
|
|
820
|
-
public get gcTestMode(): boolean {
|
|
821
|
-
return getLocalStorageFeatureGate(gcTestModeKey) ?? this.runtimeOptions.gcOptions?.runGCInTestMode === true;
|
|
822
|
-
}
|
|
823
|
-
|
|
824
789
|
private get summarizer(): Summarizer {
|
|
825
790
|
assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
|
|
826
791
|
return this._summarizer;
|
|
@@ -843,28 +808,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
843
808
|
super();
|
|
844
809
|
|
|
845
810
|
this.baseSummaryMessage = metadata?.message;
|
|
846
|
-
/**
|
|
847
|
-
* gcFeature in metadata is introduced with v1 in the metadata blob. Forced to 0/disallowed before that.
|
|
848
|
-
* For existing documents, we get this value from the metadata blob.
|
|
849
|
-
* For new documents, we get this value based on the gcAllowed flag in runtimeOptions.
|
|
850
|
-
*/
|
|
851
|
-
const prevSummaryGCVersion = existing ? getGCVersion(metadata) : undefined;
|
|
852
|
-
// Default to false for now.
|
|
853
|
-
this.latestSummaryGCVersion = prevSummaryGCVersion ??
|
|
854
|
-
(this.runtimeOptions.gcOptions.gcAllowed === true ? this.currentGCVersion : 0);
|
|
855
|
-
|
|
856
|
-
// Whether GC should run or not. Can override with localStorage flag.
|
|
857
|
-
this.shouldRunGC = getLocalStorageFeatureGate(runGCKey) ?? (
|
|
858
|
-
// GC must be enabled for the document.
|
|
859
|
-
this.gcEnabled
|
|
860
|
-
// Must not be disabled by runtime option.
|
|
861
|
-
&& !this.runtimeOptions.gcOptions.disableGC
|
|
862
|
-
);
|
|
863
|
-
|
|
864
|
-
// Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
|
|
865
|
-
// localStorage flag.
|
|
866
|
-
this.shouldRunSweep = this.shouldRunGC &&
|
|
867
|
-
(getLocalStorageFeatureGate(runSweepKey) ?? this.runtimeOptions.gcOptions.runSweep === true);
|
|
868
811
|
|
|
869
812
|
// Default to false (enabled).
|
|
870
813
|
this.disableIsolatedChannels = this.runtimeOptions.summaryOptions.disableIsolatedChannels ?? false;
|
|
@@ -877,6 +820,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
877
820
|
|
|
878
821
|
this._logger = ChildLogger.create(this.logger, "ContainerRuntime");
|
|
879
822
|
|
|
823
|
+
this.garbageCollector = GarbageCollector.create(
|
|
824
|
+
this,
|
|
825
|
+
this.runtimeOptions.gcOptions,
|
|
826
|
+
(unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
|
|
827
|
+
this._logger,
|
|
828
|
+
existing,
|
|
829
|
+
metadata,
|
|
830
|
+
);
|
|
831
|
+
|
|
880
832
|
const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
|
|
881
833
|
this.summarizerNode = createRootSummarizerNodeWithGC(
|
|
882
834
|
ChildLogger.create(this.logger, "SummarizerNode"),
|
|
@@ -893,8 +845,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
893
845
|
// Must set to true to throw on any data stores failure that was too severe to be handled.
|
|
894
846
|
// We also are not decoding the base summaries at the root.
|
|
895
847
|
throwOnFailure: true,
|
|
896
|
-
// If GC
|
|
897
|
-
gcDisabled: !this.shouldRunGC,
|
|
848
|
+
// If GC should not run, let the summarizer node know so that it does not track GC state.
|
|
849
|
+
gcDisabled: !this.garbageCollector.shouldRunGC,
|
|
898
850
|
// The max duration for which objects can be unreferenced before they are eligible for deletion.
|
|
899
851
|
maxUnreferencedDurationMs: this.runtimeOptions.gcOptions.maxUnreferencedDurationMs,
|
|
900
852
|
},
|
|
@@ -1167,7 +1119,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1167
1119
|
}
|
|
1168
1120
|
} else if (requestParser.pathParts.length > 0) {
|
|
1169
1121
|
/**
|
|
1170
|
-
* If GC
|
|
1122
|
+
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
1171
1123
|
* an error if the data store being requested is marked as unreferenced as per the data store's initial
|
|
1172
1124
|
* summary.
|
|
1173
1125
|
*
|
|
@@ -1175,7 +1127,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1175
1127
|
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
1176
1128
|
*/
|
|
1177
1129
|
const wait = typeof request.headers?.wait === "boolean" ? request.headers.wait : undefined;
|
|
1178
|
-
const dataStore = request.headers?.externalRequest && this.shouldRunGC
|
|
1130
|
+
const dataStore = request.headers?.externalRequest && this.garbageCollector.shouldRunGC
|
|
1179
1131
|
? await this.getDataStoreIfInitiallyReferenced(id, wait)
|
|
1180
1132
|
: await this.getDataStore(id, wait);
|
|
1181
1133
|
const subRequest = requestParser.createSubRequest(1);
|
|
@@ -1192,19 +1144,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1192
1144
|
}
|
|
1193
1145
|
}
|
|
1194
1146
|
|
|
1195
|
-
private get shouldWriteMetadata(): boolean {
|
|
1196
|
-
// We need the metadata blob if either isolated channels are enabled
|
|
1197
|
-
// or GC is enabled at the document level.
|
|
1198
|
-
return !this.disableIsolatedChannels || this.gcEnabled;
|
|
1199
|
-
}
|
|
1200
|
-
|
|
1201
1147
|
private formMetadata(): IContainerRuntimeMetadata {
|
|
1202
1148
|
return {
|
|
1203
1149
|
summaryFormatVersion: 1,
|
|
1204
1150
|
disableIsolatedChannels: this.disableIsolatedChannels || undefined,
|
|
1205
|
-
|
|
1206
|
-
// we always write the current GC version as that is what is used to generate the GC data.
|
|
1207
|
-
gcFeature: this.gcEnabled ? this.currentGCVersion : this.latestSummaryGCVersion,
|
|
1151
|
+
gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
|
|
1208
1152
|
// The last message processed at the time of summary. If there are no messages, nothing has changed from
|
|
1209
1153
|
// the base summary we loaded from. So, use the message from its metadata blob.
|
|
1210
1154
|
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ?? this.baseSummaryMessage,
|
|
@@ -1238,8 +1182,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1238
1182
|
* @deprecated - Use summarize to get summary of the container runtime.
|
|
1239
1183
|
*/
|
|
1240
1184
|
public async snapshot(): Promise<ITree> {
|
|
1241
|
-
if (this.shouldRunGC) {
|
|
1242
|
-
await this.collectGarbage(this.logger, true /* fullGC */);
|
|
1185
|
+
if (this.garbageCollector.shouldRunGC) {
|
|
1186
|
+
await this.collectGarbage({ logger: this.logger, fullGC: true /* fullGC */ });
|
|
1243
1187
|
}
|
|
1244
1188
|
|
|
1245
1189
|
const root: ITree = { entries: [] };
|
|
@@ -1251,9 +1195,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1251
1195
|
root.entries.push(new TreeTreeEntry(channelsTreeName, { entries }));
|
|
1252
1196
|
}
|
|
1253
1197
|
|
|
1254
|
-
|
|
1255
|
-
root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata())));
|
|
1256
|
-
}
|
|
1198
|
+
root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata())));
|
|
1257
1199
|
|
|
1258
1200
|
if (this.chunkMap.size > 0) {
|
|
1259
1201
|
root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap])));
|
|
@@ -1263,9 +1205,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1263
1205
|
}
|
|
1264
1206
|
|
|
1265
1207
|
private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
1266
|
-
|
|
1267
|
-
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
1268
|
-
}
|
|
1208
|
+
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
1269
1209
|
if (this.chunkMap.size > 0) {
|
|
1270
1210
|
const content = JSON.stringify([...this.chunkMap]);
|
|
1271
1211
|
addBlobToSummary(summaryTree, chunksBlobName, content);
|
|
@@ -1536,7 +1476,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1536
1476
|
|
|
1537
1477
|
public async createRootDataStore(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1538
1478
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
1539
|
-
fluidDataStore.
|
|
1479
|
+
fluidDataStore.bindToContext();
|
|
1540
1480
|
return fluidDataStore;
|
|
1541
1481
|
}
|
|
1542
1482
|
|
|
@@ -1560,7 +1500,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1560
1500
|
const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
|
|
1561
1501
|
Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
|
|
1562
1502
|
if (isRoot) {
|
|
1563
|
-
fluidDataStore.
|
|
1503
|
+
fluidDataStore.bindToContext();
|
|
1564
1504
|
}
|
|
1565
1505
|
return fluidDataStore;
|
|
1566
1506
|
}
|
|
@@ -1675,57 +1615,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1675
1615
|
return this.context.getAbsoluteUrl(relativeUrl);
|
|
1676
1616
|
}
|
|
1677
1617
|
|
|
1678
|
-
/**
|
|
1679
|
-
* Runs garbage collection and udpates the reference / used state of the nodes in the container.
|
|
1680
|
-
* @returns the number of data stores that have been marked as unreferenced.
|
|
1681
|
-
*/
|
|
1682
|
-
public async collectGarbage(logger: ITelemetryLogger, fullGC: boolean = false): Promise<IGCStats> {
|
|
1683
|
-
return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
1684
|
-
const gcStats: {
|
|
1685
|
-
deletedNodes?: number,
|
|
1686
|
-
totalNodes?: number,
|
|
1687
|
-
deletedDataStores?: number,
|
|
1688
|
-
totalDataStores?: number,
|
|
1689
|
-
} = {};
|
|
1690
|
-
// Get the container's GC data and run GC on the reference graph in it.
|
|
1691
|
-
const gcData = await this.dataStores.getGCData(fullGC);
|
|
1692
|
-
const { referencedNodeIds, deletedNodeIds } = runGarbageCollection(
|
|
1693
|
-
gcData.gcNodes, [ "/" ],
|
|
1694
|
-
this.logger,
|
|
1695
|
-
);
|
|
1696
|
-
|
|
1697
|
-
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1698
|
-
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1699
|
-
// always referenced, so the used routes is only self-route (empty string).
|
|
1700
|
-
this.summarizerNode.updateUsedRoutes([""]);
|
|
1701
|
-
|
|
1702
|
-
// Remove this node's route ("/") and notify data stores of routes that are used in it.
|
|
1703
|
-
const usedRoutes = referencedNodeIds.filter((id: string) => { return id !== "/"; });
|
|
1704
|
-
const { dataStoreCount, unusedDataStoreCount } = this.dataStores.updateUsedRoutes(
|
|
1705
|
-
usedRoutes,
|
|
1706
|
-
// For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
|
|
1707
|
-
// we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
1708
|
-
// of this client's connection - https://github.com/microsoft/FluidFramework/issues/7152.
|
|
1709
|
-
this.deltaManager.lastMessage?.timestamp,
|
|
1710
|
-
);
|
|
1711
|
-
|
|
1712
|
-
// Update stats to be reported in the peformance event.
|
|
1713
|
-
gcStats.deletedNodes = deletedNodeIds.length;
|
|
1714
|
-
gcStats.totalNodes = referencedNodeIds.length + deletedNodeIds.length;
|
|
1715
|
-
gcStats.deletedDataStores = unusedDataStoreCount;
|
|
1716
|
-
gcStats.totalDataStores = dataStoreCount;
|
|
1717
|
-
|
|
1718
|
-
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
1719
|
-
// involving access to deleted data.
|
|
1720
|
-
if (this.gcTestMode) {
|
|
1721
|
-
this.dataStores.deleteUnusedRoutes(deletedNodeIds);
|
|
1722
|
-
}
|
|
1723
|
-
event.end(gcStats);
|
|
1724
|
-
return gcStats as IGCStats;
|
|
1725
|
-
},
|
|
1726
|
-
{ end: true, cancel: "error" });
|
|
1727
|
-
}
|
|
1728
|
-
|
|
1729
1618
|
private async summarizeInternal(fullTree: boolean, trackState: boolean): Promise<ISummarizeInternalResult> {
|
|
1730
1619
|
const summarizeResult = await this.dataStores.summarize(fullTree, trackState);
|
|
1731
1620
|
let pathPartsForChildren: string[] | undefined;
|
|
@@ -1755,15 +1644,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1755
1644
|
trackState?: boolean,
|
|
1756
1645
|
/** True to run garbage collection before summarizing; defaults to true */
|
|
1757
1646
|
runGC?: boolean,
|
|
1758
|
-
/** True to generate full GC data
|
|
1647
|
+
/** True to generate full GC data */
|
|
1759
1648
|
fullGC?: boolean,
|
|
1760
|
-
/** True to run GC sweep phase after the mark phase
|
|
1649
|
+
/** True to run GC sweep phase after the mark phase */
|
|
1761
1650
|
runSweep?: boolean,
|
|
1762
1651
|
}): Promise<ISummaryTreeWithStats> {
|
|
1763
|
-
const { summaryLogger, fullTree = false, trackState = true, runGC = true, fullGC
|
|
1652
|
+
const { summaryLogger, fullTree = false, trackState = true, runGC = true, runSweep, fullGC } = options;
|
|
1764
1653
|
|
|
1765
1654
|
if (runGC) {
|
|
1766
|
-
await this.collectGarbage(summaryLogger, fullGC);
|
|
1655
|
+
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1767
1656
|
}
|
|
1768
1657
|
|
|
1769
1658
|
const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
|
|
@@ -1773,6 +1662,53 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1773
1662
|
return summarizeResult as ISummaryTreeWithStats;
|
|
1774
1663
|
}
|
|
1775
1664
|
|
|
1665
|
+
/**
|
|
1666
|
+
* Implementation of IGarbageCollectionRuntime::getGCData.
|
|
1667
|
+
* Generates and returns the GC data for this container.
|
|
1668
|
+
* @param fullGC - true to bypass optimizations and force full generation of GC data.
|
|
1669
|
+
*/
|
|
1670
|
+
public async getGCData(fullGC?: boolean): Promise<IGarbageCollectionData> {
|
|
1671
|
+
return this.dataStores.getGCData(fullGC);
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
/**
|
|
1675
|
+
* Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
|
|
1676
|
+
* After GC has run, called to notify this container's nodes of routes that are used in it.
|
|
1677
|
+
* @param usedRoutes - The routes that are used in all nodes in this Container.
|
|
1678
|
+
* @returns the statistics of the used state of the data stores.
|
|
1679
|
+
*/
|
|
1680
|
+
public updateUsedRoutes(usedRoutes: string[]): IUsedStateStats {
|
|
1681
|
+
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1682
|
+
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1683
|
+
// always referenced, so the used routes is only self-route (empty string).
|
|
1684
|
+
this.summarizerNode.updateUsedRoutes([""]);
|
|
1685
|
+
|
|
1686
|
+
return this.dataStores.updateUsedRoutes(
|
|
1687
|
+
usedRoutes,
|
|
1688
|
+
// For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
|
|
1689
|
+
// we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
1690
|
+
// of this client's connection - https://github.com/microsoft/FluidFramework/issues/7152.
|
|
1691
|
+
this.deltaManager.lastMessage?.timestamp,
|
|
1692
|
+
);
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
/**
|
|
1696
|
+
* Runs garbage collection and udpates the reference / used state of the nodes in the container.
|
|
1697
|
+
* @returns the statistics of the garbage collection run.
|
|
1698
|
+
*/
|
|
1699
|
+
public async collectGarbage(
|
|
1700
|
+
options: {
|
|
1701
|
+
/** Logger to use for logging GC events */
|
|
1702
|
+
logger?: ITelemetryLogger,
|
|
1703
|
+
/** True to run GC sweep phase after the mark phase */
|
|
1704
|
+
runSweep?: boolean,
|
|
1705
|
+
/** True to generate full GC data */
|
|
1706
|
+
fullGC?: boolean,
|
|
1707
|
+
},
|
|
1708
|
+
): Promise<IGCStats> {
|
|
1709
|
+
return this.garbageCollector.collectGarbage(options);
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1776
1712
|
/**
|
|
1777
1713
|
* Generates the summary tree, uploads it to storage, and then submits the summarize op.
|
|
1778
1714
|
* This is intended to be called by the summarizer, since it is the implementation of
|
|
@@ -1845,24 +1781,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1845
1781
|
return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error: continueResult.error };
|
|
1846
1782
|
}
|
|
1847
1783
|
|
|
1848
|
-
// If the GC version that this container is loaded from differs from the current GC version that this
|
|
1849
|
-
// container is running, we need to regenerate the GC data and run full summary. This is used to handle
|
|
1850
|
-
// scenarios where we upgrade the GC version because we cannot trust the data from the previous GC version.
|
|
1851
|
-
let forceRegenerateData = false;
|
|
1852
|
-
if (this.gcEnabled && this.latestSummaryGCVersion !== this.currentGCVersion) {
|
|
1853
|
-
forceRegenerateData = true;
|
|
1854
|
-
}
|
|
1855
|
-
|
|
1856
1784
|
const trace = Trace.start();
|
|
1857
1785
|
let summarizeResult: ISummaryTreeWithStats;
|
|
1858
1786
|
try {
|
|
1859
1787
|
summarizeResult = await this.summarize({
|
|
1860
1788
|
summaryLogger,
|
|
1861
|
-
|
|
1789
|
+
// If the GC version changed since the last summary was submitted, we need to regenerate summary by
|
|
1790
|
+
// running full summary. This is used to handle scenarios where we upgrade the GC version because we
|
|
1791
|
+
// cannot trust the data from the previous GC version anymore.
|
|
1792
|
+
fullTree: fullTree || this.garbageCollector.hasGCVersionChanged,
|
|
1862
1793
|
trackState: true,
|
|
1863
|
-
runGC: this.shouldRunGC,
|
|
1864
|
-
fullGC: this.runtimeOptions.gcOptions.runFullGC || forceRegenerateData,
|
|
1865
|
-
runSweep: this.shouldRunSweep,
|
|
1794
|
+
runGC: this.garbageCollector.shouldRunGC,
|
|
1866
1795
|
});
|
|
1867
1796
|
} catch (error) {
|
|
1868
1797
|
return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error };
|
|
@@ -2215,18 +2144,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2215
2144
|
summaryLogger,
|
|
2216
2145
|
);
|
|
2217
2146
|
|
|
2218
|
-
//
|
|
2219
|
-
|
|
2220
|
-
// If the summary was tracked by this client, it was the one that generated the summary in the first place.
|
|
2221
|
-
// Update the summaryGCVersion to the currentGCVersion of this client.
|
|
2222
|
-
if (result.wasSummaryTracked) {
|
|
2223
|
-
this.latestSummaryGCVersion = this.currentGCVersion;
|
|
2224
|
-
return;
|
|
2225
|
-
}
|
|
2226
|
-
// If the summary was not tracked by this client, update summaryGCVersion from the snapshot that was used
|
|
2227
|
-
// to update the latest summary.
|
|
2228
|
-
await this.updateSummaryGCVersionFromSnapshot(result.snapshot);
|
|
2229
|
-
}
|
|
2147
|
+
// Notify the garbage collector so it can update its latest summary state.
|
|
2148
|
+
await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
|
|
2230
2149
|
}
|
|
2231
2150
|
|
|
2232
2151
|
/**
|
|
@@ -2252,30 +2171,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2252
2171
|
summaryLogger,
|
|
2253
2172
|
);
|
|
2254
2173
|
|
|
2255
|
-
//
|
|
2256
|
-
|
|
2257
|
-
// Since there is not proposal handle for this summary, it should not have been tracked.
|
|
2258
|
-
assert(!result.wasSummaryTracked,
|
|
2259
|
-
0x1fd /* "Summary without proposal handle should not have been tracked" */);
|
|
2260
|
-
// Update summaryGCVersion from the snapshot that was used to update the latest summary.
|
|
2261
|
-
await this.updateSummaryGCVersionFromSnapshot(result.snapshot);
|
|
2262
|
-
}
|
|
2174
|
+
// Notify the garbage collector so it can update its latest summary state.
|
|
2175
|
+
await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
|
|
2263
2176
|
|
|
2264
2177
|
return snapshotRefSeq;
|
|
2265
2178
|
}
|
|
2266
2179
|
|
|
2267
|
-
/**
|
|
2268
|
-
* Updates the summary GC version as per the metadata blob in given snapshot.
|
|
2269
|
-
*/
|
|
2270
|
-
private async updateSummaryGCVersionFromSnapshot(snapshot: ISnapshotTree) {
|
|
2271
|
-
assert(this.gcEnabled, 0x25a /* "GC version should not be updated when GC is disabled" */);
|
|
2272
|
-
const metadataBlobId = snapshot.blobs[metadataBlobName];
|
|
2273
|
-
if (metadataBlobId) {
|
|
2274
|
-
const metadata = await readAndParse<IContainerRuntimeMetadata>(this.storage, metadataBlobId);
|
|
2275
|
-
this.latestSummaryGCVersion = getGCVersion(metadata);
|
|
2276
|
-
}
|
|
2277
|
-
}
|
|
2278
|
-
|
|
2279
2180
|
private async fetchSnapshotFromStorage(versionId: string, logger: ITelemetryLogger, event: ITelemetryGenericEvent) {
|
|
2280
2181
|
return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
2281
2182
|
const stats: { getVersionDuration?: number; getSnapshotDuration?: number } = {};
|
package/src/dataStoreContext.ts
CHANGED
package/src/dataStores.ts
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
LocalDetachedFluidDataStoreContext,
|
|
52
52
|
} from "./dataStoreContext";
|
|
53
53
|
import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summaryFormat";
|
|
54
|
+
import { IUsedStateStats } from "./garbageCollection";
|
|
54
55
|
|
|
55
56
|
/**
|
|
56
57
|
* This class encapsulates data store handling. Currently it is only used by the container runtime,
|
|
@@ -469,9 +470,9 @@ export class DataStores implements IDisposable {
|
|
|
469
470
|
* @param usedRoutes - The routes that are used in all data stores in this Container.
|
|
470
471
|
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
|
|
471
472
|
* unreferenced as part of this GC run, this should be used to update the time when it happens.
|
|
472
|
-
* @returns the
|
|
473
|
+
* @returns the statistics of the used state of the data stores.
|
|
473
474
|
*/
|
|
474
|
-
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
|
|
475
|
+
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats {
|
|
475
476
|
// Get a map of data store ids to routes used in it.
|
|
476
477
|
const usedDataStoreRoutes = getChildNodesUsedRoutes(usedRoutes);
|
|
477
478
|
|
|
@@ -488,8 +489,8 @@ export class DataStores implements IDisposable {
|
|
|
488
489
|
// Return the number of data stores that are unused.
|
|
489
490
|
const dataStoreCount = this.contexts.size;
|
|
490
491
|
return {
|
|
491
|
-
dataStoreCount,
|
|
492
|
-
|
|
492
|
+
totalNodeCount: dataStoreCount,
|
|
493
|
+
unusedNodeCount: dataStoreCount - usedDataStoreRoutes.size,
|
|
493
494
|
};
|
|
494
495
|
}
|
|
495
496
|
|
|
@@ -499,7 +500,6 @@ export class DataStores implements IDisposable {
|
|
|
499
500
|
* @param unusedRoutes - The routes that are unused in all data stores in this Container.
|
|
500
501
|
*/
|
|
501
502
|
public deleteUnusedRoutes(unusedRoutes: string[]) {
|
|
502
|
-
assert(this.runtime.gcTestMode, 0x1df /* "Data stores should be deleted only in GC test mode" */);
|
|
503
503
|
for (const route of unusedRoutes) {
|
|
504
504
|
const dataStoreId = route.split("/")[1];
|
|
505
505
|
// Delete the contexts of unused data stores.
|