@fluidframework/container-runtime 0.52.0-44610 → 0.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerHandleContext.d.ts +0 -1
- package/dist/containerHandleContext.d.ts.map +1 -1
- package/dist/containerHandleContext.js +0 -1
- package/dist/containerHandleContext.js.map +1 -1
- package/dist/containerRuntime.d.ts +18 -3
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +84 -42
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +4 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +16 -13
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +8 -8
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +20 -37
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +61 -14
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +275 -20
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -2
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +0 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +0 -36
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summarizer.d.ts +1 -3
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +0 -12
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +2 -2
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryFormat.d.ts +9 -1
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +1 -3
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/containerHandleContext.d.ts +0 -1
- package/lib/containerHandleContext.d.ts.map +1 -1
- package/lib/containerHandleContext.js +0 -1
- package/lib/containerHandleContext.js.map +1 -1
- package/lib/containerRuntime.d.ts +18 -3
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +84 -43
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +4 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +16 -13
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +8 -8
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +23 -40
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +61 -14
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +276 -21
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +0 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +0 -36
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summarizer.d.ts +1 -3
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +0 -12
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +2 -2
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryFormat.d.ts +9 -1
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +1 -3
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +16 -16
- package/src/containerHandleContext.ts +0 -1
- package/src/containerRuntime.ts +110 -53
- package/src/dataStoreContext.ts +15 -14
- package/src/dataStores.ts +32 -50
- package/src/garbageCollection.ts +390 -18
- package/src/index.ts +20 -2
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +0 -43
- package/src/summarizer.ts +0 -15
- package/src/summarizerTypes.ts +1 -2
- package/src/summaryFormat.ts +10 -1
- package/src/summaryGenerator.ts +2 -3
|
@@ -4,37 +4,96 @@
|
|
|
4
4
|
* Licensed under the MIT License.
|
|
5
5
|
*/
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.GarbageCollector = void 0;
|
|
7
|
+
exports.GarbageCollector = exports.gcBlobPrefix = exports.gcTreeKey = void 0;
|
|
8
|
+
const common_utils_1 = require("@fluidframework/common-utils");
|
|
8
9
|
const garbage_collector_1 = require("@fluidframework/garbage-collector");
|
|
10
|
+
const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
|
|
11
|
+
const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
9
12
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
13
|
+
const dataStores_1 = require("./dataStores");
|
|
10
14
|
const localStorageFeatureGates_1 = require("./localStorageFeatureGates");
|
|
11
15
|
const summaryFormat_1 = require("./summaryFormat");
|
|
12
16
|
/** This is the current version of garbage collection. */
|
|
13
17
|
const GCVersion = 1;
|
|
18
|
+
// The key for the GC tree in summary.
|
|
19
|
+
exports.gcTreeKey = "gc";
|
|
20
|
+
// They prefix for GC blobs in the GC tree in summary.
|
|
21
|
+
exports.gcBlobPrefix = "__gc";
|
|
14
22
|
// Local storage key to turn GC on / off.
|
|
15
23
|
const runGCKey = "FluidRunGC";
|
|
16
24
|
// Local storage key to turn GC test mode on / off.
|
|
17
25
|
const gcTestModeKey = "FluidGCTestMode";
|
|
18
26
|
// Local storage key to turn GC sweep on / off.
|
|
19
27
|
const runSweepKey = "FluidRunSweep";
|
|
28
|
+
const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
|
|
29
|
+
/**
|
|
30
|
+
* Helper class that tracks the state of an unreferenced node such as the time it was unreferenced. It also sets
|
|
31
|
+
* the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).
|
|
32
|
+
*/
|
|
33
|
+
class UnreferencedStateTracker {
|
|
34
|
+
constructor(unreferencedTimestampMs, inactiveTimeoutMs) {
|
|
35
|
+
this.unreferencedTimestampMs = unreferencedTimestampMs;
|
|
36
|
+
this.inactive = false;
|
|
37
|
+
// Keeps track of all inactive events that are logged. This is used to limit the log generation for each event to 1
|
|
38
|
+
// so that it is not noisy.
|
|
39
|
+
this.inactiveEventsLogged = new Set();
|
|
40
|
+
// If the timeout has already expired, the node should become inactive immediately. Otherwise, start a timer of
|
|
41
|
+
// inactiveTimeoutMs after which the node will become inactive.
|
|
42
|
+
if (inactiveTimeoutMs <= 0) {
|
|
43
|
+
this.inactive = true;
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
this.timer = new common_utils_1.Timer(inactiveTimeoutMs, () => { this.inactive = true; });
|
|
47
|
+
this.timer.start();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state. */
|
|
51
|
+
stopTracking() {
|
|
52
|
+
var _a;
|
|
53
|
+
(_a = this.timer) === null || _a === void 0 ? void 0 : _a.clear();
|
|
54
|
+
this.inactive = false;
|
|
55
|
+
}
|
|
56
|
+
/** Logs an error with the given properties if the node is inactive. */
|
|
57
|
+
logIfInactive(logger, eventName, currentTimestampMs, deleteTimeoutMs, inactiveNodeId) {
|
|
58
|
+
if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
|
|
59
|
+
logger.sendErrorEvent({
|
|
60
|
+
eventName,
|
|
61
|
+
unreferencedDuratonMs: currentTimestampMs - this.unreferencedTimestampMs,
|
|
62
|
+
deleteTimeoutMs,
|
|
63
|
+
inactiveNodeId,
|
|
64
|
+
});
|
|
65
|
+
this.inactiveEventsLogged.add(eventName);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
20
69
|
/**
|
|
21
70
|
* The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
|
|
22
71
|
* its state across summaries.
|
|
23
72
|
*/
|
|
24
73
|
class GarbageCollector {
|
|
25
74
|
constructor(provider, gcOptions,
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
var _a, _b, _c;
|
|
75
|
+
/** After GC has run, called to delete objects in the runtime whose routes are unused. */
|
|
76
|
+
deleteUnusedRoutes,
|
|
77
|
+
/** Returns the current timestamp to be assigned to nodes that become unreferenced. */
|
|
78
|
+
getCurrentTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
|
|
79
|
+
var _a, _b, _c, _d;
|
|
32
80
|
this.provider = provider;
|
|
33
81
|
this.gcOptions = gcOptions;
|
|
34
82
|
this.deleteUnusedRoutes = deleteUnusedRoutes;
|
|
83
|
+
this.getCurrentTimestampMs = getCurrentTimestampMs;
|
|
84
|
+
/**
|
|
85
|
+
* Tells whether the GC data should be written to the root of the summary tree. We do this under 2 conditions:
|
|
86
|
+
* 1. If `writeDataAtRoot` GC option is enabled.
|
|
87
|
+
* 2. If the base summary has the GC data written at the root. This is to support forward compatibility where when
|
|
88
|
+
* we start writing the GC data at root, older versions can detect that and write at root too.
|
|
89
|
+
*/
|
|
90
|
+
this._writeDataAtRoot = false;
|
|
35
91
|
// The current GC version that this container is running.
|
|
36
92
|
this.currentGCVersion = GCVersion;
|
|
93
|
+
// Map of node ids to their unreferenced state tracker.
|
|
94
|
+
this.unreferencedNodesState = new Map();
|
|
37
95
|
this.logger = telemetry_utils_1.ChildLogger.create(baseLogger, "GarbageCollector");
|
|
96
|
+
this.deleteTimeoutMs = (_a = this.gcOptions.deleteTimeoutMs) !== null && _a !== void 0 ? _a : defaultDeleteTimeoutMs;
|
|
38
97
|
let prevSummaryGCVersion;
|
|
39
98
|
// GC can only be enabled during creation. After that, it can never be enabled again. So, for existing
|
|
40
99
|
// documents, we get this information from the metadata blob.
|
|
@@ -52,7 +111,7 @@ class GarbageCollector {
|
|
|
52
111
|
// latest tracked GC version. For new documents, we will be writing the first summary with the current version.
|
|
53
112
|
this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
|
|
54
113
|
// Whether GC should run or not. Can override with localStorage flag.
|
|
55
|
-
this.shouldRunGC = (
|
|
114
|
+
this.shouldRunGC = (_b = localStorageFeatureGates_1.getLocalStorageFeatureGate(runGCKey)) !== null && _b !== void 0 ? _b : (
|
|
56
115
|
// GC must be enabled for the document.
|
|
57
116
|
this.gcEnabled
|
|
58
117
|
// GC must not be disabled via GC options.
|
|
@@ -60,16 +119,112 @@ class GarbageCollector {
|
|
|
60
119
|
// Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
|
|
61
120
|
// localStorage flag.
|
|
62
121
|
this.shouldRunSweep = this.shouldRunGC &&
|
|
63
|
-
((
|
|
122
|
+
((_c = localStorageFeatureGates_1.getLocalStorageFeatureGate(runSweepKey)) !== null && _c !== void 0 ? _c : gcOptions.runSweep === true);
|
|
64
123
|
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
65
|
-
this.testMode = (
|
|
124
|
+
this.testMode = (_d = localStorageFeatureGates_1.getLocalStorageFeatureGate(gcTestModeKey)) !== null && _d !== void 0 ? _d : gcOptions.runGCInTestMode === true;
|
|
125
|
+
// If `writeDataAtRoot` GC option is true, we should write the GC data into the root of the summary tree. This
|
|
126
|
+
// GC option is used for testing only. It will be removed once we start writing GC data into root by default.
|
|
127
|
+
this._writeDataAtRoot = this.gcOptions.writeDataAtRoot === true;
|
|
128
|
+
// Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
|
|
129
|
+
// this once since it involves fetching blobs from storage which is expensive.
|
|
130
|
+
const baseSummaryStateP = new common_utils_1.LazyPromise(async () => {
|
|
131
|
+
var _a;
|
|
132
|
+
if (baseSnapshot === undefined) {
|
|
133
|
+
return { gcNodes: {} };
|
|
134
|
+
}
|
|
135
|
+
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
136
|
+
const gcSnapshotTree = baseSnapshot.trees[exports.gcTreeKey];
|
|
137
|
+
if (gcSnapshotTree !== undefined) {
|
|
138
|
+
// forward-compat - If a newer version has written the GC tree at root, we should also do the same.
|
|
139
|
+
this._writeDataAtRoot = true;
|
|
140
|
+
return getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
141
|
+
}
|
|
142
|
+
// back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
|
|
143
|
+
// consolidate into IGarbageCollectionState format.
|
|
144
|
+
const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
|
|
145
|
+
const dataStoreSnaphotTree = dataStores_1.getSummaryForDatastores(baseSnapshot, metadata);
|
|
146
|
+
common_utils_1.assert(dataStoreSnaphotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
|
|
147
|
+
for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnaphotTree.trees)) {
|
|
148
|
+
const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcBlobKey];
|
|
149
|
+
if (blobId === undefined) {
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const gcSummaryDetails = await readAndParseBlob(blobId);
|
|
153
|
+
// If there are no nodes for this data store, skip it.
|
|
154
|
+
if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const dsRootId = `/${dsId}`;
|
|
158
|
+
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
159
|
+
// Construct that entry by adding root data store ids to its outbound routes.
|
|
160
|
+
const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[summaryFormat_1.dataStoreAttributesBlobName]);
|
|
161
|
+
if (initialSnapshotDetails.isRootDataStore) {
|
|
162
|
+
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
163
|
+
}
|
|
164
|
+
for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
|
|
165
|
+
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
166
|
+
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
167
|
+
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
168
|
+
gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
|
|
169
|
+
}
|
|
170
|
+
common_utils_1.assert(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);
|
|
171
|
+
gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
|
|
172
|
+
}
|
|
173
|
+
return gcState;
|
|
174
|
+
});
|
|
175
|
+
// Set up the initializer which initializes the base GC state from the base snapshot. Use lazy promise because
|
|
176
|
+
// we only do this once - the very first time we run GC.
|
|
177
|
+
this.initializeBaseStateP = new common_utils_1.LazyPromise(async () => {
|
|
178
|
+
const baseState = await baseSummaryStateP;
|
|
179
|
+
const gcNodes = {};
|
|
180
|
+
// Set up tracking for the nodes in the base summary state and add them to GC nodes.
|
|
181
|
+
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
182
|
+
const unreferencedTimestampMs = nodeData.unreferencedTimestampMs;
|
|
183
|
+
if (unreferencedTimestampMs !== undefined) {
|
|
184
|
+
// Get how long it has been since the node was unreferenced. Start a timeout for the remaining time
|
|
185
|
+
// left for it to be eligible for deletion.
|
|
186
|
+
const unreferencedDurationMs = this.getCurrentTimestampMs() - unreferencedTimestampMs;
|
|
187
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs - unreferencedDurationMs));
|
|
188
|
+
}
|
|
189
|
+
gcNodes[nodeId] = {
|
|
190
|
+
outboundRoutes: Array.from(nodeData.outboundRoutes),
|
|
191
|
+
unreferencedTimestampMs,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
this.currentGCState = { gcNodes };
|
|
195
|
+
});
|
|
196
|
+
// Get the GC details for each data store from the GC state in the base summary. This is returned in
|
|
197
|
+
// getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.
|
|
198
|
+
this.dataStoreGCDetailsP = new common_utils_1.LazyPromise(async () => {
|
|
199
|
+
const gcNodes = {};
|
|
200
|
+
const baseState = await baseSummaryStateP;
|
|
201
|
+
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
202
|
+
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
203
|
+
}
|
|
204
|
+
// Run GC on the nodes in the base summary to get the routes used in each node in the container.
|
|
205
|
+
// This is an optimization for space (vs performance) wherein we don't need to store the used routes of
|
|
206
|
+
// each node in the summary.
|
|
207
|
+
const usedRoutes = garbage_collector_1.runGarbageCollection(gcNodes, ["/"], this.logger).referencedNodeIds;
|
|
208
|
+
const dataStoreGCDetailsMap = garbage_collector_1.unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
|
|
209
|
+
// Currently, the data stores write the GC data. So, we need to update it's base GC details with the
|
|
210
|
+
// unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
|
|
211
|
+
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
212
|
+
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
213
|
+
const dataStoreGCDetails = dataStoreGCDetailsMap.get(nodeId.slice(1));
|
|
214
|
+
if (dataStoreGCDetails !== undefined) {
|
|
215
|
+
dataStoreGCDetails.unrefTimestamp = nodeData.unreferencedTimestampMs;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return dataStoreGCDetailsMap;
|
|
220
|
+
});
|
|
66
221
|
}
|
|
67
|
-
static create(provider, gcOptions, deleteUnusedRoutes, baseLogger, existing, metadata) {
|
|
68
|
-
return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, baseLogger, existing, metadata);
|
|
222
|
+
static create(provider, gcOptions, deleteUnusedRoutes, getCurrentTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
|
|
223
|
+
return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, getCurrentTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata);
|
|
69
224
|
}
|
|
70
225
|
/**
|
|
71
226
|
* This tracks two things:
|
|
72
|
-
* 1. Whether GC is enabled - If this is 0, GC is disabled. If this is
|
|
227
|
+
* 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.
|
|
73
228
|
* 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.
|
|
74
229
|
*/
|
|
75
230
|
get gcSummaryFeatureVersion() {
|
|
@@ -84,6 +239,9 @@ class GarbageCollector {
|
|
|
84
239
|
// 2. This client's latest summary was updated from a snapshot that has a different GC version.
|
|
85
240
|
return this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion;
|
|
86
241
|
}
|
|
242
|
+
get writeDataAtRoot() {
|
|
243
|
+
return this._writeDataAtRoot;
|
|
244
|
+
}
|
|
87
245
|
/**
|
|
88
246
|
* Runs garbage collection and udpates the reference / used state of the nodes in the container.
|
|
89
247
|
* @returns the number of data stores that have been marked as unreferenced.
|
|
@@ -91,30 +249,52 @@ class GarbageCollector {
|
|
|
91
249
|
async collectGarbage(options) {
|
|
92
250
|
const { logger = this.logger, runSweep = this.shouldRunSweep, fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged, } = options;
|
|
93
251
|
return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
252
|
+
await this.initializeBaseStateP;
|
|
94
253
|
const gcStats = {};
|
|
95
254
|
// Get the runtime's GC data and run GC on the reference graph in it.
|
|
96
255
|
const gcData = await this.provider.getGCData(fullGC);
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
256
|
+
const gcResult = garbage_collector_1.runGarbageCollection(gcData.gcNodes, ["/"], logger);
|
|
257
|
+
const currentTimestampMs = this.getCurrentTimestampMs();
|
|
258
|
+
// Update the current state of the system based on the GC run.
|
|
259
|
+
this.updateCurrentState(gcData, gcResult, currentTimestampMs);
|
|
260
|
+
const dataStoreUsedStateStats = this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);
|
|
101
261
|
if (runSweep) {
|
|
102
262
|
// Placeholder for running sweep logic.
|
|
103
263
|
}
|
|
104
264
|
// Update stats to be reported in the peformance event.
|
|
105
|
-
gcStats.deletedNodes = deletedNodeIds.length;
|
|
106
|
-
gcStats.totalNodes = referencedNodeIds.length + deletedNodeIds.length;
|
|
265
|
+
gcStats.deletedNodes = gcResult.deletedNodeIds.length;
|
|
266
|
+
gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;
|
|
107
267
|
gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;
|
|
108
268
|
gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;
|
|
109
269
|
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
110
270
|
// involving access to deleted data.
|
|
111
271
|
if (this.testMode) {
|
|
112
|
-
this.deleteUnusedRoutes(deletedNodeIds);
|
|
272
|
+
this.deleteUnusedRoutes(gcResult.deletedNodeIds);
|
|
113
273
|
}
|
|
114
274
|
event.end(gcStats);
|
|
115
275
|
return gcStats;
|
|
116
276
|
}, { end: true, cancel: "error" });
|
|
117
277
|
}
|
|
278
|
+
/**
|
|
279
|
+
* Summarizes the GC data and returns it as a summary tree.
|
|
280
|
+
* We current write the entire GC state in a single blob. This can be modified later to write multiple
|
|
281
|
+
* blobs. All the blob keys should start with `gcBlobPrefix`.
|
|
282
|
+
*/
|
|
283
|
+
summarize() {
|
|
284
|
+
if (!this.shouldRunGC || this.currentGCState === undefined) {
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
const builder = new runtime_utils_1.SummaryTreeBuilder();
|
|
288
|
+
builder.addBlob(`${exports.gcBlobPrefix}_root`, JSON.stringify(this.currentGCState));
|
|
289
|
+
return builder.getSummaryTree();
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Returns a map of data store ids to their base GC details generated from the base summary.This is used to
|
|
293
|
+
* initialize the data stores with their base GC state.
|
|
294
|
+
*/
|
|
295
|
+
async getDataStoreBaseGCDetails() {
|
|
296
|
+
return this.dataStoreGCDetailsP;
|
|
297
|
+
}
|
|
118
298
|
/**
|
|
119
299
|
* Called when the latest summary of the system has been refreshed. This will be used to update the state of the
|
|
120
300
|
* latest summary tracked.
|
|
@@ -133,6 +313,15 @@ class GarbageCollector {
|
|
|
133
313
|
// that is now the latest summary.
|
|
134
314
|
await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);
|
|
135
315
|
}
|
|
316
|
+
/**
|
|
317
|
+
* Called when a node with the given id is changed. If the node is inactive, log an error.
|
|
318
|
+
*/
|
|
319
|
+
nodeChanged(id) {
|
|
320
|
+
var _a;
|
|
321
|
+
// Prefix "/" if needed to make it relative to the root.
|
|
322
|
+
const nodeId = id.startsWith("/") ? id : `/${id}`;
|
|
323
|
+
(_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.logIfInactive(this.logger, "inactiveObjectChanged", this.getCurrentTimestampMs(), this.deleteTimeoutMs, nodeId);
|
|
324
|
+
}
|
|
136
325
|
/**
|
|
137
326
|
* Update the latest summary GC version from the metadata blob in the given snapshot.
|
|
138
327
|
*/
|
|
@@ -143,6 +332,72 @@ class GarbageCollector {
|
|
|
143
332
|
this.latestSummaryGCVersion = summaryFormat_1.getGCVersion(metadata);
|
|
144
333
|
}
|
|
145
334
|
}
|
|
335
|
+
/**
|
|
336
|
+
* Updates the state of the system as per the current GC run. It does the following:
|
|
337
|
+
* 1. Sets up the current GC state as per the gcData.
|
|
338
|
+
* 2. Starts tracking for nodes that have become unreferenced in this run.
|
|
339
|
+
* 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
|
|
340
|
+
* @param gcData - The data representing the reference graph on which GC is run.
|
|
341
|
+
* @param gcResult - The result of the GC run on the gcData.
|
|
342
|
+
* @param currentTimestampMs - The current timestamp to be used for unreferenced nodes' timestamp.
|
|
343
|
+
*/
|
|
344
|
+
updateCurrentState(gcData, gcResult, currentTimestampMs) {
|
|
345
|
+
this.currentGCState = { gcNodes: {} };
|
|
346
|
+
for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
|
|
347
|
+
this.currentGCState.gcNodes[id] = { outboundRoutes: Array.from(outboundRoutes) };
|
|
348
|
+
}
|
|
349
|
+
// Iterate through the deleted nodes and start tracking if they became unreferenced in this run.
|
|
350
|
+
for (const nodeId of gcResult.deletedNodeIds) {
|
|
351
|
+
common_utils_1.assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2aa /* "Unexpected node when running GC" */);
|
|
352
|
+
// The time when the node became unreferenced. This is added to the current GC state.
|
|
353
|
+
let unreferencedTimestampMs = currentTimestampMs;
|
|
354
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
355
|
+
if (nodeStateTracker !== undefined) {
|
|
356
|
+
unreferencedTimestampMs = nodeStateTracker.unreferencedTimestampMs;
|
|
357
|
+
}
|
|
358
|
+
else {
|
|
359
|
+
// Start tracking this node as it became unreferenced in this run.
|
|
360
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs));
|
|
361
|
+
}
|
|
362
|
+
this.currentGCState.gcNodes[nodeId].unreferencedTimestampMs = unreferencedTimestampMs;
|
|
363
|
+
}
|
|
364
|
+
// Iterate through the referenced nodes and stop tracking if they were unreferenced before.
|
|
365
|
+
for (const nodeId of gcResult.referencedNodeIds) {
|
|
366
|
+
common_utils_1.assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2ab /* "Unexpected node when running GC" */);
|
|
367
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
368
|
+
if (nodeStateTracker !== undefined) {
|
|
369
|
+
// If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
|
|
370
|
+
// log an error as this may mean the deleteTimeoutMs is not long enough.
|
|
371
|
+
nodeStateTracker.logIfInactive(this.logger, "inactiveObjectRevived", currentTimestampMs, this.deleteTimeoutMs, nodeId);
|
|
372
|
+
// Stop tracking so as to clear out any running timers.
|
|
373
|
+
nodeStateTracker.stopTracking();
|
|
374
|
+
// Delete the node as we don't need to track it any more.
|
|
375
|
+
this.unreferencedNodesState.delete(nodeId);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
146
379
|
}
|
|
147
380
|
exports.GarbageCollector = GarbageCollector;
|
|
381
|
+
/**
|
|
382
|
+
* Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
|
|
383
|
+
* Merge the GC state from all such blobs and return the merged GC state.
|
|
384
|
+
*/
|
|
385
|
+
async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
|
|
386
|
+
let rootGCState = { gcNodes: {} };
|
|
387
|
+
for (const key of Object.keys(gcSnapshotTree.blobs)) {
|
|
388
|
+
// Skip blobs that do not stsart with the GC prefix.
|
|
389
|
+
if (!key.startsWith(exports.gcBlobPrefix)) {
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
const blobId = gcSnapshotTree.blobs[key];
|
|
393
|
+
if (blobId === undefined) {
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
const gcState = await readAndParseBlob(blobId);
|
|
397
|
+
common_utils_1.assert(gcState !== undefined, 0x2ad /* "GC blob missing from snapshot" */);
|
|
398
|
+
// Merge the GC state of this blob into the root GC state.
|
|
399
|
+
rootGCState = garbage_collector_1.concatGarbageCollectionStates(rootGCState, gcState);
|
|
400
|
+
}
|
|
401
|
+
return rootGCState;
|
|
402
|
+
}
|
|
148
403
|
//# sourceMappingURL=garbageCollection.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"garbageCollection.js","sourceRoot":"","sources":["../src/garbageCollection.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,yEAAyE;AAIzE,qEAAgF;AAGhF,yEAAwE;AACxE,mDAKyB;AAEzB,yDAAyD;AACzD,MAAM,SAAS,GAAG,CAAC,CAAC;AAEpB,yCAAyC;AACzC,MAAM,QAAQ,GAAG,YAAY,CAAC;AAC9B,mDAAmD;AACnD,MAAM,aAAa,GAAG,iBAAiB,CAAC;AACxC,+CAA+C;AAC/C,MAAM,WAAW,GAAG,eAAe,CAAC;AA4CpC;;;GAGG;AACH,MAAa,gBAAgB;IAkDzB,YACqB,QAAmC,EACnC,SAA4B;IAC7C;;;OAGG;IACc,kBAAoD,EACrE,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;;QATnB,aAAQ,GAAR,QAAQ,CAA2B;QACnC,cAAS,GAAT,SAAS,CAAmB;QAK5B,uBAAkB,GAAlB,kBAAkB,CAAkC;QAZzE,yDAAyD;QACxC,qBAAgB,GAAG,SAAS,CAAC;QAgB1C,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;QAEjE,IAAI,oBAAwC,CAAC;QAC7C,sGAAsG;QACtG,6DAA6D;QAC7D,IAAI,QAAQ,EAAE;YACV,oBAAoB,GAAG,4BAAY,CAAC,QAAQ,CAAC,CAAC;YAC9C,oGAAoG;YACpG,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,oBAAoB,GAAG,CAAC,CAAC;SAC7C;aAAM;YACH,0FAA0F;YAC1F,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC;SACjD;QACD,0GAA0G;QAC1G,+GAA+G;QAC/G,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,IAAI,CAAC,gBAAgB,CAAC;QAE5E,qEAAqE;QACrE,IAAI,CAAC,WAAW,SAAG,qDAA0B,CAAC,QAAQ,CAAC,mCAAI;QACvD,uCAAuC;QACvC,IAAI,CAAC,SAAS;YACd,0CAA0C;eACvC,CAAC,SAAS,CAAC,SAAS,CAC1B,CAAC;QAEF,2GAA2G;QAC3G,qBAAqB;QACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,WAAW;YAClC,OAAC,qDAA0B,CAAC,WAAW,CAAC,mCAAI,SAAS,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAE7E,iGAAiG;QACjG,IAAI,CAAC,QAAQ,SAAG,qDAA0B,CAAC,aAAa,CAAC,mCAAI,SAAS,CAAC,eAAe,KAAK,IAAI,CAAC;IACpG,CAAC;IA9FM,MAAM,CAAC,MAAM,CAChB,QAAmC,EACnC,SAA4B,EAC5B,kBAAoD,EACpD,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;QAEpC,OAAO,IAAI,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,kBAAkB,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzG,CAAC;IAOD;;;;OAIG;IACH,IAAW,uBAAuB;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,IAAW,mBAAmB;QAC1B,+EAA+E;QAC/E,+EAA+E;QAC/E,+FAA+F;QAC/F,OAAO,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,gBAAgB,CAAC;IACrF,CAAC;IA+DD;;;OAGG;IACI,KAAK,CAAC,cAAc,CACvB,OAOC;QAED,MAAM,EACF,MAAM,GAAG,IAAI,CAAC,MAAM,EACpB,QAAQ,GAAG,IAAI,CAAC,cAAc,EAC9B,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,mBAAmB,GACzE,GAAG,OAAO,CAAC;QAEZ,OAAO,kCAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,mBAAmB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC/F,MAAM,OAAO,GAKT,EAAE,CAAC;YAEP,qEAAqE;YACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,wCAAoB,CAC9D,MAAM,CAAC,OAAO,EACd,CAAE,GAAG,CAAE,EACP,MAAM,CACT,CAAC;YAEF,uFAAuF;YACvF,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,EAAU,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpF,MAAM,uBAAuB,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAE3E,IAAI,QAAQ,EAAE;gBACV,uCAAuC;aAC1C;YAED,uDAAuD;YACvD,OAAO,CAAC,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC;YAC7C,OAAO,CAAC,UAAU,GAAG,iBAAiB,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;YACtE,OAAO,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,eAAe,CAAC;YACpE,OAAO,CAAC,eAAe,GAAG,uBAAuB,CAAC,cAAc,CAAC;YAEjE,sGAAsG;YACtG,oCAAoC;YACpC,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC;aAC3C;YACD,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,OAAmB,CAAC;QAC/B,CAAC,EACD,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,2BAA2B,CACpC,MAA4B,EAC5B,gBAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE;YACnD,OAAO;SACV;QAED,2GAA2G;QAC3G,uDAAuD;QACvD,IAAI,MAAM,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACpD,OAAO;SACV;QACD,6GAA6G;QAC7G,kCAAkC;QAClC,MAAM,IAAI,CAAC,kCAAkC,CAAC,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kCAAkC,CAAC,QAAuB,EAAE,gBAAkC;QACxG,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,gCAAgB,CAAC,CAAC;QACxD,IAAI,cAAc,EAAE;YAChB,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAA4B,cAAc,CAAC,CAAC;YACnF,IAAI,CAAC,sBAAsB,GAAG,4BAAY,CAAC,QAAQ,CAAC,CAAC;SACxD;IACL,CAAC;CACJ;AA/LD,4CA+LC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { runGarbageCollection } from \"@fluidframework/garbage-collector\";\nimport { ISnapshotTree } from \"@fluidframework/protocol-definitions\";\nimport { IGarbageCollectionData } from \"@fluidframework/runtime-definitions\";\nimport { ReadAndParseBlob, RefreshSummaryResult } from \"@fluidframework/runtime-utils\";\nimport { ChildLogger, PerformanceEvent } from \"@fluidframework/telemetry-utils\";\n\nimport { IGCRuntimeOptions } from \"./containerRuntime\";\nimport { getLocalStorageFeatureGate } from \"./localStorageFeatureGates\";\nimport {\n getGCVersion,\n GCVersion,\n IContainerRuntimeMetadata,\n metadataBlobName,\n} from \"./summaryFormat\";\n\n/** This is the current version of garbage collection. */\nconst GCVersion = 1;\n\n// Local storage key to turn GC on / off.\nconst runGCKey = \"FluidRunGC\";\n// Local storage key to turn GC test mode on / off.\nconst gcTestModeKey = \"FluidGCTestMode\";\n// Local storage key to turn GC sweep on / off.\nconst runSweepKey = \"FluidRunSweep\";\n\n/** The used state statistics of a node. */\nexport interface IUsedStateStats {\n totalNodeCount: number;\n unusedNodeCount: number;\n}\n\n/** The statistics of the system state after a garbage collection run. */\nexport interface IGCStats {\n totalNodes: number;\n deletedNodes: number;\n totalDataStores: number;\n deletedDataStores: number;\n}\n\n/** Defines the APIs for the runtime object to be passed to the garbage collector. */\nexport interface IGarbageCollectionRuntime {\n /** Returns the garbage collection data of the runtime. */\n getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;\n /** After GC has run, called to notify the runtime of routes that are used in it. */\n updateUsedRoutes(usedRoutes: string[]): IUsedStateStats;\n}\n\n/** Defines the contract for the garbage collector. */\nexport interface IGarbageCollector {\n /** Tells whether GC should run or not. */\n readonly shouldRunGC: boolean;\n /**\n * This tracks two things:\n * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is > 0, GC is enabled.\n * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n */\n readonly gcSummaryFeatureVersion: number;\n /** Tells whether the GC version has changed compared to the version in the latest summary. */\n readonly hasGCVersionChanged: boolean;\n /** Run garbage collection and update the reference / used state of the system. */\n collectGarbage(\n options: { logger?: ITelemetryLogger, runGC?: boolean, runSweep?: boolean, fullGC?: boolean },\n ): Promise<IGCStats>;\n /** Called when the latest summary of the system has been refreshed. */\n latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;\n}\n\n/**\n * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains\n * its state across summaries.\n */\nexport class GarbageCollector implements IGarbageCollector {\n public static create(\n provider: IGarbageCollectionRuntime,\n gcOptions: IGCRuntimeOptions,\n deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n baseLogger: ITelemetryLogger,\n existing: boolean,\n metadata?: IContainerRuntimeMetadata,\n ): IGarbageCollector {\n return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, baseLogger, existing, metadata);\n }\n\n /**\n * Tells whether GC should be run based on the GC options and local storage flags.\n */\n public readonly shouldRunGC: boolean;\n\n /**\n * This tracks two things:\n * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is > 0, GC is enabled.\n * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n */\n public get gcSummaryFeatureVersion(): number {\n return this.gcEnabled ? this.currentGCVersion : 0;\n }\n\n /**\n * Tells whether the GC version has changed compared to the version in the latest summary.\n */\n public get hasGCVersionChanged(): boolean {\n // The current version can differ from the latest summary version in two cases:\n // 1. The summary this client loaded with has data from a different GC version.\n // 2. This client's latest summary was updated from a snapshot that has a different GC version.\n return this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion;\n }\n\n /**\n * Tracks if GC is enabled for this document. This is specified during document creation and doesn't change\n * throughout its lifetime.\n */\n private readonly gcEnabled: boolean;\n private readonly shouldRunSweep: boolean;\n private readonly testMode: boolean;\n private readonly logger: ITelemetryLogger;\n\n // The current GC version that this container is running.\n private readonly currentGCVersion = GCVersion;\n // This is the version of GC data in the latest summary being tracked.\n private latestSummaryGCVersion: GCVersion;\n\n protected constructor(\n private readonly provider: IGarbageCollectionRuntime,\n private readonly gcOptions: IGCRuntimeOptions,\n /**\n * After GC has run, called to delete objects in the runtime whose routes are unused. This is not part of the\n * provider because its specific to this garbage collector implementation and is not part of the contract.\n */\n private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n baseLogger: ITelemetryLogger,\n existing: boolean,\n metadata?: IContainerRuntimeMetadata,\n ) {\n this.logger = ChildLogger.create(baseLogger, \"GarbageCollector\");\n\n let prevSummaryGCVersion: number | undefined;\n // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing\n // documents, we get this information from the metadata blob.\n if (existing) {\n prevSummaryGCVersion = getGCVersion(metadata);\n // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all\n // other exsiting documents, GC is enabled.\n this.gcEnabled = prevSummaryGCVersion > 0;\n } else {\n // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.\n this.gcEnabled = gcOptions.gcAllowed === true;\n }\n // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the\n // latest tracked GC version. For new documents, we will be writing the first summary with the current version.\n this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;\n\n // Whether GC should run or not. Can override with localStorage flag.\n this.shouldRunGC = getLocalStorageFeatureGate(runGCKey) ?? (\n // GC must be enabled for the document.\n this.gcEnabled\n // GC must not be disabled via GC options.\n && !gcOptions.disableGC\n );\n\n // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with\n // localStorage flag.\n this.shouldRunSweep = this.shouldRunGC &&\n (getLocalStorageFeatureGate(runSweepKey) ?? gcOptions.runSweep === true);\n\n // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.\n this.testMode = getLocalStorageFeatureGate(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;\n }\n\n /**\n * Runs garbage collection and udpates the reference / used state of the nodes in the container.\n * @returns the number of data stores that have been marked as unreferenced.\n */\n public async collectGarbage(\n options: {\n /** Logger to use for logging GC events */\n logger?: ITelemetryLogger,\n /** True to run GC sweep phase after the mark phase */\n runSweep?: boolean,\n /** True to generate full GC data */\n fullGC?: boolean,\n },\n ): Promise<IGCStats> {\n const {\n logger = this.logger,\n runSweep = this.shouldRunSweep,\n fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged,\n } = options;\n\n return PerformanceEvent.timedExecAsync(logger, { eventName: \"GarbageCollection\" }, async (event) => {\n const gcStats: {\n deletedNodes?: number,\n totalNodes?: number,\n deletedDataStores?: number,\n totalDataStores?: number,\n } = {};\n\n // Get the runtime's GC data and run GC on the reference graph in it.\n const gcData = await this.provider.getGCData(fullGC);\n const { referencedNodeIds, deletedNodeIds } = runGarbageCollection(\n gcData.gcNodes,\n [ \"/\" ],\n logger,\n );\n\n // Remove this node's route (\"/\") and notify data stores of routes that are used in it.\n const usedRoutes = referencedNodeIds.filter((id: string) => { return id !== \"/\"; });\n const dataStoreUsedStateStats = this.provider.updateUsedRoutes(usedRoutes);\n\n if (runSweep) {\n // Placeholder for running sweep logic.\n }\n\n // Update stats to be reported in the peformance event.\n gcStats.deletedNodes = deletedNodeIds.length;\n gcStats.totalNodes = referencedNodeIds.length + deletedNodeIds.length;\n gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;\n gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;\n\n // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios\n // involving access to deleted data.\n if (this.testMode) {\n this.deleteUnusedRoutes(deletedNodeIds);\n }\n event.end(gcStats);\n return gcStats as IGCStats;\n },\n { end: true, cancel: \"error\" });\n }\n\n /**\n * Called when the latest summary of the system has been refreshed. This will be used to update the state of the\n * latest summary tracked.\n */\n public async latestSummaryStateRefreshed(\n result: RefreshSummaryResult,\n readAndParseBlob: ReadAndParseBlob,\n ): Promise<void> {\n if (!this.shouldRunGC || !result.latestSummaryUpdated) {\n return;\n }\n\n // If the summary was tracked by this client, it was the one that generated the summary in the first place.\n // Basically, it was written in the current GC version.\n if (result.wasSummaryTracked) {\n this.latestSummaryGCVersion = this.currentGCVersion;\n return;\n }\n // If the summary was not tracked by this client, update latest GC version from the snapshot in the result as\n // that is now the latest summary.\n await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);\n }\n\n /**\n * Update the latest summary GC version from the metadata blob in the given snapshot.\n */\n private async updateSummaryGCVersionFromSnapshot(snapshot: ISnapshotTree, readAndParseBlob: ReadAndParseBlob) {\n const metadataBlobId = snapshot.blobs[metadataBlobName];\n if (metadataBlobId) {\n const metadata = await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId);\n this.latestSummaryGCVersion = getGCVersion(metadata);\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"garbageCollection.js","sourceRoot":"","sources":["../src/garbageCollection.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAA0E;AAC1E,yEAK2C;AAE3C,6EAO6C;AAC7C,iEAIuC;AACvC,qEAAgF;AAGhF,6CAAuD;AACvD,yEAAwE;AACxE,mDAOyB;AAEzB,yDAAyD;AACzD,MAAM,SAAS,GAAG,CAAC,CAAC;AAEpB,sCAAsC;AACzB,QAAA,SAAS,GAAG,IAAI,CAAC;AAC9B,sDAAsD;AACzC,QAAA,YAAY,GAAG,MAAM,CAAC;AAEnC,yCAAyC;AACzC,MAAM,QAAQ,GAAG,YAAY,CAAC;AAC9B,mDAAmD;AACnD,MAAM,aAAa,GAAG,iBAAiB,CAAC;AACxC,+CAA+C;AAC/C,MAAM,WAAW,GAAG,eAAe,CAAC;AAEpC,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAoDjE;;;GAGG;AACH,MAAM,wBAAwB;IAO1B,YACoB,uBAA+B,EAC/C,iBAAyB;QADT,4BAAuB,GAAvB,uBAAuB,CAAQ;QAP3C,aAAQ,GAAY,KAAK,CAAC;QAClC,mHAAmH;QACnH,2BAA2B;QACV,yBAAoB,GAAgB,IAAI,GAAG,EAAE,CAAC;QAO3D,+GAA+G;QAC/G,+DAA+D;QAC/D,IAAI,iBAAiB,IAAI,CAAC,EAAE;YACxB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;SACxB;aAAM;YACH,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAK,CAAC,iBAAiB,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3E,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;SACtB;IACL,CAAC;IAED,+FAA+F;IACxF,YAAY;;QACf,MAAA,IAAI,CAAC,KAAK,0CAAE,KAAK,GAAG;QACpB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,wEAAwE;IACjE,aAAa,CAChB,MAAwB,EACxB,SAAiB,EACjB,kBAA0B,EAC1B,eAAuB,EACvB,cAAsB;QAEtB,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC5D,MAAM,CAAC,cAAc,CAAC;gBAClB,SAAS;gBACT,qBAAqB,EAAE,kBAAkB,GAAG,IAAI,CAAC,uBAAuB;gBACxE,eAAe;gBACf,cAAc;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;SAC5C;IACL,CAAC;CACJ;AAED;;;GAGG;AACH,MAAa,gBAAgB;IAsFzB,YACqB,QAAmC,EACnC,SAA4B;IAC7C,yFAAyF;IACxE,kBAAoD;IACrE,sFAAsF;IACrE,qBAAmC,EACpD,YAAuC,EACvC,gBAAkC,EAClC,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;;QAVnB,aAAQ,GAAR,QAAQ,CAA2B;QACnC,cAAS,GAAT,SAAS,CAAmB;QAE5B,uBAAkB,GAAlB,kBAAkB,CAAkC;QAEpD,0BAAqB,GAArB,qBAAqB,CAAc;QAlCxD;;;;;WAKG;QACK,qBAAgB,GAAY,KAAK,CAAC;QAK1C,yDAAyD;QACxC,qBAAgB,GAAG,SAAS,CAAC;QAa9C,uDAAuD;QACtC,2BAAsB,GAA0C,IAAI,GAAG,EAAE,CAAC;QAevF,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;QAEjE,IAAI,CAAC,eAAe,SAAG,IAAI,CAAC,SAAS,CAAC,eAAe,mCAAI,sBAAsB,CAAC;QAEhF,IAAI,oBAAwC,CAAC;QAC7C,sGAAsG;QACtG,6DAA6D;QAC7D,IAAI,QAAQ,EAAE;YACV,oBAAoB,GAAG,4BAAY,CAAC,QAAQ,CAAC,CAAC;YAC9C,oGAAoG;YACpG,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,oBAAoB,GAAG,CAAC,CAAC;SAC7C;aAAM;YACH,0FAA0F;YAC1F,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC;SACjD;QACD,0GAA0G;QAC1G,+GAA+G;QAC/G,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,IAAI,CAAC,gBAAgB,CAAC;QAE5E,qEAAqE;QACrE,IAAI,CAAC,WAAW,SAAG,qDAA0B,CAAC,QAAQ,CAAC,mCAAI;QACvD,uCAAuC;QACvC,IAAI,CAAC,SAAS;YACd,0CAA0C;eACvC,CAAC,SAAS,CAAC,SAAS,CAC1B,CAAC;QAEF,2GAA2G;QAC3G,qBAAqB;QACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,WAAW;YAClC,OAAC,qDAA0B,CAAC,WAAW,CAAC,mCAAI,SAAS,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QAE7E,iGAAiG;QACjG,IAAI,CAAC,QAAQ,SAAG,qDAA0B,CAAC,aAAa,CAAC,mCAAI,SAAS,CAAC,eAAe,KAAK,IAAI,CAAC;QAEhG,8GAA8G;QAC9G,6GAA6G;QAC7G,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,eAAe,KAAK,IAAI,CAAC;QAEhE,qGAAqG;QACrG,8EAA8E;QAC9E,MAAM,iBAAiB,GAAG,IAAI,0BAAW,CAA0B,KAAK,IAAI,EAAE;;YAC1E,IAAI,YAAY,KAAK,SAAS,EAAE;gBAC5B,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;aAC1B;YAED,6FAA6F;YAC7F,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,iBAAS,CAAC,CAAC;YACrD,IAAI,cAAc,KAAK,SAAS,EAAE;gBAC9B,mGAAmG;gBACnG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,OAAO,sBAAsB,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;aACnE;YAED,uGAAuG;YACvG,mDAAmD;YACnD,MAAM,OAAO,GAA4B,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;YACtF,MAAM,oBAAoB,GAAG,oCAAuB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;YAC7E,qBAAM,CAAC,oBAAoB,KAAK,SAAS,EACrC,KAAK,CAAC,0DAA0D,CAAC,CAAC;YACtE,KAAK,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;gBAC7E,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,+BAAS,CAAC,CAAC;gBAC/C,IAAI,MAAM,KAAK,SAAS,EAAE;oBACtB,SAAS;iBACZ;gBAED,MAAM,gBAAgB,GAAG,MAAM,gBAAgB,CAAmC,MAAM,CAAC,CAAC;gBAC1F,sDAAsD;gBACtD,IAAI,OAAA,gBAAgB,CAAC,MAAM,0CAAE,OAAO,MAAK,SAAS,EAAE;oBAChD,SAAS;iBACZ;gBAED,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC5B,iGAAiG;gBACjG,6EAA6E;gBAC7E,MAAM,sBAAsB,GAAG,MAAM,gBAAgB,CACjD,cAAc,CAAC,KAAK,CAAC,2CAA2B,CAAC,CACpD,CAAC;gBACF,IAAI,sBAAsB,CAAC,eAAe,EAAE;oBACxC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;iBACtD;gBAED,KAAK,MAAM,CAAC,EAAE,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;oBAChF,2FAA2F;oBAC3F,+EAA+E;oBAC/E,MAAM,MAAM,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,EAAE,EAAE,CAAC;oBAC1D,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;iBAC5E;gBACD,qBAAM,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,SAAS,EAC1C,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBAClE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,uBAAuB,GAAG,gBAAgB,CAAC,cAAc,CAAC;aACvF;YACD,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,8GAA8G;QAC9G,wDAAwD;QACxD,IAAI,CAAC,oBAAoB,GAAG,IAAI,0BAAW,CAAO,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC;YAE1C,MAAM,OAAO,GAAmD,EAAE,CAAC;YACnE,oFAAoF;YACpF,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;gBAChE,MAAM,uBAAuB,GAAG,QAAQ,CAAC,uBAAuB,CAAC;gBACjE,IAAI,uBAAuB,KAAK,SAAS,EAAE;oBACvC,mGAAmG;oBACnG,2CAA2C;oBAC3C,MAAM,sBAAsB,GAAG,IAAI,CAAC,qBAAqB,EAAE,GAAG,uBAAuB,CAAC;oBACtF,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAC3B,MAAM,EACN,IAAI,wBAAwB,CACxB,uBAAuB,EACvB,IAAI,CAAC,eAAe,GAAG,sBAAsB,CAChD,CACJ,CAAC;iBACL;gBAED,OAAO,CAAC,MAAM,CAAC,GAAG;oBACd,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;oBACnD,uBAAuB;iBAC1B,CAAC;aACL;YACD,IAAI,CAAC,cAAc,GAAG,EAAE,OAAO,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,oGAAoG;QACpG,yFAAyF;QACzF,IAAI,CAAC,mBAAmB,GAAG,IAAI,0BAAW,CAAgD,KAAK,IAAI,EAAE;YACjG,MAAM,OAAO,GAAiC,EAAE,CAAC;YACjD,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC;YAC1C,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;gBAChE,OAAO,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;aACzD;YACD,gGAAgG;YAChG,uGAAuG;YACvG,4BAA4B;YAC5B,MAAM,UAAU,GAAG,wCAAoB,CACnC,OAAO,EACP,CAAE,GAAG,CAAE,EACP,IAAI,CAAC,MAAM,CACd,CAAC,iBAAiB,CAAC;YAEpB,MAAM,qBAAqB,GAAG,6CAAyB,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;YAC7F,oGAAoG;YACpG,oGAAoG;YACpG,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;gBAChE,IAAI,QAAQ,CAAC,uBAAuB,KAAK,SAAS,EAAE;oBAChD,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBACtE,IAAI,kBAAkB,KAAK,SAAS,EAAE;wBAClC,kBAAkB,CAAC,cAAc,GAAG,QAAQ,CAAC,uBAAuB,CAAC;qBACxE;iBACJ;aACJ;YACD,OAAO,qBAAqB,CAAC;QACjC,CAAC,CAAC,CAAC;IACP,CAAC;IA9PM,MAAM,CAAC,MAAM,CAChB,QAAmC,EACnC,SAA4B,EAC5B,kBAAoD,EACpD,qBAAmC,EACnC,YAAuC,EACvC,gBAAkC,EAClC,UAA4B,EAC5B,QAAiB,EACjB,QAAoC;QAEpC,OAAO,IAAI,gBAAgB,CACvB,QAAQ,EACR,SAAS,EACT,kBAAkB,EAClB,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,UAAU,EACV,QAAQ,EACR,QAAQ,CACX,CAAC;IACN,CAAC;IAOD;;;;OAIG;IACH,IAAW,uBAAuB;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,IAAW,mBAAmB;QAC1B,+EAA+E;QAC/E,+EAA+E;QAC/E,+FAA+F;QAC/F,OAAO,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,gBAAgB,CAAC;IACrF,CAAC;IAkBD,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IA8LD;;;OAGG;IACI,KAAK,CAAC,cAAc,CACvB,OAOC;QAED,MAAM,EACF,MAAM,GAAG,IAAI,CAAC,MAAM,EACpB,QAAQ,GAAG,IAAI,CAAC,cAAc,EAC9B,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,mBAAmB,GACzE,GAAG,OAAO,CAAC;QAEZ,OAAO,kCAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,mBAAmB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC/F,MAAM,IAAI,CAAC,oBAAoB,CAAC;YAEhC,MAAM,OAAO,GAKT,EAAE,CAAC;YAEP,qEAAqE;YACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,wCAAoB,CACjC,MAAM,CAAC,OAAO,EACd,CAAE,GAAG,CAAE,EACP,MAAM,CACT,CAAC;YAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACxD,8DAA8D;YAC9D,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;YAE9D,MAAM,uBAAuB,GACzB,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;YAEnF,IAAI,QAAQ,EAAE;gBACV,uCAAuC;aAC1C;YAED,uDAAuD;YACvD,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC;YACtD,OAAO,CAAC,UAAU,GAAG,QAAQ,CAAC,iBAAiB,CAAC,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC;YACxF,OAAO,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,eAAe,CAAC;YACpE,OAAO,CAAC,eAAe,GAAG,uBAAuB,CAAC,cAAc,CAAC;YAEjE,sGAAsG;YACtG,oCAAoC;YACpC,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;aACpD;YACD,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,OAAmB,CAAC;QAC/B,CAAC,EACD,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACK,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE;YACxD,OAAO;SACV;QAED,MAAM,OAAO,GAAG,IAAI,kCAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,OAAO,CAAC,GAAG,oBAAY,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QAC7E,OAAO,OAAO,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,yBAAyB;QAClC,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,2BAA2B,CACpC,MAA4B,EAC5B,gBAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE;YACnD,OAAO;SACV;QAED,2GAA2G;QAC3G,uDAAuD;QACvD,IAAI,MAAM,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACpD,OAAO;SACV;QACD,6GAA6G;QAC7G,kCAAkC;QAClC,MAAM,IAAI,CAAC,kCAAkC,CAAC,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACI,WAAW,CAAC,EAAU;;QACzB,wDAAwD;QACxD,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClD,MAAA,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,0CAAE,aAAa,CAClD,IAAI,CAAC,MAAM,EACX,uBAAuB,EACvB,IAAI,CAAC,qBAAqB,EAAE,EAC5B,IAAI,CAAC,eAAe,EACpB,MAAM,EACR;IACN,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kCAAkC,CAAC,QAAuB,EAAE,gBAAkC;QACxG,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,gCAAgB,CAAC,CAAC;QACxD,IAAI,cAAc,EAAE;YAChB,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAA4B,cAAc,CAAC,CAAC;YACnF,IAAI,CAAC,sBAAsB,GAAG,4BAAY,CAAC,QAAQ,CAAC,CAAC;SACxD;IACL,CAAC;IAED;;;;;;;;OAQG;IACK,kBAAkB,CAAC,MAA8B,EAAE,QAAmB,EAAE,kBAA0B;QACtG,IAAI,CAAC,cAAc,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,EAAE,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YAC/D,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;SACpF;QAED,gGAAgG;QAChG,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,cAAc,EAAE;YAC1C,qBAAM,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEzG,qFAAqF;YACrF,IAAI,uBAAuB,GAAW,kBAAkB,CAAC;YACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAChC,uBAAuB,GAAG,gBAAgB,CAAC,uBAAuB,CAAC;aACtE;iBAAM;gBACH,kEAAkE;gBAClE,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAC3B,MAAM,EACN,IAAI,wBAAwB,CAAC,uBAAuB,EAAE,IAAI,CAAC,eAAe,CAAC,CAC9E,CAAC;aACL;YACD,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;SACzF;QAED,2FAA2F;QAC3F,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,iBAAiB,EAAE;YAC7C,qBAAM,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACzG,MAAM,gBAAgB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAChC,8FAA8F;gBAC9F,wEAAwE;gBACxE,gBAAgB,CAAC,aAAa,CAC1B,IAAI,CAAC,MAAM,EACX,uBAAuB,EACvB,kBAAkB,EAClB,IAAI,CAAC,eAAe,EACpB,MAAM,CACT,CAAC;gBACF,uDAAuD;gBACvD,gBAAgB,CAAC,YAAY,EAAE,CAAC;gBAChC,yDAAyD;gBACzD,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aAC9C;SACJ;IACL,CAAC;CACJ;AAlcD,4CAkcC;AAED;;;EAGE;AACF,KAAK,UAAU,sBAAsB,CACjC,cAA6B,EAC7B,gBAAkC;IAElC,IAAI,WAAW,GAA4B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC3D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE;QACjD,oDAAoD;QACpD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,oBAAY,CAAC,EAAE;YAC/B,SAAS;SACZ;QAED,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAA0B,MAAM,CAAC,CAAC;QACxE,qBAAM,CAAC,OAAO,KAAK,SAAS,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC3E,0DAA0D;QAC1D,WAAW,GAAG,iDAA6B,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;KACrE;IACD,OAAO,WAAW,CAAC;AACvB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, LazyPromise, Timer } from \"@fluidframework/common-utils\";\nimport {\n concatGarbageCollectionStates,\n unpackChildNodesGCDetails,\n IGCResult,\n runGarbageCollection,\n} from \"@fluidframework/garbage-collector\";\nimport { ISnapshotTree } from \"@fluidframework/protocol-definitions\";\nimport {\n gcBlobKey,\n IGarbageCollectionData,\n IGarbageCollectionNodeData,\n IGarbageCollectionState,\n IGarbageCollectionSummaryDetails,\n ISummaryTreeWithStats,\n} from \"@fluidframework/runtime-definitions\";\nimport {\n ReadAndParseBlob,\n RefreshSummaryResult,\n SummaryTreeBuilder,\n} from \"@fluidframework/runtime-utils\";\nimport { ChildLogger, PerformanceEvent } from \"@fluidframework/telemetry-utils\";\n\nimport { IGCRuntimeOptions } from \"./containerRuntime\";\nimport { getSummaryForDatastores } from \"./dataStores\";\nimport { getLocalStorageFeatureGate } from \"./localStorageFeatureGates\";\nimport {\n getGCVersion,\n GCVersion,\n IContainerRuntimeMetadata,\n metadataBlobName,\n ReadFluidDataStoreAttributes,\n dataStoreAttributesBlobName,\n} from \"./summaryFormat\";\n\n/** This is the current version of garbage collection. */\nconst GCVersion = 1;\n\n// The key for the GC tree in summary.\nexport const gcTreeKey = \"gc\";\n// They prefix for GC blobs in the GC tree in summary.\nexport const gcBlobPrefix = \"__gc\";\n\n// Local storage key to turn GC on / off.\nconst runGCKey = \"FluidRunGC\";\n// Local storage key to turn GC test mode on / off.\nconst gcTestModeKey = \"FluidGCTestMode\";\n// Local storage key to turn GC sweep on / off.\nconst runSweepKey = \"FluidRunSweep\";\n\nconst defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days\n\n/** The used state statistics of a node. */\nexport interface IUsedStateStats {\n totalNodeCount: number;\n unusedNodeCount: number;\n}\n\n/** The statistics of the system state after a garbage collection run. */\nexport interface IGCStats {\n totalNodes: number;\n deletedNodes: number;\n totalDataStores: number;\n deletedDataStores: number;\n}\n\n/** Defines the APIs for the runtime object to be passed to the garbage collector. */\nexport interface IGarbageCollectionRuntime {\n /** Returns the garbage collection data of the runtime. */\n getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;\n /** After GC has run, called to notify the runtime of routes that are used in it. */\n updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats;\n}\n\n/** Defines the contract for the garbage collector. */\nexport interface IGarbageCollector {\n /** Tells whether GC should run or not. */\n readonly shouldRunGC: boolean;\n /**\n * This tracks two things:\n * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.\n * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n */\n readonly gcSummaryFeatureVersion: number;\n /** Tells whether the GC version has changed compared to the version in the latest summary. */\n readonly hasGCVersionChanged: boolean;\n /** Tells whether GC data should be written to the root of the summary tree. */\n readonly writeDataAtRoot: boolean;\n /** Run garbage collection and update the reference / used state of the system. */\n collectGarbage(\n options: { logger?: ITelemetryLogger, runGC?: boolean, runSweep?: boolean, fullGC?: boolean },\n ): Promise<IGCStats>;\n /** Summarizes the GC data and returns it as a summary tree. */\n summarize(): ISummaryTreeWithStats | undefined;\n /** Returns a map of each data store id to its GC details in the base summary. */\n getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>>;\n /** Called when the latest summary of the system has been refreshed. */\n latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;\n /** Called when a node is changed. Used to detect and log when an inactive node is changed. */\n nodeChanged(id: string): void;\n}\n\n/**\n * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced. It also sets\n * the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).\n */\nclass UnreferencedStateTracker {\n private inactive: boolean = false;\n // Keeps track of all inactive events that are logged. This is used to limit the log generation for each event to 1\n // so that it is not noisy.\n private readonly inactiveEventsLogged: Set<string> = new Set();\n private readonly timer: Timer | undefined;\n\n constructor(\n public readonly unreferencedTimestampMs: number,\n inactiveTimeoutMs: number,\n ) {\n // If the timeout has already expired, the node should become inactive immediately. Otherwise, start a timer of\n // inactiveTimeoutMs after which the node will become inactive.\n if (inactiveTimeoutMs <= 0) {\n this.inactive = true;\n } else {\n this.timer = new Timer(inactiveTimeoutMs, () => { this.inactive = true; });\n this.timer.start();\n }\n }\n\n /** Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state. */\n public stopTracking() {\n this.timer?.clear();\n this.inactive = false;\n }\n\n /** Logs an error with the given properties if the node is inactive. */\n public logIfInactive(\n logger: ITelemetryLogger,\n eventName: string,\n currentTimestampMs: number,\n deleteTimeoutMs: number,\n inactiveNodeId: string,\n ) {\n if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {\n logger.sendErrorEvent({\n eventName,\n unreferencedDuratonMs: currentTimestampMs - this.unreferencedTimestampMs,\n deleteTimeoutMs,\n inactiveNodeId,\n });\n this.inactiveEventsLogged.add(eventName);\n }\n }\n}\n\n/**\n * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains\n * its state across summaries.\n */\nexport class GarbageCollector implements IGarbageCollector {\n public static create(\n provider: IGarbageCollectionRuntime,\n gcOptions: IGCRuntimeOptions,\n deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n getCurrentTimestampMs: () => number,\n baseSnapshot: ISnapshotTree | undefined,\n readAndParseBlob: ReadAndParseBlob,\n baseLogger: ITelemetryLogger,\n existing: boolean,\n metadata?: IContainerRuntimeMetadata,\n ): IGarbageCollector {\n return new GarbageCollector(\n provider,\n gcOptions,\n deleteUnusedRoutes,\n getCurrentTimestampMs,\n baseSnapshot,\n readAndParseBlob,\n baseLogger,\n existing,\n metadata,\n );\n }\n\n /**\n * Tells whether GC should be run based on the GC options and local storage flags.\n */\n public readonly shouldRunGC: boolean;\n\n /**\n * This tracks two things:\n * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.\n * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.\n */\n public get gcSummaryFeatureVersion(): number {\n return this.gcEnabled ? this.currentGCVersion : 0;\n }\n\n /**\n * Tells whether the GC version has changed compared to the version in the latest summary.\n */\n public get hasGCVersionChanged(): boolean {\n // The current version can differ from the latest summary version in two cases:\n // 1. The summary this client loaded with has data from a different GC version.\n // 2. This client's latest summary was updated from a snapshot that has a different GC version.\n return this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion;\n }\n\n /**\n * Tracks if GC is enabled for this document. This is specified during document creation and doesn't change\n * throughout its lifetime.\n */\n private readonly gcEnabled: boolean;\n private readonly shouldRunSweep: boolean;\n private readonly testMode: boolean;\n private readonly logger: ITelemetryLogger;\n\n /**\n * Tells whether the GC data should be written to the root of the summary tree. We do this under 2 conditions:\n * 1. If `writeDataAtRoot` GC option is enabled.\n * 2. If the base summary has the GC data written at the root. This is to support forward compatibility where when\n * we start writing the GC data at root, older versions can detect that and write at root too.\n */\n private _writeDataAtRoot: boolean = false;\n public get writeDataAtRoot(): boolean {\n return this._writeDataAtRoot;\n }\n\n // The current GC version that this container is running.\n private readonly currentGCVersion = GCVersion;\n // This is the version of GC data in the latest summary being tracked.\n private latestSummaryGCVersion: GCVersion;\n\n // The current state - each node's GC data and unreferenced timestamp.\n private currentGCState: IGarbageCollectionState | undefined;\n\n // Promise when resolved initializes the base state of the nodes from the base summary state.\n private readonly initializeBaseStateP: Promise<void>;\n // The map of data store ids to their GC details in the base summary returned in getDataStoreGCDetails().\n private readonly dataStoreGCDetailsP: Promise<Map<string, IGarbageCollectionSummaryDetails>>;\n // The time after which an unreferenced node can be deleted. Currently, we only set the node's state to expired.\n private readonly deleteTimeoutMs: number;\n // Map of node ids to their unreferenced state tracker.\n private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();\n\n protected constructor(\n private readonly provider: IGarbageCollectionRuntime,\n private readonly gcOptions: IGCRuntimeOptions,\n /** After GC has run, called to delete objects in the runtime whose routes are unused. */\n private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,\n /** Returns the current timestamp to be assigned to nodes that become unreferenced. */\n private readonly getCurrentTimestampMs: () => number,\n baseSnapshot: ISnapshotTree | undefined,\n readAndParseBlob: ReadAndParseBlob,\n baseLogger: ITelemetryLogger,\n existing: boolean,\n metadata?: IContainerRuntimeMetadata,\n ) {\n this.logger = ChildLogger.create(baseLogger, \"GarbageCollector\");\n\n this.deleteTimeoutMs = this.gcOptions.deleteTimeoutMs ?? defaultDeleteTimeoutMs;\n\n let prevSummaryGCVersion: number | undefined;\n // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing\n // documents, we get this information from the metadata blob.\n if (existing) {\n prevSummaryGCVersion = getGCVersion(metadata);\n // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all\n // other exsiting documents, GC is enabled.\n this.gcEnabled = prevSummaryGCVersion > 0;\n } else {\n // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.\n this.gcEnabled = gcOptions.gcAllowed === true;\n }\n // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the\n // latest tracked GC version. For new documents, we will be writing the first summary with the current version.\n this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;\n\n // Whether GC should run or not. Can override with localStorage flag.\n this.shouldRunGC = getLocalStorageFeatureGate(runGCKey) ?? (\n // GC must be enabled for the document.\n this.gcEnabled\n // GC must not be disabled via GC options.\n && !gcOptions.disableGC\n );\n\n // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with\n // localStorage flag.\n this.shouldRunSweep = this.shouldRunGC &&\n (getLocalStorageFeatureGate(runSweepKey) ?? gcOptions.runSweep === true);\n\n // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.\n this.testMode = getLocalStorageFeatureGate(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;\n\n // If `writeDataAtRoot` GC option is true, we should write the GC data into the root of the summary tree. This\n // GC option is used for testing only. It will be removed once we start writing GC data into root by default.\n this._writeDataAtRoot = this.gcOptions.writeDataAtRoot === true;\n\n // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do\n // this once since it involves fetching blobs from storage which is expensive.\n const baseSummaryStateP = new LazyPromise<IGarbageCollectionState>(async () => {\n if (baseSnapshot === undefined) {\n return { gcNodes: {} };\n }\n\n // For newer documents, GC data should be present in the GC tree in the root of the snapshot.\n const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];\n if (gcSnapshotTree !== undefined) {\n // forward-compat - If a newer version has written the GC tree at root, we should also do the same.\n this._writeDataAtRoot = true;\n return getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);\n }\n\n // back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and\n // consolidate into IGarbageCollectionState format.\n const gcState: IGarbageCollectionState = { gcNodes: { \"/\": { outboundRoutes: [] } } };\n const dataStoreSnaphotTree = getSummaryForDatastores(baseSnapshot, metadata);\n assert(dataStoreSnaphotTree !== undefined,\n 0x2a8 /* \"Expected data store snapshot tree in base snapshot\" */);\n for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnaphotTree.trees)) {\n const blobId = dsSnapshotTree.blobs[gcBlobKey];\n if (blobId === undefined) {\n continue;\n }\n\n const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionSummaryDetails>(blobId);\n // If there are no nodes for this data store, skip it.\n if (gcSummaryDetails.gcData?.gcNodes === undefined) {\n continue;\n }\n\n const dsRootId = `/${dsId}`;\n // Since we used to write GC data at data store level, we won't have an entry for the root (\"/\").\n // Construct that entry by adding root data store ids to its outbound routes.\n const initialSnapshotDetails = await readAndParseBlob<ReadFluidDataStoreAttributes>(\n dsSnapshotTree.blobs[dataStoreAttributesBlobName],\n );\n if (initialSnapshotDetails.isRootDataStore) {\n gcState.gcNodes[\"/\"].outboundRoutes.push(dsRootId);\n }\n\n for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {\n // Prefix the data store id to the GC node ids to make them relative to the root from being\n // relative to the data store. Similar to how its done in DataStore::getGCData.\n const rootId = id === \"/\" ? dsRootId : `${dsRootId}${id}`;\n gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };\n }\n assert(gcState.gcNodes[dsRootId] !== undefined,\n 0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);\n gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;\n }\n return gcState;\n });\n\n // Set up the initializer which initializes the base GC state from the base snapshot. Use lazy promise because\n // we only do this once - the very first time we run GC.\n this.initializeBaseStateP = new LazyPromise<void>(async () => {\n const baseState = await baseSummaryStateP;\n\n const gcNodes: { [ id: string ]: IGarbageCollectionNodeData } = {};\n // Set up tracking for the nodes in the base summary state and add them to GC nodes.\n for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {\n const unreferencedTimestampMs = nodeData.unreferencedTimestampMs;\n if (unreferencedTimestampMs !== undefined) {\n // Get how long it has been since the node was unreferenced. Start a timeout for the remaining time\n // left for it to be eligible for deletion.\n const unreferencedDurationMs = this.getCurrentTimestampMs() - unreferencedTimestampMs;\n this.unreferencedNodesState.set(\n nodeId,\n new UnreferencedStateTracker(\n unreferencedTimestampMs,\n this.deleteTimeoutMs - unreferencedDurationMs,\n ),\n );\n }\n\n gcNodes[nodeId] = {\n outboundRoutes: Array.from(nodeData.outboundRoutes),\n unreferencedTimestampMs,\n };\n }\n this.currentGCState = { gcNodes };\n });\n\n // Get the GC details for each data store from the GC state in the base summary. This is returned in\n // getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.\n this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionSummaryDetails>>(async () => {\n const gcNodes: { [ id: string ]: string[] } = {};\n const baseState = await baseSummaryStateP;\n for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {\n gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);\n }\n // Run GC on the nodes in the base summary to get the routes used in each node in the container.\n // This is an optimization for space (vs performance) wherein we don't need to store the used routes of\n // each node in the summary.\n const usedRoutes = runGarbageCollection(\n gcNodes,\n [ \"/\" ],\n this.logger,\n ).referencedNodeIds;\n\n const dataStoreGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });\n // Currently, the data stores write the GC data. So, we need to update it's base GC details with the\n // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.\n for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {\n if (nodeData.unreferencedTimestampMs !== undefined) {\n const dataStoreGCDetails = dataStoreGCDetailsMap.get(nodeId.slice(1));\n if (dataStoreGCDetails !== undefined) {\n dataStoreGCDetails.unrefTimestamp = nodeData.unreferencedTimestampMs;\n }\n }\n }\n return dataStoreGCDetailsMap;\n });\n }\n\n /**\n * Runs garbage collection and udpates the reference / used state of the nodes in the container.\n * @returns the number of data stores that have been marked as unreferenced.\n */\n public async collectGarbage(\n options: {\n /** Logger to use for logging GC events */\n logger?: ITelemetryLogger,\n /** True to run GC sweep phase after the mark phase */\n runSweep?: boolean,\n /** True to generate full GC data */\n fullGC?: boolean,\n },\n ): Promise<IGCStats> {\n const {\n logger = this.logger,\n runSweep = this.shouldRunSweep,\n fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged,\n } = options;\n\n return PerformanceEvent.timedExecAsync(logger, { eventName: \"GarbageCollection\" }, async (event) => {\n await this.initializeBaseStateP;\n\n const gcStats: {\n deletedNodes?: number,\n totalNodes?: number,\n deletedDataStores?: number,\n totalDataStores?: number,\n } = {};\n\n // Get the runtime's GC data and run GC on the reference graph in it.\n const gcData = await this.provider.getGCData(fullGC);\n const gcResult = runGarbageCollection(\n gcData.gcNodes,\n [ \"/\" ],\n logger,\n );\n\n const currentTimestampMs = this.getCurrentTimestampMs();\n // Update the current state of the system based on the GC run.\n this.updateCurrentState(gcData, gcResult, currentTimestampMs);\n\n const dataStoreUsedStateStats =\n this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);\n\n if (runSweep) {\n // Placeholder for running sweep logic.\n }\n\n // Update stats to be reported in the peformance event.\n gcStats.deletedNodes = gcResult.deletedNodeIds.length;\n gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;\n gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;\n gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;\n\n // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios\n // involving access to deleted data.\n if (this.testMode) {\n this.deleteUnusedRoutes(gcResult.deletedNodeIds);\n }\n event.end(gcStats);\n return gcStats as IGCStats;\n },\n { end: true, cancel: \"error\" });\n }\n\n /**\n * Summarizes the GC data and returns it as a summary tree.\n * We current write the entire GC state in a single blob. This can be modified later to write multiple\n * blobs. All the blob keys should start with `gcBlobPrefix`.\n */\n public summarize(): ISummaryTreeWithStats | undefined {\n if (!this.shouldRunGC || this.currentGCState === undefined) {\n return;\n }\n\n const builder = new SummaryTreeBuilder();\n builder.addBlob(`${gcBlobPrefix}_root`, JSON.stringify(this.currentGCState));\n return builder.getSummaryTree();\n }\n\n /**\n * Returns a map of data store ids to their base GC details generated from the base summary.This is used to\n * initialize the data stores with their base GC state.\n */\n public async getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>> {\n return this.dataStoreGCDetailsP;\n }\n\n /**\n * Called when the latest summary of the system has been refreshed. This will be used to update the state of the\n * latest summary tracked.\n */\n public async latestSummaryStateRefreshed(\n result: RefreshSummaryResult,\n readAndParseBlob: ReadAndParseBlob,\n ): Promise<void> {\n if (!this.shouldRunGC || !result.latestSummaryUpdated) {\n return;\n }\n\n // If the summary was tracked by this client, it was the one that generated the summary in the first place.\n // Basically, it was written in the current GC version.\n if (result.wasSummaryTracked) {\n this.latestSummaryGCVersion = this.currentGCVersion;\n return;\n }\n // If the summary was not tracked by this client, update latest GC version from the snapshot in the result as\n // that is now the latest summary.\n await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);\n }\n\n /**\n * Called when a node with the given id is changed. If the node is inactive, log an error.\n */\n public nodeChanged(id: string) {\n // Prefix \"/\" if needed to make it relative to the root.\n const nodeId = id.startsWith(\"/\") ? id : `/${id}`;\n this.unreferencedNodesState.get(nodeId)?.logIfInactive(\n this.logger,\n \"inactiveObjectChanged\",\n this.getCurrentTimestampMs(),\n this.deleteTimeoutMs,\n nodeId,\n );\n }\n\n /**\n * Update the latest summary GC version from the metadata blob in the given snapshot.\n */\n private async updateSummaryGCVersionFromSnapshot(snapshot: ISnapshotTree, readAndParseBlob: ReadAndParseBlob) {\n const metadataBlobId = snapshot.blobs[metadataBlobName];\n if (metadataBlobId) {\n const metadata = await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId);\n this.latestSummaryGCVersion = getGCVersion(metadata);\n }\n }\n\n /**\n * Updates the state of the system as per the current GC run. It does the following:\n * 1. Sets up the current GC state as per the gcData.\n * 2. Starts tracking for nodes that have become unreferenced in this run.\n * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.\n * @param gcData - The data representing the reference graph on which GC is run.\n * @param gcResult - The result of the GC run on the gcData.\n * @param currentTimestampMs - The current timestamp to be used for unreferenced nodes' timestamp.\n */\n private updateCurrentState(gcData: IGarbageCollectionData, gcResult: IGCResult, currentTimestampMs: number) {\n this.currentGCState = { gcNodes: {} };\n for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {\n this.currentGCState.gcNodes[id] = { outboundRoutes: Array.from(outboundRoutes) };\n }\n\n // Iterate through the deleted nodes and start tracking if they became unreferenced in this run.\n for (const nodeId of gcResult.deletedNodeIds) {\n assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2aa /* \"Unexpected node when running GC\" */);\n\n // The time when the node became unreferenced. This is added to the current GC state.\n let unreferencedTimestampMs: number = currentTimestampMs;\n const nodeStateTracker = this.unreferencedNodesState.get(nodeId);\n if (nodeStateTracker !== undefined) {\n unreferencedTimestampMs = nodeStateTracker.unreferencedTimestampMs;\n } else {\n // Start tracking this node as it became unreferenced in this run.\n this.unreferencedNodesState.set(\n nodeId,\n new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs),\n );\n }\n this.currentGCState.gcNodes[nodeId].unreferencedTimestampMs = unreferencedTimestampMs;\n }\n\n // Iterate through the referenced nodes and stop tracking if they were unreferenced before.\n for (const nodeId of gcResult.referencedNodeIds) {\n assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2ab /* \"Unexpected node when running GC\" */);\n const nodeStateTracker = this.unreferencedNodesState.get(nodeId);\n if (nodeStateTracker !== undefined) {\n // If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,\n // log an error as this may mean the deleteTimeoutMs is not long enough.\n nodeStateTracker.logIfInactive(\n this.logger,\n \"inactiveObjectRevived\",\n currentTimestampMs,\n this.deleteTimeoutMs,\n nodeId,\n );\n // Stop tracking so as to clear out any running timers.\n nodeStateTracker.stopTracking();\n // Delete the node as we don't need to track it any more.\n this.unreferencedNodesState.delete(nodeId);\n }\n }\n }\n}\n\n/**\n * Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.\n * Merge the GC state from all such blobs and return the merged GC state.\n*/\nasync function getGCStateFromSnapshot(\n gcSnapshotTree: ISnapshotTree,\n readAndParseBlob: ReadAndParseBlob,\n): Promise<IGarbageCollectionState> {\n let rootGCState: IGarbageCollectionState = { gcNodes: {} };\n for (const key of Object.keys(gcSnapshotTree.blobs)) {\n // Skip blobs that do not stsart with the GC prefix.\n if (!key.startsWith(gcBlobPrefix)) {\n continue;\n }\n\n const blobId = gcSnapshotTree.blobs[key];\n if (blobId === undefined) {\n continue;\n }\n const gcState = await readAndParseBlob<IGarbageCollectionState>(blobId);\n assert(gcState !== undefined, 0x2ad /* \"GC blob missing from snapshot\" */);\n // Merge the GC state of this blob into the root GC state.\n rootGCState = concatGarbageCollectionStates(rootGCState, gcState);\n }\n return rootGCState;\n}\n"]}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
|
-
export
|
|
5
|
+
export { ContainerMessageType, IChunkedOp, ContainerRuntimeMessage, IGCRuntimeOptions, ISummaryRuntimeOptions, IContainerRuntimeOptions, isRuntimeMessage, unpackRuntimeMessage, ScheduleManager, agentSchedulerId, ContainerRuntime, } from "./containerRuntime";
|
|
6
6
|
export * from "./deltaScheduler";
|
|
7
7
|
export * from "./dataStoreRegistry";
|
|
8
|
-
export { IGarbageCollectionRuntime, IGCStats, IUsedStateStats } from "./garbageCollection";
|
|
8
|
+
export { gcBlobPrefix, gcTreeKey, IGarbageCollectionRuntime, IGCStats, IUsedStateStats, } from "./garbageCollection";
|
|
9
9
|
export * from "./pendingStateManager";
|
|
10
10
|
export * from "./summarizer";
|
|
11
11
|
export * from "./summarizerTypes";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACH,oBAAoB,EACpB,UAAU,EACV,uBAAuB,EACvB,iBAAiB,EACjB,sBAAsB,EACtB,wBAAwB,EACxB,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACnB,MAAM,oBAAoB,CAAC;AAC5B,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC;AACpC,OAAO,EACH,YAAY,EACZ,SAAS,EACT,yBAAyB,EACzB,QAAQ,EACR,eAAe,GAClB,MAAM,qBAAqB,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,OAAO,EAAE,gCAAgC,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -14,10 +14,19 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.neverCancelledSummaryToken = void 0;
|
|
18
|
-
|
|
17
|
+
exports.neverCancelledSummaryToken = exports.gcTreeKey = exports.gcBlobPrefix = exports.ContainerRuntime = exports.agentSchedulerId = exports.ScheduleManager = exports.unpackRuntimeMessage = exports.isRuntimeMessage = exports.ContainerMessageType = void 0;
|
|
18
|
+
var containerRuntime_1 = require("./containerRuntime");
|
|
19
|
+
Object.defineProperty(exports, "ContainerMessageType", { enumerable: true, get: function () { return containerRuntime_1.ContainerMessageType; } });
|
|
20
|
+
Object.defineProperty(exports, "isRuntimeMessage", { enumerable: true, get: function () { return containerRuntime_1.isRuntimeMessage; } });
|
|
21
|
+
Object.defineProperty(exports, "unpackRuntimeMessage", { enumerable: true, get: function () { return containerRuntime_1.unpackRuntimeMessage; } });
|
|
22
|
+
Object.defineProperty(exports, "ScheduleManager", { enumerable: true, get: function () { return containerRuntime_1.ScheduleManager; } });
|
|
23
|
+
Object.defineProperty(exports, "agentSchedulerId", { enumerable: true, get: function () { return containerRuntime_1.agentSchedulerId; } });
|
|
24
|
+
Object.defineProperty(exports, "ContainerRuntime", { enumerable: true, get: function () { return containerRuntime_1.ContainerRuntime; } });
|
|
19
25
|
__exportStar(require("./deltaScheduler"), exports);
|
|
20
26
|
__exportStar(require("./dataStoreRegistry"), exports);
|
|
27
|
+
var garbageCollection_1 = require("./garbageCollection");
|
|
28
|
+
Object.defineProperty(exports, "gcBlobPrefix", { enumerable: true, get: function () { return garbageCollection_1.gcBlobPrefix; } });
|
|
29
|
+
Object.defineProperty(exports, "gcTreeKey", { enumerable: true, get: function () { return garbageCollection_1.gcTreeKey; } });
|
|
21
30
|
__exportStar(require("./pendingStateManager"), exports);
|
|
22
31
|
__exportStar(require("./summarizer"), exports);
|
|
23
32
|
__exportStar(require("./summarizerTypes"), exports);
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;AAEH,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;AAEH,uDAY4B;AAXxB,wHAAA,oBAAoB,OAAA;AAMpB,oHAAA,gBAAgB,OAAA;AAChB,wHAAA,oBAAoB,OAAA;AACpB,mHAAA,eAAe,OAAA;AACf,oHAAA,gBAAgB,OAAA;AAChB,oHAAA,gBAAgB,OAAA;AAEpB,mDAAiC;AACjC,sDAAoC;AACpC,yDAM6B;AALzB,iHAAA,YAAY,OAAA;AACZ,8GAAA,SAAS,OAAA;AAKb,wDAAsC;AACtC,+CAA6B;AAC7B,oDAAkC;AAClC,sDAAoC;AACpC,+EAA8G;AAAnE,0IAAA,0BAA0B,OAAA","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport {\n ContainerMessageType,\n IChunkedOp,\n ContainerRuntimeMessage,\n IGCRuntimeOptions,\n ISummaryRuntimeOptions,\n IContainerRuntimeOptions,\n isRuntimeMessage,\n unpackRuntimeMessage,\n ScheduleManager,\n agentSchedulerId,\n ContainerRuntime,\n} from \"./containerRuntime\";\nexport * from \"./deltaScheduler\";\nexport * from \"./dataStoreRegistry\";\nexport {\n gcBlobPrefix,\n gcTreeKey,\n IGarbageCollectionRuntime,\n IGCStats,\n IUsedStateStats,\n} from \"./garbageCollection\";\nexport * from \"./pendingStateManager\";\nexport * from \"./summarizer\";\nexport * from \"./summarizerTypes\";\nexport * from \"./summaryCollection\";\nexport { ICancellableSummarizerController, neverCancelledSummaryToken } from \"./runWhileConnectedCoordinator\";\n"]}
|
package/dist/packageVersion.d.ts
CHANGED
|
@@ -5,5 +5,5 @@
|
|
|
5
5
|
* THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
|
|
6
6
|
*/
|
|
7
7
|
export declare const pkgName = "@fluidframework/container-runtime";
|
|
8
|
-
export declare const pkgVersion = "0.
|
|
8
|
+
export declare const pkgVersion = "0.53.0";
|
|
9
9
|
//# sourceMappingURL=packageVersion.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,sCAAsC,CAAC;AAC3D,eAAO,MAAM,UAAU,
|
|
1
|
+
{"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,sCAAsC,CAAC;AAC3D,eAAO,MAAM,UAAU,WAAW,CAAC"}
|
package/dist/packageVersion.js
CHANGED
|
@@ -8,5 +8,5 @@
|
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
9
|
exports.pkgVersion = exports.pkgName = void 0;
|
|
10
10
|
exports.pkgName = "@fluidframework/container-runtime";
|
|
11
|
-
exports.pkgVersion = "0.
|
|
11
|
+
exports.pkgVersion = "0.53.0";
|
|
12
12
|
//# sourceMappingURL=packageVersion.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,OAAO,GAAG,mCAAmC,CAAC;AAC9C,QAAA,UAAU,GAAG,
|
|
1
|
+
{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,OAAO,GAAG,mCAAmC,CAAC;AAC9C,QAAA,UAAU,GAAG,QAAQ,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"0.53.0\";\n"]}
|
|
@@ -61,7 +61,6 @@ export declare class PendingStateManager implements IDisposable {
|
|
|
61
61
|
private readonly initialStates;
|
|
62
62
|
private readonly previousClientIds;
|
|
63
63
|
private readonly firstStashedCSN;
|
|
64
|
-
private stashedCount;
|
|
65
64
|
private readonly disposeOnce;
|
|
66
65
|
private pendingMessagesCount;
|
|
67
66
|
private isProcessingBatch;
|