@fluidframework/container-runtime 0.56.7 → 0.57.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +9 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +6 -6
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +68 -28
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +148 -89
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +27 -0
- package/dist/dataStore.d.ts.map +1 -0
- package/dist/dataStore.js +113 -0
- package/dist/dataStore.js.map +1 -0
- package/dist/dataStoreContext.d.ts +1 -7
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +10 -6
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +9 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +14 -19
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +66 -27
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +272 -97
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +1 -0
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +23 -15
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +4 -6
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryGenerator.d.ts +2 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +46 -29
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +9 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +6 -6
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +68 -28
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +149 -90
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +27 -0
- package/lib/dataStore.d.ts.map +1 -0
- package/lib/dataStore.js +108 -0
- package/lib/dataStore.js.map +1 -0
- package/lib/dataStoreContext.d.ts +1 -7
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +10 -6
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +9 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +13 -18
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +66 -27
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +274 -99
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +1 -0
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +23 -15
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +4 -6
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryGenerator.d.ts +2 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +46 -29
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +13 -13
- package/src/blobManager.ts +12 -1
- package/src/connectionTelemetry.ts +7 -6
- package/src/containerRuntime.ts +244 -115
- package/src/dataStore.ts +151 -0
- package/src/dataStoreContext.ts +11 -14
- package/src/dataStores.ts +23 -38
- package/src/garbageCollection.ts +385 -150
- package/src/index.ts +2 -1
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +25 -16
- package/src/summarizerTypes.ts +4 -8
- package/src/summaryGenerator.ts +71 -23
|
@@ -11,6 +11,7 @@ const garbage_collector_1 = require("@fluidframework/garbage-collector");
|
|
|
11
11
|
const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
|
|
12
12
|
const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
13
13
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
14
|
+
const _1 = require(".");
|
|
14
15
|
const dataStores_1 = require("./dataStores");
|
|
15
16
|
const summaryFormat_1 = require("./summaryFormat");
|
|
16
17
|
/** This is the current version of garbage collection. */
|
|
@@ -27,45 +28,55 @@ const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
|
27
28
|
const runSweepKey = "Fluid.GarbageCollection.RunSweep";
|
|
28
29
|
// Feature gate key to write GC data at the root of the summary tree.
|
|
29
30
|
const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
|
|
31
|
+
// Feature gate key to expire a session after a set period of time.
|
|
32
|
+
const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
|
|
30
33
|
const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
|
|
34
|
+
const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
35
|
+
;
|
|
31
36
|
/**
|
|
32
37
|
* Helper class that tracks the state of an unreferenced node such as the time it was unreferenced. It also sets
|
|
33
38
|
* the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).
|
|
34
39
|
*/
|
|
35
40
|
class UnreferencedStateTracker {
|
|
36
|
-
constructor(unreferencedTimestampMs, inactiveTimeoutMs) {
|
|
41
|
+
constructor(unreferencedTimestampMs, inactiveTimeoutMs, currentReferenceTimestampMs) {
|
|
37
42
|
this.unreferencedTimestampMs = unreferencedTimestampMs;
|
|
38
|
-
this.
|
|
39
|
-
|
|
40
|
-
//
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
if (inactiveTimeoutMs <= 0) {
|
|
45
|
-
this.inactive = true;
|
|
43
|
+
this.inactiveTimeoutMs = inactiveTimeoutMs;
|
|
44
|
+
this._inactive = false;
|
|
45
|
+
// If there is no current reference timestamp, don't track the node's inactive state. This will happen later
|
|
46
|
+
// when updateTracking is called with a reference timestamp.
|
|
47
|
+
if (currentReferenceTimestampMs !== undefined) {
|
|
48
|
+
this.updateTracking(currentReferenceTimestampMs);
|
|
46
49
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
+
}
|
|
51
|
+
get inactive() {
|
|
52
|
+
return this._inactive;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Updates the tracking state based on the provided timestamp.
|
|
56
|
+
*/
|
|
57
|
+
updateTracking(currentReferenceTimestampMs) {
|
|
58
|
+
var _a;
|
|
59
|
+
const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
|
|
60
|
+
// If the timeout has already expired, the node has become inactive.
|
|
61
|
+
if (unreferencedDurationMs > this.inactiveTimeoutMs) {
|
|
62
|
+
this._inactive = true;
|
|
63
|
+
(_a = this.timer) === null || _a === void 0 ? void 0 : _a.clear();
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
// The node isn't inactive yet. Restart a timer for the duration remaining for it to become inactive.
|
|
67
|
+
const remainingDurationMs = this.inactiveTimeoutMs - unreferencedDurationMs;
|
|
68
|
+
if (this.timer === undefined) {
|
|
69
|
+
this.timer = new common_utils_1.Timer(remainingDurationMs, () => { this._inactive = true; });
|
|
50
70
|
}
|
|
71
|
+
this.timer.restart(remainingDurationMs);
|
|
51
72
|
}
|
|
52
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state.
|
|
75
|
+
*/
|
|
53
76
|
stopTracking() {
|
|
54
77
|
var _a;
|
|
55
78
|
(_a = this.timer) === null || _a === void 0 ? void 0 : _a.clear();
|
|
56
|
-
this.
|
|
57
|
-
}
|
|
58
|
-
/** Logs an error with the given properties if the node is inactive. */
|
|
59
|
-
logIfInactive(logger, eventName, currentTimestampMs, deleteTimeoutMs, inactiveNodeId) {
|
|
60
|
-
if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
|
|
61
|
-
logger.sendErrorEvent({
|
|
62
|
-
eventName,
|
|
63
|
-
age: currentTimestampMs - this.unreferencedTimestampMs,
|
|
64
|
-
timeout: deleteTimeoutMs,
|
|
65
|
-
id: inactiveNodeId,
|
|
66
|
-
});
|
|
67
|
-
this.inactiveEventsLogged.add(eventName);
|
|
68
|
-
}
|
|
79
|
+
this._inactive = false;
|
|
69
80
|
}
|
|
70
81
|
}
|
|
71
82
|
/**
|
|
@@ -76,14 +87,23 @@ class GarbageCollector {
|
|
|
76
87
|
constructor(provider, gcOptions,
|
|
77
88
|
/** After GC has run, called to delete objects in the runtime whose routes are unused. */
|
|
78
89
|
deleteUnusedRoutes,
|
|
79
|
-
/**
|
|
80
|
-
|
|
90
|
+
/** For a given node path, returns the node's package path. */
|
|
91
|
+
getNodePackagePath,
|
|
92
|
+
/**
|
|
93
|
+
* Returns a referenced timestamp to be used to track unreferenced nodes. This is a server generated timestamp
|
|
94
|
+
* and may not be available if there aren't any ops processed yet. If so, we skip tracking unreferenced state
|
|
95
|
+
* such as time when node becomes unreferenced or inactive.
|
|
96
|
+
*/
|
|
97
|
+
getCurrentReferenceTimestampMs,
|
|
98
|
+
/** Returns the timestamp of the last summary generated for this container. */
|
|
99
|
+
getLastSummaryTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
|
|
81
100
|
var _a, _b, _c, _d, _e;
|
|
82
101
|
this.provider = provider;
|
|
83
102
|
this.gcOptions = gcOptions;
|
|
84
103
|
this.deleteUnusedRoutes = deleteUnusedRoutes;
|
|
85
|
-
this.
|
|
86
|
-
this.
|
|
104
|
+
this.getNodePackagePath = getNodePackagePath;
|
|
105
|
+
this.getCurrentReferenceTimestampMs = getCurrentReferenceTimestampMs;
|
|
106
|
+
this.getLastSummaryTimestampMs = getLastSummaryTimestampMs;
|
|
87
107
|
/**
|
|
88
108
|
* Tells whether the GC data should be written to the root of the summary tree.
|
|
89
109
|
*/
|
|
@@ -106,6 +126,11 @@ class GarbageCollector {
|
|
|
106
126
|
this.referencesSinceLastRun = new Map();
|
|
107
127
|
// Map of node ids to their unreferenced state tracker.
|
|
108
128
|
this.unreferencedNodesState = new Map();
|
|
129
|
+
// Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
|
|
130
|
+
// per event per node.
|
|
131
|
+
this.loggedUnreferencedEvents = new Set();
|
|
132
|
+
// Queue for unreferenced events that should be logged the next time GC runs.
|
|
133
|
+
this.pendingEventsQueue = [];
|
|
109
134
|
this.mc = telemetry_utils_1.loggerToMonitoringContext(telemetry_utils_1.ChildLogger.create(baseLogger, "GarbageCollector"));
|
|
110
135
|
this.deleteTimeoutMs = (_a = this.gcOptions.deleteTimeoutMs) !== null && _a !== void 0 ? _a : defaultDeleteTimeoutMs;
|
|
111
136
|
let prevSummaryGCVersion;
|
|
@@ -122,12 +147,19 @@ class GarbageCollector {
|
|
|
122
147
|
else {
|
|
123
148
|
// For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.
|
|
124
149
|
this.gcEnabled = gcOptions.gcAllowed === true;
|
|
125
|
-
|
|
150
|
+
// Set the Session Expiry only if the flag is enabled or the test option is set.
|
|
151
|
+
if (this.mc.config.getBoolean(runSessionExpiry) && this.gcEnabled) {
|
|
152
|
+
this.sessionExpiryTimeoutMs = defaultSessionExpiryDurationMs;
|
|
153
|
+
}
|
|
126
154
|
}
|
|
127
155
|
// If session expiry is enabled, we need to close the container when the timeout expires
|
|
128
156
|
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
129
|
-
const
|
|
130
|
-
|
|
157
|
+
const timeoutMs = this.sessionExpiryTimeoutMs;
|
|
158
|
+
setLongTimeout(timeoutMs, () => {
|
|
159
|
+
this.provider.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
160
|
+
}, (timer) => {
|
|
161
|
+
this.sessionExpiryTimer = timer;
|
|
162
|
+
});
|
|
131
163
|
}
|
|
132
164
|
// For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
|
|
133
165
|
// latest tracked GC version. For new documents, we will be writing the first summary with the current version.
|
|
@@ -207,22 +239,21 @@ class GarbageCollector {
|
|
|
207
239
|
// very first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
208
240
|
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
209
241
|
});
|
|
210
|
-
|
|
211
|
-
|
|
242
|
+
/**
|
|
243
|
+
* Set up the initializer which initializes the base GC state from the base snapshot. Note that the reference
|
|
244
|
+
* timestamp maybe from old ops which were not summarized and stored in the file. So, the unreferenced state
|
|
245
|
+
* may be out of date. This is fine because the state is updated every time GC runs based on the time then.
|
|
246
|
+
*/
|
|
212
247
|
this.initializeBaseStateP = new common_utils_1.LazyPromise(async () => {
|
|
213
|
-
const
|
|
248
|
+
const currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs();
|
|
214
249
|
const baseState = await baseSummaryStateP;
|
|
215
250
|
if (baseState === undefined) {
|
|
216
251
|
return;
|
|
217
252
|
}
|
|
218
253
|
const gcNodes = {};
|
|
219
254
|
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
// Get how long it has been since the node was unreferenced. Start a timeout for the remaining time
|
|
223
|
-
// left for it to be eligible for deletion.
|
|
224
|
-
const unreferencedDurationMs = currentTimestampMs - unreferencedTimestampMs;
|
|
225
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs - unreferencedDurationMs));
|
|
255
|
+
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
256
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.deleteTimeoutMs, currentReferenceTimestampMs));
|
|
226
257
|
}
|
|
227
258
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
228
259
|
}
|
|
@@ -256,9 +287,22 @@ class GarbageCollector {
|
|
|
256
287
|
}
|
|
257
288
|
return dataStoreGCDetailsMap;
|
|
258
289
|
});
|
|
290
|
+
// Initialize the base state. The base GC data is used to detect and log when inactive / deleted objects are
|
|
291
|
+
// used in the container.
|
|
292
|
+
if (this.shouldRunGC) {
|
|
293
|
+
this.initializeBaseStateP.catch((error) => {
|
|
294
|
+
throw new container_utils_1.DataProcessingError(error === null || error === void 0 ? void 0 : error.message, "FailedToInitializeGC", {
|
|
295
|
+
gcEnabled: this.gcEnabled,
|
|
296
|
+
runSweep: this.shouldRunSweep,
|
|
297
|
+
writeAtRoot: this._writeDataAtRoot,
|
|
298
|
+
testMode: this.testMode,
|
|
299
|
+
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
300
|
+
});
|
|
301
|
+
});
|
|
302
|
+
}
|
|
259
303
|
}
|
|
260
|
-
static create(provider, gcOptions, deleteUnusedRoutes,
|
|
261
|
-
return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes,
|
|
304
|
+
static create(provider, gcOptions, deleteUnusedRoutes, getNodePackagePath, getCurrentReferenceTimestampMs, getLastSummaryTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
|
|
305
|
+
return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, getNodePackagePath, getCurrentReferenceTimestampMs, getLastSummaryTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata);
|
|
262
306
|
}
|
|
263
307
|
/**
|
|
264
308
|
* This tracks two things:
|
|
@@ -293,29 +337,26 @@ class GarbageCollector {
|
|
|
293
337
|
await this.initializeBaseStateP;
|
|
294
338
|
// Let the runtime update its pending state before GC runs.
|
|
295
339
|
await this.provider.updateStateBeforeGC();
|
|
296
|
-
const gcStats = {};
|
|
297
340
|
// Get the runtime's GC data and run GC on the reference graph in it.
|
|
298
341
|
const gcData = await this.provider.getGCData(fullGC);
|
|
299
|
-
this.updateStateSinceLatestRun(gcData);
|
|
300
342
|
const gcResult = garbage_collector_1.runGarbageCollection(gcData.gcNodes, ["/"], logger);
|
|
301
|
-
const
|
|
343
|
+
const gcStats = this.generateStatsAndLogEvents(gcResult);
|
|
344
|
+
// Update the state since the last GC run. There can be nodes that were referenced between the last and
|
|
345
|
+
// the current run. We need to identify than and update their unreferenced state if needed.
|
|
346
|
+
this.updateStateSinceLastRun(gcData);
|
|
302
347
|
// Update the current state of the system based on the GC run.
|
|
303
|
-
this.
|
|
304
|
-
|
|
348
|
+
const currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs();
|
|
349
|
+
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
350
|
+
this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentReferenceTimestampMs);
|
|
305
351
|
if (runSweep) {
|
|
306
352
|
// Placeholder for running sweep logic.
|
|
307
353
|
}
|
|
308
|
-
// Update stats to be reported in the peformance event.
|
|
309
|
-
gcStats.deletedNodes = gcResult.deletedNodeIds.length;
|
|
310
|
-
gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;
|
|
311
|
-
gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;
|
|
312
|
-
gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;
|
|
313
354
|
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
314
355
|
// involving access to deleted data.
|
|
315
356
|
if (this.testMode) {
|
|
316
357
|
this.deleteUnusedRoutes(gcResult.deletedNodeIds);
|
|
317
358
|
}
|
|
318
|
-
event.end(gcStats);
|
|
359
|
+
event.end(Object.assign({}, gcStats));
|
|
319
360
|
return gcStats;
|
|
320
361
|
}, { end: true, cancel: "error" });
|
|
321
362
|
}
|
|
@@ -369,32 +410,42 @@ class GarbageCollector {
|
|
|
369
410
|
await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);
|
|
370
411
|
}
|
|
371
412
|
/**
|
|
372
|
-
* Called when a node with the given id is
|
|
413
|
+
* Called when a node with the given id is updated. If the node is inactive, log an error.
|
|
414
|
+
* @param nodePath - The id of the node that changed.
|
|
415
|
+
* @param reason - Whether the node was loaded or changed.
|
|
416
|
+
* @param timestampMs - The timestamp when the node changed.
|
|
417
|
+
* @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
|
|
418
|
+
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
373
419
|
*/
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
const nodeId = id.startsWith("/") ? id : `/${id}`;
|
|
378
|
-
(_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.logIfInactive(this.mc.logger, "inactiveObjectChanged", this.getCurrentTimestampMs(), this.deleteTimeoutMs, nodeId);
|
|
379
|
-
}
|
|
380
|
-
dispose() {
|
|
381
|
-
if (this.sessionExpiryTimer !== undefined) {
|
|
382
|
-
clearTimeout(this.sessionExpiryTimer);
|
|
383
|
-
this.sessionExpiryTimer = undefined;
|
|
420
|
+
nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
|
|
421
|
+
if (!this.shouldRunGC) {
|
|
422
|
+
return;
|
|
384
423
|
}
|
|
424
|
+
this.logIfInactive(reason, nodePath, timestampMs, packagePath, requestHeaders);
|
|
385
425
|
}
|
|
386
426
|
/**
|
|
387
427
|
* Called when an outbound reference is added to a node. This is used to identify all nodes that have been
|
|
388
428
|
* referenced between summaries so that their unreferenced timestamp can be reset.
|
|
389
429
|
*
|
|
390
|
-
* @param
|
|
391
|
-
* @param
|
|
430
|
+
* @param fromNodePath - The node from which the reference is added.
|
|
431
|
+
* @param toNodePath - The node to which the reference is added.
|
|
392
432
|
*/
|
|
393
|
-
addedOutboundReference(
|
|
433
|
+
addedOutboundReference(fromNodePath, toNodePath) {
|
|
394
434
|
var _a;
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
435
|
+
if (!this.shouldRunGC) {
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
const outboundRoutes = (_a = this.referencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
|
|
439
|
+
outboundRoutes.push(toNodePath);
|
|
440
|
+
this.referencesSinceLastRun.set(fromNodePath, outboundRoutes);
|
|
441
|
+
// If the node that got referenced is inactive, log an event as that may indicate use-after-delete.
|
|
442
|
+
this.logIfInactive("Revived", toNodePath);
|
|
443
|
+
}
|
|
444
|
+
dispose() {
|
|
445
|
+
if (this.sessionExpiryTimer !== undefined) {
|
|
446
|
+
clearTimeout(this.sessionExpiryTimer);
|
|
447
|
+
this.sessionExpiryTimer = undefined;
|
|
448
|
+
}
|
|
398
449
|
}
|
|
399
450
|
/**
|
|
400
451
|
* Update the latest summary GC version from the metadata blob in the given snapshot.
|
|
@@ -413,37 +464,43 @@ class GarbageCollector {
|
|
|
413
464
|
* 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
|
|
414
465
|
* @param gcData - The data representing the reference graph on which GC is run.
|
|
415
466
|
* @param gcResult - The result of the GC run on the gcData.
|
|
416
|
-
* @param
|
|
467
|
+
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
417
468
|
*/
|
|
418
|
-
updateCurrentState(gcData, gcResult,
|
|
469
|
+
updateCurrentState(gcData, gcResult, currentReferenceTimestampMs) {
|
|
419
470
|
this.gcDataFromLastRun = garbage_collector_1.cloneGCData(gcData);
|
|
420
471
|
this.referencesSinceLastRun.clear();
|
|
421
|
-
// Iterate through the deleted nodes and start tracking if they became unreferenced in this run.
|
|
422
|
-
for (const nodeId of gcResult.deletedNodeIds) {
|
|
423
|
-
// The time when the node became unreferenced. This is added to the current GC state.
|
|
424
|
-
let unreferencedTimestampMs = currentTimestampMs;
|
|
425
|
-
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
426
|
-
if (nodeStateTracker !== undefined) {
|
|
427
|
-
unreferencedTimestampMs = nodeStateTracker.unreferencedTimestampMs;
|
|
428
|
-
}
|
|
429
|
-
else {
|
|
430
|
-
// Start tracking this node as it became unreferenced in this run.
|
|
431
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs));
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
472
|
// Iterate through the referenced nodes and stop tracking if they were unreferenced before.
|
|
435
473
|
for (const nodeId of gcResult.referencedNodeIds) {
|
|
436
474
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
437
475
|
if (nodeStateTracker !== undefined) {
|
|
438
|
-
// If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
|
|
439
|
-
// log an error as this may mean the deleteTimeoutMs is not long enough.
|
|
440
|
-
nodeStateTracker.logIfInactive(this.mc.logger, "inactiveObjectRevived", currentTimestampMs, this.deleteTimeoutMs, nodeId);
|
|
441
476
|
// Stop tracking so as to clear out any running timers.
|
|
442
477
|
nodeStateTracker.stopTracking();
|
|
443
478
|
// Delete the node as we don't need to track it any more.
|
|
444
479
|
this.unreferencedNodesState.delete(nodeId);
|
|
445
480
|
}
|
|
446
481
|
}
|
|
482
|
+
/**
|
|
483
|
+
* If there is no current reference time, skip tracking when a node becomes unreferenced. This would happen
|
|
484
|
+
* if no ops have been processed ever and we still try to run GC. If so, there is nothing interesting to track
|
|
485
|
+
* anyway.
|
|
486
|
+
*/
|
|
487
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* If a node became unreferenced in this run, start tracking it.
|
|
492
|
+
* If a node was already unreferenced, update its tracking information. Since the current reference time is
|
|
493
|
+
* from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
|
|
494
|
+
*/
|
|
495
|
+
for (const nodeId of gcResult.deletedNodeIds) {
|
|
496
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
497
|
+
if (nodeStateTracker === undefined) {
|
|
498
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.deleteTimeoutMs, currentReferenceTimestampMs));
|
|
499
|
+
}
|
|
500
|
+
else {
|
|
501
|
+
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
447
504
|
}
|
|
448
505
|
/**
|
|
449
506
|
* Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
|
|
@@ -453,7 +510,7 @@ class GarbageCollector {
|
|
|
453
510
|
* This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
|
|
454
511
|
* If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
|
|
455
512
|
*/
|
|
456
|
-
|
|
513
|
+
updateStateSinceLastRun(currentGCData) {
|
|
457
514
|
// If we haven't run GC before or no references were added since the last run, there is nothing to do.
|
|
458
515
|
if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
|
|
459
516
|
return;
|
|
@@ -509,8 +566,7 @@ class GarbageCollector {
|
|
|
509
566
|
* @param currentGCData - The GC data (reference graph) from the current GC run.
|
|
510
567
|
*/
|
|
511
568
|
validateReferenceCorrectness(currentGCData) {
|
|
512
|
-
common_utils_1.assert(this.gcDataFromLastRun !== undefined, 0x2b7
|
|
513
|
-
/* "Can't validate correctness without GC data from last run" */ );
|
|
569
|
+
common_utils_1.assert(this.gcDataFromLastRun !== undefined, 0x2b7);
|
|
514
570
|
// Get a list of all the outbound routes (or references) in the current GC data.
|
|
515
571
|
const currentReferences = [];
|
|
516
572
|
for (const [nodeId, outboundRoutes] of Object.entries(currentGCData.gcNodes)) {
|
|
@@ -534,9 +590,9 @@ class GarbageCollector {
|
|
|
534
590
|
// Validate that the current reference graph doesn't have references that we are not already aware of. If this
|
|
535
591
|
// happens, it might indicate data corruption since we may delete objects prematurely.
|
|
536
592
|
currentReferences.forEach((route) => {
|
|
537
|
-
// Validate references for data stores only
|
|
538
|
-
//
|
|
539
|
-
if (route
|
|
593
|
+
// Validate references for data stores only. Currently, layers below data stores don't have GC implemented
|
|
594
|
+
// so there is no guarantee their references will be notified.
|
|
595
|
+
if (isDataStoreNode(route) && !explicitReferences.includes(route)) {
|
|
540
596
|
/**
|
|
541
597
|
* The following log will be enabled once this issue is resolved:
|
|
542
598
|
* https://github.com/microsoft/FluidFramework/issues/8878.
|
|
@@ -550,12 +606,104 @@ class GarbageCollector {
|
|
|
550
606
|
}
|
|
551
607
|
});
|
|
552
608
|
}
|
|
609
|
+
/**
|
|
610
|
+
* Generates the stats of a garbage collection run from the given results of the run. Also, logs any pending events
|
|
611
|
+
* in the pendingEventsQueue.
|
|
612
|
+
* @param gcResult - The result of a GC run.
|
|
613
|
+
* @returns the GC stats of the GC run.
|
|
614
|
+
*/
|
|
615
|
+
generateStatsAndLogEvents(gcResult) {
|
|
616
|
+
// Log pending events for unreferenced nodes after GC has run. We should have the package data available for
|
|
617
|
+
// them now since the GC run should have loaded these nodes.
|
|
618
|
+
let event = this.pendingEventsQueue.shift();
|
|
619
|
+
while (event !== undefined) {
|
|
620
|
+
const pkg = this.getNodePackagePath(event.id);
|
|
621
|
+
this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, event), { pkg: pkg ? { value: `/${pkg.join("/")}`, tag: telemetry_utils_1.TelemetryDataTag.PackageData } : undefined }));
|
|
622
|
+
event = this.pendingEventsQueue.shift();
|
|
623
|
+
}
|
|
624
|
+
const gcStats = {
|
|
625
|
+
nodeCount: 0,
|
|
626
|
+
dataStoreCount: 0,
|
|
627
|
+
unrefNodeCount: 0,
|
|
628
|
+
unrefDataStoreCount: 0,
|
|
629
|
+
updatedNodeCount: 0,
|
|
630
|
+
updatedDataStoreCount: 0,
|
|
631
|
+
};
|
|
632
|
+
for (const nodeId of gcResult.referencedNodeIds) {
|
|
633
|
+
gcStats.nodeCount++;
|
|
634
|
+
const isDataStore = isDataStoreNode(nodeId);
|
|
635
|
+
if (isDataStore) {
|
|
636
|
+
gcStats.dataStoreCount++;
|
|
637
|
+
}
|
|
638
|
+
// If a referenced node has an entry in `unreferencedNodesState`, it was previously unreferenced. So, its
|
|
639
|
+
// reference state updated from the last GC run.
|
|
640
|
+
if (this.unreferencedNodesState.has(nodeId)) {
|
|
641
|
+
gcStats.updatedNodeCount++;
|
|
642
|
+
if (isDataStore) {
|
|
643
|
+
gcStats.updatedDataStoreCount++;
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
for (const nodeId of gcResult.deletedNodeIds) {
|
|
648
|
+
gcStats.nodeCount++;
|
|
649
|
+
gcStats.unrefNodeCount++;
|
|
650
|
+
const isDataStore = isDataStoreNode(nodeId);
|
|
651
|
+
if (isDataStore) {
|
|
652
|
+
gcStats.dataStoreCount++;
|
|
653
|
+
gcStats.unrefDataStoreCount++;
|
|
654
|
+
}
|
|
655
|
+
// If an unreferenced node doesn't an entry in `unreferencedNodesState`, it was previously referenced. So,
|
|
656
|
+
// its reference state updated from the last GC run.
|
|
657
|
+
if (!this.unreferencedNodesState.has(nodeId)) {
|
|
658
|
+
gcStats.updatedNodeCount++;
|
|
659
|
+
if (isDataStore) {
|
|
660
|
+
gcStats.updatedDataStoreCount++;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
return gcStats;
|
|
665
|
+
}
|
|
666
|
+
/**
|
|
667
|
+
* Logs an event if a node is inactive and is used.
|
|
668
|
+
*/
|
|
669
|
+
logIfInactive(eventSuffix, nodeId, currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs(), packagePath, requestHeaders) {
|
|
670
|
+
// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
|
|
671
|
+
// logging as nothing interesting would have happened worth logging.
|
|
672
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
const eventName = `inactiveObject_${eventSuffix}`;
|
|
676
|
+
// We log a particular event for a given node only once so that it is not too noisy.
|
|
677
|
+
const uniqueEventId = `${nodeId}-${eventName}`;
|
|
678
|
+
const nodeState = this.unreferencedNodesState.get(nodeId);
|
|
679
|
+
if ((nodeState === null || nodeState === void 0 ? void 0 : nodeState.inactive) && !this.loggedUnreferencedEvents.has(uniqueEventId)) {
|
|
680
|
+
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
681
|
+
const event = {
|
|
682
|
+
eventName,
|
|
683
|
+
id: nodeId,
|
|
684
|
+
age: currentReferenceTimestampMs - nodeState.unreferencedTimestampMs,
|
|
685
|
+
timeout: this.deleteTimeoutMs,
|
|
686
|
+
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
687
|
+
externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[_1.RuntimeHeaders.externalRequest],
|
|
688
|
+
viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[_1.RuntimeHeaders.viaHandle],
|
|
689
|
+
};
|
|
690
|
+
// If the package data for the node exists, log immediately. Otherwise, queue it and it will be logged the
|
|
691
|
+
// next time GC runs as the package data should be available then.
|
|
692
|
+
const pkg = packagePath !== null && packagePath !== void 0 ? packagePath : this.getNodePackagePath(nodeId);
|
|
693
|
+
if (pkg !== undefined) {
|
|
694
|
+
this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, event), { pkg: { value: `/${pkg.join("/")}`, tag: telemetry_utils_1.TelemetryDataTag.PackageData } }));
|
|
695
|
+
}
|
|
696
|
+
else {
|
|
697
|
+
this.pendingEventsQueue.push(event);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
553
701
|
}
|
|
554
702
|
exports.GarbageCollector = GarbageCollector;
|
|
555
703
|
/**
|
|
556
704
|
* Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
|
|
557
705
|
* Merge the GC state from all such blobs and return the merged GC state.
|
|
558
|
-
*/
|
|
706
|
+
*/
|
|
559
707
|
async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
|
|
560
708
|
let rootGCState = { gcNodes: {} };
|
|
561
709
|
for (const key of Object.keys(gcSnapshotTree.blobs)) {
|
|
@@ -574,4 +722,31 @@ async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
|
|
|
574
722
|
}
|
|
575
723
|
return rootGCState;
|
|
576
724
|
}
|
|
725
|
+
/**
|
|
726
|
+
* setLongTimeout is used for timeouts longer than setTimeout's ~24.8 day max
|
|
727
|
+
* @param timeoutMs - the total time the timeout needs to last in ms
|
|
728
|
+
* @param timeoutFn - the function to execute when the timer ends
|
|
729
|
+
* @param setTimerFn - the function used to update your timer variable
|
|
730
|
+
*/
|
|
731
|
+
function setLongTimeout(timeoutMs, timeoutFn, setTimerFn) {
|
|
732
|
+
// The setTimeout max is 24.8 days before looping occurs.
|
|
733
|
+
const maxTimeout = 2147483647;
|
|
734
|
+
let timer;
|
|
735
|
+
if (timeoutMs > maxTimeout) {
|
|
736
|
+
const newTimeoutMs = timeoutMs - maxTimeout;
|
|
737
|
+
timer = setTimeout(() => setLongTimeout(newTimeoutMs, timeoutFn, setTimerFn), maxTimeout);
|
|
738
|
+
}
|
|
739
|
+
else {
|
|
740
|
+
timer = setTimeout(() => timeoutFn(), timeoutMs);
|
|
741
|
+
}
|
|
742
|
+
setTimerFn(timer);
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* Given a GC nodeId, tells whether it belongs to a data store or not.
|
|
746
|
+
*/
|
|
747
|
+
function isDataStoreNode(nodeId) {
|
|
748
|
+
const pathParts = nodeId.split("/");
|
|
749
|
+
// Data store ids are in the format "/dataStoreId".
|
|
750
|
+
return pathParts.length === 2 && pathParts[1] !== "" ? true : false;
|
|
751
|
+
}
|
|
577
752
|
//# sourceMappingURL=garbageCollection.js.map
|