@fluidframework/container-runtime 1.2.7 → 2.0.0-dev.1.3.0.96595
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mocharc.js +12 -0
- package/dist/batchManager.d.ts +37 -0
- package/dist/batchManager.d.ts.map +1 -0
- package/dist/batchManager.js +73 -0
- package/dist/batchManager.js.map +1 -0
- package/dist/batchTracker.d.ts +1 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +2 -3
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +87 -25
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +317 -99
- package/dist/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +109 -124
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +349 -542
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.js +29 -24
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +20 -14
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +49 -58
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +12 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +21 -20
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaScheduler.d.ts +6 -4
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +6 -4
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/garbageCollection.d.ts +74 -14
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +249 -170
- package/dist/garbageCollection.js.map +1 -1
- package/dist/gcSweepReadyUsageDetection.d.ts +53 -0
- package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/dist/gcSweepReadyUsageDetection.js +126 -0
- package/dist/gcSweepReadyUsageDetection.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/opProperties.d.ts +7 -0
- package/dist/opProperties.d.ts.map +1 -0
- package/dist/opProperties.js +20 -0
- package/dist/opProperties.js.map +1 -0
- package/dist/orderedClientElection.d.ts +28 -10
- package/dist/orderedClientElection.d.ts.map +1 -1
- package/dist/orderedClientElection.js +14 -4
- package/dist/orderedClientElection.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +0 -11
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +24 -46
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/runningSummarizer.d.ts +14 -4
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +68 -26
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/scheduleManager.d.ts +31 -0
- package/dist/scheduleManager.d.ts.map +1 -0
- package/dist/scheduleManager.js +243 -0
- package/dist/scheduleManager.js.map +1 -0
- package/dist/summarizer.d.ts +0 -2
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +1 -12
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts +26 -4
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +95 -18
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +45 -18
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryCollection.d.ts +1 -0
- package/dist/summaryCollection.d.ts.map +1 -1
- package/dist/summaryCollection.js +31 -15
- package/dist/summaryCollection.js.map +1 -1
- package/dist/summaryFormat.d.ts +0 -5
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts +1 -0
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +11 -9
- package/dist/summaryGenerator.js.map +1 -1
- package/dist/summaryManager.d.ts +2 -2
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +22 -7
- package/dist/summaryManager.js.map +1 -1
- package/lib/batchManager.d.ts +37 -0
- package/lib/batchManager.d.ts.map +1 -0
- package/lib/batchManager.js +69 -0
- package/lib/batchManager.js.map +1 -0
- package/lib/batchTracker.d.ts +1 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +2 -3
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +87 -25
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +319 -101
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +109 -124
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +355 -547
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.js +29 -24
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +20 -14
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +46 -55
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +12 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +21 -20
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaScheduler.d.ts +6 -4
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +6 -4
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/garbageCollection.d.ts +74 -14
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +238 -160
- package/lib/garbageCollection.js.map +1 -1
- package/lib/gcSweepReadyUsageDetection.d.ts +53 -0
- package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/lib/gcSweepReadyUsageDetection.js +121 -0
- package/lib/gcSweepReadyUsageDetection.js.map +1 -0
- package/lib/index.d.ts +2 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/opProperties.d.ts +7 -0
- package/lib/opProperties.d.ts.map +1 -0
- package/lib/opProperties.js +16 -0
- package/lib/opProperties.js.map +1 -0
- package/lib/orderedClientElection.d.ts +28 -10
- package/lib/orderedClientElection.d.ts.map +1 -1
- package/lib/orderedClientElection.js +14 -4
- package/lib/orderedClientElection.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +0 -11
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +24 -46
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/runningSummarizer.d.ts +14 -4
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +68 -26
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/scheduleManager.d.ts +31 -0
- package/lib/scheduleManager.d.ts.map +1 -0
- package/lib/scheduleManager.js +239 -0
- package/lib/scheduleManager.js.map +1 -0
- package/lib/summarizer.d.ts +0 -2
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +1 -12
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts +26 -4
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +95 -18
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +45 -18
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryCollection.d.ts +1 -0
- package/lib/summaryCollection.d.ts.map +1 -1
- package/lib/summaryCollection.js +31 -15
- package/lib/summaryCollection.js.map +1 -1
- package/lib/summaryFormat.d.ts +0 -5
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts +1 -0
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +11 -9
- package/lib/summaryGenerator.js.map +1 -1
- package/lib/summaryManager.d.ts +2 -2
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +22 -7
- package/lib/summaryManager.js.map +1 -1
- package/package.json +65 -24
- package/src/batchManager.ts +91 -0
- package/src/batchTracker.ts +2 -3
- package/src/blobManager.ts +385 -118
- package/src/containerRuntime.ts +529 -740
- package/src/dataStore.ts +49 -37
- package/src/dataStoreContext.ts +44 -56
- package/src/dataStores.ts +34 -30
- package/src/deltaScheduler.ts +6 -4
- package/src/garbageCollection.ts +297 -206
- package/src/gcSweepReadyUsageDetection.ts +139 -0
- package/src/index.ts +1 -2
- package/src/opProperties.ts +19 -0
- package/src/orderedClientElection.ts +31 -10
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +27 -59
- package/src/runningSummarizer.ts +75 -22
- package/src/scheduleManager.ts +314 -0
- package/src/summarizer.ts +1 -18
- package/src/summarizerHeuristics.ts +133 -19
- package/src/summarizerTypes.ts +53 -18
- package/src/summaryCollection.ts +33 -18
- package/src/summaryFormat.ts +0 -6
- package/src/summaryGenerator.ts +40 -22
- package/src/summaryManager.ts +22 -7
- package/dist/opTelemetry.d.ts +0 -22
- package/dist/opTelemetry.d.ts.map +0 -1
- package/dist/opTelemetry.js +0 -59
- package/dist/opTelemetry.js.map +0 -1
- package/lib/opTelemetry.d.ts +0 -22
- package/lib/opTelemetry.d.ts.map +0 -1
- package/lib/opTelemetry.js +0 -55
- package/lib/opTelemetry.js.map +0 -1
- package/src/opTelemetry.ts +0 -71
package/src/garbageCollection.ts
CHANGED
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
|
|
43
43
|
import { IGCRuntimeOptions, RuntimeHeaders } from "./containerRuntime";
|
|
44
44
|
import { getSummaryForDatastores } from "./dataStores";
|
|
45
|
+
import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
|
|
45
46
|
import {
|
|
46
47
|
getGCVersion,
|
|
47
48
|
GCVersion,
|
|
@@ -61,26 +62,26 @@ export const gcTreeKey = "gc";
|
|
|
61
62
|
export const gcBlobPrefix = "__gc";
|
|
62
63
|
|
|
63
64
|
// Feature gate key to turn GC on / off.
|
|
64
|
-
const runGCKey = "Fluid.GarbageCollection.RunGC";
|
|
65
|
+
export const runGCKey = "Fluid.GarbageCollection.RunGC";
|
|
65
66
|
// Feature gate key to turn GC sweep on / off.
|
|
66
|
-
const runSweepKey = "Fluid.GarbageCollection.RunSweep";
|
|
67
|
+
export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
|
|
67
68
|
// Feature gate key to turn GC test mode on / off.
|
|
68
|
-
const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
69
|
+
export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
69
70
|
// Feature gate key to write GC data at the root of the summary tree.
|
|
70
71
|
const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
|
|
71
72
|
// Feature gate key to expire a session after a set period of time.
|
|
72
73
|
export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
|
|
73
|
-
// Feature gate key to disable expiring session after a set period of time, even if expiry value is present
|
|
74
|
+
// Feature gate key to disable expiring session after a set period of time, even if expiry value is present.
|
|
74
75
|
export const disableSessionExpiryKey = "Fluid.GarbageCollection.DisableSessionExpiry";
|
|
75
76
|
// Feature gate key to write the gc blob as a handle if the data is the same.
|
|
76
77
|
export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
|
|
77
78
|
// Feature gate key to turn GC sweep log off.
|
|
78
|
-
const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
|
|
79
|
+
export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
|
|
79
80
|
|
|
80
81
|
// One day in milliseconds.
|
|
81
82
|
export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
|
|
82
83
|
|
|
83
|
-
const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
|
|
84
|
+
export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
|
|
84
85
|
export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days
|
|
85
86
|
|
|
86
87
|
/** The statistics of the system state after a garbage collection run. */
|
|
@@ -133,7 +134,7 @@ export interface IGarbageCollectionRuntime {
|
|
|
133
134
|
/** Returns the type of the GC node. */
|
|
134
135
|
getNodeType(nodePath: string): GCNodeType;
|
|
135
136
|
/** Called when the runtime should close because of an error. */
|
|
136
|
-
closeFn(error?: ICriticalContainerError)
|
|
137
|
+
closeFn: (error?: ICriticalContainerError) => void;
|
|
137
138
|
}
|
|
138
139
|
|
|
139
140
|
/** Defines the contract for the garbage collector. */
|
|
@@ -148,7 +149,7 @@ export interface IGarbageCollector {
|
|
|
148
149
|
/** Run garbage collection and update the reference / used state of the system. */
|
|
149
150
|
collectGarbage(
|
|
150
151
|
options: { logger?: ITelemetryLogger; runSweep?: boolean; fullGC?: boolean; },
|
|
151
|
-
): Promise<IGCStats>;
|
|
152
|
+
): Promise<IGCStats | undefined>;
|
|
152
153
|
/** Summarizes the GC data and returns it as a summary tree. */
|
|
153
154
|
summarize(
|
|
154
155
|
fullTree: boolean,
|
|
@@ -171,6 +172,7 @@ export interface IGarbageCollector {
|
|
|
171
172
|
): void;
|
|
172
173
|
/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
|
|
173
174
|
addedOutboundReference(fromNodePath: string, toNodePath: string): void;
|
|
175
|
+
setConnectionState(connected: boolean, clientId?: string): void;
|
|
174
176
|
dispose(): void;
|
|
175
177
|
}
|
|
176
178
|
|
|
@@ -186,18 +188,20 @@ export interface IGarbageCollectorCreateParams {
|
|
|
186
188
|
readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
|
|
187
189
|
readonly getLastSummaryTimestampMs: () => number | undefined;
|
|
188
190
|
readonly readAndParseBlob: ReadAndParseBlob;
|
|
191
|
+
readonly activeConnection: () => boolean;
|
|
192
|
+
readonly getContainerDiagnosticId: () => string;
|
|
189
193
|
readonly snapshotCacheExpiryMs?: number;
|
|
190
194
|
}
|
|
191
195
|
|
|
192
196
|
/** The state of node that is unreferenced. */
|
|
193
|
-
const UnreferencedState = {
|
|
197
|
+
export const UnreferencedState = {
|
|
194
198
|
/** The node is active, i.e., it can become referenced again. */
|
|
195
199
|
Active: "Active",
|
|
196
200
|
/** The node is inactive, i.e., it should not become referenced. */
|
|
197
201
|
Inactive: "Inactive",
|
|
198
202
|
/** The node is ready to be deleted by the sweep phase. */
|
|
199
203
|
SweepReady: "SweepReady",
|
|
200
|
-
};
|
|
204
|
+
} as const;
|
|
201
205
|
export type UnreferencedState = typeof UnreferencedState[keyof typeof UnreferencedState];
|
|
202
206
|
|
|
203
207
|
/** The event that is logged when unreferenced node is used after a certain time. */
|
|
@@ -220,29 +224,47 @@ interface IUnreferencedEventProps {
|
|
|
220
224
|
* Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
|
|
221
225
|
* be deleted by the sweep phase.
|
|
222
226
|
*/
|
|
223
|
-
class UnreferencedStateTracker {
|
|
227
|
+
export class UnreferencedStateTracker {
|
|
224
228
|
private _state: UnreferencedState = UnreferencedState.Active;
|
|
225
229
|
public get state(): UnreferencedState {
|
|
226
230
|
return this._state;
|
|
227
231
|
}
|
|
228
232
|
|
|
229
|
-
|
|
230
|
-
private
|
|
233
|
+
/** Timer to indicate when an unreferenced object is considered Inactive */
|
|
234
|
+
private readonly inactiveTimer: TimerWithNoDefaultTimeout;
|
|
235
|
+
/** Timer to indicate when an unreferenced object is Sweep-Ready */
|
|
236
|
+
private readonly sweepTimer: TimerWithNoDefaultTimeout;
|
|
231
237
|
|
|
232
238
|
constructor(
|
|
233
239
|
public readonly unreferencedTimestampMs: number,
|
|
234
240
|
/** The time after which node transitions to Inactive state. */
|
|
235
241
|
private readonly inactiveTimeoutMs: number,
|
|
242
|
+
/** The current reference timestamp used to track how long this node has been unreferenced for. */
|
|
243
|
+
currentReferenceTimestampMs: number,
|
|
236
244
|
/** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
|
|
237
|
-
private readonly sweepTimeoutMs
|
|
238
|
-
/** The current reference timestamp; undefined if no ops have ever been processed which can happen in tests. */
|
|
239
|
-
currentReferenceTimestampMs?: number,
|
|
245
|
+
private readonly sweepTimeoutMs: number | undefined,
|
|
240
246
|
) {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
this.updateTracking(currentReferenceTimestampMs);
|
|
247
|
+
if (this.sweepTimeoutMs !== undefined) {
|
|
248
|
+
assert(this.inactiveTimeoutMs <= this.sweepTimeoutMs,
|
|
249
|
+
0x3b0 /* inactive timeout must not be greater than the sweep timeout */);
|
|
245
250
|
}
|
|
251
|
+
|
|
252
|
+
this.sweepTimer = new TimerWithNoDefaultTimeout(
|
|
253
|
+
() => {
|
|
254
|
+
this._state = UnreferencedState.SweepReady;
|
|
255
|
+
assert(!this.inactiveTimer.hasTimer, 0x3b1 /* inactiveTimer still running after sweepTimer fired! */);
|
|
256
|
+
},
|
|
257
|
+
);
|
|
258
|
+
|
|
259
|
+
this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
|
|
260
|
+
this._state = UnreferencedState.Inactive;
|
|
261
|
+
|
|
262
|
+
// After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
|
|
263
|
+
if (this.sweepTimeoutMs !== undefined) {
|
|
264
|
+
this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
this.updateTracking(currentReferenceTimestampMs);
|
|
246
268
|
}
|
|
247
269
|
|
|
248
270
|
/* Updates the unreferenced state based on the provided timestamp. */
|
|
@@ -260,42 +282,21 @@ class UnreferencedStateTracker {
|
|
|
260
282
|
// Also, start a timer for the sweep timeout.
|
|
261
283
|
if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
|
|
262
284
|
this._state = UnreferencedState.Inactive;
|
|
263
|
-
this.
|
|
285
|
+
this.inactiveTimer.clear();
|
|
264
286
|
|
|
265
287
|
if (this.sweepTimeoutMs !== undefined) {
|
|
266
|
-
|
|
267
|
-
this.sweepTimeoutMs - unreferencedDurationMs,
|
|
268
|
-
() => { this._state = UnreferencedState.SweepReady; },
|
|
269
|
-
(timer) => { this.sweepTimer = timer; },
|
|
270
|
-
);
|
|
288
|
+
this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
|
|
271
289
|
}
|
|
272
290
|
return;
|
|
273
291
|
}
|
|
274
292
|
|
|
275
|
-
// The node is still active.
|
|
276
|
-
|
|
277
|
-
if (this.inactiveTimer === undefined) {
|
|
278
|
-
const inactiveTimeoutHandler = () => {
|
|
279
|
-
this._state = UnreferencedState.Inactive;
|
|
280
|
-
// After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
|
|
281
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
282
|
-
setLongTimeout(
|
|
283
|
-
this.sweepTimeoutMs - this.inactiveTimeoutMs,
|
|
284
|
-
() => { this._state = UnreferencedState.SweepReady; },
|
|
285
|
-
(timer) => { this.sweepTimer = timer; },
|
|
286
|
-
);
|
|
287
|
-
}
|
|
288
|
-
};
|
|
289
|
-
this.inactiveTimer = new Timer(remainingDurationMs, () => inactiveTimeoutHandler());
|
|
290
|
-
}
|
|
291
|
-
this.inactiveTimer.restart(remainingDurationMs);
|
|
293
|
+
// The node is still active. Ensure the inactive timer is running with the proper remaining duration.
|
|
294
|
+
this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
|
|
292
295
|
}
|
|
293
296
|
|
|
294
297
|
private clearTimers() {
|
|
295
|
-
this.inactiveTimer
|
|
296
|
-
|
|
297
|
-
clearTimeout(this.sweepTimer);
|
|
298
|
-
}
|
|
298
|
+
this.inactiveTimer.clear();
|
|
299
|
+
this.sweepTimer.clear();
|
|
299
300
|
}
|
|
300
301
|
|
|
301
302
|
/** Stop tracking this node. Reset the unreferenced timers and state, if any. */
|
|
@@ -310,8 +311,12 @@ class UnreferencedStateTracker {
|
|
|
310
311
|
* its state across summaries.
|
|
311
312
|
*
|
|
312
313
|
* Node - represented as nodeId, it's a node on the GC graph
|
|
314
|
+
*
|
|
313
315
|
* Outbound Route - a path from one node to another node, think `nodeA` -\> `nodeB`
|
|
316
|
+
*
|
|
314
317
|
* Graph - all nodes with their respective routes
|
|
318
|
+
*
|
|
319
|
+
* ```
|
|
315
320
|
* GC Graph
|
|
316
321
|
*
|
|
317
322
|
* Node
|
|
@@ -321,6 +326,7 @@ class UnreferencedStateTracker {
|
|
|
321
326
|
* / \\
|
|
322
327
|
* Node Node
|
|
323
328
|
* NodeId = "dds1" NodeId = "dds2"
|
|
329
|
+
* ```
|
|
324
330
|
*/
|
|
325
331
|
export class GarbageCollector implements IGarbageCollector {
|
|
326
332
|
public static create(createParams: IGarbageCollectorCreateParams): IGarbageCollector {
|
|
@@ -329,11 +335,16 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
329
335
|
|
|
330
336
|
/**
|
|
331
337
|
* Tells whether the GC state needs to be reset in the next summary. We need to do this if:
|
|
338
|
+
*
|
|
332
339
|
* 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
|
|
340
|
+
*
|
|
333
341
|
* 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
|
|
342
|
+
*
|
|
334
343
|
* 3. The GC version in the latest summary is different from the current GC version. This can happen if:
|
|
335
|
-
*
|
|
336
|
-
*
|
|
344
|
+
*
|
|
345
|
+
* 3.1. The summary this client loaded with has data from a different GC version.
|
|
346
|
+
*
|
|
347
|
+
* 3.2. This client's latest summary was updated from a snapshot that has a different GC version.
|
|
337
348
|
*/
|
|
338
349
|
public get summaryStateNeedsReset(): boolean {
|
|
339
350
|
return this.initialStateNeedsReset ||
|
|
@@ -377,10 +388,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
377
388
|
|
|
378
389
|
/**
|
|
379
390
|
* Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
|
|
391
|
+
*
|
|
380
392
|
* 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
|
|
381
|
-
*
|
|
393
|
+
* it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
|
|
394
|
+
*
|
|
382
395
|
* 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
|
|
383
|
-
*
|
|
396
|
+
* a document and the first time GC is enabled after is was disabled before.
|
|
384
397
|
*
|
|
385
398
|
* Note that the state needs reset only for the very first time summary is generated by this client. After that, the
|
|
386
399
|
* state will be up-to-date and this flag will be reset.
|
|
@@ -412,8 +425,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
412
425
|
private readonly baseGCDetailsP: Promise<Map<string, IGarbageCollectionDetailsBase>>;
|
|
413
426
|
// Map of node ids to their unreferenced state tracker.
|
|
414
427
|
private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();
|
|
415
|
-
// The
|
|
416
|
-
private sessionExpiryTimer
|
|
428
|
+
// The Timer responsible for closing the container when the session has expired
|
|
429
|
+
private sessionExpiryTimer: Timer | undefined;
|
|
417
430
|
|
|
418
431
|
// Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
|
|
419
432
|
// per event per node.
|
|
@@ -439,6 +452,27 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
439
452
|
private readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
|
|
440
453
|
/** Returns the timestamp of the last summary generated for this container. */
|
|
441
454
|
private readonly getLastSummaryTimestampMs: () => number | undefined;
|
|
455
|
+
/** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
|
|
456
|
+
private readonly activeConnection: () => boolean;
|
|
457
|
+
|
|
458
|
+
/** Returns a list of all the configurations for garbage collection. */
|
|
459
|
+
private get configs() {
|
|
460
|
+
return {
|
|
461
|
+
gcEnabled: this.gcEnabled,
|
|
462
|
+
sweepEnabled: this.sweepEnabled,
|
|
463
|
+
runGC: this.shouldRunGC,
|
|
464
|
+
runSweep: this.shouldRunSweep,
|
|
465
|
+
writeAtRoot: this._writeDataAtRoot,
|
|
466
|
+
testMode: this.testMode,
|
|
467
|
+
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
468
|
+
inactiveTimeout: this.inactiveTimeoutMs,
|
|
469
|
+
trackGCState: this.trackGCState,
|
|
470
|
+
...this.gcOptions,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** Handler to respond to when a SweepReady object is used */
|
|
475
|
+
private readonly sweepReadyUsageHandler: SweepReadyUsageDetectionHandler;
|
|
442
476
|
|
|
443
477
|
protected constructor(createParams: IGarbageCollectorCreateParams) {
|
|
444
478
|
this.runtime = createParams.runtime;
|
|
@@ -446,6 +480,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
446
480
|
this.gcOptions = createParams.gcOptions;
|
|
447
481
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
448
482
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
483
|
+
this.activeConnection = createParams.activeConnection;
|
|
449
484
|
|
|
450
485
|
const baseSnapshot = createParams.baseSnapshot;
|
|
451
486
|
const metadata = createParams.metadata;
|
|
@@ -455,6 +490,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
455
490
|
createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } },
|
|
456
491
|
));
|
|
457
492
|
|
|
493
|
+
this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(
|
|
494
|
+
createParams.getContainerDiagnosticId(),
|
|
495
|
+
this.mc,
|
|
496
|
+
this.runtime.closeFn,
|
|
497
|
+
);
|
|
498
|
+
|
|
458
499
|
let prevSummaryGCVersion: number | undefined;
|
|
459
500
|
|
|
460
501
|
/**
|
|
@@ -484,9 +525,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
484
525
|
// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
|
|
485
526
|
this.sweepEnabled = this.gcOptions.sweepAllowed === true;
|
|
486
527
|
|
|
487
|
-
// Set the Session Expiry only if the flag is enabled
|
|
528
|
+
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
488
529
|
if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
|
|
489
|
-
this.sessionExpiryTimeoutMs = defaultSessionExpiryDurationMs;
|
|
530
|
+
this.sessionExpiryTimeoutMs = this.gcOptions.sessionExpiryTimeoutMs ?? defaultSessionExpiryDurationMs;
|
|
490
531
|
}
|
|
491
532
|
}
|
|
492
533
|
|
|
@@ -497,11 +538,16 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
497
538
|
this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
|
|
498
539
|
const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.sessionExpiryTimeoutMs;
|
|
499
540
|
|
|
500
|
-
|
|
541
|
+
this.sessionExpiryTimer = new Timer(
|
|
501
542
|
timeoutMs,
|
|
502
543
|
() => { this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs)); },
|
|
503
|
-
(timer) => { this.sessionExpiryTimer = timer; },
|
|
504
544
|
);
|
|
545
|
+
this.sessionExpiryTimer.start();
|
|
546
|
+
|
|
547
|
+
// TEMPORARY: Hardcode a default of 2 days which is the value used in the ODSP driver.
|
|
548
|
+
// This unblocks the Sweep Log (see logSweepEvents function).
|
|
549
|
+
// This will be removed before sweep is fully implemented.
|
|
550
|
+
const snapshotCacheExpiryMs = createParams.snapshotCacheExpiryMs ?? 2 * 24 * 60 * 60 * 1000;
|
|
505
551
|
|
|
506
552
|
/**
|
|
507
553
|
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
@@ -509,8 +555,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
509
555
|
* added to account for any clock skew. We use server timestamps throughout so the skew should be minimal
|
|
510
556
|
* but make it one day to be safe.
|
|
511
557
|
*/
|
|
512
|
-
if (
|
|
513
|
-
this.sweepTimeoutMs = this.sessionExpiryTimeoutMs +
|
|
558
|
+
if (snapshotCacheExpiryMs !== undefined) {
|
|
559
|
+
this.sweepTimeoutMs = this.sessionExpiryTimeoutMs + snapshotCacheExpiryMs + oneDayMs;
|
|
514
560
|
}
|
|
515
561
|
}
|
|
516
562
|
|
|
@@ -520,8 +566,11 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
520
566
|
|
|
521
567
|
/**
|
|
522
568
|
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
569
|
+
*
|
|
523
570
|
* 1. GC should be enabled for this container.
|
|
571
|
+
*
|
|
524
572
|
* 2. GC should not be disabled via disableGC GC option.
|
|
573
|
+
*
|
|
525
574
|
* These conditions can be overridden via runGCKey feature flag.
|
|
526
575
|
*/
|
|
527
576
|
this.shouldRunGC = this.mc.config.getBoolean(runGCKey) ?? (
|
|
@@ -533,14 +582,18 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
533
582
|
|
|
534
583
|
/**
|
|
535
584
|
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
585
|
+
*
|
|
536
586
|
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
587
|
+
*
|
|
537
588
|
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
589
|
+
*
|
|
538
590
|
* 3. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
539
|
-
*
|
|
591
|
+
* feature flag.
|
|
540
592
|
*/
|
|
541
|
-
this.shouldRunSweep =
|
|
542
|
-
|
|
543
|
-
&&
|
|
593
|
+
this.shouldRunSweep = false; // disable while TEMPORARY measure hardcoding snapshotCacheExpiryMs is here
|
|
594
|
+
// this.shouldRunGC
|
|
595
|
+
// && this.sweepTimeoutMs !== undefined
|
|
596
|
+
// && (this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
|
|
544
597
|
|
|
545
598
|
this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
|
|
546
599
|
|
|
@@ -551,7 +604,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
551
604
|
|
|
552
605
|
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
553
606
|
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
554
|
-
throw new UsageError("inactive timeout should not be
|
|
607
|
+
throw new UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
555
608
|
}
|
|
556
609
|
|
|
557
610
|
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
@@ -574,64 +627,72 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
574
627
|
return undefined;
|
|
575
628
|
}
|
|
576
629
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
this.
|
|
630
|
+
try {
|
|
631
|
+
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
632
|
+
const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
|
|
633
|
+
if (gcSnapshotTree !== undefined) {
|
|
634
|
+
// If the GC tree is written at root, we should also do the same.
|
|
635
|
+
this._writeDataAtRoot = true;
|
|
636
|
+
const baseGCState = await getGCStateFromSnapshot(
|
|
637
|
+
gcSnapshotTree,
|
|
638
|
+
readAndParseBlob,
|
|
639
|
+
);
|
|
640
|
+
if (this.trackGCState) {
|
|
641
|
+
this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(baseGCState));
|
|
642
|
+
}
|
|
643
|
+
return baseGCState;
|
|
588
644
|
}
|
|
589
|
-
return baseGCState;
|
|
590
|
-
}
|
|
591
645
|
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
646
|
+
// back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
|
|
647
|
+
// consolidate into IGarbageCollectionState format.
|
|
648
|
+
// Add a node for the root node that is not present in older snapshot format.
|
|
649
|
+
const gcState: IGarbageCollectionState = { gcNodes: { "/": { outboundRoutes: [] } } };
|
|
650
|
+
const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
|
|
651
|
+
assert(dataStoreSnapshotTree !== undefined,
|
|
652
|
+
0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
|
|
653
|
+
for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
|
|
654
|
+
const blobId = dsSnapshotTree.blobs[gcBlobKey];
|
|
655
|
+
if (blobId === undefined) {
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
604
658
|
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
659
|
+
const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionDetailsBase>(blobId);
|
|
660
|
+
// If there are no nodes for this data store, skip it.
|
|
661
|
+
if (gcSummaryDetails.gcData?.gcNodes === undefined) {
|
|
662
|
+
continue;
|
|
663
|
+
}
|
|
610
664
|
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
665
|
+
const dsRootId = `/${dsId}`;
|
|
666
|
+
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
667
|
+
// Construct that entry by adding root data store ids to its outbound routes.
|
|
668
|
+
const initialSnapshotDetails = await readAndParseBlob<ReadFluidDataStoreAttributes>(
|
|
669
|
+
dsSnapshotTree.blobs[dataStoreAttributesBlobName],
|
|
670
|
+
);
|
|
671
|
+
if (initialSnapshotDetails.isRootDataStore) {
|
|
672
|
+
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
673
|
+
}
|
|
620
674
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
675
|
+
for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
|
|
676
|
+
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
677
|
+
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
678
|
+
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
679
|
+
gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
|
|
680
|
+
}
|
|
681
|
+
assert(gcState.gcNodes[dsRootId] !== undefined,
|
|
682
|
+
0x2a9 /* GC nodes for data store not in GC blob */);
|
|
683
|
+
gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
|
|
626
684
|
}
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
gcState.gcNodes
|
|
685
|
+
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
686
|
+
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
687
|
+
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
688
|
+
} catch (error) {
|
|
689
|
+
const dpe = DataProcessingError.wrapIfUnrecognized(
|
|
690
|
+
error,
|
|
691
|
+
"FailedToInitializeGC",
|
|
692
|
+
);
|
|
693
|
+
dpe.addTelemetryProperties({ gcConfigs: JSON.stringify(this.configs) });
|
|
694
|
+
throw dpe;
|
|
630
695
|
}
|
|
631
|
-
|
|
632
|
-
// If there is only one node (root node just added above), either GC is disabled or we are loading from the
|
|
633
|
-
// very first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
634
|
-
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
635
696
|
});
|
|
636
697
|
|
|
637
698
|
/**
|
|
@@ -641,7 +702,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
641
702
|
*/
|
|
642
703
|
this.initializeBaseStateP = new LazyPromise<void>(async () => {
|
|
643
704
|
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
705
|
+
/**
|
|
706
|
+
* If there is no current reference timestamp, skip initialization. We need the current timestamp to track
|
|
707
|
+
* how long objects have been unreferenced and if they can be deleted.
|
|
708
|
+
*
|
|
709
|
+
* Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
|
|
710
|
+
* for this container and it is in read mode. In this scenario, there is no point in running GC anyway
|
|
711
|
+
* because references in the container do not change without any ops, i.e., there is nothing to collect.
|
|
712
|
+
*/
|
|
713
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
714
|
+
// Log an event so we can evaluate how often we run into this scenario.
|
|
715
|
+
this.mc.logger.sendErrorEvent({
|
|
716
|
+
eventName: "GarbageCollectorInitializedWithoutTimestamp",
|
|
717
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
718
|
+
});
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
|
|
644
722
|
const baseState = await baseSummaryStateP;
|
|
723
|
+
/**
|
|
724
|
+
* The base state will not be present if the container is loaded from:
|
|
725
|
+
* 1. The first summary created by the detached container.
|
|
726
|
+
* 2. A summary that was generated with GC disabled.
|
|
727
|
+
* 3. A summary that was generated before GC even existed.
|
|
728
|
+
*/
|
|
645
729
|
if (baseState === undefined) {
|
|
646
730
|
return;
|
|
647
731
|
}
|
|
@@ -654,8 +738,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
654
738
|
new UnreferencedStateTracker(
|
|
655
739
|
nodeData.unreferencedTimestampMs,
|
|
656
740
|
this.inactiveTimeoutMs,
|
|
657
|
-
this.sweepTimeoutMs,
|
|
658
741
|
currentReferenceTimestampMs,
|
|
742
|
+
this.sweepTimeoutMs,
|
|
659
743
|
),
|
|
660
744
|
);
|
|
661
745
|
}
|
|
@@ -682,7 +766,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
682
766
|
const usedRoutes = runGarbageCollection(gcNodes, ["/"]).referencedNodeIds;
|
|
683
767
|
|
|
684
768
|
const baseGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
|
|
685
|
-
// Currently, the nodes may write the GC data. So, we need to update
|
|
769
|
+
// Currently, the nodes may write the GC data. So, we need to update its base GC details with the
|
|
686
770
|
// unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
|
|
687
771
|
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
688
772
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
@@ -697,42 +781,38 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
697
781
|
|
|
698
782
|
// Log all the GC options and the state determined by the garbage collector. This is interesting only for the
|
|
699
783
|
// summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
|
|
700
|
-
const gcConfigProps = JSON.stringify({
|
|
701
|
-
gcEnabled: this.gcEnabled,
|
|
702
|
-
sweepEnabled: this.sweepEnabled,
|
|
703
|
-
runGC: this.shouldRunGC,
|
|
704
|
-
runSweep: this.shouldRunSweep,
|
|
705
|
-
writeAtRoot: this._writeDataAtRoot,
|
|
706
|
-
testMode: this.testMode,
|
|
707
|
-
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
708
|
-
inactiveTimeout: this.inactiveTimeoutMs,
|
|
709
|
-
existing: createParams.existing,
|
|
710
|
-
trackGCState: this.trackGCState,
|
|
711
|
-
...this.gcOptions,
|
|
712
|
-
});
|
|
713
784
|
if (this.isSummarizerClient) {
|
|
714
785
|
this.mc.logger.sendTelemetryEvent({
|
|
715
786
|
eventName: "GarbageCollectorLoaded",
|
|
716
|
-
gcConfigs:
|
|
787
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
717
788
|
});
|
|
718
789
|
}
|
|
790
|
+
}
|
|
719
791
|
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
792
|
+
/**
|
|
793
|
+
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
794
|
+
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
|
|
795
|
+
* @param connected - Whether the runtime connected / disconnected.
|
|
796
|
+
* @param clientId - The clientId of this runtime.
|
|
797
|
+
*/
|
|
798
|
+
public setConnectionState(connected: boolean, clientId?: string | undefined): void {
|
|
799
|
+
/**
|
|
800
|
+
* For non-summarizer clients, initialize the base state when the container becomes active, i.e., it transitions
|
|
801
|
+
* to "write" mode. This will ensure that the container's own join op is processed and there is a recent
|
|
802
|
+
* reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
|
|
803
|
+
* could affect the GC state will have been processed.
|
|
804
|
+
*
|
|
805
|
+
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
806
|
+
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
807
|
+
*/
|
|
808
|
+
if (this.activeConnection() && !this.isSummarizerClient && this.shouldRunGC) {
|
|
809
|
+
this.initializeBaseStateP.catch((error) => {});
|
|
730
810
|
}
|
|
731
811
|
}
|
|
732
812
|
|
|
733
813
|
/**
|
|
734
814
|
* Runs garbage collection and updates the reference / used state of the nodes in the container.
|
|
735
|
-
* @returns the
|
|
815
|
+
* @returns stats of the GC run or undefined if GC did not run.
|
|
736
816
|
*/
|
|
737
817
|
public async collectGarbage(
|
|
738
818
|
options: {
|
|
@@ -743,15 +823,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
743
823
|
/** True to generate full GC data */
|
|
744
824
|
fullGC?: boolean;
|
|
745
825
|
},
|
|
746
|
-
): Promise<IGCStats> {
|
|
747
|
-
const
|
|
748
|
-
fullGC = this.gcOptions.runFullGC === true || this.summaryStateNeedsReset,
|
|
749
|
-
} = options;
|
|
750
|
-
|
|
826
|
+
): Promise<IGCStats | undefined> {
|
|
827
|
+
const fullGC = options.fullGC ?? (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
|
|
751
828
|
const logger = options.logger
|
|
752
829
|
? ChildLogger.create(options.logger, undefined, { all: { completedGCRuns: () => this.completedRuns } })
|
|
753
830
|
: this.mc.logger;
|
|
754
831
|
|
|
832
|
+
/**
|
|
833
|
+
* If there is no current reference timestamp, skip running GC. We need the current timestamp to track
|
|
834
|
+
* how long objects have been unreferenced and if they should be deleted.
|
|
835
|
+
*
|
|
836
|
+
* Note that the only scenario where GC is called and there is no reference timestamp is when no ops have ever
|
|
837
|
+
* been processed for this container and it is in read mode. In this scenario, there is no point in running GC
|
|
838
|
+
* anyway because references in the container do not change without any ops, i.e., there is nothing to collect.
|
|
839
|
+
*/
|
|
840
|
+
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
841
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
842
|
+
// Log an event so we can evaluate how often we run into this scenario.
|
|
843
|
+
logger.sendErrorEvent({
|
|
844
|
+
eventName: "CollectGarbageCalledWithoutTimestamp",
|
|
845
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
846
|
+
});
|
|
847
|
+
return undefined;
|
|
848
|
+
}
|
|
849
|
+
|
|
755
850
|
return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
756
851
|
await this.runPreGCSteps();
|
|
757
852
|
|
|
@@ -759,8 +854,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
759
854
|
const gcData = await this.runtime.getGCData(fullGC);
|
|
760
855
|
const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
|
|
761
856
|
|
|
762
|
-
const gcStats = await this.runPostGCSteps(gcData, gcResult, logger);
|
|
763
|
-
event.end({ ...gcStats });
|
|
857
|
+
const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
|
|
858
|
+
event.end({ ...gcStats, timestamp: currentReferenceTimestampMs });
|
|
764
859
|
this.completedRuns++;
|
|
765
860
|
return gcStats;
|
|
766
861
|
}, { end: true, cancel: "error" });
|
|
@@ -773,7 +868,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
773
868
|
await this.runtime.updateStateBeforeGC();
|
|
774
869
|
}
|
|
775
870
|
|
|
776
|
-
private async runPostGCSteps(
|
|
871
|
+
private async runPostGCSteps(
|
|
872
|
+
gcData: IGarbageCollectionData,
|
|
873
|
+
gcResult: IGCResult,
|
|
874
|
+
logger: ITelemetryLogger,
|
|
875
|
+
currentReferenceTimestampMs: number,
|
|
876
|
+
): Promise<IGCStats> {
|
|
777
877
|
// Generate statistics from the current run. This is done before updating the current state because it
|
|
778
878
|
// generates some of its data based on previous state of the system.
|
|
779
879
|
const gcStats = this.generateStats(gcResult);
|
|
@@ -783,7 +883,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
783
883
|
this.updateStateSinceLastRun(gcData, logger);
|
|
784
884
|
|
|
785
885
|
// Update the current state and update the runtime of all routes or ids that used as per the GC run.
|
|
786
|
-
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
787
886
|
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
788
887
|
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds, currentReferenceTimestampMs);
|
|
789
888
|
|
|
@@ -980,10 +1079,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
980
1079
|
}
|
|
981
1080
|
|
|
982
1081
|
public dispose(): void {
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
this.sessionExpiryTimer = undefined;
|
|
986
|
-
}
|
|
1082
|
+
this.sessionExpiryTimer?.clear();
|
|
1083
|
+
this.sessionExpiryTimer = undefined;
|
|
987
1084
|
}
|
|
988
1085
|
|
|
989
1086
|
/**
|
|
@@ -998,7 +1095,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
998
1095
|
private updateCurrentState(
|
|
999
1096
|
gcData: IGarbageCollectionData,
|
|
1000
1097
|
gcResult: IGCResult,
|
|
1001
|
-
currentReferenceTimestampMs
|
|
1098
|
+
currentReferenceTimestampMs: number,
|
|
1002
1099
|
) {
|
|
1003
1100
|
this.previousGCDataFromLastRun = cloneGCData(gcData);
|
|
1004
1101
|
this.newReferencesSinceLastRun.clear();
|
|
@@ -1014,15 +1111,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1014
1111
|
}
|
|
1015
1112
|
}
|
|
1016
1113
|
|
|
1017
|
-
/**
|
|
1018
|
-
* If there is no current reference time, skip tracking when a node becomes unreferenced. This would happen
|
|
1019
|
-
* if no ops have been processed ever and we still try to run GC. If so, there is nothing interesting to track
|
|
1020
|
-
* anyway.
|
|
1021
|
-
*/
|
|
1022
|
-
if (currentReferenceTimestampMs === undefined) {
|
|
1023
|
-
return;
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
1114
|
/**
|
|
1027
1115
|
* If a node became unreferenced in this run, start tracking it.
|
|
1028
1116
|
* If a node was already unreferenced, update its tracking information. Since the current reference time is
|
|
@@ -1036,8 +1124,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1036
1124
|
new UnreferencedStateTracker(
|
|
1037
1125
|
currentReferenceTimestampMs,
|
|
1038
1126
|
this.inactiveTimeoutMs,
|
|
1039
|
-
this.sweepTimeoutMs,
|
|
1040
1127
|
currentReferenceTimestampMs,
|
|
1128
|
+
this.sweepTimeoutMs,
|
|
1041
1129
|
),
|
|
1042
1130
|
);
|
|
1043
1131
|
} else {
|
|
@@ -1090,14 +1178,19 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1090
1178
|
* run, and then add the references since last run.
|
|
1091
1179
|
*
|
|
1092
1180
|
* Note on why we need to combine the data from previous run, current run and all references in between -
|
|
1181
|
+
*
|
|
1093
1182
|
* 1. We need data from last run because some of its references may have been deleted since then. If those
|
|
1094
|
-
*
|
|
1183
|
+
* references added new outbound references before getting deleted, we need to detect them.
|
|
1184
|
+
*
|
|
1095
1185
|
* 2. We need new outbound references since last run because some of them may have been deleted later. If those
|
|
1096
|
-
*
|
|
1186
|
+
* references added new outbound references before getting deleted, we need to detect them.
|
|
1187
|
+
*
|
|
1097
1188
|
* 3. We need data from the current run because currently we may not detect when DDSes are referenced:
|
|
1098
|
-
*
|
|
1099
|
-
*
|
|
1100
|
-
*
|
|
1189
|
+
*
|
|
1190
|
+
* - We don't require DDSes handles to be stored in a referenced DDS. For this, we need GC at DDS level
|
|
1191
|
+
* which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
|
|
1192
|
+
*
|
|
1193
|
+
* - A new data store may have "root" DDSes already created and we don't detect them today.
|
|
1101
1194
|
*/
|
|
1102
1195
|
const gcDataSuperSet = concatGarbageCollectionData(this.previousGCDataFromLastRun, currentGCData);
|
|
1103
1196
|
this.newReferencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
|
|
@@ -1243,10 +1336,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1243
1336
|
* For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
|
|
1244
1337
|
* this will give us a view into how much deleted content a container has.
|
|
1245
1338
|
*/
|
|
1246
|
-
private logSweepEvents(logger: ITelemetryLogger, currentReferenceTimestampMs
|
|
1247
|
-
if (this.mc.config.getBoolean(disableSweepLogKey) === true
|
|
1248
|
-
|| currentReferenceTimestampMs === undefined
|
|
1249
|
-
|| this.sweepTimeoutMs === undefined) {
|
|
1339
|
+
private logSweepEvents(logger: ITelemetryLogger, currentReferenceTimestampMs: number) {
|
|
1340
|
+
if (this.mc.config.getBoolean(disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
|
|
1250
1341
|
return;
|
|
1251
1342
|
}
|
|
1252
1343
|
|
|
@@ -1297,12 +1388,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1297
1388
|
return;
|
|
1298
1389
|
}
|
|
1299
1390
|
|
|
1300
|
-
// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
|
|
1301
|
-
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects).
|
|
1302
|
-
if (!this.isSummarizerClient && usageType !== "Loaded") {
|
|
1303
|
-
return;
|
|
1304
|
-
}
|
|
1305
|
-
|
|
1306
1391
|
// We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
|
|
1307
1392
|
// as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
|
|
1308
1393
|
const nodeType = this.runtime.getNodeType(nodeId);
|
|
@@ -1338,11 +1423,22 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1338
1423
|
if (this.isSummarizerClient) {
|
|
1339
1424
|
this.pendingEventsQueue.push({ ...propsToLog, usageType, state });
|
|
1340
1425
|
} else {
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1426
|
+
// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
|
|
1427
|
+
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
|
|
1428
|
+
if (usageType === "Loaded") {
|
|
1429
|
+
this.mc.logger.sendErrorEvent({
|
|
1430
|
+
...propsToLog,
|
|
1431
|
+
eventName: `${state}Object_${usageType}`,
|
|
1432
|
+
pkg: packagePath ? { value: packagePath.join("/"), tag: TelemetryDataTag.CodeArtifact } : undefined,
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
// If SweepReady Usage Detection is enabed, the handler may close the interactive container.
|
|
1437
|
+
// Once Sweep is fully implemented, this will be removed since the objects will be gone
|
|
1438
|
+
// and errors will arise elsewhere in the runtime
|
|
1439
|
+
if (state === UnreferencedState.SweepReady) {
|
|
1440
|
+
this.sweepReadyUsageHandler.usageDetectedInInteractiveClient({ ...propsToLog, usageType });
|
|
1441
|
+
}
|
|
1346
1442
|
}
|
|
1347
1443
|
}
|
|
1348
1444
|
|
|
@@ -1410,25 +1506,20 @@ function generateSortedGCState(gcState: IGarbageCollectionState): IGarbageCollec
|
|
|
1410
1506
|
return sortedGCState;
|
|
1411
1507
|
}
|
|
1412
1508
|
|
|
1413
|
-
/**
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
)
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
const newTimeoutMs = timeoutMs - maxTimeout;
|
|
1429
|
-
timer = setTimeout(() => setLongTimeout(newTimeoutMs, timeoutFn, setTimerFn), maxTimeout);
|
|
1430
|
-
} else {
|
|
1431
|
-
timer = setTimeout(() => timeoutFn(), timeoutMs);
|
|
1509
|
+
/** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
|
|
1510
|
+
class TimerWithNoDefaultTimeout extends Timer {
|
|
1511
|
+
constructor(
|
|
1512
|
+
private readonly callback: () => void,
|
|
1513
|
+
) {
|
|
1514
|
+
// The default timeout/handlers will never be used since start/restart pass overrides below
|
|
1515
|
+
super(0, () => { throw new Error("DefaultHandler should not be used"); });
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
start(timeoutMs: number) {
|
|
1519
|
+
super.start(timeoutMs, this.callback);
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
restart(timeoutMs: number): void {
|
|
1523
|
+
super.restart(timeoutMs, this.callback);
|
|
1432
1524
|
}
|
|
1433
|
-
setTimerFn(timer);
|
|
1434
1525
|
}
|