@fluidframework/container-runtime 2.0.0-internal.1.2.0.93071 → 2.0.0-internal.1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/batchManager.d.ts +7 -2
- package/dist/batchManager.d.ts.map +1 -1
- package/dist/batchManager.js +19 -17
- package/dist/batchManager.js.map +1 -1
- package/dist/batchTracker.d.ts +1 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +1 -2
- package/dist/batchTracker.js.map +1 -1
- package/dist/containerRuntime.d.ts +10 -5
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +87 -63
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +14 -5
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +19 -11
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +6 -2
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +7 -9
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaScheduler.d.ts +6 -4
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +6 -4
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/garbageCollection.d.ts +41 -12
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +176 -98
- package/dist/garbageCollection.js.map +1 -1
- package/dist/gcSweepReadyUsageDetection.d.ts +53 -0
- package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/dist/gcSweepReadyUsageDetection.js +135 -0
- package/dist/gcSweepReadyUsageDetection.js.map +1 -0
- package/dist/orderedClientElection.d.ts +28 -10
- package/dist/orderedClientElection.d.ts.map +1 -1
- package/dist/orderedClientElection.js +14 -4
- package/dist/orderedClientElection.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +4 -2
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts +6 -3
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +21 -13
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summarizerTypes.d.ts +13 -6
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryCollection.d.ts.map +1 -1
- package/dist/summaryCollection.js +3 -6
- package/dist/summaryCollection.js.map +1 -1
- package/dist/summaryManager.d.ts +2 -2
- package/dist/summaryManager.js +2 -2
- package/dist/summaryManager.js.map +1 -1
- package/lib/batchManager.d.ts +7 -2
- package/lib/batchManager.d.ts.map +1 -1
- package/lib/batchManager.js +19 -17
- package/lib/batchManager.js.map +1 -1
- package/lib/batchTracker.d.ts +1 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +1 -2
- package/lib/batchTracker.js.map +1 -1
- package/lib/containerRuntime.d.ts +10 -5
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +87 -63
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +14 -5
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +20 -12
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +6 -2
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +7 -9
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaScheduler.d.ts +6 -4
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +6 -4
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/garbageCollection.d.ts +41 -12
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +175 -97
- package/lib/garbageCollection.js.map +1 -1
- package/lib/gcSweepReadyUsageDetection.d.ts +53 -0
- package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/lib/gcSweepReadyUsageDetection.js +130 -0
- package/lib/gcSweepReadyUsageDetection.js.map +1 -0
- package/lib/orderedClientElection.d.ts +28 -10
- package/lib/orderedClientElection.d.ts.map +1 -1
- package/lib/orderedClientElection.js +14 -4
- package/lib/orderedClientElection.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +4 -2
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts +6 -3
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +22 -14
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summarizerTypes.d.ts +13 -6
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryCollection.d.ts.map +1 -1
- package/lib/summaryCollection.js +3 -6
- package/lib/summaryCollection.js.map +1 -1
- package/lib/summaryManager.d.ts +2 -2
- package/lib/summaryManager.js +2 -2
- package/lib/summaryManager.js.map +1 -1
- package/package.json +19 -16
- package/src/batchManager.ts +22 -19
- package/src/batchTracker.ts +1 -2
- package/src/containerRuntime.ts +118 -80
- package/src/dataStoreContext.ts +20 -10
- package/src/dataStores.ts +7 -8
- package/src/deltaScheduler.ts +6 -4
- package/src/garbageCollection.ts +224 -134
- package/src/gcSweepReadyUsageDetection.ts +147 -0
- package/src/orderedClientElection.ts +31 -10
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +4 -2
- package/src/scheduleManager.ts +30 -10
- package/src/summarizerTypes.ts +14 -6
- package/src/summaryCollection.ts +3 -5
- package/src/summaryManager.ts +2 -2
package/src/garbageCollection.ts
CHANGED
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
|
|
43
43
|
import { IGCRuntimeOptions, RuntimeHeaders } from "./containerRuntime";
|
|
44
44
|
import { getSummaryForDatastores } from "./dataStores";
|
|
45
|
+
import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
|
|
45
46
|
import {
|
|
46
47
|
getGCVersion,
|
|
47
48
|
GCVersion,
|
|
@@ -70,12 +71,12 @@ export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
|
70
71
|
const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
|
|
71
72
|
// Feature gate key to expire a session after a set period of time.
|
|
72
73
|
export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
|
|
73
|
-
// Feature gate key to disable expiring session after a set period of time, even if expiry value is present
|
|
74
|
+
// Feature gate key to disable expiring session after a set period of time, even if expiry value is present.
|
|
74
75
|
export const disableSessionExpiryKey = "Fluid.GarbageCollection.DisableSessionExpiry";
|
|
75
76
|
// Feature gate key to write the gc blob as a handle if the data is the same.
|
|
76
77
|
export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
|
|
77
78
|
// Feature gate key to turn GC sweep log off.
|
|
78
|
-
const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
|
|
79
|
+
export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
|
|
79
80
|
|
|
80
81
|
// One day in milliseconds.
|
|
81
82
|
export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
|
|
@@ -133,7 +134,7 @@ export interface IGarbageCollectionRuntime {
|
|
|
133
134
|
/** Returns the type of the GC node. */
|
|
134
135
|
getNodeType(nodePath: string): GCNodeType;
|
|
135
136
|
/** Called when the runtime should close because of an error. */
|
|
136
|
-
closeFn(error?: ICriticalContainerError)
|
|
137
|
+
closeFn: (error?: ICriticalContainerError) => void;
|
|
137
138
|
}
|
|
138
139
|
|
|
139
140
|
/** Defines the contract for the garbage collector. */
|
|
@@ -148,7 +149,7 @@ export interface IGarbageCollector {
|
|
|
148
149
|
/** Run garbage collection and update the reference / used state of the system. */
|
|
149
150
|
collectGarbage(
|
|
150
151
|
options: { logger?: ITelemetryLogger; runSweep?: boolean; fullGC?: boolean; },
|
|
151
|
-
): Promise<IGCStats>;
|
|
152
|
+
): Promise<IGCStats | undefined>;
|
|
152
153
|
/** Summarizes the GC data and returns it as a summary tree. */
|
|
153
154
|
summarize(
|
|
154
155
|
fullTree: boolean,
|
|
@@ -171,6 +172,7 @@ export interface IGarbageCollector {
|
|
|
171
172
|
): void;
|
|
172
173
|
/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
|
|
173
174
|
addedOutboundReference(fromNodePath: string, toNodePath: string): void;
|
|
175
|
+
setConnectionState(connected: boolean, clientId?: string): void;
|
|
174
176
|
dispose(): void;
|
|
175
177
|
}
|
|
176
178
|
|
|
@@ -186,6 +188,8 @@ export interface IGarbageCollectorCreateParams {
|
|
|
186
188
|
readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
|
|
187
189
|
readonly getLastSummaryTimestampMs: () => number | undefined;
|
|
188
190
|
readonly readAndParseBlob: ReadAndParseBlob;
|
|
191
|
+
readonly activeConnection: () => boolean;
|
|
192
|
+
readonly getContainerDiagnosticId: () => string;
|
|
189
193
|
readonly snapshotCacheExpiryMs?: number;
|
|
190
194
|
}
|
|
191
195
|
|
|
@@ -235,10 +239,10 @@ export class UnreferencedStateTracker {
|
|
|
235
239
|
public readonly unreferencedTimestampMs: number,
|
|
236
240
|
/** The time after which node transitions to Inactive state. */
|
|
237
241
|
private readonly inactiveTimeoutMs: number,
|
|
242
|
+
/** The current reference timestamp used to track how long this node has been unreferenced for. */
|
|
243
|
+
currentReferenceTimestampMs: number,
|
|
238
244
|
/** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
|
|
239
|
-
private readonly sweepTimeoutMs
|
|
240
|
-
/** The current reference timestamp; undefined if no ops have ever been processed which can happen in tests. */
|
|
241
|
-
currentReferenceTimestampMs?: number,
|
|
245
|
+
private readonly sweepTimeoutMs: number | undefined,
|
|
242
246
|
) {
|
|
243
247
|
if (this.sweepTimeoutMs !== undefined) {
|
|
244
248
|
assert(this.inactiveTimeoutMs <= this.sweepTimeoutMs,
|
|
@@ -260,12 +264,7 @@ export class UnreferencedStateTracker {
|
|
|
260
264
|
this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
|
|
261
265
|
}
|
|
262
266
|
});
|
|
263
|
-
|
|
264
|
-
// If there is no current reference timestamp, can't track the node's unreferenced state at this time.
|
|
265
|
-
// This will happen later when updateTracking is called with a reference timestamp.
|
|
266
|
-
if (currentReferenceTimestampMs !== undefined) {
|
|
267
|
-
this.updateTracking(currentReferenceTimestampMs);
|
|
268
|
-
}
|
|
267
|
+
this.updateTracking(currentReferenceTimestampMs);
|
|
269
268
|
}
|
|
270
269
|
|
|
271
270
|
/* Updates the unreferenced state based on the provided timestamp. */
|
|
@@ -312,8 +311,12 @@ export class UnreferencedStateTracker {
|
|
|
312
311
|
* its state across summaries.
|
|
313
312
|
*
|
|
314
313
|
* Node - represented as nodeId, it's a node on the GC graph
|
|
314
|
+
*
|
|
315
315
|
* Outbound Route - a path from one node to another node, think `nodeA` -\> `nodeB`
|
|
316
|
+
*
|
|
316
317
|
* Graph - all nodes with their respective routes
|
|
318
|
+
*
|
|
319
|
+
* ```
|
|
317
320
|
* GC Graph
|
|
318
321
|
*
|
|
319
322
|
* Node
|
|
@@ -323,6 +326,7 @@ export class UnreferencedStateTracker {
|
|
|
323
326
|
* / \\
|
|
324
327
|
* Node Node
|
|
325
328
|
* NodeId = "dds1" NodeId = "dds2"
|
|
329
|
+
* ```
|
|
326
330
|
*/
|
|
327
331
|
export class GarbageCollector implements IGarbageCollector {
|
|
328
332
|
public static create(createParams: IGarbageCollectorCreateParams): IGarbageCollector {
|
|
@@ -331,11 +335,16 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
331
335
|
|
|
332
336
|
/**
|
|
333
337
|
* Tells whether the GC state needs to be reset in the next summary. We need to do this if:
|
|
338
|
+
*
|
|
334
339
|
* 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
|
|
340
|
+
*
|
|
335
341
|
* 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
|
|
342
|
+
*
|
|
336
343
|
* 3. The GC version in the latest summary is different from the current GC version. This can happen if:
|
|
337
|
-
*
|
|
338
|
-
*
|
|
344
|
+
*
|
|
345
|
+
* 3.1. The summary this client loaded with has data from a different GC version.
|
|
346
|
+
*
|
|
347
|
+
* 3.2. This client's latest summary was updated from a snapshot that has a different GC version.
|
|
339
348
|
*/
|
|
340
349
|
public get summaryStateNeedsReset(): boolean {
|
|
341
350
|
return this.initialStateNeedsReset ||
|
|
@@ -379,10 +388,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
379
388
|
|
|
380
389
|
/**
|
|
381
390
|
* Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
|
|
391
|
+
*
|
|
382
392
|
* 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
|
|
383
|
-
*
|
|
393
|
+
* it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
|
|
394
|
+
*
|
|
384
395
|
* 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
|
|
385
|
-
*
|
|
396
|
+
* a document and the first time GC is enabled after is was disabled before.
|
|
386
397
|
*
|
|
387
398
|
* Note that the state needs reset only for the very first time summary is generated by this client. After that, the
|
|
388
399
|
* state will be up-to-date and this flag will be reset.
|
|
@@ -441,6 +452,27 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
441
452
|
private readonly getNodePackagePath: (nodePath: string) => Promise<readonly string[] | undefined>;
|
|
442
453
|
/** Returns the timestamp of the last summary generated for this container. */
|
|
443
454
|
private readonly getLastSummaryTimestampMs: () => number | undefined;
|
|
455
|
+
/** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
|
|
456
|
+
private readonly activeConnection: () => boolean;
|
|
457
|
+
|
|
458
|
+
/** Returns a list of all the configurations for garbage collection. */
|
|
459
|
+
private get configs() {
|
|
460
|
+
return {
|
|
461
|
+
gcEnabled: this.gcEnabled,
|
|
462
|
+
sweepEnabled: this.sweepEnabled,
|
|
463
|
+
runGC: this.shouldRunGC,
|
|
464
|
+
runSweep: this.shouldRunSweep,
|
|
465
|
+
writeAtRoot: this._writeDataAtRoot,
|
|
466
|
+
testMode: this.testMode,
|
|
467
|
+
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
468
|
+
inactiveTimeout: this.inactiveTimeoutMs,
|
|
469
|
+
trackGCState: this.trackGCState,
|
|
470
|
+
...this.gcOptions,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** Handler to respond to when a SweepReady object is used */
|
|
475
|
+
private readonly sweepReadyUsageHandler: SweepReadyUsageDetectionHandler;
|
|
444
476
|
|
|
445
477
|
protected constructor(createParams: IGarbageCollectorCreateParams) {
|
|
446
478
|
this.runtime = createParams.runtime;
|
|
@@ -448,6 +480,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
448
480
|
this.gcOptions = createParams.gcOptions;
|
|
449
481
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
450
482
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
483
|
+
this.activeConnection = createParams.activeConnection;
|
|
451
484
|
|
|
452
485
|
const baseSnapshot = createParams.baseSnapshot;
|
|
453
486
|
const metadata = createParams.metadata;
|
|
@@ -457,6 +490,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
457
490
|
createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } },
|
|
458
491
|
));
|
|
459
492
|
|
|
493
|
+
this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(
|
|
494
|
+
createParams.getContainerDiagnosticId(),
|
|
495
|
+
this.mc,
|
|
496
|
+
this.runtime.closeFn,
|
|
497
|
+
);
|
|
498
|
+
|
|
460
499
|
let prevSummaryGCVersion: number | undefined;
|
|
461
500
|
|
|
462
501
|
/**
|
|
@@ -527,8 +566,11 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
527
566
|
|
|
528
567
|
/**
|
|
529
568
|
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
569
|
+
*
|
|
530
570
|
* 1. GC should be enabled for this container.
|
|
571
|
+
*
|
|
531
572
|
* 2. GC should not be disabled via disableGC GC option.
|
|
573
|
+
*
|
|
532
574
|
* These conditions can be overridden via runGCKey feature flag.
|
|
533
575
|
*/
|
|
534
576
|
this.shouldRunGC = this.mc.config.getBoolean(runGCKey) ?? (
|
|
@@ -540,10 +582,13 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
540
582
|
|
|
541
583
|
/**
|
|
542
584
|
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
585
|
+
*
|
|
543
586
|
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
587
|
+
*
|
|
544
588
|
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
589
|
+
*
|
|
545
590
|
* 3. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
546
|
-
*
|
|
591
|
+
* feature flag.
|
|
547
592
|
*/
|
|
548
593
|
this.shouldRunSweep = false; // disable while TEMPORARY measure hardcoding snapshotCacheExpiryMs is here
|
|
549
594
|
// this.shouldRunGC
|
|
@@ -582,64 +627,72 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
582
627
|
return undefined;
|
|
583
628
|
}
|
|
584
629
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
this.
|
|
630
|
+
try {
|
|
631
|
+
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
632
|
+
const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
|
|
633
|
+
if (gcSnapshotTree !== undefined) {
|
|
634
|
+
// If the GC tree is written at root, we should also do the same.
|
|
635
|
+
this._writeDataAtRoot = true;
|
|
636
|
+
const baseGCState = await getGCStateFromSnapshot(
|
|
637
|
+
gcSnapshotTree,
|
|
638
|
+
readAndParseBlob,
|
|
639
|
+
);
|
|
640
|
+
if (this.trackGCState) {
|
|
641
|
+
this.latestSerializedSummaryState = JSON.stringify(generateSortedGCState(baseGCState));
|
|
642
|
+
}
|
|
643
|
+
return baseGCState;
|
|
596
644
|
}
|
|
597
|
-
return baseGCState;
|
|
598
|
-
}
|
|
599
645
|
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
646
|
+
// back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
|
|
647
|
+
// consolidate into IGarbageCollectionState format.
|
|
648
|
+
// Add a node for the root node that is not present in older snapshot format.
|
|
649
|
+
const gcState: IGarbageCollectionState = { gcNodes: { "/": { outboundRoutes: [] } } };
|
|
650
|
+
const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
|
|
651
|
+
assert(dataStoreSnapshotTree !== undefined,
|
|
652
|
+
0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
|
|
653
|
+
for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
|
|
654
|
+
const blobId = dsSnapshotTree.blobs[gcBlobKey];
|
|
655
|
+
if (blobId === undefined) {
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
612
658
|
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
659
|
+
const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionDetailsBase>(blobId);
|
|
660
|
+
// If there are no nodes for this data store, skip it.
|
|
661
|
+
if (gcSummaryDetails.gcData?.gcNodes === undefined) {
|
|
662
|
+
continue;
|
|
663
|
+
}
|
|
618
664
|
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
665
|
+
const dsRootId = `/${dsId}`;
|
|
666
|
+
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
667
|
+
// Construct that entry by adding root data store ids to its outbound routes.
|
|
668
|
+
const initialSnapshotDetails = await readAndParseBlob<ReadFluidDataStoreAttributes>(
|
|
669
|
+
dsSnapshotTree.blobs[dataStoreAttributesBlobName],
|
|
670
|
+
);
|
|
671
|
+
if (initialSnapshotDetails.isRootDataStore) {
|
|
672
|
+
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
673
|
+
}
|
|
628
674
|
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
675
|
+
for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
|
|
676
|
+
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
677
|
+
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
678
|
+
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
679
|
+
gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
|
|
680
|
+
}
|
|
681
|
+
assert(gcState.gcNodes[dsRootId] !== undefined,
|
|
682
|
+
0x2a9 /* GC nodes for data store not in GC blob */);
|
|
683
|
+
gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
|
|
634
684
|
}
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
gcState.gcNodes
|
|
685
|
+
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
686
|
+
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
687
|
+
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
688
|
+
} catch (error) {
|
|
689
|
+
const dpe = DataProcessingError.wrapIfUnrecognized(
|
|
690
|
+
error,
|
|
691
|
+
"FailedToInitializeGC",
|
|
692
|
+
);
|
|
693
|
+
dpe.addTelemetryProperties({ gcConfigs: JSON.stringify(this.configs) });
|
|
694
|
+
throw dpe;
|
|
638
695
|
}
|
|
639
|
-
|
|
640
|
-
// If there is only one node (root node just added above), either GC is disabled or we are loading from the
|
|
641
|
-
// very first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
642
|
-
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
643
696
|
});
|
|
644
697
|
|
|
645
698
|
/**
|
|
@@ -649,7 +702,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
649
702
|
*/
|
|
650
703
|
this.initializeBaseStateP = new LazyPromise<void>(async () => {
|
|
651
704
|
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
705
|
+
/**
|
|
706
|
+
* If there is no current reference timestamp, skip initialization. We need the current timestamp to track
|
|
707
|
+
* how long objects have been unreferenced and if they can be deleted.
|
|
708
|
+
*
|
|
709
|
+
* Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
|
|
710
|
+
* for this container and it is in read mode. In this scenario, there is no point in running GC anyway
|
|
711
|
+
* because references in the container do not change without any ops, i.e., there is nothing to collect.
|
|
712
|
+
*/
|
|
713
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
714
|
+
// Log an event so we can evaluate how often we run into this scenario.
|
|
715
|
+
this.mc.logger.sendErrorEvent({
|
|
716
|
+
eventName: "GarbageCollectorInitializedWithoutTimestamp",
|
|
717
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
718
|
+
});
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
|
|
652
722
|
const baseState = await baseSummaryStateP;
|
|
723
|
+
/**
|
|
724
|
+
* The base state will not be present if the container is loaded from:
|
|
725
|
+
* 1. The first summary created by the detached container.
|
|
726
|
+
* 2. A summary that was generated with GC disabled.
|
|
727
|
+
* 3. A summary that was generated before GC even existed.
|
|
728
|
+
*/
|
|
653
729
|
if (baseState === undefined) {
|
|
654
730
|
return;
|
|
655
731
|
}
|
|
@@ -662,8 +738,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
662
738
|
new UnreferencedStateTracker(
|
|
663
739
|
nodeData.unreferencedTimestampMs,
|
|
664
740
|
this.inactiveTimeoutMs,
|
|
665
|
-
this.sweepTimeoutMs,
|
|
666
741
|
currentReferenceTimestampMs,
|
|
742
|
+
this.sweepTimeoutMs,
|
|
667
743
|
),
|
|
668
744
|
);
|
|
669
745
|
}
|
|
@@ -705,42 +781,38 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
705
781
|
|
|
706
782
|
// Log all the GC options and the state determined by the garbage collector. This is interesting only for the
|
|
707
783
|
// summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
|
|
708
|
-
const gcConfigProps = JSON.stringify({
|
|
709
|
-
gcEnabled: this.gcEnabled,
|
|
710
|
-
sweepEnabled: this.sweepEnabled,
|
|
711
|
-
runGC: this.shouldRunGC,
|
|
712
|
-
runSweep: this.shouldRunSweep,
|
|
713
|
-
writeAtRoot: this._writeDataAtRoot,
|
|
714
|
-
testMode: this.testMode,
|
|
715
|
-
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
716
|
-
inactiveTimeout: this.inactiveTimeoutMs,
|
|
717
|
-
existing: createParams.existing,
|
|
718
|
-
trackGCState: this.trackGCState,
|
|
719
|
-
...this.gcOptions,
|
|
720
|
-
});
|
|
721
784
|
if (this.isSummarizerClient) {
|
|
722
785
|
this.mc.logger.sendTelemetryEvent({
|
|
723
786
|
eventName: "GarbageCollectorLoaded",
|
|
724
|
-
gcConfigs:
|
|
787
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
725
788
|
});
|
|
726
789
|
}
|
|
790
|
+
}
|
|
727
791
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
792
|
+
/**
|
|
793
|
+
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
794
|
+
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
|
|
795
|
+
* @param connected - Whether the runtime connected / disconnected.
|
|
796
|
+
* @param clientId - The clientId of this runtime.
|
|
797
|
+
*/
|
|
798
|
+
public setConnectionState(connected: boolean, clientId?: string | undefined): void {
|
|
799
|
+
/**
|
|
800
|
+
* For non-summarizer clients, initialize the base state when the container becomes active, i.e., it transitions
|
|
801
|
+
* to "write" mode. This will ensure that the container's own join op is processed and there is a recent
|
|
802
|
+
* reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
|
|
803
|
+
* could affect the GC state will have been processed.
|
|
804
|
+
*
|
|
805
|
+
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
806
|
+
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
807
|
+
*/
|
|
808
|
+
if (this.activeConnection() && !this.isSummarizerClient && this.shouldRunGC) {
|
|
809
|
+
this.initializeBaseStateP.catch((error) => {});
|
|
738
810
|
}
|
|
739
811
|
}
|
|
740
812
|
|
|
741
813
|
/**
|
|
742
814
|
* Runs garbage collection and updates the reference / used state of the nodes in the container.
|
|
743
|
-
* @returns the
|
|
815
|
+
* @returns stats of the GC run or undefined if GC did not run.
|
|
744
816
|
*/
|
|
745
817
|
public async collectGarbage(
|
|
746
818
|
options: {
|
|
@@ -751,15 +823,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
751
823
|
/** True to generate full GC data */
|
|
752
824
|
fullGC?: boolean;
|
|
753
825
|
},
|
|
754
|
-
): Promise<IGCStats> {
|
|
755
|
-
const
|
|
756
|
-
fullGC = this.gcOptions.runFullGC === true || this.summaryStateNeedsReset,
|
|
757
|
-
} = options;
|
|
758
|
-
|
|
826
|
+
): Promise<IGCStats | undefined> {
|
|
827
|
+
const fullGC = options.fullGC ?? (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
|
|
759
828
|
const logger = options.logger
|
|
760
829
|
? ChildLogger.create(options.logger, undefined, { all: { completedGCRuns: () => this.completedRuns } })
|
|
761
830
|
: this.mc.logger;
|
|
762
831
|
|
|
832
|
+
/**
|
|
833
|
+
* If there is no current reference timestamp, skip running GC. We need the current timestamp to track
|
|
834
|
+
* how long objects have been unreferenced and if they should be deleted.
|
|
835
|
+
*
|
|
836
|
+
* Note that the only scenario where GC is called and there is no reference timestamp is when no ops have ever
|
|
837
|
+
* been processed for this container and it is in read mode. In this scenario, there is no point in running GC
|
|
838
|
+
* anyway because references in the container do not change without any ops, i.e., there is nothing to collect.
|
|
839
|
+
*/
|
|
840
|
+
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
841
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
842
|
+
// Log an event so we can evaluate how often we run into this scenario.
|
|
843
|
+
logger.sendErrorEvent({
|
|
844
|
+
eventName: "CollectGarbageCalledWithoutTimestamp",
|
|
845
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
846
|
+
});
|
|
847
|
+
return undefined;
|
|
848
|
+
}
|
|
849
|
+
|
|
763
850
|
return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
764
851
|
await this.runPreGCSteps();
|
|
765
852
|
|
|
@@ -767,7 +854,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
767
854
|
const gcData = await this.runtime.getGCData(fullGC);
|
|
768
855
|
const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
|
|
769
856
|
|
|
770
|
-
const gcStats = await this.runPostGCSteps(gcData, gcResult, logger);
|
|
857
|
+
const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
|
|
771
858
|
event.end({ ...gcStats });
|
|
772
859
|
this.completedRuns++;
|
|
773
860
|
return gcStats;
|
|
@@ -781,7 +868,12 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
781
868
|
await this.runtime.updateStateBeforeGC();
|
|
782
869
|
}
|
|
783
870
|
|
|
784
|
-
private async runPostGCSteps(
|
|
871
|
+
private async runPostGCSteps(
|
|
872
|
+
gcData: IGarbageCollectionData,
|
|
873
|
+
gcResult: IGCResult,
|
|
874
|
+
logger: ITelemetryLogger,
|
|
875
|
+
currentReferenceTimestampMs: number,
|
|
876
|
+
): Promise<IGCStats> {
|
|
785
877
|
// Generate statistics from the current run. This is done before updating the current state because it
|
|
786
878
|
// generates some of its data based on previous state of the system.
|
|
787
879
|
const gcStats = this.generateStats(gcResult);
|
|
@@ -791,7 +883,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
791
883
|
this.updateStateSinceLastRun(gcData, logger);
|
|
792
884
|
|
|
793
885
|
// Update the current state and update the runtime of all routes or ids that used as per the GC run.
|
|
794
|
-
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
795
886
|
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
796
887
|
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds, currentReferenceTimestampMs);
|
|
797
888
|
|
|
@@ -1004,7 +1095,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1004
1095
|
private updateCurrentState(
|
|
1005
1096
|
gcData: IGarbageCollectionData,
|
|
1006
1097
|
gcResult: IGCResult,
|
|
1007
|
-
currentReferenceTimestampMs
|
|
1098
|
+
currentReferenceTimestampMs: number,
|
|
1008
1099
|
) {
|
|
1009
1100
|
this.previousGCDataFromLastRun = cloneGCData(gcData);
|
|
1010
1101
|
this.newReferencesSinceLastRun.clear();
|
|
@@ -1020,15 +1111,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1020
1111
|
}
|
|
1021
1112
|
}
|
|
1022
1113
|
|
|
1023
|
-
/**
|
|
1024
|
-
* If there is no current reference time, skip tracking when a node becomes unreferenced. This would happen
|
|
1025
|
-
* if no ops have been processed ever and we still try to run GC. If so, there is nothing interesting to track
|
|
1026
|
-
* anyway.
|
|
1027
|
-
*/
|
|
1028
|
-
if (currentReferenceTimestampMs === undefined) {
|
|
1029
|
-
return;
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
1114
|
/**
|
|
1033
1115
|
* If a node became unreferenced in this run, start tracking it.
|
|
1034
1116
|
* If a node was already unreferenced, update its tracking information. Since the current reference time is
|
|
@@ -1042,8 +1124,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1042
1124
|
new UnreferencedStateTracker(
|
|
1043
1125
|
currentReferenceTimestampMs,
|
|
1044
1126
|
this.inactiveTimeoutMs,
|
|
1045
|
-
this.sweepTimeoutMs,
|
|
1046
1127
|
currentReferenceTimestampMs,
|
|
1128
|
+
this.sweepTimeoutMs,
|
|
1047
1129
|
),
|
|
1048
1130
|
);
|
|
1049
1131
|
} else {
|
|
@@ -1096,14 +1178,19 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1096
1178
|
* run, and then add the references since last run.
|
|
1097
1179
|
*
|
|
1098
1180
|
* Note on why we need to combine the data from previous run, current run and all references in between -
|
|
1181
|
+
*
|
|
1099
1182
|
* 1. We need data from last run because some of its references may have been deleted since then. If those
|
|
1100
|
-
*
|
|
1183
|
+
* references added new outbound references before getting deleted, we need to detect them.
|
|
1184
|
+
*
|
|
1101
1185
|
* 2. We need new outbound references since last run because some of them may have been deleted later. If those
|
|
1102
|
-
*
|
|
1186
|
+
* references added new outbound references before getting deleted, we need to detect them.
|
|
1187
|
+
*
|
|
1103
1188
|
* 3. We need data from the current run because currently we may not detect when DDSes are referenced:
|
|
1104
|
-
*
|
|
1105
|
-
*
|
|
1106
|
-
*
|
|
1189
|
+
*
|
|
1190
|
+
* - We don't require DDSes handles to be stored in a referenced DDS. For this, we need GC at DDS level
|
|
1191
|
+
* which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
|
|
1192
|
+
*
|
|
1193
|
+
* - A new data store may have "root" DDSes already created and we don't detect them today.
|
|
1107
1194
|
*/
|
|
1108
1195
|
const gcDataSuperSet = concatGarbageCollectionData(this.previousGCDataFromLastRun, currentGCData);
|
|
1109
1196
|
this.newReferencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
|
|
@@ -1249,10 +1336,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1249
1336
|
* For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
|
|
1250
1337
|
* this will give us a view into how much deleted content a container has.
|
|
1251
1338
|
*/
|
|
1252
|
-
private logSweepEvents(logger: ITelemetryLogger, currentReferenceTimestampMs
|
|
1253
|
-
if (this.mc.config.getBoolean(disableSweepLogKey) === true
|
|
1254
|
-
|| currentReferenceTimestampMs === undefined
|
|
1255
|
-
|| this.sweepTimeoutMs === undefined) {
|
|
1339
|
+
private logSweepEvents(logger: ITelemetryLogger, currentReferenceTimestampMs: number) {
|
|
1340
|
+
if (this.mc.config.getBoolean(disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
|
|
1256
1341
|
return;
|
|
1257
1342
|
}
|
|
1258
1343
|
|
|
@@ -1303,12 +1388,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1303
1388
|
return;
|
|
1304
1389
|
}
|
|
1305
1390
|
|
|
1306
|
-
// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
|
|
1307
|
-
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects).
|
|
1308
|
-
if (!this.isSummarizerClient && usageType !== "Loaded") {
|
|
1309
|
-
return;
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
1391
|
// We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
|
|
1313
1392
|
// as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
|
|
1314
1393
|
const nodeType = this.runtime.getNodeType(nodeId);
|
|
@@ -1344,11 +1423,22 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1344
1423
|
if (this.isSummarizerClient) {
|
|
1345
1424
|
this.pendingEventsQueue.push({ ...propsToLog, usageType, state });
|
|
1346
1425
|
} else {
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1426
|
+
// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
|
|
1427
|
+
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
|
|
1428
|
+
if (usageType === "Loaded") {
|
|
1429
|
+
this.mc.logger.sendErrorEvent({
|
|
1430
|
+
...propsToLog,
|
|
1431
|
+
eventName: `${state}Object_${usageType}`,
|
|
1432
|
+
pkg: packagePath ? { value: packagePath.join("/"), tag: TelemetryDataTag.CodeArtifact } : undefined,
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
// If SweepReady Usage Detection is enabed, the handler may close the interactive container.
|
|
1437
|
+
// Once Sweep is fully implemented, this will be removed since the objects will be gone
|
|
1438
|
+
// and errors will arise elsewhere in the runtime
|
|
1439
|
+
if (state === UnreferencedState.SweepReady) {
|
|
1440
|
+
this.sweepReadyUsageHandler.usageDetectedInInteractiveClient({ ...propsToLog, usageType });
|
|
1441
|
+
}
|
|
1352
1442
|
}
|
|
1353
1443
|
}
|
|
1354
1444
|
|