@fluidframework/container-runtime 0.56.0 → 0.57.0-51086
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +9 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +6 -6
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +65 -25
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +149 -79
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +62 -0
- package/dist/dataStore.d.ts.map +1 -0
- package/dist/dataStore.js +135 -0
- package/dist/dataStore.js.map +1 -0
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +9 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +14 -19
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +47 -21
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +195 -61
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +1 -0
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +23 -15
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +6 -6
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryGenerator.d.ts +2 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +46 -28
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +9 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +6 -6
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +65 -25
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +150 -80
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +62 -0
- package/lib/dataStore.d.ts.map +1 -0
- package/lib/dataStore.js +130 -0
- package/lib/dataStore.js.map +1 -0
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +9 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +13 -18
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +47 -21
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +197 -63
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +3 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +1 -0
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +23 -15
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +6 -6
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryGenerator.d.ts +2 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +46 -28
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +13 -13
- package/src/blobManager.ts +12 -1
- package/src/connectionTelemetry.ts +7 -6
- package/src/containerRuntime.ts +231 -103
- package/src/dataStore.ts +187 -0
- package/src/dataStoreContext.ts +1 -1
- package/src/dataStores.ts +18 -38
- package/src/garbageCollection.ts +283 -105
- package/src/index.ts +3 -1
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +25 -16
- package/src/summarizerTypes.ts +6 -8
- package/src/summaryGenerator.ts +72 -23
package/src/garbageCollection.ts
CHANGED
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
import { ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
7
|
import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
|
|
8
8
|
import { ICriticalContainerError } from "@fluidframework/container-definitions";
|
|
9
|
-
import { ClientSessionExpiredError } from "@fluidframework/container-utils";
|
|
9
|
+
import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
|
|
10
|
+
import { IRequestHeader } from "@fluidframework/core-interfaces";
|
|
10
11
|
import {
|
|
11
12
|
cloneGCData,
|
|
12
13
|
concatGarbageCollectionStates,
|
|
@@ -33,7 +34,9 @@ import {
|
|
|
33
34
|
loggerToMonitoringContext,
|
|
34
35
|
MonitoringContext,
|
|
35
36
|
PerformanceEvent,
|
|
37
|
+
TelemetryDataTag,
|
|
36
38
|
} from "@fluidframework/telemetry-utils";
|
|
39
|
+
import { RuntimeHeaders } from ".";
|
|
37
40
|
|
|
38
41
|
import { IGCRuntimeOptions } from "./containerRuntime";
|
|
39
42
|
import { getSummaryForDatastores } from "./dataStores";
|
|
@@ -62,23 +65,38 @@ const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
|
62
65
|
const runSweepKey = "Fluid.GarbageCollection.RunSweep";
|
|
63
66
|
// Feature gate key to write GC data at the root of the summary tree.
|
|
64
67
|
const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
|
|
68
|
+
// Feature gate key to expire a session after a set period of time.
|
|
69
|
+
const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
|
|
65
70
|
|
|
66
71
|
const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
|
|
67
|
-
|
|
68
|
-
/** The used state statistics of a node. */
|
|
69
|
-
export interface IUsedStateStats {
|
|
70
|
-
totalNodeCount: number;
|
|
71
|
-
unusedNodeCount: number;
|
|
72
|
-
}
|
|
72
|
+
const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
73
73
|
|
|
74
74
|
/** The statistics of the system state after a garbage collection run. */
|
|
75
75
|
export interface IGCStats {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
/** The number of nodes in the container. */
|
|
77
|
+
nodeCount: number;
|
|
78
|
+
/** The number of data stores in the container. */
|
|
79
|
+
dataStoreCount: number;
|
|
80
|
+
/** The number of unreferenced nodes in the container. */
|
|
81
|
+
unrefNodeCount: number;
|
|
82
|
+
/** The number of unreferenced data stores in the container. */
|
|
83
|
+
unrefDataStoreCount: number;
|
|
84
|
+
/** The number of nodes whose reference state updated since last GC run. */
|
|
85
|
+
updatedNodeCount: number;
|
|
86
|
+
/** The number of data stores whose reference state updated since last GC run. */
|
|
87
|
+
updatedDataStoreCount: number;
|
|
80
88
|
}
|
|
81
89
|
|
|
90
|
+
/** The event that is logged when unreferenced node is used after a certain time. */
|
|
91
|
+
interface IUnreferencedEvent {
|
|
92
|
+
eventName: string;
|
|
93
|
+
id: string;
|
|
94
|
+
age: number;
|
|
95
|
+
timeout: number;
|
|
96
|
+
externalRequest?: boolean;
|
|
97
|
+
viaHandle?: boolean;
|
|
98
|
+
};
|
|
99
|
+
|
|
82
100
|
/** Defines the APIs for the runtime object to be passed to the garbage collector. */
|
|
83
101
|
export interface IGarbageCollectionRuntime {
|
|
84
102
|
/** Before GC runs, called to notify the runtime to update any pending GC state. */
|
|
@@ -86,7 +104,7 @@ export interface IGarbageCollectionRuntime {
|
|
|
86
104
|
/** Returns the garbage collection data of the runtime. */
|
|
87
105
|
getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
|
|
88
106
|
/** After GC has run, called to notify the runtime of routes that are used in it. */
|
|
89
|
-
updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number):
|
|
107
|
+
updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): void;
|
|
90
108
|
}
|
|
91
109
|
|
|
92
110
|
/** Defines the contract for the garbage collector. */
|
|
@@ -115,10 +133,15 @@ export interface IGarbageCollector {
|
|
|
115
133
|
getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionDetailsBase>>;
|
|
116
134
|
/** Called when the latest summary of the system has been refreshed. */
|
|
117
135
|
latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;
|
|
118
|
-
/** Called when a node is
|
|
119
|
-
|
|
136
|
+
/** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
|
|
137
|
+
nodeUpdated(
|
|
138
|
+
nodePath: string,
|
|
139
|
+
reason: "Loaded" | "Changed",
|
|
140
|
+
packagePath?: readonly string[],
|
|
141
|
+
requestHeaders?: IRequestHeader,
|
|
142
|
+
): void;
|
|
120
143
|
/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
|
|
121
|
-
addedOutboundReference(
|
|
144
|
+
addedOutboundReference(fromNodePath: string, toNodePath: string): void;
|
|
122
145
|
dispose(): void;
|
|
123
146
|
}
|
|
124
147
|
|
|
@@ -127,10 +150,11 @@ export interface IGarbageCollector {
|
|
|
127
150
|
* the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).
|
|
128
151
|
*/
|
|
129
152
|
class UnreferencedStateTracker {
|
|
130
|
-
private
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
153
|
+
private _inactive: boolean = false;
|
|
154
|
+
public get inactive(): boolean {
|
|
155
|
+
return this._inactive;
|
|
156
|
+
}
|
|
157
|
+
|
|
134
158
|
private readonly timer: Timer | undefined;
|
|
135
159
|
|
|
136
160
|
constructor(
|
|
@@ -140,9 +164,9 @@ class UnreferencedStateTracker {
|
|
|
140
164
|
// If the timeout has already expired, the node should become inactive immediately. Otherwise, start a timer of
|
|
141
165
|
// inactiveTimeoutMs after which the node will become inactive.
|
|
142
166
|
if (inactiveTimeoutMs <= 0) {
|
|
143
|
-
this.
|
|
167
|
+
this._inactive = true;
|
|
144
168
|
} else {
|
|
145
|
-
this.timer = new Timer(inactiveTimeoutMs, () => { this.
|
|
169
|
+
this.timer = new Timer(inactiveTimeoutMs, () => { this._inactive = true; });
|
|
146
170
|
this.timer.start();
|
|
147
171
|
}
|
|
148
172
|
}
|
|
@@ -150,26 +174,7 @@ class UnreferencedStateTracker {
|
|
|
150
174
|
/** Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state. */
|
|
151
175
|
public stopTracking() {
|
|
152
176
|
this.timer?.clear();
|
|
153
|
-
this.
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
/** Logs an error with the given properties if the node is inactive. */
|
|
157
|
-
public logIfInactive(
|
|
158
|
-
logger: ITelemetryLogger,
|
|
159
|
-
eventName: string,
|
|
160
|
-
currentTimestampMs: number,
|
|
161
|
-
deleteTimeoutMs: number,
|
|
162
|
-
inactiveNodeId: string,
|
|
163
|
-
) {
|
|
164
|
-
if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
|
|
165
|
-
logger.sendErrorEvent({
|
|
166
|
-
eventName,
|
|
167
|
-
age: currentTimestampMs - this.unreferencedTimestampMs,
|
|
168
|
-
timeout: deleteTimeoutMs,
|
|
169
|
-
id: inactiveNodeId,
|
|
170
|
-
});
|
|
171
|
-
this.inactiveEventsLogged.add(eventName);
|
|
172
|
-
}
|
|
177
|
+
this._inactive = false;
|
|
173
178
|
}
|
|
174
179
|
}
|
|
175
180
|
|
|
@@ -182,6 +187,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
182
187
|
provider: IGarbageCollectionRuntime,
|
|
183
188
|
gcOptions: IGCRuntimeOptions,
|
|
184
189
|
deleteUnusedRoutes: (unusedRoutes: string[]) => void,
|
|
190
|
+
getNodePackagePath: (nodeId: string) => readonly string[] | undefined,
|
|
185
191
|
getCurrentTimestampMs: () => number,
|
|
186
192
|
closeFn: (error?: ICriticalContainerError) => void,
|
|
187
193
|
baseSnapshot: ISnapshotTree | undefined,
|
|
@@ -194,6 +200,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
194
200
|
provider,
|
|
195
201
|
gcOptions,
|
|
196
202
|
deleteUnusedRoutes,
|
|
203
|
+
getNodePackagePath,
|
|
197
204
|
getCurrentTimestampMs,
|
|
198
205
|
closeFn,
|
|
199
206
|
baseSnapshot,
|
|
@@ -287,11 +294,19 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
287
294
|
// The timeout responsible for closing the container when the session has expired
|
|
288
295
|
private sessionExpiryTimer?: ReturnType<typeof setTimeout>;
|
|
289
296
|
|
|
297
|
+
// Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
|
|
298
|
+
// per event per node.
|
|
299
|
+
private readonly loggedUnreferencedEvents: Set<string> = new Set();
|
|
300
|
+
// Queue for unreferenced events that should be logged the next time GC runs.
|
|
301
|
+
private pendingEventsQueue: IUnreferencedEvent[] = [];
|
|
302
|
+
|
|
290
303
|
protected constructor(
|
|
291
304
|
private readonly provider: IGarbageCollectionRuntime,
|
|
292
305
|
private readonly gcOptions: IGCRuntimeOptions,
|
|
293
306
|
/** After GC has run, called to delete objects in the runtime whose routes are unused. */
|
|
294
307
|
private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,
|
|
308
|
+
/** For a given node path, returns the node's package path. */
|
|
309
|
+
private readonly getNodePackagePath: (nodePath: string) => readonly string[] | undefined,
|
|
295
310
|
/** Returns the current timestamp to be assigned to nodes that become unreferenced. */
|
|
296
311
|
private readonly getCurrentTimestampMs: () => number,
|
|
297
312
|
private readonly closeFn: (error?: ICriticalContainerError) => void,
|
|
@@ -320,14 +335,22 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
320
335
|
} else {
|
|
321
336
|
// For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.
|
|
322
337
|
this.gcEnabled = gcOptions.gcAllowed === true;
|
|
323
|
-
|
|
338
|
+
// Set the Session Expiry only if the flag is enabled or the test option is set.
|
|
339
|
+
if (this.mc.config.getBoolean(runSessionExpiry) && this.gcEnabled) {
|
|
340
|
+
this.sessionExpiryTimeoutMs = defaultSessionExpiryDurationMs;
|
|
341
|
+
}
|
|
324
342
|
}
|
|
325
343
|
|
|
326
344
|
// If session expiry is enabled, we need to close the container when the timeout expires
|
|
327
345
|
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
328
|
-
const
|
|
329
|
-
|
|
330
|
-
|
|
346
|
+
const timeoutMs = this.sessionExpiryTimeoutMs;
|
|
347
|
+
setLongTimeout(timeoutMs,
|
|
348
|
+
() => {
|
|
349
|
+
this.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
350
|
+
},
|
|
351
|
+
(timer) => {
|
|
352
|
+
this.sessionExpiryTimer = timer;
|
|
353
|
+
});
|
|
331
354
|
}
|
|
332
355
|
|
|
333
356
|
// For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
|
|
@@ -429,7 +452,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
429
452
|
// we only do this once - the very first time we run GC.
|
|
430
453
|
this.initializeBaseStateP = new LazyPromise<void>(async () => {
|
|
431
454
|
const currentTimestampMs = this.getCurrentTimestampMs();
|
|
432
|
-
const baseState =
|
|
455
|
+
const baseState = await baseSummaryStateP;
|
|
433
456
|
if (baseState === undefined) {
|
|
434
457
|
return;
|
|
435
458
|
}
|
|
@@ -488,6 +511,24 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
488
511
|
}
|
|
489
512
|
return dataStoreGCDetailsMap;
|
|
490
513
|
});
|
|
514
|
+
|
|
515
|
+
// Initialize the base state. The base GC data is used to detect and log when inactive / deleted objects are
|
|
516
|
+
// used in the container.
|
|
517
|
+
if (this.shouldRunGC) {
|
|
518
|
+
this.initializeBaseStateP.catch((error) => {
|
|
519
|
+
throw new DataProcessingError(
|
|
520
|
+
error?.message,
|
|
521
|
+
"FailedToInitializeGC",
|
|
522
|
+
{
|
|
523
|
+
gcEnabled: this.gcEnabled,
|
|
524
|
+
runSweep: this.shouldRunSweep,
|
|
525
|
+
writeAtRoot: this._writeDataAtRoot,
|
|
526
|
+
testMode: this.testMode,
|
|
527
|
+
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
528
|
+
},
|
|
529
|
+
);
|
|
530
|
+
});
|
|
531
|
+
}
|
|
491
532
|
}
|
|
492
533
|
|
|
493
534
|
/**
|
|
@@ -516,48 +557,36 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
516
557
|
// Let the runtime update its pending state before GC runs.
|
|
517
558
|
await this.provider.updateStateBeforeGC();
|
|
518
559
|
|
|
519
|
-
const gcStats: {
|
|
520
|
-
deletedNodes?: number,
|
|
521
|
-
totalNodes?: number,
|
|
522
|
-
deletedDataStores?: number,
|
|
523
|
-
totalDataStores?: number,
|
|
524
|
-
} = {};
|
|
525
|
-
|
|
526
560
|
// Get the runtime's GC data and run GC on the reference graph in it.
|
|
527
561
|
const gcData = await this.provider.getGCData(fullGC);
|
|
528
|
-
|
|
529
|
-
this.updateStateSinceLatestRun(gcData);
|
|
530
|
-
|
|
531
562
|
const gcResult = runGarbageCollection(
|
|
532
563
|
gcData.gcNodes,
|
|
533
564
|
[ "/" ],
|
|
534
565
|
logger,
|
|
535
566
|
);
|
|
567
|
+
const gcStats = this.generateStatsAndLogEvents(gcResult);
|
|
568
|
+
|
|
569
|
+
// Update the state since the last GC run. There can be nodes that were referenced between the last and
|
|
570
|
+
// the current run. We need to identify than and update their unreferenced state if needed.
|
|
571
|
+
this.updateStateSinceLastRun(gcData);
|
|
536
572
|
|
|
537
|
-
const currentTimestampMs = this.getCurrentTimestampMs();
|
|
538
573
|
// Update the current state of the system based on the GC run.
|
|
574
|
+
const currentTimestampMs = this.getCurrentTimestampMs();
|
|
539
575
|
this.updateCurrentState(gcData, gcResult, currentTimestampMs);
|
|
540
576
|
|
|
541
|
-
|
|
542
|
-
this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);
|
|
577
|
+
this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);
|
|
543
578
|
|
|
544
579
|
if (runSweep) {
|
|
545
580
|
// Placeholder for running sweep logic.
|
|
546
581
|
}
|
|
547
582
|
|
|
548
|
-
// Update stats to be reported in the peformance event.
|
|
549
|
-
gcStats.deletedNodes = gcResult.deletedNodeIds.length;
|
|
550
|
-
gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;
|
|
551
|
-
gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;
|
|
552
|
-
gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;
|
|
553
|
-
|
|
554
583
|
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
555
584
|
// involving access to deleted data.
|
|
556
585
|
if (this.testMode) {
|
|
557
586
|
this.deleteUnusedRoutes(gcResult.deletedNodeIds);
|
|
558
587
|
}
|
|
559
|
-
event.end(gcStats);
|
|
560
|
-
return gcStats
|
|
588
|
+
event.end({ ...gcStats });
|
|
589
|
+
return gcStats;
|
|
561
590
|
},
|
|
562
591
|
{ end: true, cancel: "error" });
|
|
563
592
|
}
|
|
@@ -621,38 +650,61 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
621
650
|
}
|
|
622
651
|
|
|
623
652
|
/**
|
|
624
|
-
* Called when a node with the given id is
|
|
653
|
+
* Called when a node with the given id is updated. If the node is inactive, log an error.
|
|
654
|
+
* @param nodePath - The id of the node that changed.
|
|
655
|
+
* @param reason - Whether the node was loaded or changed.
|
|
656
|
+
* @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
|
|
657
|
+
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
625
658
|
*/
|
|
626
|
-
public
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
659
|
+
public nodeUpdated(
|
|
660
|
+
nodePath: string,
|
|
661
|
+
reason: "Loaded" | "Changed",
|
|
662
|
+
packagePath?: readonly string[],
|
|
663
|
+
requestHeaders?: IRequestHeader,
|
|
664
|
+
) {
|
|
665
|
+
if (!this.shouldRunGC) {
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
this.logIfInactive(
|
|
670
|
+
reason,
|
|
671
|
+
nodePath,
|
|
632
672
|
this.getCurrentTimestampMs(),
|
|
633
|
-
|
|
634
|
-
|
|
673
|
+
packagePath,
|
|
674
|
+
requestHeaders,
|
|
635
675
|
);
|
|
636
676
|
}
|
|
637
677
|
|
|
638
|
-
public dispose(): void {
|
|
639
|
-
if (this.sessionExpiryTimer !== undefined) {
|
|
640
|
-
clearTimeout(this.sessionExpiryTimer);
|
|
641
|
-
this.sessionExpiryTimer = undefined;
|
|
642
|
-
}
|
|
643
|
-
}
|
|
644
|
-
|
|
645
678
|
/**
|
|
646
679
|
* Called when an outbound reference is added to a node. This is used to identify all nodes that have been
|
|
647
680
|
* referenced between summaries so that their unreferenced timestamp can be reset.
|
|
648
681
|
*
|
|
649
|
-
* @param
|
|
650
|
-
* @param
|
|
682
|
+
* @param fromNodePath - The node from which the reference is added.
|
|
683
|
+
* @param toNodePath - The node to which the reference is added.
|
|
651
684
|
*/
|
|
652
|
-
public addedOutboundReference(
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
685
|
+
public addedOutboundReference(fromNodePath: string, toNodePath: string) {
|
|
686
|
+
if (!this.shouldRunGC) {
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
const outboundRoutes = this.referencesSinceLastRun.get(fromNodePath) ?? [];
|
|
691
|
+
outboundRoutes.push(toNodePath);
|
|
692
|
+
this.referencesSinceLastRun.set(fromNodePath, outboundRoutes);
|
|
693
|
+
|
|
694
|
+
// If the node that got referenced is inactive, log an event as that may indicate use-after-delete.
|
|
695
|
+
this.logIfInactive(
|
|
696
|
+
"Revived",
|
|
697
|
+
toNodePath,
|
|
698
|
+
this.getCurrentTimestampMs(),
|
|
699
|
+
this.getNodePackagePath(toNodePath),
|
|
700
|
+
);
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
public dispose(): void {
|
|
704
|
+
if (this.sessionExpiryTimer !== undefined) {
|
|
705
|
+
clearTimeout(this.sessionExpiryTimer);
|
|
706
|
+
this.sessionExpiryTimer = undefined;
|
|
707
|
+
}
|
|
656
708
|
}
|
|
657
709
|
|
|
658
710
|
/**
|
|
@@ -699,15 +751,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
699
751
|
for (const nodeId of gcResult.referencedNodeIds) {
|
|
700
752
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
701
753
|
if (nodeStateTracker !== undefined) {
|
|
702
|
-
// If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
|
|
703
|
-
// log an error as this may mean the deleteTimeoutMs is not long enough.
|
|
704
|
-
nodeStateTracker.logIfInactive(
|
|
705
|
-
this.mc.logger,
|
|
706
|
-
"inactiveObjectRevived",
|
|
707
|
-
currentTimestampMs,
|
|
708
|
-
this.deleteTimeoutMs,
|
|
709
|
-
nodeId,
|
|
710
|
-
);
|
|
711
754
|
// Stop tracking so as to clear out any running timers.
|
|
712
755
|
nodeStateTracker.stopTracking();
|
|
713
756
|
// Delete the node as we don't need to track it any more.
|
|
@@ -724,7 +767,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
724
767
|
* This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
|
|
725
768
|
* If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
|
|
726
769
|
*/
|
|
727
|
-
private
|
|
770
|
+
private updateStateSinceLastRun(currentGCData: IGarbageCollectionData) {
|
|
728
771
|
// If we haven't run GC before or no references were added since the last run, there is nothing to do.
|
|
729
772
|
if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
|
|
730
773
|
return;
|
|
@@ -783,8 +826,10 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
783
826
|
* @param currentGCData - The GC data (reference graph) from the current GC run.
|
|
784
827
|
*/
|
|
785
828
|
private validateReferenceCorrectness(currentGCData: IGarbageCollectionData) {
|
|
786
|
-
assert(
|
|
787
|
-
|
|
829
|
+
assert(
|
|
830
|
+
this.gcDataFromLastRun !== undefined,
|
|
831
|
+
0x2b7, /* "Can't validate correctness without GC data from last run" */
|
|
832
|
+
);
|
|
788
833
|
|
|
789
834
|
// Get a list of all the outbound routes (or references) in the current GC data.
|
|
790
835
|
const currentReferences: string[] = [];
|
|
@@ -811,9 +856,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
811
856
|
// Validate that the current reference graph doesn't have references that we are not already aware of. If this
|
|
812
857
|
// happens, it might indicate data corruption since we may delete objects prematurely.
|
|
813
858
|
currentReferences.forEach((route: string) => {
|
|
814
|
-
// Validate references for data stores only
|
|
815
|
-
//
|
|
816
|
-
if (route
|
|
859
|
+
// Validate references for data stores only. Currently, layers below data stores don't have GC implemented
|
|
860
|
+
// so there is no guarantee their references will be notified.
|
|
861
|
+
if (isDataStoreNode(route) && !explicitReferences.includes(route)) {
|
|
817
862
|
/**
|
|
818
863
|
* The following log will be enabled once this issue is resolved:
|
|
819
864
|
* https://github.com/microsoft/FluidFramework/issues/8878.
|
|
@@ -827,12 +872,113 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
827
872
|
}
|
|
828
873
|
});
|
|
829
874
|
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* Generates the stats of a garbage collection run from the given results of the run. Also, logs any pending events
|
|
878
|
+
* in the pendingEventsQueue.
|
|
879
|
+
* @param gcResult - The result of a GC run.
|
|
880
|
+
* @returns the GC stats of the GC run.
|
|
881
|
+
*/
|
|
882
|
+
private generateStatsAndLogEvents(gcResult: IGCResult): IGCStats {
|
|
883
|
+
// Log pending events for unreferenced nodes after GC has run. We should have the package data available for
|
|
884
|
+
// them now since the GC run should have loaded these nodes.
|
|
885
|
+
let event = this.pendingEventsQueue.shift();
|
|
886
|
+
while (event !== undefined) {
|
|
887
|
+
const pkg = this.getNodePackagePath(event.id);
|
|
888
|
+
this.mc.logger.sendErrorEvent({
|
|
889
|
+
...event,
|
|
890
|
+
pkg: pkg ? { value: `/${pkg.join("/")}`, tag: TelemetryDataTag.PackageData } : undefined,
|
|
891
|
+
});
|
|
892
|
+
event = this.pendingEventsQueue.shift();
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
const gcStats: IGCStats = {
|
|
896
|
+
nodeCount: 0,
|
|
897
|
+
dataStoreCount: 0,
|
|
898
|
+
unrefNodeCount: 0,
|
|
899
|
+
unrefDataStoreCount: 0,
|
|
900
|
+
updatedNodeCount: 0,
|
|
901
|
+
updatedDataStoreCount: 0,
|
|
902
|
+
};
|
|
903
|
+
|
|
904
|
+
for (const nodeId of gcResult.referencedNodeIds) {
|
|
905
|
+
gcStats.nodeCount++;
|
|
906
|
+
const isDataStore = isDataStoreNode(nodeId);
|
|
907
|
+
if (isDataStore) {
|
|
908
|
+
gcStats.dataStoreCount++;
|
|
909
|
+
}
|
|
910
|
+
// If a referenced node has an entry in `unreferencedNodesState`, it was previously unreferenced. So, its
|
|
911
|
+
// reference state updated from the last GC run.
|
|
912
|
+
if (this.unreferencedNodesState.has(nodeId)) {
|
|
913
|
+
gcStats.updatedNodeCount++;
|
|
914
|
+
if (isDataStore) {
|
|
915
|
+
gcStats.updatedDataStoreCount++;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
for (const nodeId of gcResult.deletedNodeIds) {
|
|
921
|
+
gcStats.nodeCount++;
|
|
922
|
+
gcStats.unrefNodeCount++;
|
|
923
|
+
const isDataStore = isDataStoreNode(nodeId);
|
|
924
|
+
if (isDataStore) {
|
|
925
|
+
gcStats.dataStoreCount++;
|
|
926
|
+
gcStats.unrefDataStoreCount++;
|
|
927
|
+
}
|
|
928
|
+
// If an unreferenced node doesn't an entry in `unreferencedNodesState`, it was previously referenced. So,
|
|
929
|
+
// its reference state updated from the last GC run.
|
|
930
|
+
if (!this.unreferencedNodesState.has(nodeId)) {
|
|
931
|
+
gcStats.updatedNodeCount++;
|
|
932
|
+
if (isDataStore) {
|
|
933
|
+
gcStats.updatedDataStoreCount++;
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
return gcStats;
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
/**
|
|
942
|
+
* Logs an event if a node is inactive and is used. If the package data for the node exists, log immediately. Else,
|
|
943
|
+
* queue it and it will be logged the next time GC runs as the package data should be available then.
|
|
944
|
+
*/
|
|
945
|
+
private logIfInactive(
|
|
946
|
+
eventSuffix: "Changed" | "Loaded" | "Revived",
|
|
947
|
+
nodeId: string,
|
|
948
|
+
timestampMs: number,
|
|
949
|
+
packagePath?: readonly string[],
|
|
950
|
+
requestHeaders?: IRequestHeader,
|
|
951
|
+
) {
|
|
952
|
+
const eventName = `inactiveObject_${eventSuffix}`;
|
|
953
|
+
// We log a particular event for a given node only once so that it is not too noisy.
|
|
954
|
+
const uniqueEventId = `${nodeId}-${eventName}`;
|
|
955
|
+
const nodeState = this.unreferencedNodesState.get(nodeId);
|
|
956
|
+
if (nodeState?.inactive && !this.loggedUnreferencedEvents.has(uniqueEventId)) {
|
|
957
|
+
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
958
|
+
const event: IUnreferencedEvent = {
|
|
959
|
+
eventName,
|
|
960
|
+
id: nodeId,
|
|
961
|
+
age: timestampMs - nodeState.unreferencedTimestampMs,
|
|
962
|
+
timeout: this.deleteTimeoutMs,
|
|
963
|
+
externalRequest: requestHeaders?.[RuntimeHeaders.externalRequest],
|
|
964
|
+
viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
|
|
965
|
+
};
|
|
966
|
+
if (packagePath !== undefined) {
|
|
967
|
+
this.mc.logger.sendErrorEvent({
|
|
968
|
+
...event,
|
|
969
|
+
pkg: { value: `/${packagePath.join("/")}`, tag: TelemetryDataTag.PackageData },
|
|
970
|
+
});
|
|
971
|
+
} else {
|
|
972
|
+
this.pendingEventsQueue.push(event);
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
}
|
|
830
976
|
}
|
|
831
977
|
|
|
832
978
|
/**
|
|
833
979
|
* Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
|
|
834
980
|
* Merge the GC state from all such blobs and return the merged GC state.
|
|
835
|
-
*/
|
|
981
|
+
*/
|
|
836
982
|
async function getGCStateFromSnapshot(
|
|
837
983
|
gcSnapshotTree: ISnapshotTree,
|
|
838
984
|
readAndParseBlob: ReadAndParseBlob,
|
|
@@ -855,3 +1001,35 @@ async function getGCStateFromSnapshot(
|
|
|
855
1001
|
}
|
|
856
1002
|
return rootGCState;
|
|
857
1003
|
}
|
|
1004
|
+
|
|
1005
|
+
/**
|
|
1006
|
+
* setLongTimeout is used for timeouts longer than setTimeout's ~24.8 day max
|
|
1007
|
+
* @param timeoutMs - the total time the timeout needs to last in ms
|
|
1008
|
+
* @param timeoutFn - the function to execute when the timer ends
|
|
1009
|
+
* @param setTimerFn - the function used to update your timer variable
|
|
1010
|
+
*/
|
|
1011
|
+
function setLongTimeout(
|
|
1012
|
+
timeoutMs: number,
|
|
1013
|
+
timeoutFn: () => void,
|
|
1014
|
+
setTimerFn: (timer: ReturnType<typeof setTimeout>) => void,
|
|
1015
|
+
) {
|
|
1016
|
+
// The setTimeout max is 24.8 days before looping occurs.
|
|
1017
|
+
const maxTimeout = 2147483647;
|
|
1018
|
+
let timer: ReturnType<typeof setTimeout>;
|
|
1019
|
+
if (timeoutMs > maxTimeout) {
|
|
1020
|
+
const newTimeoutMs = timeoutMs - maxTimeout;
|
|
1021
|
+
timer = setTimeout(() => setLongTimeout(newTimeoutMs, timeoutFn, setTimerFn), maxTimeout);
|
|
1022
|
+
} else {
|
|
1023
|
+
timer = setTimeout(() => timeoutFn(), timeoutMs);
|
|
1024
|
+
}
|
|
1025
|
+
setTimerFn(timer);
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
/**
|
|
1029
|
+
* Given a GC nodeId, tells whether it belongs to a data store or not.
|
|
1030
|
+
*/
|
|
1031
|
+
function isDataStoreNode(nodeId: string): boolean {
|
|
1032
|
+
const pathParts = nodeId.split("/");
|
|
1033
|
+
// Data store ids are in the format "/dataStoreId".
|
|
1034
|
+
return pathParts.length === 2 && pathParts[1] !== "" ? true : false;
|
|
1035
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -10,13 +10,16 @@ export {
|
|
|
10
10
|
IGCRuntimeOptions,
|
|
11
11
|
ISummaryRuntimeOptions,
|
|
12
12
|
IContainerRuntimeOptions,
|
|
13
|
+
IRootSummaryTreeWithStats,
|
|
13
14
|
isRuntimeMessage,
|
|
14
15
|
RuntimeMessage,
|
|
15
16
|
unpackRuntimeMessage,
|
|
16
17
|
ScheduleManager,
|
|
17
18
|
agentSchedulerId,
|
|
18
19
|
ContainerRuntime,
|
|
20
|
+
RuntimeHeaders,
|
|
19
21
|
} from "./containerRuntime";
|
|
22
|
+
export { IDataStore, AliasResult } from "./dataStore";
|
|
20
23
|
export { DeltaScheduler } from "./deltaScheduler";
|
|
21
24
|
export { FluidDataStoreRegistry } from "./dataStoreRegistry";
|
|
22
25
|
export {
|
|
@@ -24,7 +27,6 @@ export {
|
|
|
24
27
|
gcTreeKey,
|
|
25
28
|
IGarbageCollectionRuntime,
|
|
26
29
|
IGCStats,
|
|
27
|
-
IUsedStateStats,
|
|
28
30
|
} from "./garbageCollection";
|
|
29
31
|
export {
|
|
30
32
|
IPendingFlush,
|
package/src/packageVersion.ts
CHANGED