@fluidframework/container-runtime 0.53.0 → 0.54.0-47413
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerRuntime.d.ts +25 -16
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +125 -77
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +29 -3
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +29 -4
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +7 -3
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +54 -5
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +22 -2
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +112 -34
- package/dist/garbageCollection.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +3 -2
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +6 -6
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizer.d.ts +22 -0
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +135 -33
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +1 -8
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryFormat.d.ts +1 -0
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js +2 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryManager.d.ts +0 -15
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +1 -35
- package/dist/summaryManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +25 -16
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +131 -83
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +29 -3
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +29 -4
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +7 -3
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +54 -5
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +22 -2
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +114 -36
- package/lib/garbageCollection.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +3 -2
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +6 -6
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizer.d.ts +22 -0
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +135 -33
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +1 -8
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryFormat.d.ts +1 -0
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js +1 -0
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryManager.d.ts +0 -15
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +1 -34
- package/lib/summaryManager.js.map +1 -1
- package/package.json +13 -13
- package/src/containerRuntime.ts +176 -93
- package/src/dataStoreContext.ts +44 -6
- package/src/dataStores.ts +84 -4
- package/src/garbageCollection.ts +137 -46
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +12 -10
- package/src/summarizer.ts +154 -38
- package/src/summarizerTypes.ts +2 -9
- package/src/summaryFormat.ts +1 -0
- package/src/summaryManager.ts +2 -49
- package/dist/localStorageFeatureGates.d.ts +0 -13
- package/dist/localStorageFeatureGates.d.ts.map +0 -1
- package/dist/localStorageFeatureGates.js +0 -31
- package/dist/localStorageFeatureGates.js.map +0 -1
- package/lib/localStorageFeatureGates.d.ts +0 -13
- package/lib/localStorageFeatureGates.d.ts.map +0 -1
- package/lib/localStorageFeatureGates.js +0 -27
- package/lib/localStorageFeatureGates.js.map +0 -1
- package/src/localStorageFeatureGates.ts +0 -27
package/src/dataStoreContext.ts
CHANGED
|
@@ -30,7 +30,7 @@ import { BlobTreeEntry } from "@fluidframework/protocol-base";
|
|
|
30
30
|
import {
|
|
31
31
|
IClientDetails,
|
|
32
32
|
IDocumentMessage,
|
|
33
|
-
|
|
33
|
+
IQuorumClients,
|
|
34
34
|
ISequencedDocumentMessage,
|
|
35
35
|
ISnapshotTree,
|
|
36
36
|
ITreeEntry,
|
|
@@ -67,6 +67,7 @@ import {
|
|
|
67
67
|
ThresholdCounter,
|
|
68
68
|
} from "@fluidframework/telemetry-utils";
|
|
69
69
|
import { CreateProcessingError } from "@fluidframework/container-utils";
|
|
70
|
+
|
|
70
71
|
import { ContainerRuntime } from "./containerRuntime";
|
|
71
72
|
import {
|
|
72
73
|
dataStoreAttributesBlobName,
|
|
@@ -386,7 +387,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
386
387
|
this.channel?.processSignal(message, local);
|
|
387
388
|
}
|
|
388
389
|
|
|
389
|
-
public getQuorum():
|
|
390
|
+
public getQuorum(): IQuorumClients {
|
|
390
391
|
return this._containerRuntime.getQuorum();
|
|
391
392
|
}
|
|
392
393
|
|
|
@@ -496,9 +497,19 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
496
497
|
}
|
|
497
498
|
}
|
|
498
499
|
|
|
500
|
+
/**
|
|
501
|
+
* Called when a new outbound reference is added to another node. This is used by garbage collection to identify
|
|
502
|
+
* all references added in the system.
|
|
503
|
+
* @param srcHandle - The handle of the node that added the reference.
|
|
504
|
+
* @param outboundHandle - The handle of the outbound node that is referenced.
|
|
505
|
+
*/
|
|
506
|
+
public addedGCOutboundReference(srcHandle: IFluidHandle, outboundHandle: IFluidHandle) {
|
|
507
|
+
this._containerRuntime.addedGCOutboundReference(srcHandle, outboundHandle);
|
|
508
|
+
}
|
|
509
|
+
|
|
499
510
|
/**
|
|
500
511
|
* Updates the used routes of the channel and its child contexts. The channel must be loaded before calling this.
|
|
501
|
-
* It is called in these two
|
|
512
|
+
* It is called in these two scenarios:
|
|
502
513
|
* 1. When the used routes of the data store is updated and the data store is loaded.
|
|
503
514
|
* 2. When the data store is realized. This updates the channel's used routes as per last GC run.
|
|
504
515
|
*/
|
|
@@ -582,7 +593,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
582
593
|
try
|
|
583
594
|
{
|
|
584
595
|
assert(!this.detachedRuntimeCreation, 0x148 /* "Detached runtime creation on runtime bind" */);
|
|
585
|
-
assert(this.channelDeferred !== undefined, 0x149 /* "Undefined channel
|
|
596
|
+
assert(this.channelDeferred !== undefined, 0x149 /* "Undefined channel deferral" */);
|
|
586
597
|
assert(this.pkg !== undefined, 0x14a /* "Undefined package path" */);
|
|
587
598
|
|
|
588
599
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
@@ -634,6 +645,13 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
634
645
|
|
|
635
646
|
protected abstract getInitialSnapshotDetails(): Promise<ISnapshotDetails>;
|
|
636
647
|
|
|
648
|
+
/**
|
|
649
|
+
* @deprecated - Sets the datastore as root, for aliasing purposes: #7948
|
|
650
|
+
* This method should not be used outside of the aliasing context.
|
|
651
|
+
* It will be removed, as the source of truth for this flag will be the aliasing blob.
|
|
652
|
+
*/
|
|
653
|
+
public abstract setRoot(): void;
|
|
654
|
+
|
|
637
655
|
public abstract getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails>;
|
|
638
656
|
|
|
639
657
|
public reSubmit(contents: any, localOpMetadata: unknown) {
|
|
@@ -679,6 +697,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
679
697
|
}
|
|
680
698
|
|
|
681
699
|
export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
|
|
700
|
+
private isRootDataStore: boolean | undefined;
|
|
701
|
+
|
|
682
702
|
constructor(
|
|
683
703
|
id: string,
|
|
684
704
|
private readonly initSnapshotValue: ISnapshotTree | string | undefined,
|
|
@@ -750,7 +770,7 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
750
770
|
* data stores in older documents are not garbage collected incorrectly. This may lead to additional
|
|
751
771
|
* roots in the document but they won't break.
|
|
752
772
|
*/
|
|
753
|
-
isRootDataStore = attributes.isRootDataStore ?? true;
|
|
773
|
+
isRootDataStore = this.isRootDataStore === true || (attributes.isRootDataStore ?? true);
|
|
754
774
|
|
|
755
775
|
if (hasIsolatedChannels(attributes)) {
|
|
756
776
|
tree = tree.trees[channelsTreeName];
|
|
@@ -782,6 +802,15 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
782
802
|
public generateAttachMessage(): IAttachMessage {
|
|
783
803
|
throw new Error("Cannot attach remote store");
|
|
784
804
|
}
|
|
805
|
+
|
|
806
|
+
/**
|
|
807
|
+
* @deprecated - Sets the datastore as root, for aliasing purposes: #7948
|
|
808
|
+
* This method should not be used outside of the aliasing context.
|
|
809
|
+
* It will be removed, as the source of truth for this flag will be the aliasing blob.
|
|
810
|
+
*/
|
|
811
|
+
public setRoot(): void {
|
|
812
|
+
this.isRootDataStore = true;
|
|
813
|
+
}
|
|
785
814
|
}
|
|
786
815
|
|
|
787
816
|
/**
|
|
@@ -883,7 +912,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
|
|
|
883
912
|
// If there is no isRootDataStore in the attributes blob, set it to true. This ensures that data
|
|
884
913
|
// stores in older documents are not garbage collected incorrectly. This may lead to additional
|
|
885
914
|
// roots in the document but they won't break.
|
|
886
|
-
this.isRootDataStore = attributes.isRootDataStore ?? true;
|
|
915
|
+
this.isRootDataStore = this.isRootDataStore || (attributes.isRootDataStore ?? true);
|
|
887
916
|
}
|
|
888
917
|
}
|
|
889
918
|
assert(this.pkg !== undefined, 0x152 /* "pkg should be available in local data store" */);
|
|
@@ -901,6 +930,15 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
|
|
|
901
930
|
// Local data store does not have initial summary.
|
|
902
931
|
return {};
|
|
903
932
|
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* @deprecated - Sets the datastore as root, for aliasing purposes: #7948
|
|
936
|
+
* This method should not be used outside of the aliasing context.
|
|
937
|
+
* It will be removed, as the source of truth for this flag will be the aliasing blob.
|
|
938
|
+
*/
|
|
939
|
+
public setRoot(): void {
|
|
940
|
+
this.isRootDataStore = true;
|
|
941
|
+
}
|
|
904
942
|
}
|
|
905
943
|
|
|
906
944
|
/**
|
package/src/dataStores.ts
CHANGED
|
@@ -50,6 +50,32 @@ import {
|
|
|
50
50
|
import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summaryFormat";
|
|
51
51
|
import { IUsedStateStats } from "./garbageCollection";
|
|
52
52
|
|
|
53
|
+
type PendingAliasResolve = (success: boolean) => void;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Interface for an op to be used for assigning an
|
|
57
|
+
* alias to a datastore
|
|
58
|
+
*/
|
|
59
|
+
interface IDataStoreAliasMessage {
|
|
60
|
+
/** The internal id of the datastore */
|
|
61
|
+
readonly internalId: string;
|
|
62
|
+
/** The alias name to be assigned to the datastore */
|
|
63
|
+
readonly alias: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Type guard that returns true if the given alias message is actually an instance of
|
|
68
|
+
* a class which implements @see IDataStoreAliasMessage
|
|
69
|
+
* @param maybeDataStoreAliasMessage - message object to be validated
|
|
70
|
+
* @returns True if the @see IDataStoreAliasMessage is fully implemented, false otherwise
|
|
71
|
+
*/
|
|
72
|
+
const isDataStoreAliasMessage = (
|
|
73
|
+
maybeDataStoreAliasMessage: any,
|
|
74
|
+
): maybeDataStoreAliasMessage is IDataStoreAliasMessage => {
|
|
75
|
+
return typeof maybeDataStoreAliasMessage?.internalId === "string"
|
|
76
|
+
&& typeof maybeDataStoreAliasMessage?.alias === "string";
|
|
77
|
+
};
|
|
78
|
+
|
|
53
79
|
/**
|
|
54
80
|
* This class encapsulates data store handling. Currently it is only used by the container runtime,
|
|
55
81
|
* but eventually could be hosted on any channel once we formalize the channel api boundary.
|
|
@@ -81,6 +107,7 @@ export class DataStores implements IDisposable {
|
|
|
81
107
|
baseLogger: ITelemetryBaseLogger,
|
|
82
108
|
getDataStoreBaseGCDetails: () => Promise<Map<string, IGarbageCollectionSummaryDetails>>,
|
|
83
109
|
private readonly dataStoreChanged: (id: string) => void,
|
|
110
|
+
private readonly aliasMap: Map<string, string>,
|
|
84
111
|
private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
|
|
85
112
|
) {
|
|
86
113
|
this.logger = ChildLogger.create(baseLogger);
|
|
@@ -146,6 +173,10 @@ export class DataStores implements IDisposable {
|
|
|
146
173
|
};
|
|
147
174
|
}
|
|
148
175
|
|
|
176
|
+
public aliases(): ReadonlyMap<string, string> {
|
|
177
|
+
return this.aliasMap;
|
|
178
|
+
}
|
|
179
|
+
|
|
149
180
|
public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
|
|
150
181
|
const attachMessage = message.contents as InboundAttachMessage;
|
|
151
182
|
// The local object has already been attached
|
|
@@ -205,7 +236,6 @@ export class DataStores implements IDisposable {
|
|
|
205
236
|
}),
|
|
206
237
|
pkg);
|
|
207
238
|
|
|
208
|
-
// Resolve pending gets and store off any new ones
|
|
209
239
|
this.contexts.addBoundOrRemoted(remotedFluidDataStoreContext);
|
|
210
240
|
|
|
211
241
|
// Equivalent of nextTick() - Prefetch once all current ops have completed
|
|
@@ -213,6 +243,55 @@ export class DataStores implements IDisposable {
|
|
|
213
243
|
Promise.resolve().then(async () => remotedFluidDataStoreContext.realize());
|
|
214
244
|
}
|
|
215
245
|
|
|
246
|
+
public processAliasMessage(
|
|
247
|
+
message: ISequencedDocumentMessage,
|
|
248
|
+
localOpMetadata: unknown,
|
|
249
|
+
local: boolean,
|
|
250
|
+
): void {
|
|
251
|
+
const aliasMessage = message.contents as IDataStoreAliasMessage;
|
|
252
|
+
if (!isDataStoreAliasMessage(aliasMessage)) {
|
|
253
|
+
throw new DataCorruptionError(
|
|
254
|
+
"malformedDataStoreAliasMessage",
|
|
255
|
+
{
|
|
256
|
+
...extractSafePropertiesFromMessage(message),
|
|
257
|
+
},
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const resolve = localOpMetadata as PendingAliasResolve;
|
|
262
|
+
const aliasResult = this.processAliasMessageCore(aliasMessage);
|
|
263
|
+
if (local) {
|
|
264
|
+
resolve(aliasResult);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
private processAliasMessageCore(aliasMessage: IDataStoreAliasMessage): boolean {
|
|
269
|
+
const existingMapping = this.aliasMap.get(aliasMessage.alias);
|
|
270
|
+
if (existingMapping !== undefined) {
|
|
271
|
+
return false;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Unlikely scenario, but we may receive an alias OP with the alias value
|
|
275
|
+
// equal to one of the ids supplied to `createRootDataStore` in the past
|
|
276
|
+
const maybeContextWithAliasAsId = this.contexts.get(aliasMessage.alias);
|
|
277
|
+
if (maybeContextWithAliasAsId !== undefined) {
|
|
278
|
+
return false;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const currentContext = this.contexts.get(aliasMessage.internalId);
|
|
282
|
+
if (currentContext === undefined) {
|
|
283
|
+
this.logger.sendErrorEvent({
|
|
284
|
+
eventName: "AliasFluidDataStoreNotFound",
|
|
285
|
+
fluidDataStoreId: aliasMessage.internalId,
|
|
286
|
+
});
|
|
287
|
+
return false;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
this.aliasMap.set(aliasMessage.alias, currentContext.id);
|
|
291
|
+
currentContext.setRoot();
|
|
292
|
+
return true;
|
|
293
|
+
}
|
|
294
|
+
|
|
216
295
|
public bindFluidDataStore(fluidDataStoreRuntime: IFluidDataStoreChannel): void {
|
|
217
296
|
const id = fluidDataStoreRuntime.id;
|
|
218
297
|
const localContext = this.contexts.getUnbound(id);
|
|
@@ -304,8 +383,9 @@ export class DataStores implements IDisposable {
|
|
|
304
383
|
}
|
|
305
384
|
|
|
306
385
|
public async getDataStore(id: string, wait: boolean): Promise<FluidDataStoreContext> {
|
|
307
|
-
const
|
|
386
|
+
const internalId = this.aliasMap.get(id) ?? id;
|
|
308
387
|
|
|
388
|
+
const context = await this.contexts.getBoundOrRemoted(internalId, wait);
|
|
309
389
|
if (context === undefined) {
|
|
310
390
|
// The requested data store does not exits. Throw a 404 response exception.
|
|
311
391
|
const request = { url: id };
|
|
@@ -421,8 +501,8 @@ export class DataStores implements IDisposable {
|
|
|
421
501
|
/**
|
|
422
502
|
* Generates data used for garbage collection. It does the following:
|
|
423
503
|
* 1. Calls into each child data store context to get its GC data.
|
|
424
|
-
* 2.
|
|
425
|
-
*
|
|
504
|
+
* 2. Prefixes the child context's id to the GC nodes in the child's GC data. This makes sure that the node can be
|
|
505
|
+
* identified as belonging to the child.
|
|
426
506
|
* 3. Adds a GC node for this channel to the nodes received from the children. All these nodes together represent
|
|
427
507
|
* the GC data of this channel.
|
|
428
508
|
* @param fullGC - true to bypass optimizations and force full generation of GC data.
|
package/src/garbageCollection.ts
CHANGED
|
@@ -6,16 +6,17 @@
|
|
|
6
6
|
import { ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
7
|
import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
|
|
8
8
|
import {
|
|
9
|
+
cloneGCData,
|
|
9
10
|
concatGarbageCollectionStates,
|
|
10
|
-
|
|
11
|
+
concatGarbageCollectionData,
|
|
11
12
|
IGCResult,
|
|
12
13
|
runGarbageCollection,
|
|
14
|
+
unpackChildNodesGCDetails,
|
|
13
15
|
} from "@fluidframework/garbage-collector";
|
|
14
16
|
import { ISnapshotTree } from "@fluidframework/protocol-definitions";
|
|
15
17
|
import {
|
|
16
18
|
gcBlobKey,
|
|
17
19
|
IGarbageCollectionData,
|
|
18
|
-
IGarbageCollectionNodeData,
|
|
19
20
|
IGarbageCollectionState,
|
|
20
21
|
IGarbageCollectionSummaryDetails,
|
|
21
22
|
ISummaryTreeWithStats,
|
|
@@ -25,11 +26,15 @@ import {
|
|
|
25
26
|
RefreshSummaryResult,
|
|
26
27
|
SummaryTreeBuilder,
|
|
27
28
|
} from "@fluidframework/runtime-utils";
|
|
28
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
ChildLogger,
|
|
31
|
+
loggerToMonitoringContext,
|
|
32
|
+
MonitoringContext,
|
|
33
|
+
PerformanceEvent,
|
|
34
|
+
} from "@fluidframework/telemetry-utils";
|
|
29
35
|
|
|
30
36
|
import { IGCRuntimeOptions } from "./containerRuntime";
|
|
31
37
|
import { getSummaryForDatastores } from "./dataStores";
|
|
32
|
-
import { getLocalStorageFeatureGate } from "./localStorageFeatureGates";
|
|
33
38
|
import {
|
|
34
39
|
getGCVersion,
|
|
35
40
|
GCVersion,
|
|
@@ -48,11 +53,11 @@ export const gcTreeKey = "gc";
|
|
|
48
53
|
export const gcBlobPrefix = "__gc";
|
|
49
54
|
|
|
50
55
|
// Local storage key to turn GC on / off.
|
|
51
|
-
const runGCKey = "
|
|
56
|
+
const runGCKey = "Fluid.GarbageCollection.RunGC";
|
|
52
57
|
// Local storage key to turn GC test mode on / off.
|
|
53
|
-
const gcTestModeKey = "
|
|
58
|
+
const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
54
59
|
// Local storage key to turn GC sweep on / off.
|
|
55
|
-
const runSweepKey = "
|
|
60
|
+
const runSweepKey = "Fluid.GarbageCollection.RunSweep";
|
|
56
61
|
|
|
57
62
|
const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
|
|
58
63
|
|
|
@@ -104,6 +109,8 @@ export interface IGarbageCollector {
|
|
|
104
109
|
latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;
|
|
105
110
|
/** Called when a node is changed. Used to detect and log when an inactive node is changed. */
|
|
106
111
|
nodeChanged(id: string): void;
|
|
112
|
+
/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
|
|
113
|
+
addedOutboundReference(fromNodeId: string, toNodeId: string): void;
|
|
107
114
|
}
|
|
108
115
|
|
|
109
116
|
/**
|
|
@@ -148,9 +155,9 @@ class UnreferencedStateTracker {
|
|
|
148
155
|
if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
|
|
149
156
|
logger.sendErrorEvent({
|
|
150
157
|
eventName,
|
|
151
|
-
|
|
152
|
-
deleteTimeoutMs,
|
|
153
|
-
inactiveNodeId,
|
|
158
|
+
age: currentTimestampMs - this.unreferencedTimestampMs,
|
|
159
|
+
timeout: deleteTimeoutMs,
|
|
160
|
+
id: inactiveNodeId,
|
|
154
161
|
});
|
|
155
162
|
this.inactiveEventsLogged.add(eventName);
|
|
156
163
|
}
|
|
@@ -217,7 +224,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
217
224
|
private readonly gcEnabled: boolean;
|
|
218
225
|
private readonly shouldRunSweep: boolean;
|
|
219
226
|
private readonly testMode: boolean;
|
|
220
|
-
private readonly
|
|
227
|
+
private readonly mc: MonitoringContext;
|
|
221
228
|
|
|
222
229
|
/**
|
|
223
230
|
* Tells whether the GC data should be written to the root of the summary tree. We do this under 2 conditions:
|
|
@@ -235,8 +242,11 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
235
242
|
// This is the version of GC data in the latest summary being tracked.
|
|
236
243
|
private latestSummaryGCVersion: GCVersion;
|
|
237
244
|
|
|
238
|
-
//
|
|
239
|
-
private
|
|
245
|
+
// Keeps track of the GC state from the last run.
|
|
246
|
+
private gcDataFromLastRun: IGarbageCollectionData | undefined;
|
|
247
|
+
// Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
|
|
248
|
+
// outbound routes from that node.
|
|
249
|
+
private readonly referencesSinceLastRun: Map<string, string[]> = new Map();
|
|
240
250
|
|
|
241
251
|
// Promise when resolved initializes the base state of the nodes from the base summary state.
|
|
242
252
|
private readonly initializeBaseStateP: Promise<void>;
|
|
@@ -260,7 +270,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
260
270
|
existing: boolean,
|
|
261
271
|
metadata?: IContainerRuntimeMetadata,
|
|
262
272
|
) {
|
|
263
|
-
this.
|
|
273
|
+
this.mc = loggerToMonitoringContext(
|
|
274
|
+
ChildLogger.create(baseLogger, "GarbageCollector"));
|
|
264
275
|
|
|
265
276
|
this.deleteTimeoutMs = this.gcOptions.deleteTimeoutMs ?? defaultDeleteTimeoutMs;
|
|
266
277
|
|
|
@@ -281,7 +292,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
281
292
|
this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;
|
|
282
293
|
|
|
283
294
|
// Whether GC should run or not. Can override with localStorage flag.
|
|
284
|
-
this.shouldRunGC =
|
|
295
|
+
this.shouldRunGC = this.mc.config.getBoolean(runGCKey) ?? (
|
|
285
296
|
// GC must be enabled for the document.
|
|
286
297
|
this.gcEnabled
|
|
287
298
|
// GC must not be disabled via GC options.
|
|
@@ -291,10 +302,10 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
291
302
|
// Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
|
|
292
303
|
// localStorage flag.
|
|
293
304
|
this.shouldRunSweep = this.shouldRunGC &&
|
|
294
|
-
(
|
|
305
|
+
(this.mc.config.getBoolean(runSweepKey) ?? gcOptions.runSweep === true);
|
|
295
306
|
|
|
296
307
|
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
297
|
-
this.testMode =
|
|
308
|
+
this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;
|
|
298
309
|
|
|
299
310
|
// If `writeDataAtRoot` GC option is true, we should write the GC data into the root of the summary tree. This
|
|
300
311
|
// GC option is used for testing only. It will be removed once we start writing GC data into root by default.
|
|
@@ -302,9 +313,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
302
313
|
|
|
303
314
|
// Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
|
|
304
315
|
// this once since it involves fetching blobs from storage which is expensive.
|
|
305
|
-
const baseSummaryStateP = new LazyPromise<IGarbageCollectionState>(async () => {
|
|
316
|
+
const baseSummaryStateP = new LazyPromise<IGarbageCollectionState | undefined>(async () => {
|
|
306
317
|
if (baseSnapshot === undefined) {
|
|
307
|
-
return
|
|
318
|
+
return undefined;
|
|
308
319
|
}
|
|
309
320
|
|
|
310
321
|
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
@@ -317,6 +328,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
317
328
|
|
|
318
329
|
// back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
|
|
319
330
|
// consolidate into IGarbageCollectionState format.
|
|
331
|
+
// Add a node for the root node that is not present in older snapshot format.
|
|
320
332
|
const gcState: IGarbageCollectionState = { gcNodes: { "/": { outboundRoutes: [] } } };
|
|
321
333
|
const dataStoreSnaphotTree = getSummaryForDatastores(baseSnapshot, metadata);
|
|
322
334
|
assert(dataStoreSnaphotTree !== undefined,
|
|
@@ -353,22 +365,28 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
353
365
|
0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);
|
|
354
366
|
gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
|
|
355
367
|
}
|
|
356
|
-
|
|
368
|
+
|
|
369
|
+
// If there is only one node (root node just added above), either GC is disabled or we are loading from the
|
|
370
|
+
// very first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
371
|
+
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
357
372
|
});
|
|
358
373
|
|
|
359
374
|
// Set up the initializer which initializes the base GC state from the base snapshot. Use lazy promise because
|
|
360
375
|
// we only do this once - the very first time we run GC.
|
|
361
376
|
this.initializeBaseStateP = new LazyPromise<void>(async () => {
|
|
362
|
-
const
|
|
377
|
+
const currentTimestampMs = this.getCurrentTimestampMs();
|
|
378
|
+
const baseState = await baseSummaryStateP;
|
|
379
|
+
if (baseState === undefined) {
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
363
382
|
|
|
364
|
-
const gcNodes: { [ id: string ]:
|
|
365
|
-
// Set up tracking for the nodes in the base summary state and add them to GC nodes.
|
|
383
|
+
const gcNodes: { [ id: string ]: string[] } = {};
|
|
366
384
|
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
367
385
|
const unreferencedTimestampMs = nodeData.unreferencedTimestampMs;
|
|
368
386
|
if (unreferencedTimestampMs !== undefined) {
|
|
369
387
|
// Get how long it has been since the node was unreferenced. Start a timeout for the remaining time
|
|
370
388
|
// left for it to be eligible for deletion.
|
|
371
|
-
const unreferencedDurationMs =
|
|
389
|
+
const unreferencedDurationMs = currentTimestampMs - unreferencedTimestampMs;
|
|
372
390
|
this.unreferencedNodesState.set(
|
|
373
391
|
nodeId,
|
|
374
392
|
new UnreferencedStateTracker(
|
|
@@ -377,20 +395,20 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
377
395
|
),
|
|
378
396
|
);
|
|
379
397
|
}
|
|
380
|
-
|
|
381
|
-
gcNodes[nodeId] = {
|
|
382
|
-
outboundRoutes: Array.from(nodeData.outboundRoutes),
|
|
383
|
-
unreferencedTimestampMs,
|
|
384
|
-
};
|
|
398
|
+
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
385
399
|
}
|
|
386
|
-
this.
|
|
400
|
+
this.gcDataFromLastRun = { gcNodes };
|
|
387
401
|
});
|
|
388
402
|
|
|
389
403
|
// Get the GC details for each data store from the GC state in the base summary. This is returned in
|
|
390
404
|
// getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.
|
|
391
405
|
this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionSummaryDetails>>(async () => {
|
|
392
|
-
const gcNodes: { [ id: string ]: string[] } = {};
|
|
393
406
|
const baseState = await baseSummaryStateP;
|
|
407
|
+
if (baseState === undefined) {
|
|
408
|
+
return new Map();
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const gcNodes: { [ id: string ]: string[] } = {};
|
|
394
412
|
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
395
413
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
396
414
|
}
|
|
@@ -400,7 +418,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
400
418
|
const usedRoutes = runGarbageCollection(
|
|
401
419
|
gcNodes,
|
|
402
420
|
[ "/" ],
|
|
403
|
-
this.logger,
|
|
421
|
+
this.mc.logger,
|
|
404
422
|
).referencedNodeIds;
|
|
405
423
|
|
|
406
424
|
const dataStoreGCDetailsMap = unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
|
|
@@ -433,7 +451,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
433
451
|
},
|
|
434
452
|
): Promise<IGCStats> {
|
|
435
453
|
const {
|
|
436
|
-
logger = this.logger,
|
|
454
|
+
logger = this.mc.logger,
|
|
437
455
|
runSweep = this.shouldRunSweep,
|
|
438
456
|
fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged,
|
|
439
457
|
} = options;
|
|
@@ -450,6 +468,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
450
468
|
|
|
451
469
|
// Get the runtime's GC data and run GC on the reference graph in it.
|
|
452
470
|
const gcData = await this.provider.getGCData(fullGC);
|
|
471
|
+
|
|
472
|
+
this.updateStateSinceLatestRun(gcData);
|
|
473
|
+
|
|
453
474
|
const gcResult = runGarbageCollection(
|
|
454
475
|
gcData.gcNodes,
|
|
455
476
|
[ "/" ],
|
|
@@ -489,13 +510,21 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
489
510
|
* We current write the entire GC state in a single blob. This can be modified later to write multiple
|
|
490
511
|
* blobs. All the blob keys should start with `gcBlobPrefix`.
|
|
491
512
|
*/
|
|
492
|
-
|
|
493
|
-
if (!this.shouldRunGC || this.
|
|
513
|
+
public summarize(): ISummaryTreeWithStats | undefined {
|
|
514
|
+
if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
494
515
|
return;
|
|
495
516
|
}
|
|
496
517
|
|
|
518
|
+
const gcState: IGarbageCollectionState = { gcNodes: {} };
|
|
519
|
+
for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
|
|
520
|
+
gcState.gcNodes[nodeId] = {
|
|
521
|
+
outboundRoutes,
|
|
522
|
+
unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
|
|
497
526
|
const builder = new SummaryTreeBuilder();
|
|
498
|
-
builder.addBlob(`${gcBlobPrefix}_root`, JSON.stringify(
|
|
527
|
+
builder.addBlob(`${gcBlobPrefix}_root`, JSON.stringify(gcState));
|
|
499
528
|
return builder.getSummaryTree();
|
|
500
529
|
}
|
|
501
530
|
|
|
@@ -537,7 +566,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
537
566
|
// Prefix "/" if needed to make it relative to the root.
|
|
538
567
|
const nodeId = id.startsWith("/") ? id : `/${id}`;
|
|
539
568
|
this.unreferencedNodesState.get(nodeId)?.logIfInactive(
|
|
540
|
-
this.logger,
|
|
569
|
+
this.mc.logger,
|
|
541
570
|
"inactiveObjectChanged",
|
|
542
571
|
this.getCurrentTimestampMs(),
|
|
543
572
|
this.deleteTimeoutMs,
|
|
@@ -545,6 +574,19 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
545
574
|
);
|
|
546
575
|
}
|
|
547
576
|
|
|
577
|
+
/**
|
|
578
|
+
* Called when an outbound reference is added to a node. This is used to identify all nodes that have been
|
|
579
|
+
* referenced between summaries so that their unreferenced timestamp can be reset.
|
|
580
|
+
*
|
|
581
|
+
* @param fromNodeId - The node from which the reference is added.
|
|
582
|
+
* @param toNodeId - The node to which the reference is added.
|
|
583
|
+
*/
|
|
584
|
+
public addedOutboundReference(fromNodeId: string, toNodeId: string) {
|
|
585
|
+
const outboundRoutes = this.referencesSinceLastRun.get(fromNodeId) ?? [];
|
|
586
|
+
outboundRoutes.push(toNodeId);
|
|
587
|
+
this.referencesSinceLastRun.set(fromNodeId, outboundRoutes);
|
|
588
|
+
}
|
|
589
|
+
|
|
548
590
|
/**
|
|
549
591
|
* Update the latest summary GC version from the metadata blob in the given snapshot.
|
|
550
592
|
*/
|
|
@@ -566,15 +608,11 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
566
608
|
* @param currentTimestampMs - The current timestamp to be used for unreferenced nodes' timestamp.
|
|
567
609
|
*/
|
|
568
610
|
private updateCurrentState(gcData: IGarbageCollectionData, gcResult: IGCResult, currentTimestampMs: number) {
|
|
569
|
-
this.
|
|
570
|
-
|
|
571
|
-
this.currentGCState.gcNodes[id] = { outboundRoutes: Array.from(outboundRoutes) };
|
|
572
|
-
}
|
|
611
|
+
this.gcDataFromLastRun = cloneGCData(gcData);
|
|
612
|
+
this.referencesSinceLastRun.clear();
|
|
573
613
|
|
|
574
614
|
// Iterate through the deleted nodes and start tracking if they became unreferenced in this run.
|
|
575
615
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
576
|
-
assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2aa /* "Unexpected node when running GC" */);
|
|
577
|
-
|
|
578
616
|
// The time when the node became unreferenced. This is added to the current GC state.
|
|
579
617
|
let unreferencedTimestampMs: number = currentTimestampMs;
|
|
580
618
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
@@ -587,18 +625,16 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
587
625
|
new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs),
|
|
588
626
|
);
|
|
589
627
|
}
|
|
590
|
-
this.currentGCState.gcNodes[nodeId].unreferencedTimestampMs = unreferencedTimestampMs;
|
|
591
628
|
}
|
|
592
629
|
|
|
593
630
|
// Iterate through the referenced nodes and stop tracking if they were unreferenced before.
|
|
594
631
|
for (const nodeId of gcResult.referencedNodeIds) {
|
|
595
|
-
assert(this.currentGCState.gcNodes[nodeId] !== undefined, 0x2ab /* "Unexpected node when running GC" */);
|
|
596
632
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
597
633
|
if (nodeStateTracker !== undefined) {
|
|
598
634
|
// If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
|
|
599
635
|
// log an error as this may mean the deleteTimeoutMs is not long enough.
|
|
600
636
|
nodeStateTracker.logIfInactive(
|
|
601
|
-
this.logger,
|
|
637
|
+
this.mc.logger,
|
|
602
638
|
"inactiveObjectRevived",
|
|
603
639
|
currentTimestampMs,
|
|
604
640
|
this.deleteTimeoutMs,
|
|
@@ -611,6 +647,61 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
611
647
|
}
|
|
612
648
|
}
|
|
613
649
|
}
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
|
|
653
|
+
* time. It's possible that nodes transition from `unreferenced -> referenced -> unreferenced` between two runs. The
|
|
654
|
+
* unreferenced timestamp of such nodes needs to be reset as they may have been accessed when they were referenced.
|
|
655
|
+
*
|
|
656
|
+
* This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
|
|
657
|
+
* If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
|
|
658
|
+
*/
|
|
659
|
+
private updateStateSinceLatestRun(currentGCData: IGarbageCollectionData) {
|
|
660
|
+
// If we haven't run GC before or no references were added since the last run, there is nothing to do.
|
|
661
|
+
if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
|
|
662
|
+
return;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
|
|
667
|
+
* edges that have been added since then. To do this, combine the GC data from the last run and the current
|
|
668
|
+
* run, and then add the references since last run.
|
|
669
|
+
*
|
|
670
|
+
* Note on why we need to combine the data from previous run, current run and all references in between -
|
|
671
|
+
* 1. We need data from last run because some of its references may have been deleted since then. If those
|
|
672
|
+
* references added new outbound references before getting deleted, we need to detect them.
|
|
673
|
+
* 2. We need new outbound references since last run because some of them may have been deleted later. If those
|
|
674
|
+
* references added new outbound references before getting deleted, we need to detect them.
|
|
675
|
+
* 3. We need data from the current run because currently we may not detect when DDSs are referenced:
|
|
676
|
+
* - We don't require DDSs handles to be stored in a referenced DDS. For this, we need GC at DDS level
|
|
677
|
+
* which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
|
|
678
|
+
* - A new data store may have "root" DDSs already created and we don't detect them today.
|
|
679
|
+
*/
|
|
680
|
+
const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
|
|
681
|
+
this.referencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
|
|
682
|
+
if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
|
|
683
|
+
gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
|
|
684
|
+
} else {
|
|
685
|
+
gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
|
|
686
|
+
}
|
|
687
|
+
});
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Run GC on the above reference graph to find all nodes that are referenced. For each one, if they are
|
|
691
|
+
* unreferenced, stop tracking them and remove from unreferenced list.
|
|
692
|
+
* Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
|
|
693
|
+
*/
|
|
694
|
+
const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/"], this.mc.logger);
|
|
695
|
+
for (const nodeId of gcResult.referencedNodeIds) {
|
|
696
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
697
|
+
if (nodeStateTracker !== undefined) {
|
|
698
|
+
// Stop tracking so as to clear out any running timers.
|
|
699
|
+
nodeStateTracker.stopTracking();
|
|
700
|
+
// Delete the node as we don't need to track it any more.
|
|
701
|
+
this.unreferencedNodesState.delete(nodeId);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
}
|
|
614
705
|
}
|
|
615
706
|
|
|
616
707
|
/**
|
package/src/packageVersion.ts
CHANGED