@fluidframework/container-runtime 2.0.0-internal.1.4.2 → 2.0.0-internal.2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/batchManager.d.ts +2 -3
- package/dist/batchManager.d.ts.map +1 -1
- package/dist/batchManager.js +3 -8
- package/dist/batchManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +43 -16
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +107 -83
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +4 -20
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +17 -47
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +2 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +3 -11
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +1 -10
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +43 -51
- package/dist/garbageCollection.js.map +1 -1
- package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
- package/dist/gcSweepReadyUsageDetection.js +3 -12
- package/dist/gcSweepReadyUsageDetection.js.map +1 -1
- package/dist/index.d.ts +3 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -5
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +6 -26
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +42 -62
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summarizer.js +7 -2
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +19 -2
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryFormat.d.ts +4 -2
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +10 -6
- package/dist/summaryManager.js.map +1 -1
- package/lib/batchManager.d.ts +2 -3
- package/lib/batchManager.d.ts.map +1 -1
- package/lib/batchManager.js +3 -8
- package/lib/batchManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +43 -16
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +108 -84
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +4 -20
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +18 -48
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +2 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +3 -11
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +1 -10
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +42 -50
- package/lib/garbageCollection.js.map +1 -1
- package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
- package/lib/gcSweepReadyUsageDetection.js +3 -12
- package/lib/gcSweepReadyUsageDetection.js.map +1 -1
- package/lib/index.d.ts +3 -5
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +0 -2
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +6 -26
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +42 -62
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summarizer.js +7 -2
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +19 -2
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryFormat.d.ts +4 -2
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +10 -6
- package/lib/summaryManager.js.map +1 -1
- package/package.json +40 -38
- package/src/batchManager.ts +7 -11
- package/src/containerRuntime.ts +149 -102
- package/src/dataStoreContext.ts +20 -62
- package/src/dataStores.ts +2 -10
- package/src/garbageCollection.ts +45 -55
- package/src/gcSweepReadyUsageDetection.ts +2 -10
- package/src/index.ts +2 -3
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +57 -96
- package/src/scheduleManager.ts +1 -0
- package/src/summarizer.ts +6 -6
- package/src/summarizerTypes.ts +20 -7
- package/src/summaryFormat.ts +4 -2
- package/src/summaryManager.ts +18 -7
package/src/dataStoreContext.ts
CHANGED
|
@@ -42,7 +42,6 @@ import {
|
|
|
42
42
|
CreateChildSummarizerNodeFn,
|
|
43
43
|
CreateChildSummarizerNodeParam,
|
|
44
44
|
FluidDataStoreRegistryEntry,
|
|
45
|
-
gcBlobKey,
|
|
46
45
|
IAttachMessage,
|
|
47
46
|
IFluidDataStoreChannel,
|
|
48
47
|
IFluidDataStoreContext,
|
|
@@ -51,7 +50,6 @@ import {
|
|
|
51
50
|
IFluidDataStoreRegistry,
|
|
52
51
|
IGarbageCollectionData,
|
|
53
52
|
IGarbageCollectionDetailsBase,
|
|
54
|
-
IGarbageCollectionSummaryDetails,
|
|
55
53
|
IInboundSignalMessage,
|
|
56
54
|
IProvideFluidDataStoreFactory,
|
|
57
55
|
ISummarizeInternalResult,
|
|
@@ -117,7 +115,6 @@ export interface IFluidDataStoreContextProps {
|
|
|
117
115
|
readonly storage: IDocumentStorageService;
|
|
118
116
|
readonly scope: FluidObject;
|
|
119
117
|
readonly createSummarizerNodeFn: CreateChildSummarizerNodeFn;
|
|
120
|
-
readonly writeGCDataAtRoot: boolean;
|
|
121
118
|
readonly pkg?: Readonly<string[]>;
|
|
122
119
|
}
|
|
123
120
|
|
|
@@ -135,7 +132,7 @@ export interface ILocalFluidDataStoreContextProps extends IFluidDataStoreContext
|
|
|
135
132
|
|
|
136
133
|
/** Properties necessary for creating a remote FluidDataStoreContext */
|
|
137
134
|
export interface IRemoteFluidDataStoreContextProps extends IFluidDataStoreContextProps {
|
|
138
|
-
readonly snapshotTree: ISnapshotTree |
|
|
135
|
+
readonly snapshotTree: ISnapshotTree | undefined;
|
|
139
136
|
readonly getBaseGCDetails: () => Promise<IGarbageCollectionDetailsBase | undefined>;
|
|
140
137
|
}
|
|
141
138
|
|
|
@@ -239,15 +236,14 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
239
236
|
private readonly thresholdOpsCounter: ThresholdCounter;
|
|
240
237
|
private static readonly pendingOpsCountThreshold = 1000;
|
|
241
238
|
|
|
242
|
-
// The used
|
|
239
|
+
// The used routes of this node as per the last GC run. This is used to update the used routes of the channel
|
|
243
240
|
// if it realizes after GC is run.
|
|
244
|
-
private
|
|
241
|
+
private lastUsedRoutes: string[] | undefined;
|
|
245
242
|
|
|
246
243
|
public readonly id: string;
|
|
247
244
|
private readonly _containerRuntime: ContainerRuntime;
|
|
248
245
|
public readonly storage: IDocumentStorageService;
|
|
249
246
|
public readonly scope: FluidObject;
|
|
250
|
-
private readonly writeGCDataAtRoot: boolean;
|
|
251
247
|
protected pkg?: readonly string[];
|
|
252
248
|
|
|
253
249
|
constructor(
|
|
@@ -263,7 +259,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
263
259
|
this.id = props.id;
|
|
264
260
|
this.storage = props.storage;
|
|
265
261
|
this.scope = props.scope;
|
|
266
|
-
this.writeGCDataAtRoot = props.writeGCDataAtRoot;
|
|
267
262
|
this.pkg = props.pkg;
|
|
268
263
|
|
|
269
264
|
// URIs use slashes as delimiters. Handles use URIs.
|
|
@@ -474,11 +469,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
474
469
|
const attributes = createAttributes(pkg, isRoot);
|
|
475
470
|
addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes));
|
|
476
471
|
|
|
477
|
-
// Add GC data to the summary if it's not written at the root.
|
|
478
|
-
if (!this.writeGCDataAtRoot) {
|
|
479
|
-
addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
|
|
480
|
-
}
|
|
481
|
-
|
|
482
472
|
// If we are not referenced, mark the summary tree as unreferenced. Also, update unreferenced blob
|
|
483
473
|
// size in the summary stats with the blobs size of this data store.
|
|
484
474
|
if (!this.summarizerNode.isReferenced()) {
|
|
@@ -532,21 +522,17 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
532
522
|
* 5. To update the timestamp when this data store or any children are marked as unreferenced.
|
|
533
523
|
*
|
|
534
524
|
* @param usedRoutes - The routes that are used in this data store.
|
|
535
|
-
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node becomes unreferenced
|
|
536
|
-
* as part of this GC run, this should be used to update the time when it happens.
|
|
537
525
|
*/
|
|
538
|
-
public updateUsedRoutes(usedRoutes: string[]
|
|
526
|
+
public updateUsedRoutes(usedRoutes: string[]) {
|
|
539
527
|
// Update the used routes in this data store's summarizer node.
|
|
540
|
-
this.summarizerNode.updateUsedRoutes(usedRoutes
|
|
528
|
+
this.summarizerNode.updateUsedRoutes(usedRoutes);
|
|
541
529
|
|
|
542
530
|
/**
|
|
543
|
-
*
|
|
544
|
-
*
|
|
545
|
-
*
|
|
546
|
-
* child's reference state changes, the gcTimestamp has to be used to update its unreferencedTimestamp. Since
|
|
547
|
-
* it will result in a change in this data store's used routes, it will be realized to regenerate its summary.
|
|
531
|
+
* Store the used routes to update the channel if the data store is not loaded yet. If the used routes changed
|
|
532
|
+
* since the previous run, the data store will be loaded during summarize since the used state changed. So, it's
|
|
533
|
+
* safe to only store the last used routes.
|
|
548
534
|
*/
|
|
549
|
-
this.
|
|
535
|
+
this.lastUsedRoutes = usedRoutes;
|
|
550
536
|
|
|
551
537
|
// If we are loaded, call the channel so it can update the used routes of the child contexts.
|
|
552
538
|
// If we are not loaded, we will update this when we are realized.
|
|
@@ -575,16 +561,16 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
575
561
|
assert(this.loaded, 0x144 /* "Channel should be loaded when updating used routes" */);
|
|
576
562
|
assert(this.channel !== undefined, 0x145 /* "Channel should be present when data store is loaded" */);
|
|
577
563
|
|
|
578
|
-
// If there is no
|
|
579
|
-
if (this.
|
|
564
|
+
// If there is no lastUsedRoutes, GC has not run up until this point.
|
|
565
|
+
if (this.lastUsedRoutes === undefined) {
|
|
580
566
|
return;
|
|
581
567
|
}
|
|
582
568
|
|
|
583
569
|
// Remove the route to this data store, if it exists.
|
|
584
|
-
const usedChannelRoutes = this.
|
|
570
|
+
const usedChannelRoutes = this.lastUsedRoutes.filter(
|
|
585
571
|
(id: string) => { return id !== "/" && id !== ""; },
|
|
586
572
|
);
|
|
587
|
-
this.channel.updateUsedRoutes(usedChannelRoutes
|
|
573
|
+
this.channel.updateUsedRoutes(usedChannelRoutes);
|
|
588
574
|
}
|
|
589
575
|
|
|
590
576
|
/**
|
|
@@ -718,11 +704,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
718
704
|
this._isInMemoryRoot = true;
|
|
719
705
|
}
|
|
720
706
|
|
|
721
|
-
/**
|
|
722
|
-
* @deprecated Renamed to `{@link FluidDataStoreContext.getBaseGCDetails}()`.
|
|
723
|
-
*/
|
|
724
|
-
public abstract getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails>;
|
|
725
|
-
|
|
726
707
|
public abstract getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
|
|
727
708
|
|
|
728
709
|
public reSubmit(contents: any, localOpMetadata: unknown) {
|
|
@@ -779,7 +760,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
779
760
|
}
|
|
780
761
|
|
|
781
762
|
export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
782
|
-
private readonly initSnapshotValue: ISnapshotTree |
|
|
763
|
+
private readonly initSnapshotValue: ISnapshotTree | undefined;
|
|
783
764
|
private readonly baseGCDetailsP: Promise<IGarbageCollectionDetailsBase>;
|
|
784
765
|
|
|
785
766
|
constructor(props: IRemoteFluidDataStoreContextProps) {
|
|
@@ -797,28 +778,20 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
797
778
|
this.baseGCDetailsP = new LazyPromise<IGarbageCollectionDetailsBase>(async () => {
|
|
798
779
|
return (await props.getBaseGCDetails()) ?? {};
|
|
799
780
|
});
|
|
781
|
+
|
|
782
|
+
if (props.snapshotTree !== undefined) {
|
|
783
|
+
this.summarizerNode.updateBaseSummaryState(props.snapshotTree);
|
|
784
|
+
}
|
|
800
785
|
}
|
|
801
786
|
|
|
802
787
|
private readonly initialSnapshotDetailsP = new LazyPromise<ISnapshotDetails>(async () => {
|
|
803
|
-
let tree
|
|
788
|
+
let tree = this.initSnapshotValue;
|
|
804
789
|
let isRootDataStore = true;
|
|
805
790
|
|
|
806
|
-
if (typeof this.initSnapshotValue === "string") {
|
|
807
|
-
const commit = (await this.storage.getVersions(this.initSnapshotValue, 1))[0];
|
|
808
|
-
tree = await this.storage.getSnapshotTree(commit) ?? undefined;
|
|
809
|
-
} else {
|
|
810
|
-
tree = this.initSnapshotValue;
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
const localReadAndParse = async <T>(id: string) => readAndParse<T>(this.storage, id);
|
|
814
|
-
if (tree) {
|
|
815
|
-
tree = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
|
|
816
|
-
}
|
|
817
|
-
|
|
818
791
|
if (!!tree && tree.blobs[dataStoreAttributesBlobName] !== undefined) {
|
|
819
792
|
// Need to get through snapshot and use that to populate extraBlobs
|
|
820
793
|
const attributes =
|
|
821
|
-
await
|
|
794
|
+
await readAndParse<ReadFluidDataStoreAttributes>(this.storage, tree.blobs[dataStoreAttributesBlobName]);
|
|
822
795
|
|
|
823
796
|
let pkgFromSnapshot: string[];
|
|
824
797
|
// Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot.
|
|
@@ -859,13 +832,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
859
832
|
return this.initialSnapshotDetailsP;
|
|
860
833
|
}
|
|
861
834
|
|
|
862
|
-
/**
|
|
863
|
-
* @deprecated Renamed to {@link RemoteFluidDataStoreContext.getBaseGCDetails}.
|
|
864
|
-
*/
|
|
865
|
-
public async getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails> {
|
|
866
|
-
return this.getBaseGCDetails();
|
|
867
|
-
}
|
|
868
|
-
|
|
869
835
|
public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
|
|
870
836
|
return this.baseGCDetailsP;
|
|
871
837
|
}
|
|
@@ -974,14 +940,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
|
|
|
974
940
|
};
|
|
975
941
|
}
|
|
976
942
|
|
|
977
|
-
/**
|
|
978
|
-
* @deprecated Renamed to {@link LocalFluidDataStoreContextBase.getBaseGCDetails}.
|
|
979
|
-
*/
|
|
980
|
-
public async getInitialGCSummaryDetails(): Promise<IGarbageCollectionSummaryDetails> {
|
|
981
|
-
// Local data store does not have initial summary.
|
|
982
|
-
return {};
|
|
983
|
-
}
|
|
984
|
-
|
|
985
943
|
public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
|
|
986
944
|
// Local data store does not have initial summary.
|
|
987
945
|
return {};
|
package/src/dataStores.ts
CHANGED
|
@@ -97,7 +97,6 @@ export class DataStores implements IDisposable {
|
|
|
97
97
|
private readonly gcNodeUpdated: (
|
|
98
98
|
nodePath: string, timestampMs: number, packagePath?: readonly string[]) => void,
|
|
99
99
|
private readonly aliasMap: Map<string, string>,
|
|
100
|
-
private readonly writeGCDataAtRoot: boolean,
|
|
101
100
|
private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
|
|
102
101
|
) {
|
|
103
102
|
this.logger = ChildLogger.create(baseLogger);
|
|
@@ -142,7 +141,6 @@ export class DataStores implements IDisposable {
|
|
|
142
141
|
key,
|
|
143
142
|
{ type: CreateSummarizerNodeSource.FromSummary },
|
|
144
143
|
),
|
|
145
|
-
writeGCDataAtRoot: this.writeGCDataAtRoot,
|
|
146
144
|
});
|
|
147
145
|
} else {
|
|
148
146
|
if (typeof value !== "object") {
|
|
@@ -162,7 +160,6 @@ export class DataStores implements IDisposable {
|
|
|
162
160
|
makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(key),
|
|
163
161
|
snapshotTree,
|
|
164
162
|
isRootDataStore: undefined,
|
|
165
|
-
writeGCDataAtRoot: this.writeGCDataAtRoot,
|
|
166
163
|
});
|
|
167
164
|
}
|
|
168
165
|
this.contexts.addBoundOrRemoted(dataStoreContext);
|
|
@@ -247,7 +244,6 @@ export class DataStores implements IDisposable {
|
|
|
247
244
|
},
|
|
248
245
|
},
|
|
249
246
|
),
|
|
250
|
-
writeGCDataAtRoot: this.writeGCDataAtRoot,
|
|
251
247
|
pkg,
|
|
252
248
|
});
|
|
253
249
|
|
|
@@ -351,7 +347,6 @@ export class DataStores implements IDisposable {
|
|
|
351
347
|
makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
|
|
352
348
|
snapshotTree: undefined,
|
|
353
349
|
isRootDataStore: isRoot,
|
|
354
|
-
writeGCDataAtRoot: this.writeGCDataAtRoot,
|
|
355
350
|
});
|
|
356
351
|
this.contexts.addUnbound(context);
|
|
357
352
|
return context;
|
|
@@ -372,7 +367,6 @@ export class DataStores implements IDisposable {
|
|
|
372
367
|
makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
|
|
373
368
|
snapshotTree: undefined,
|
|
374
369
|
isRootDataStore: false,
|
|
375
|
-
writeGCDataAtRoot: this.writeGCDataAtRoot,
|
|
376
370
|
createProps: props,
|
|
377
371
|
});
|
|
378
372
|
this.contexts.addUnbound(context);
|
|
@@ -595,10 +589,8 @@ export class DataStores implements IDisposable {
|
|
|
595
589
|
/**
|
|
596
590
|
* After GC has run, called to notify this Container's data stores of routes that are used in it.
|
|
597
591
|
* @param usedRoutes - The routes that are used in all data stores in this Container.
|
|
598
|
-
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
|
|
599
|
-
* unreferenced as part of this GC run, this should be used to update the time when it happens.
|
|
600
592
|
*/
|
|
601
|
-
public updateUsedRoutes(usedRoutes: string[]
|
|
593
|
+
public updateUsedRoutes(usedRoutes: string[]) {
|
|
602
594
|
// Get a map of data store ids to routes used in it.
|
|
603
595
|
const usedDataStoreRoutes = unpackChildNodesUsedRoutes(usedRoutes);
|
|
604
596
|
|
|
@@ -609,7 +601,7 @@ export class DataStores implements IDisposable {
|
|
|
609
601
|
|
|
610
602
|
// Update the used routes in each data store. Used routes is empty for unused data stores.
|
|
611
603
|
for (const [contextId, context] of this.contexts) {
|
|
612
|
-
context.updateUsedRoutes(usedDataStoreRoutes.get(contextId) ?? []
|
|
604
|
+
context.updateUsedRoutes(usedDataStoreRoutes.get(contextId) ?? []);
|
|
613
605
|
}
|
|
614
606
|
}
|
|
615
607
|
|
package/src/garbageCollection.ts
CHANGED
|
@@ -67,12 +67,8 @@ export const runGCKey = "Fluid.GarbageCollection.RunGC";
|
|
|
67
67
|
export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
|
|
68
68
|
// Feature gate key to turn GC test mode on / off.
|
|
69
69
|
export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
|
|
70
|
-
// Feature gate key to write GC data at the root of the summary tree.
|
|
71
|
-
const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
|
|
72
70
|
// Feature gate key to expire a session after a set period of time.
|
|
73
71
|
export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
|
|
74
|
-
// Feature gate key to disable expiring session after a set period of time, even if expiry value is present.
|
|
75
|
-
export const disableSessionExpiryKey = "Fluid.GarbageCollection.DisableSessionExpiry";
|
|
76
72
|
// Feature gate key to write the gc blob as a handle if the data is the same.
|
|
77
73
|
export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
|
|
78
74
|
// Feature gate key to turn GC sweep log off.
|
|
@@ -126,7 +122,7 @@ export interface IGarbageCollectionRuntime {
|
|
|
126
122
|
/** Returns the garbage collection data of the runtime. */
|
|
127
123
|
getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
|
|
128
124
|
/** After GC has run, called to notify the runtime of routes that are used in it. */
|
|
129
|
-
updateUsedRoutes(usedRoutes: string[]
|
|
125
|
+
updateUsedRoutes(usedRoutes: string[]): void;
|
|
130
126
|
/** After GC has run, called to delete objects in the runtime whose routes are unused. */
|
|
131
127
|
deleteUnusedRoutes(unusedRoutes: string[]): void;
|
|
132
128
|
/** Returns a referenced timestamp to be used to track unreferenced nodes. */
|
|
@@ -143,8 +139,6 @@ export interface IGarbageCollector {
|
|
|
143
139
|
readonly shouldRunGC: boolean;
|
|
144
140
|
/** Tells whether the GC state in summary needs to be reset in the next summary. */
|
|
145
141
|
readonly summaryStateNeedsReset: boolean;
|
|
146
|
-
/** Tells whether GC data should be written to the root of the summary tree. */
|
|
147
|
-
readonly writeDataAtRoot: boolean;
|
|
148
142
|
readonly trackGCState: boolean;
|
|
149
143
|
/** Run garbage collection and update the reference / used state of the system. */
|
|
150
144
|
collectGarbage(
|
|
@@ -190,7 +184,6 @@ export interface IGarbageCollectorCreateParams {
|
|
|
190
184
|
readonly readAndParseBlob: ReadAndParseBlob;
|
|
191
185
|
readonly activeConnection: () => boolean;
|
|
192
186
|
readonly getContainerDiagnosticId: () => string;
|
|
193
|
-
readonly snapshotCacheExpiryMs?: number;
|
|
194
187
|
}
|
|
195
188
|
|
|
196
189
|
/** The state of node that is unreferenced. */
|
|
@@ -378,14 +371,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
378
371
|
private readonly testMode: boolean;
|
|
379
372
|
private readonly mc: MonitoringContext;
|
|
380
373
|
|
|
381
|
-
/**
|
|
382
|
-
* Tells whether the GC data should be written to the root of the summary tree.
|
|
383
|
-
*/
|
|
384
|
-
private _writeDataAtRoot: boolean = true;
|
|
385
|
-
public get writeDataAtRoot(): boolean {
|
|
386
|
-
return this._writeDataAtRoot;
|
|
387
|
-
}
|
|
388
|
-
|
|
389
374
|
/**
|
|
390
375
|
* Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
|
|
391
376
|
*
|
|
@@ -462,9 +447,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
462
447
|
sweepEnabled: this.sweepEnabled,
|
|
463
448
|
runGC: this.shouldRunGC,
|
|
464
449
|
runSweep: this.shouldRunSweep,
|
|
465
|
-
writeAtRoot: this._writeDataAtRoot,
|
|
466
450
|
testMode: this.testMode,
|
|
467
451
|
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
452
|
+
sweepTimeout: this.sweepTimeoutMs,
|
|
468
453
|
inactiveTimeout: this.inactiveTimeoutMs,
|
|
469
454
|
trackGCState: this.trackGCState,
|
|
470
455
|
...this.gcOptions,
|
|
@@ -498,6 +483,21 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
498
483
|
|
|
499
484
|
let prevSummaryGCVersion: number | undefined;
|
|
500
485
|
|
|
486
|
+
/**
|
|
487
|
+
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
488
|
+
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
|
|
489
|
+
*
|
|
490
|
+
* The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
|
|
491
|
+
* The buffer is added to account for any clock skew or other edge cases.
|
|
492
|
+
* We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
|
|
493
|
+
*/
|
|
494
|
+
function computeSweepTimeout(sessionExpiryTimeoutMs: number | undefined) {
|
|
495
|
+
const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
|
|
496
|
+
const bufferMs = oneDayMs;
|
|
497
|
+
return sessionExpiryTimeoutMs &&
|
|
498
|
+
(sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
501
|
/**
|
|
502
502
|
* The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
|
|
503
503
|
* 1. Whether running GC mark phase is allowed or not.
|
|
@@ -512,6 +512,9 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
512
512
|
this.gcEnabled = prevSummaryGCVersion > 0;
|
|
513
513
|
this.sweepEnabled = metadata?.sweepEnabled ?? false;
|
|
514
514
|
this.sessionExpiryTimeoutMs = metadata?.sessionExpiryTimeoutMs;
|
|
515
|
+
this.sweepTimeoutMs =
|
|
516
|
+
metadata?.sweepTimeoutMs
|
|
517
|
+
?? computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
|
|
515
518
|
} else {
|
|
516
519
|
// Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
|
|
517
520
|
// scenario but explicitly failing makes it clearer and promotes correct usage.
|
|
@@ -519,20 +522,28 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
519
522
|
throw new UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
|
|
520
523
|
}
|
|
521
524
|
|
|
525
|
+
// This Test Override only applies for new containers
|
|
526
|
+
const testOverrideSweepTimeoutMs =
|
|
527
|
+
this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
|
|
528
|
+
|
|
522
529
|
// For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
|
|
523
530
|
// flag in GC options to false.
|
|
524
531
|
this.gcEnabled = this.gcOptions.gcAllowed !== false;
|
|
525
532
|
// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
|
|
526
|
-
|
|
533
|
+
// ...unless we're using the TestOverride
|
|
534
|
+
this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
|
|
527
535
|
|
|
528
536
|
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
529
537
|
if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
|
|
530
538
|
this.sessionExpiryTimeoutMs = this.gcOptions.sessionExpiryTimeoutMs ?? defaultSessionExpiryDurationMs;
|
|
531
539
|
}
|
|
540
|
+
this.sweepTimeoutMs =
|
|
541
|
+
testOverrideSweepTimeoutMs
|
|
542
|
+
?? computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
532
543
|
}
|
|
533
544
|
|
|
534
545
|
// If session expiry is enabled, we need to close the container when the session expiry timeout expires.
|
|
535
|
-
if (this.sessionExpiryTimeoutMs !== undefined
|
|
546
|
+
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
536
547
|
// If Test Override config is set, override Session Expiry timeout.
|
|
537
548
|
const overrideSessionExpiryTimeoutMs =
|
|
538
549
|
this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
|
|
@@ -543,21 +554,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
543
554
|
() => { this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs)); },
|
|
544
555
|
);
|
|
545
556
|
this.sessionExpiryTimer.start();
|
|
546
|
-
|
|
547
|
-
// TEMPORARY: Hardcode a default of 5 days which will be >= the policy's value in the ODSP driver.
|
|
548
|
-
// This unblocks the Sweep Log (see logSweepEvents function).
|
|
549
|
-
// This will be removed before sweep is fully implemented.
|
|
550
|
-
const snapshotCacheExpiryMs = createParams.snapshotCacheExpiryMs ?? 5 * 24 * 60 * 60 * 1000;
|
|
551
|
-
|
|
552
|
-
/**
|
|
553
|
-
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
554
|
-
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer. The buffer is
|
|
555
|
-
* added to account for any clock skew. We use server timestamps throughout so the skew should be minimal
|
|
556
|
-
* but make it one day to be safe.
|
|
557
|
-
*/
|
|
558
|
-
if (snapshotCacheExpiryMs !== undefined) {
|
|
559
|
-
this.sweepTimeoutMs = this.sessionExpiryTimeoutMs + snapshotCacheExpiryMs + oneDayMs;
|
|
560
|
-
}
|
|
561
557
|
}
|
|
562
558
|
|
|
563
559
|
// For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
|
|
@@ -584,16 +580,16 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
584
580
|
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
585
581
|
*
|
|
586
582
|
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
587
|
-
*
|
|
588
583
|
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
589
|
-
*
|
|
590
|
-
*
|
|
584
|
+
* 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
|
|
585
|
+
* the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
|
|
586
|
+
* 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
591
587
|
* feature flag.
|
|
592
588
|
*/
|
|
593
|
-
this.shouldRunSweep =
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
589
|
+
this.shouldRunSweep =
|
|
590
|
+
this.shouldRunGC
|
|
591
|
+
&& this.sweepTimeoutMs !== undefined
|
|
592
|
+
&& (this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
|
|
597
593
|
|
|
598
594
|
this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
|
|
599
595
|
|
|
@@ -610,15 +606,10 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
610
606
|
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
611
607
|
this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? this.gcOptions.runGCInTestMode === true;
|
|
612
608
|
|
|
613
|
-
// GC state
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
// The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
|
|
618
|
-
// contain GC tree and GC is enabled.
|
|
619
|
-
const gcTreePresent = baseSnapshot?.trees[gcTreeKey] !== undefined;
|
|
620
|
-
this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
|
|
621
|
-
}
|
|
609
|
+
// The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
|
|
610
|
+
// contain GC tree and GC is enabled.
|
|
611
|
+
const gcTreePresent = baseSnapshot?.trees[gcTreeKey] !== undefined;
|
|
612
|
+
this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
|
|
622
613
|
|
|
623
614
|
// Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
|
|
624
615
|
// this once since it involves fetching blobs from storage which is expensive.
|
|
@@ -631,8 +622,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
631
622
|
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
632
623
|
const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
|
|
633
624
|
if (gcSnapshotTree !== undefined) {
|
|
634
|
-
// If the GC tree is written at root, we should also do the same.
|
|
635
|
-
this._writeDataAtRoot = true;
|
|
636
625
|
const baseGCState = await getGCStateFromSnapshot(
|
|
637
626
|
gcSnapshotTree,
|
|
638
627
|
readAndParseBlob,
|
|
@@ -855,7 +844,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
855
844
|
const gcResult = runGarbageCollection(gcData.gcNodes, ["/"]);
|
|
856
845
|
|
|
857
846
|
const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
|
|
858
|
-
event.end({ ...gcStats });
|
|
847
|
+
event.end({ ...gcStats, timestamp: currentReferenceTimestampMs });
|
|
859
848
|
this.completedRuns++;
|
|
860
849
|
return gcStats;
|
|
861
850
|
}, { end: true, cancel: "error" });
|
|
@@ -884,7 +873,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
884
873
|
|
|
885
874
|
// Update the current state and update the runtime of all routes or ids that used as per the GC run.
|
|
886
875
|
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
887
|
-
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds
|
|
876
|
+
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
|
|
888
877
|
|
|
889
878
|
// Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
|
|
890
879
|
// delete these objects here instead.
|
|
@@ -967,6 +956,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
967
956
|
gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
|
|
968
957
|
sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
|
|
969
958
|
sweepEnabled: this.sweepEnabled,
|
|
959
|
+
sweepTimeoutMs: this.sweepTimeoutMs,
|
|
970
960
|
};
|
|
971
961
|
}
|
|
972
962
|
|
|
@@ -1155,7 +1145,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1155
1145
|
this.newReferencesSinceLastRun,
|
|
1156
1146
|
);
|
|
1157
1147
|
|
|
1158
|
-
if (
|
|
1148
|
+
if (missingExplicitReferences.length > 0) {
|
|
1159
1149
|
missingExplicitReferences.forEach((missingExplicitReference) => {
|
|
1160
1150
|
const event: ITelemetryPerformanceEvent = {
|
|
1161
1151
|
eventName: "gcUnknownOutboundReferences",
|
|
@@ -74,16 +74,8 @@ export class SweepReadyUsageDetectionHandler {
|
|
|
74
74
|
localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
|
|
75
75
|
) {
|
|
76
76
|
const noopStorage = { getItem: () => null, setItem: () => {} };
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
} else {
|
|
80
|
-
try {
|
|
81
|
-
// localStorage is not defined in Node environment so this throws
|
|
82
|
-
this.localStorage = localStorage ?? noopStorage;
|
|
83
|
-
} catch (error) {
|
|
84
|
-
this.localStorage = noopStorage;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
77
|
+
// localStorage is not defined in Node environment, so fall back to noopStorage if needed.
|
|
78
|
+
this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
|
|
87
79
|
|
|
88
80
|
if (this.localStorage === noopStorage) {
|
|
89
81
|
// This means the Skip Closure Period logic will not work.
|
package/src/index.ts
CHANGED
|
@@ -23,8 +23,8 @@ export {
|
|
|
23
23
|
RuntimeHeaders,
|
|
24
24
|
ISummaryConfiguration,
|
|
25
25
|
DefaultSummaryConfiguration,
|
|
26
|
+
ICompressionRuntimeOptions,
|
|
26
27
|
} from "./containerRuntime";
|
|
27
|
-
export { DeltaScheduler } from "./deltaScheduler";
|
|
28
28
|
export { FluidDataStoreRegistry } from "./dataStoreRegistry";
|
|
29
29
|
export {
|
|
30
30
|
gcBlobPrefix,
|
|
@@ -34,12 +34,10 @@ export {
|
|
|
34
34
|
} from "./garbageCollection";
|
|
35
35
|
export {
|
|
36
36
|
IPendingFlush,
|
|
37
|
-
IPendingFlushMode,
|
|
38
37
|
IPendingLocalState,
|
|
39
38
|
IPendingMessage,
|
|
40
39
|
IPendingState,
|
|
41
40
|
} from "./pendingStateManager";
|
|
42
|
-
export { ScheduleManager } from "./scheduleManager";
|
|
43
41
|
export { Summarizer } from "./summarizer";
|
|
44
42
|
export {
|
|
45
43
|
EnqueueSummarizeResult,
|
|
@@ -54,6 +52,7 @@ export {
|
|
|
54
52
|
INackSummaryResult,
|
|
55
53
|
IOnDemandSummarizeOptions,
|
|
56
54
|
IProvideSummarizer,
|
|
55
|
+
IRefreshSummaryAckOptions,
|
|
57
56
|
ISubmitSummaryOpResult,
|
|
58
57
|
ISubmitSummaryOptions,
|
|
59
58
|
ISummarizeOptions,
|
package/src/packageVersion.ts
CHANGED