@fluidframework/container-runtime 0.52.1 → 0.53.0-46105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/containerHandleContext.d.ts +0 -1
- package/dist/containerHandleContext.d.ts.map +1 -1
- package/dist/containerHandleContext.js +0 -1
- package/dist/containerHandleContext.js.map +1 -1
- package/dist/containerRuntime.d.ts +18 -3
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +84 -42
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +4 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +16 -13
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +8 -8
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +20 -37
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +61 -14
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +275 -20
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -2
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/summarizer.d.ts +1 -3
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +0 -12
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +2 -2
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryFormat.d.ts +9 -1
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +1 -3
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/containerHandleContext.d.ts +0 -1
- package/lib/containerHandleContext.d.ts.map +1 -1
- package/lib/containerHandleContext.js +0 -1
- package/lib/containerHandleContext.js.map +1 -1
- package/lib/containerRuntime.d.ts +18 -3
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +84 -43
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +4 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +16 -13
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +8 -8
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +23 -40
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +61 -14
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +276 -21
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/summarizer.d.ts +1 -3
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +0 -12
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +2 -2
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryFormat.d.ts +9 -1
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +1 -3
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +13 -13
- package/src/containerHandleContext.ts +0 -1
- package/src/containerRuntime.ts +110 -53
- package/src/dataStoreContext.ts +15 -14
- package/src/dataStores.ts +32 -50
- package/src/garbageCollection.ts +390 -18
- package/src/index.ts +20 -2
- package/src/packageVersion.ts +1 -1
- package/src/summarizer.ts +0 -15
- package/src/summarizerTypes.ts +1 -2
- package/src/summaryFormat.ts +10 -1
- package/src/summaryGenerator.ts +2 -3
package/src/containerRuntime.ts
CHANGED
|
@@ -6,15 +6,14 @@
|
|
|
6
6
|
import { EventEmitter } from "events";
|
|
7
7
|
import { ITelemetryGenericEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
8
8
|
import {
|
|
9
|
+
FluidObject,
|
|
10
|
+
IFluidConfiguration,
|
|
11
|
+
IFluidHandle,
|
|
12
|
+
IFluidHandleContext,
|
|
9
13
|
IFluidObject,
|
|
10
14
|
IFluidRouter,
|
|
11
|
-
IFluidHandleContext,
|
|
12
|
-
IFluidSerializer,
|
|
13
15
|
IRequest,
|
|
14
16
|
IResponse,
|
|
15
|
-
IFluidHandle,
|
|
16
|
-
IFluidConfiguration,
|
|
17
|
-
FluidObject,
|
|
18
17
|
} from "@fluidframework/core-interfaces";
|
|
19
18
|
import {
|
|
20
19
|
IAudience,
|
|
@@ -49,10 +48,6 @@ import {
|
|
|
49
48
|
import { IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
50
49
|
import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
|
|
51
50
|
import { DataCorruptionError, GenericError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
|
|
52
|
-
import {
|
|
53
|
-
BlobTreeEntry,
|
|
54
|
-
TreeTreeEntry,
|
|
55
|
-
} from "@fluidframework/protocol-base";
|
|
56
51
|
import {
|
|
57
52
|
IClientDetails,
|
|
58
53
|
IDocumentMessage,
|
|
@@ -90,13 +85,13 @@ import {
|
|
|
90
85
|
addTreeToSummary,
|
|
91
86
|
convertToSummaryTree,
|
|
92
87
|
createRootSummarizerNodeWithGC,
|
|
93
|
-
FluidSerializer,
|
|
94
88
|
IRootSummarizerNodeWithGC,
|
|
95
89
|
RequestParser,
|
|
96
90
|
create404Response,
|
|
97
91
|
exceptionToResponse,
|
|
98
92
|
responseToException,
|
|
99
93
|
seqFromTree,
|
|
94
|
+
convertSummaryTreeToITree,
|
|
100
95
|
} from "@fluidframework/runtime-utils";
|
|
101
96
|
import { v4 as uuid } from "uuid";
|
|
102
97
|
import { ContainerFluidHandleContext } from "./containerHandleContext";
|
|
@@ -115,6 +110,7 @@ import {
|
|
|
115
110
|
electedSummarizerBlobName,
|
|
116
111
|
extractSummaryMetadataMessage,
|
|
117
112
|
IContainerRuntimeMetadata,
|
|
113
|
+
ICreateContainerMetadata,
|
|
118
114
|
ISummaryMetadataMessage,
|
|
119
115
|
metadataBlobName,
|
|
120
116
|
wrapSummaryInChannelsTree,
|
|
@@ -137,6 +133,7 @@ import { formExponentialFn, Throttler } from "./throttler";
|
|
|
137
133
|
import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
|
|
138
134
|
import {
|
|
139
135
|
GarbageCollector,
|
|
136
|
+
gcTreeKey,
|
|
140
137
|
IGarbageCollectionRuntime,
|
|
141
138
|
IGarbageCollector,
|
|
142
139
|
IGCStats,
|
|
@@ -584,6 +581,19 @@ export class ScheduleManager {
|
|
|
584
581
|
*/
|
|
585
582
|
export const agentSchedulerId = "_scheduler";
|
|
586
583
|
|
|
584
|
+
// safely check navigator and get the hardware spec value
|
|
585
|
+
export function getDeviceSpec() {
|
|
586
|
+
try {
|
|
587
|
+
if (typeof navigator === "object" && navigator !== null) {
|
|
588
|
+
return {
|
|
589
|
+
deviceMemory: (navigator as any).deviceMemory,
|
|
590
|
+
hardwareConcurrency: navigator.hardwareConcurrency,
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
} catch {
|
|
594
|
+
}
|
|
595
|
+
return {};
|
|
596
|
+
}
|
|
587
597
|
/**
|
|
588
598
|
* Represents the runtime of the container. Contains helper functions/state of the container.
|
|
589
599
|
* It will define the store level mappings.
|
|
@@ -805,9 +815,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
805
815
|
return this.context.attachState;
|
|
806
816
|
}
|
|
807
817
|
|
|
808
|
-
// Back compat: 0.28, can be removed in 0.29
|
|
809
|
-
public readonly IFluidSerializer: IFluidSerializer;
|
|
810
|
-
|
|
811
818
|
public readonly IFluidHandleContext: IFluidHandleContext;
|
|
812
819
|
|
|
813
820
|
// internal logger for ContainerRuntime. Use this.logger for stores, summaries, etc.
|
|
@@ -874,6 +881,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
874
881
|
|
|
875
882
|
private readonly dataStores: DataStores;
|
|
876
883
|
|
|
884
|
+
/**
|
|
885
|
+
* True, if GC data should be written at root of the summary tree.
|
|
886
|
+
* False, if data stores should write GC blobs in their summary tree.
|
|
887
|
+
*/
|
|
888
|
+
public get writeGCDataAtRoot(): boolean {
|
|
889
|
+
return this.garbageCollector.writeDataAtRoot;
|
|
890
|
+
}
|
|
891
|
+
|
|
877
892
|
/**
|
|
878
893
|
* True if generating summaries with isolated channels is
|
|
879
894
|
* explicitly disabled. This only affects how summaries are written,
|
|
@@ -892,6 +907,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
892
907
|
return this._summarizer;
|
|
893
908
|
}
|
|
894
909
|
|
|
910
|
+
private readonly createContainerMetadata: ICreateContainerMetadata;
|
|
911
|
+
private summaryCount: number | undefined;
|
|
912
|
+
|
|
895
913
|
private constructor(
|
|
896
914
|
private readonly context: IContainerContext,
|
|
897
915
|
private readonly registry: IFluidDataStoreRegistry,
|
|
@@ -907,9 +925,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
907
925
|
private _storage?: IDocumentStorageService,
|
|
908
926
|
) {
|
|
909
927
|
super();
|
|
910
|
-
|
|
911
928
|
this.baseSummaryMessage = metadata?.message;
|
|
912
929
|
|
|
930
|
+
// If this is an existing container, we get values from metadata.
|
|
931
|
+
// otherwise, we initialize them.
|
|
932
|
+
if (existing) {
|
|
933
|
+
this.createContainerMetadata = {
|
|
934
|
+
createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
|
|
935
|
+
createContainerTimestamp: metadata?.createContainerTimestamp,
|
|
936
|
+
};
|
|
937
|
+
this.summaryCount = metadata?.summaryCount;
|
|
938
|
+
} else {
|
|
939
|
+
this.createContainerMetadata = {
|
|
940
|
+
createContainerRuntimeVersion: pkgVersion,
|
|
941
|
+
createContainerTimestamp: Date.now(),
|
|
942
|
+
};
|
|
943
|
+
}
|
|
944
|
+
|
|
913
945
|
// Default to false (enabled).
|
|
914
946
|
this.disableIsolatedChannels = this.runtimeOptions.summaryOptions.disableIsolatedChannels ?? false;
|
|
915
947
|
|
|
@@ -917,14 +949,26 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
917
949
|
this.chunkMap = new Map<string, string[]>(chunks);
|
|
918
950
|
|
|
919
951
|
this.IFluidHandleContext = new ContainerFluidHandleContext("", this);
|
|
920
|
-
this.IFluidSerializer = new FluidSerializer(this.IFluidHandleContext);
|
|
921
952
|
|
|
922
953
|
this._logger = ChildLogger.create(this.logger, "ContainerRuntime");
|
|
923
954
|
|
|
955
|
+
/**
|
|
956
|
+
* Function that return the current server timestamp. This is used by the garbage collector to set the
|
|
957
|
+
* time when a node becomes unreferenced.
|
|
958
|
+
* For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
|
|
959
|
+
* we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
960
|
+
* of this client's connection - https://github.com/microsoft/FluidFramework/issues/8375.
|
|
961
|
+
*/
|
|
962
|
+
const getCurrentTimestamp = () => {
|
|
963
|
+
return this.deltaManager.lastMessage?.timestamp ?? Date.now();
|
|
964
|
+
};
|
|
924
965
|
this.garbageCollector = GarbageCollector.create(
|
|
925
966
|
this,
|
|
926
967
|
this.runtimeOptions.gcOptions,
|
|
927
968
|
(unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
|
|
969
|
+
getCurrentTimestamp,
|
|
970
|
+
context.baseSnapshot,
|
|
971
|
+
async <T>(id: string) => readAndParse<T>(this.storage, id),
|
|
928
972
|
this._logger,
|
|
929
973
|
existing,
|
|
930
974
|
metadata,
|
|
@@ -948,8 +992,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
948
992
|
throwOnFailure: true,
|
|
949
993
|
// If GC should not run, let the summarizer node know so that it does not track GC state.
|
|
950
994
|
gcDisabled: !this.garbageCollector.shouldRunGC,
|
|
951
|
-
// The max duration for which objects can be unreferenced before they are eligible for deletion.
|
|
952
|
-
maxUnreferencedDurationMs: this.runtimeOptions.gcOptions.maxUnreferencedDurationMs,
|
|
953
995
|
},
|
|
954
996
|
);
|
|
955
997
|
|
|
@@ -974,7 +1016,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
974
1016
|
getInitialGCSummaryDetailsFn,
|
|
975
1017
|
),
|
|
976
1018
|
(id: string) => this.summarizerNode.deleteChild(id),
|
|
977
|
-
this._logger
|
|
1019
|
+
this._logger,
|
|
1020
|
+
async () => this.garbageCollector.getDataStoreBaseGCDetails(),
|
|
1021
|
+
(id: string) => this.garbageCollector.nodeChanged(id),
|
|
1022
|
+
);
|
|
978
1023
|
|
|
979
1024
|
this.blobManager = new BlobManager(
|
|
980
1025
|
this.IFluidHandleContext,
|
|
@@ -1002,12 +1047,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1002
1047
|
this.clearPartialChunks(clientId);
|
|
1003
1048
|
});
|
|
1004
1049
|
|
|
1005
|
-
this.context.quorum.on("addProposal", (proposal) => {
|
|
1006
|
-
if (proposal.key === "code" || proposal.key === "code2") {
|
|
1007
|
-
this.emit("codeDetailsProposed", proposal.value, proposal);
|
|
1008
|
-
}
|
|
1009
|
-
});
|
|
1010
|
-
|
|
1011
1050
|
this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
|
|
1012
1051
|
|
|
1013
1052
|
// Only create a SummaryManager if summaries are enabled and we are not the summarizer client
|
|
@@ -1104,7 +1143,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1104
1143
|
this.deltaManager.on("readonly", (readonly: boolean) => {
|
|
1105
1144
|
// we accumulate ops while being in read-only state.
|
|
1106
1145
|
// once user gets write permissions and we have active connection, flush all pending ops.
|
|
1107
|
-
|
|
1146
|
+
// eslint-disable-next-line max-len
|
|
1147
|
+
assert(readonly === this.deltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
|
|
1108
1148
|
|
|
1109
1149
|
// We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
|
|
1110
1150
|
// when we either never send an op, or attempted to send it but we know for sure it was not
|
|
@@ -1127,6 +1167,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1127
1167
|
this.deltaManager.on("op", this.onOp);
|
|
1128
1168
|
}
|
|
1129
1169
|
|
|
1170
|
+
// logging hardware telemetry
|
|
1171
|
+
logger.sendTelemetryEvent({
|
|
1172
|
+
eventName:"DeviceSpec",
|
|
1173
|
+
...getDeviceSpec(),
|
|
1174
|
+
});
|
|
1175
|
+
|
|
1176
|
+
// logging container load stats
|
|
1177
|
+
this.logger.sendTelemetryEvent({
|
|
1178
|
+
eventName: "ContainerLoadStats",
|
|
1179
|
+
...this.createContainerMetadata,
|
|
1180
|
+
...this.dataStores.containerLoadStats,
|
|
1181
|
+
summaryCount: this.summaryCount,
|
|
1182
|
+
summaryFormatVersion: metadata?.summaryFormatVersion,
|
|
1183
|
+
disableIsolatedChannels: metadata?.disableIsolatedChannels,
|
|
1184
|
+
gcVersion: metadata?.gcFeature,
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1130
1187
|
ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
|
|
1131
1188
|
}
|
|
1132
1189
|
|
|
@@ -1251,6 +1308,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1251
1308
|
|
|
1252
1309
|
private formMetadata(): IContainerRuntimeMetadata {
|
|
1253
1310
|
return {
|
|
1311
|
+
...this.createContainerMetadata,
|
|
1312
|
+
summaryCount: this.summaryCount,
|
|
1254
1313
|
summaryFormatVersion: 1,
|
|
1255
1314
|
disableIsolatedChannels: this.disableIsolatedChannels || undefined,
|
|
1256
1315
|
gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
|
|
@@ -1287,26 +1346,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1287
1346
|
* @deprecated - Use summarize to get summary of the container runtime.
|
|
1288
1347
|
*/
|
|
1289
1348
|
public async snapshot(): Promise<ITree> {
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
root.entries = root.entries.concat(entries);
|
|
1299
|
-
} else {
|
|
1300
|
-
root.entries.push(new TreeTreeEntry(channelsTreeName, { entries }));
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata())));
|
|
1304
|
-
|
|
1305
|
-
if (this.chunkMap.size > 0) {
|
|
1306
|
-
root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap])));
|
|
1307
|
-
}
|
|
1308
|
-
|
|
1309
|
-
return root;
|
|
1349
|
+
const summaryResult = await this.summarize({
|
|
1350
|
+
summaryLogger: this.logger,
|
|
1351
|
+
fullTree: true,
|
|
1352
|
+
trackState: false,
|
|
1353
|
+
runGC: this.garbageCollector.shouldRunGC,
|
|
1354
|
+
fullGC: true,
|
|
1355
|
+
});
|
|
1356
|
+
return convertSummaryTreeToITree(summaryResult.summary);
|
|
1310
1357
|
}
|
|
1311
1358
|
|
|
1312
1359
|
private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
@@ -1328,6 +1375,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1328
1375
|
const blobsTree = convertToSummaryTree(snapshot, false);
|
|
1329
1376
|
addTreeToSummary(summaryTree, blobsTreeName, blobsTree);
|
|
1330
1377
|
}
|
|
1378
|
+
|
|
1379
|
+
if (this.writeGCDataAtRoot) {
|
|
1380
|
+
const gcSummary = this.garbageCollector.summarize();
|
|
1381
|
+
if (gcSummary !== undefined) {
|
|
1382
|
+
addTreeToSummary(summaryTree, gcTreeKey, gcSummary);
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1331
1385
|
}
|
|
1332
1386
|
|
|
1333
1387
|
private replayPendingStates() {
|
|
@@ -1777,21 +1831,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1777
1831
|
* Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
|
|
1778
1832
|
* After GC has run, called to notify this container's nodes of routes that are used in it.
|
|
1779
1833
|
* @param usedRoutes - The routes that are used in all nodes in this Container.
|
|
1834
|
+
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
|
|
1835
|
+
* unreferenced as part of this GC run, this should be used to update the time when it happens.
|
|
1780
1836
|
* @returns the statistics of the used state of the data stores.
|
|
1781
1837
|
*/
|
|
1782
|
-
public updateUsedRoutes(usedRoutes: string[]): IUsedStateStats {
|
|
1838
|
+
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats {
|
|
1783
1839
|
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1784
1840
|
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1785
1841
|
// always referenced, so the used routes is only self-route (empty string).
|
|
1786
1842
|
this.summarizerNode.updateUsedRoutes([""]);
|
|
1787
1843
|
|
|
1788
|
-
return this.dataStores.updateUsedRoutes(
|
|
1789
|
-
usedRoutes,
|
|
1790
|
-
// For now, we use the timestamp of the last op for gcTimestamp. However, there can be cases where
|
|
1791
|
-
// we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
1792
|
-
// of this client's connection - https://github.com/microsoft/FluidFramework/issues/7152.
|
|
1793
|
-
this.deltaManager.lastMessage?.timestamp,
|
|
1794
|
-
);
|
|
1844
|
+
return this.dataStores.updateUsedRoutes(usedRoutes, gcTimestamp);
|
|
1795
1845
|
}
|
|
1796
1846
|
|
|
1797
1847
|
/**
|
|
@@ -1883,6 +1933,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1883
1933
|
return { stage: "base", referenceSequenceNumber: summaryRefSeqNum, error: continueResult.error };
|
|
1884
1934
|
}
|
|
1885
1935
|
|
|
1936
|
+
// increment summary count
|
|
1937
|
+
if (this.summaryCount !== undefined) {
|
|
1938
|
+
this.summaryCount++;
|
|
1939
|
+
} else {
|
|
1940
|
+
this.summaryCount = 1;
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1886
1943
|
const trace = Trace.start();
|
|
1887
1944
|
let summarizeResult: ISummaryTreeWithStats;
|
|
1888
1945
|
try {
|
package/src/dataStoreContext.ts
CHANGED
|
@@ -188,6 +188,11 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
188
188
|
return this._containerRuntime.disableIsolatedChannels;
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
+
/** Tells whether GC data will be written at the root of the summary tree. If so, data store should not write it. */
|
|
192
|
+
protected get writeGCDataAtRoot(): boolean {
|
|
193
|
+
return this._containerRuntime.writeGCDataAtRoot;
|
|
194
|
+
}
|
|
195
|
+
|
|
191
196
|
protected registry: IFluidDataStoreRegistry | undefined;
|
|
192
197
|
|
|
193
198
|
protected detachedRuntimeCreation = false;
|
|
@@ -416,8 +421,10 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
416
421
|
const attributes = createAttributes(pkg, isRootDataStore, this.disableIsolatedChannels);
|
|
417
422
|
addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes));
|
|
418
423
|
|
|
419
|
-
// Add GC
|
|
420
|
-
|
|
424
|
+
// Add GC data to the summary if it's not written at the root.
|
|
425
|
+
if (!this.writeGCDataAtRoot) {
|
|
426
|
+
addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
|
|
427
|
+
}
|
|
421
428
|
|
|
422
429
|
// If we are not referenced, mark the summary tree as unreferenced. Also, update unreferenced blob
|
|
423
430
|
// size in the summary stats with the blobs size of this data store.
|
|
@@ -675,6 +682,7 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
675
682
|
constructor(
|
|
676
683
|
id: string,
|
|
677
684
|
private readonly initSnapshotValue: ISnapshotTree | string | undefined,
|
|
685
|
+
private readonly getBaseSummaryGCDetails: () => Promise<IGarbageCollectionSummaryDetails | undefined>,
|
|
678
686
|
runtime: ContainerRuntime,
|
|
679
687
|
storage: IDocumentStorageService,
|
|
680
688
|
scope: FluidObject,
|
|
@@ -760,16 +768,7 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
760
768
|
});
|
|
761
769
|
|
|
762
770
|
private readonly gcDetailsInInitialSummaryP = new LazyPromise<IGarbageCollectionSummaryDetails>(async () => {
|
|
763
|
-
|
|
764
|
-
if (!(!this.initSnapshotValue || typeof this.initSnapshotValue === "string")
|
|
765
|
-
&& this.initSnapshotValue.blobs[gcBlobKey] !== undefined) {
|
|
766
|
-
return readAndParse<IGarbageCollectionSummaryDetails>(
|
|
767
|
-
this.storage,
|
|
768
|
-
this.initSnapshotValue.blobs[gcBlobKey],
|
|
769
|
-
);
|
|
770
|
-
} else {
|
|
771
|
-
return {};
|
|
772
|
-
}
|
|
771
|
+
return (await this.getBaseSummaryGCDetails()) ?? {};
|
|
773
772
|
});
|
|
774
773
|
|
|
775
774
|
protected async getInitialSnapshotDetails(): Promise<ISnapshotDetails> {
|
|
@@ -850,8 +849,10 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
|
|
|
850
849
|
);
|
|
851
850
|
addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes));
|
|
852
851
|
|
|
853
|
-
// Add GC
|
|
854
|
-
|
|
852
|
+
// Add GC data to the summary if it's not written at the root.
|
|
853
|
+
if (!this.writeGCDataAtRoot) {
|
|
854
|
+
addBlobToSummary(summarizeResult, gcBlobKey, JSON.stringify(this.summarizerNode.getGCSummaryDetails()));
|
|
855
|
+
}
|
|
855
856
|
|
|
856
857
|
// Attach message needs the summary in ITree format. Convert the ISummaryTree into an ITree.
|
|
857
858
|
const snapshot = convertSummaryTreeToITree(summarizeResult.summary);
|
package/src/dataStores.ts
CHANGED
|
@@ -8,8 +8,6 @@ import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidfra
|
|
|
8
8
|
import {
|
|
9
9
|
ISequencedDocumentMessage,
|
|
10
10
|
ISnapshotTree,
|
|
11
|
-
ITreeEntry,
|
|
12
|
-
SummaryType,
|
|
13
11
|
} from "@fluidframework/protocol-definitions";
|
|
14
12
|
import {
|
|
15
13
|
channelsTreeName,
|
|
@@ -21,6 +19,7 @@ import {
|
|
|
21
19
|
IFluidDataStoreChannel,
|
|
22
20
|
IFluidDataStoreContextDetached,
|
|
23
21
|
IGarbageCollectionData,
|
|
22
|
+
IGarbageCollectionSummaryDetails,
|
|
24
23
|
IInboundSignalMessage,
|
|
25
24
|
InboundAttachMessage,
|
|
26
25
|
ISummarizeResult,
|
|
@@ -28,7 +27,6 @@ import {
|
|
|
28
27
|
} from "@fluidframework/runtime-definitions";
|
|
29
28
|
import {
|
|
30
29
|
convertSnapshotTreeToSummaryTree,
|
|
31
|
-
convertSummaryTreeToITree,
|
|
32
30
|
convertToSummaryTree,
|
|
33
31
|
create404Response,
|
|
34
32
|
responseToException,
|
|
@@ -37,10 +35,9 @@ import {
|
|
|
37
35
|
import { ChildLogger, TelemetryDataTag } from "@fluidframework/telemetry-utils";
|
|
38
36
|
import { AttachState } from "@fluidframework/container-definitions";
|
|
39
37
|
import { BlobCacheStorageService, buildSnapshotTree } from "@fluidframework/driver-utils";
|
|
40
|
-
import { assert, Lazy } from "@fluidframework/common-utils";
|
|
38
|
+
import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
|
|
41
39
|
import { v4 as uuid } from "uuid";
|
|
42
|
-
import {
|
|
43
|
-
import { GCDataBuilder, getChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
|
|
40
|
+
import { GCDataBuilder, unpackChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
|
|
44
41
|
import { DataStoreContexts } from "./dataStoreContexts";
|
|
45
42
|
import { ContainerRuntime } from "./containerRuntime";
|
|
46
43
|
import {
|
|
@@ -67,6 +64,13 @@ export class DataStores implements IDisposable {
|
|
|
67
64
|
|
|
68
65
|
private readonly disposeOnce = new Lazy<void>(() => this.contexts.dispose());
|
|
69
66
|
|
|
67
|
+
public readonly containerLoadStats: {
|
|
68
|
+
// number of dataStores during loadContainer
|
|
69
|
+
readonly containerLoadDataStoreCount: number;
|
|
70
|
+
// number of unreferenced dataStores during loadContainer
|
|
71
|
+
readonly referencedDataStoreCount: number;
|
|
72
|
+
};
|
|
73
|
+
|
|
70
74
|
constructor(
|
|
71
75
|
private readonly baseSnapshot: ISnapshotTree | undefined,
|
|
72
76
|
private readonly runtime: ContainerRuntime,
|
|
@@ -75,12 +79,23 @@ export class DataStores implements IDisposable {
|
|
|
75
79
|
(id: string, createParam: CreateChildSummarizerNodeParam) => CreateChildSummarizerNodeFn,
|
|
76
80
|
private readonly deleteChildSummarizerNodeFn: (id: string) => void,
|
|
77
81
|
baseLogger: ITelemetryBaseLogger,
|
|
82
|
+
getDataStoreBaseGCDetails: () => Promise<Map<string, IGarbageCollectionSummaryDetails>>,
|
|
83
|
+
private readonly dataStoreChanged: (id: string) => void,
|
|
78
84
|
private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
|
|
79
85
|
) {
|
|
80
86
|
this.logger = ChildLogger.create(baseLogger);
|
|
87
|
+
|
|
88
|
+
const baseDataStoresGCDetailsP = new LazyPromise(async () => {
|
|
89
|
+
return getDataStoreBaseGCDetails();
|
|
90
|
+
});
|
|
91
|
+
// Returns the base summary GC details for the data store with the given id.
|
|
92
|
+
const dataStoreBaseGCDetails = async (dataStoreId: string) => {
|
|
93
|
+
const baseGCDetails = await baseDataStoresGCDetailsP;
|
|
94
|
+
return baseGCDetails.get(dataStoreId);
|
|
95
|
+
};
|
|
96
|
+
|
|
81
97
|
// Extract stores stored inside the snapshot
|
|
82
98
|
const fluidDataStores = new Map<string, ISnapshotTree>();
|
|
83
|
-
|
|
84
99
|
if (baseSnapshot) {
|
|
85
100
|
for (const [key, value] of Object.entries(baseSnapshot.trees)) {
|
|
86
101
|
fluidDataStores.set(key, value);
|
|
@@ -101,6 +116,7 @@ export class DataStores implements IDisposable {
|
|
|
101
116
|
dataStoreContext = new RemotedFluidDataStoreContext(
|
|
102
117
|
key,
|
|
103
118
|
value,
|
|
119
|
+
async () => dataStoreBaseGCDetails(key),
|
|
104
120
|
this.runtime,
|
|
105
121
|
this.runtime.storage,
|
|
106
122
|
this.runtime.scope,
|
|
@@ -124,11 +140,10 @@ export class DataStores implements IDisposable {
|
|
|
124
140
|
}
|
|
125
141
|
this.contexts.addBoundOrRemoted(dataStoreContext);
|
|
126
142
|
}
|
|
127
|
-
this.
|
|
128
|
-
|
|
129
|
-
dataStoreCount: fluidDataStores.size,
|
|
143
|
+
this.containerLoadStats = {
|
|
144
|
+
containerLoadDataStoreCount: fluidDataStores.size,
|
|
130
145
|
referencedDataStoreCount: fluidDataStores.size - unreferencedDataStoreCount,
|
|
131
|
-
}
|
|
146
|
+
};
|
|
132
147
|
}
|
|
133
148
|
|
|
134
149
|
public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
|
|
@@ -170,6 +185,8 @@ export class DataStores implements IDisposable {
|
|
|
170
185
|
const remotedFluidDataStoreContext = new RemotedFluidDataStoreContext(
|
|
171
186
|
attachMessage.id,
|
|
172
187
|
snapshotTree,
|
|
188
|
+
// New data stores begin with empty GC details since GC hasn't run on them yet.
|
|
189
|
+
async () => { return {}; },
|
|
173
190
|
this.runtime,
|
|
174
191
|
new BlobCacheStorageService(this.runtime.storage, flatBlobs),
|
|
175
192
|
this.runtime.scope,
|
|
@@ -281,6 +298,9 @@ export class DataStores implements IDisposable {
|
|
|
281
298
|
const context = this.contexts.get(envelope.address);
|
|
282
299
|
assert(!!context, 0x162 /* "There should be a store context for the op" */);
|
|
283
300
|
context.process(transformed, local, localMessageMetadata);
|
|
301
|
+
|
|
302
|
+
// Notify that a data store changed. This is used to detect if a deleted data store is being used.
|
|
303
|
+
this.dataStoreChanged(envelope.address);
|
|
284
304
|
}
|
|
285
305
|
|
|
286
306
|
public async getDataStore(id: string, wait: boolean): Promise<FluidDataStoreContext> {
|
|
@@ -339,44 +359,6 @@ export class DataStores implements IDisposable {
|
|
|
339
359
|
}
|
|
340
360
|
}
|
|
341
361
|
|
|
342
|
-
/**
|
|
343
|
-
* Notifies this object to take the snapshot of the container.
|
|
344
|
-
* @deprecated - Use summarize to get summary of the container runtime.
|
|
345
|
-
*/
|
|
346
|
-
public async snapshot(): Promise<ITreeEntry[]> {
|
|
347
|
-
// Iterate over each store and ask it to snapshot
|
|
348
|
-
const fluidDataStoreSnapshotsP = Array.from(this.contexts).map(async ([fluidDataStoreId, value]) => {
|
|
349
|
-
const summaryTree = await value.summarize(true /* fullTree */, false /* trackState */);
|
|
350
|
-
assert(
|
|
351
|
-
summaryTree.summary.type === SummaryType.Tree,
|
|
352
|
-
0x164 /* "summarize should always return a tree when fullTree is true" */);
|
|
353
|
-
// back-compat summary - Remove this once snapshot is removed.
|
|
354
|
-
const snapshot = convertSummaryTreeToITree(summaryTree.summary);
|
|
355
|
-
|
|
356
|
-
// If ID exists then previous commit is still valid
|
|
357
|
-
return {
|
|
358
|
-
fluidDataStoreId,
|
|
359
|
-
snapshot,
|
|
360
|
-
};
|
|
361
|
-
});
|
|
362
|
-
|
|
363
|
-
const entries: ITreeEntry[] = [];
|
|
364
|
-
|
|
365
|
-
// Add in module references to the store snapshots
|
|
366
|
-
const fluidDataStoreSnapshots = await Promise.all(fluidDataStoreSnapshotsP);
|
|
367
|
-
|
|
368
|
-
// Sort for better diffing of snapshots (in replay tool, used to find bugs in snapshotting logic)
|
|
369
|
-
fluidDataStoreSnapshots.sort((a, b) => a?.fluidDataStoreId.localeCompare(b.fluidDataStoreId));
|
|
370
|
-
|
|
371
|
-
for (const fluidDataStoreSnapshot of fluidDataStoreSnapshots) {
|
|
372
|
-
entries.push(new TreeTreeEntry(
|
|
373
|
-
fluidDataStoreSnapshot.fluidDataStoreId,
|
|
374
|
-
fluidDataStoreSnapshot.snapshot,
|
|
375
|
-
));
|
|
376
|
-
}
|
|
377
|
-
return entries;
|
|
378
|
-
}
|
|
379
|
-
|
|
380
362
|
public get size(): number {
|
|
381
363
|
return this.contexts.size;
|
|
382
364
|
}
|
|
@@ -474,7 +456,7 @@ export class DataStores implements IDisposable {
|
|
|
474
456
|
*/
|
|
475
457
|
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats {
|
|
476
458
|
// Get a map of data store ids to routes used in it.
|
|
477
|
-
const usedDataStoreRoutes =
|
|
459
|
+
const usedDataStoreRoutes = unpackChildNodesUsedRoutes(usedRoutes);
|
|
478
460
|
|
|
479
461
|
// Verify that the used routes are correct.
|
|
480
462
|
for (const [id] of usedDataStoreRoutes) {
|