@fluidframework/container-runtime 0.59.2001 → 0.59.3000
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +0 -1
- package/dist/batchTracker.js +1 -1
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +8 -1
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +27 -17
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.js +8 -8
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerHandleContext.js +1 -1
- package/dist/containerHandleContext.js.map +1 -1
- package/dist/containerRuntime.d.ts +27 -17
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +152 -176
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.js +1 -1
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +44 -44
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts +2 -2
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +8 -8
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStores.d.ts +6 -4
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +50 -37
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +23 -23
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +81 -50
- package/dist/garbageCollection.js.map +1 -1
- package/dist/opTelemetry.js +1 -1
- package/dist/opTelemetry.js.map +1 -1
- package/dist/orderedClientElection.d.ts.map +1 -1
- package/dist/orderedClientElection.js +2 -2
- package/dist/orderedClientElection.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.js +17 -17
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/runWhileConnectedCoordinator.js +1 -1
- package/dist/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +7 -6
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +4 -3
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerClientElection.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts +1 -1
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +1 -1
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +4 -2
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryCollection.js +2 -2
- package/dist/summaryCollection.js.map +1 -1
- package/dist/summaryFormat.d.ts +37 -11
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js +12 -4
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +6 -4
- package/dist/summaryGenerator.js.map +1 -1
- package/dist/summaryManager.js +5 -5
- package/dist/summaryManager.js.map +1 -1
- package/dist/throttler.js +2 -2
- package/dist/throttler.js.map +1 -1
- package/lib/blobManager.d.ts +8 -1
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +19 -9
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +27 -17
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +71 -95
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts +2 -2
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +2 -2
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStores.d.ts +6 -4
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +27 -14
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +23 -23
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +68 -37
- package/lib/garbageCollection.js.map +1 -1
- package/lib/opTelemetry.js.map +1 -1
- package/lib/orderedClientElection.d.ts.map +1 -1
- package/lib/orderedClientElection.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +4 -3
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +1 -0
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerClientElection.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts +1 -1
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +1 -1
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +4 -2
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryCollection.js.map +1 -1
- package/lib/summaryFormat.d.ts +37 -11
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js +10 -2
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +2 -0
- package/lib/summaryGenerator.js.map +1 -1
- package/lib/summaryManager.js.map +1 -1
- package/lib/throttler.js.map +1 -1
- package/package.json +26 -20
- package/src/blobManager.ts +23 -11
- package/src/containerRuntime.ts +111 -139
- package/src/dataStoreContext.ts +8 -11
- package/src/dataStoreContexts.ts +5 -5
- package/src/dataStores.ts +35 -17
- package/src/garbageCollection.ts +100 -57
- package/src/orderedClientElection.ts +5 -10
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +2 -2
- package/src/runningSummarizer.ts +8 -9
- package/src/summarizer.ts +2 -2
- package/src/summarizerHeuristics.ts +1 -1
- package/src/summarizerTypes.ts +8 -6
- package/src/summaryFormat.ts +38 -11
- package/src/summaryGenerator.ts +7 -5
- package/src/summaryManager.ts +2 -2
- package/src/throttler.ts +1 -1
package/src/containerRuntime.ts
CHANGED
|
@@ -48,7 +48,7 @@ import {
|
|
|
48
48
|
TelemetryDataTag,
|
|
49
49
|
} from "@fluidframework/telemetry-utils";
|
|
50
50
|
import { DriverHeader, IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
51
|
-
import { readAndParse
|
|
51
|
+
import { readAndParse } from "@fluidframework/driver-utils";
|
|
52
52
|
import {
|
|
53
53
|
DataCorruptionError,
|
|
54
54
|
GenericError,
|
|
@@ -208,29 +208,34 @@ const DefaultSummaryConfiguration: ISummaryConfiguration = {
|
|
|
208
208
|
};
|
|
209
209
|
|
|
210
210
|
export interface IGCRuntimeOptions {
|
|
211
|
-
/* Flag that will disable garbage collection if set to true. */
|
|
212
|
-
disableGC?: boolean;
|
|
213
|
-
|
|
214
211
|
/**
|
|
215
|
-
* Flag
|
|
216
|
-
*
|
|
217
|
-
*
|
|
218
|
-
*
|
|
219
|
-
*
|
|
212
|
+
* Flag that if true, will enable running garbage collection (GC) in a container. GC has mark phase and sweep phase.
|
|
213
|
+
* In mark phase, unreferenced objects are identified and marked as such in the summary. This option enables the
|
|
214
|
+
* mark phase.
|
|
215
|
+
* In sweep phase, unreferenced objects are eventually deleted from the container if they meet certain conditions.
|
|
216
|
+
* Sweep phase can be enabled via the "sweepAllowed" option.
|
|
217
|
+
* Note: This setting becomes part of the container's summary and cannot be changed.
|
|
220
218
|
*/
|
|
221
219
|
gcAllowed?: boolean;
|
|
222
220
|
|
|
223
221
|
/**
|
|
224
|
-
* Flag that
|
|
225
|
-
*
|
|
222
|
+
* Flag that if true, enables GC's sweep phase which will eventually delete unreferenced objects from the container.
|
|
223
|
+
* This flag should only be set to true if "gcAllowed" is true.
|
|
224
|
+
* Note: This setting becomes part of the container's summary and cannot be changed.
|
|
226
225
|
*/
|
|
227
|
-
|
|
226
|
+
sweepAllowed?: boolean;
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Flag that will disable garbage collection if set to true. Can be used to disable running GC on container where
|
|
230
|
+
* is allowed via the gcAllowed option.
|
|
231
|
+
*/
|
|
232
|
+
disableGC?: boolean;
|
|
228
233
|
|
|
229
234
|
/**
|
|
230
|
-
* Flag that
|
|
231
|
-
*
|
|
235
|
+
* Flag that will bypass optimizations and generate GC data for all nodes irrespective of whether a node
|
|
236
|
+
* changed or not.
|
|
232
237
|
*/
|
|
233
|
-
|
|
238
|
+
runFullGC?: boolean;
|
|
234
239
|
|
|
235
240
|
/**
|
|
236
241
|
* Allows additional GC options to be passed.
|
|
@@ -706,8 +711,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
706
711
|
IGarbageCollectionRuntime,
|
|
707
712
|
IRuntime,
|
|
708
713
|
ISummarizerRuntime,
|
|
709
|
-
ISummarizerInternalsProvider
|
|
710
|
-
{
|
|
714
|
+
ISummarizerInternalsProvider {
|
|
711
715
|
public get IContainerRuntime() { return this; }
|
|
712
716
|
public get IFluidRouter() { return this; }
|
|
713
717
|
|
|
@@ -746,33 +750,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
746
750
|
flushMode = defaultFlushMode,
|
|
747
751
|
} = runtimeOptions;
|
|
748
752
|
|
|
749
|
-
|
|
750
|
-
// then there are no .channel layers, we pack at level 1, otherwise we pack at level 2
|
|
751
|
-
const packingLevel = summaryOptions.disableIsolatedChannels ? 1 : 2;
|
|
752
|
-
|
|
753
|
-
let storage = context.storage;
|
|
754
|
-
if (context.baseSnapshot) {
|
|
755
|
-
// This will patch snapshot in place!
|
|
756
|
-
// If storage is provided, it will wrap storage with BlobAggregationStorage that can
|
|
757
|
-
// pack & unpack aggregated blobs.
|
|
758
|
-
// Note that if storage is provided later by loader layer, we will wrap storage in this.storage getter.
|
|
759
|
-
// BlobAggregationStorage is smart enough for double-wrapping to be no-op
|
|
760
|
-
if (context.attachState === AttachState.Attached) {
|
|
761
|
-
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
762
|
-
// So once we release 0.40 container-defn package we can remove this check.
|
|
763
|
-
assert(context.storage !== undefined, 0x1f4 /* "Attached state should have storage" */);
|
|
764
|
-
const aggrStorage = BlobAggregationStorage.wrap(
|
|
765
|
-
context.storage,
|
|
766
|
-
logger,
|
|
767
|
-
undefined /* allowPacking */,
|
|
768
|
-
packingLevel,
|
|
769
|
-
);
|
|
770
|
-
await aggrStorage.unpackSnapshot(context.baseSnapshot);
|
|
771
|
-
storage = aggrStorage;
|
|
772
|
-
} else {
|
|
773
|
-
await BlobAggregationStorage.unpackSnapshot(context.baseSnapshot);
|
|
774
|
-
}
|
|
775
|
-
}
|
|
753
|
+
const storage = context.storage;
|
|
776
754
|
|
|
777
755
|
const registry = new FluidDataStoreRegistry(registryEntries);
|
|
778
756
|
|
|
@@ -846,7 +824,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
846
824
|
loadExisting,
|
|
847
825
|
blobManagerSnapshot,
|
|
848
826
|
requestHandler,
|
|
849
|
-
storage,
|
|
850
827
|
);
|
|
851
828
|
|
|
852
829
|
return runtime;
|
|
@@ -869,25 +846,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
869
846
|
}
|
|
870
847
|
|
|
871
848
|
public get storage(): IDocumentStorageService {
|
|
872
|
-
|
|
873
|
-
// All callers should be fixed, as this API is called in detached state of container when we have
|
|
874
|
-
// no storage and it's passed down the stack without right typing.
|
|
875
|
-
// back-compat 0.40 NoStorageInDetachedMode. Also, IContainerContext storage api return type still
|
|
876
|
-
// has undefined in 0.39 package version.
|
|
877
|
-
// So once we release 0.40 container-defn package we can remove this check.
|
|
878
|
-
if (!this._storage && this.context.storage) {
|
|
879
|
-
// Note: BlobAggregationStorage is smart enough for double-wrapping to be no-op
|
|
880
|
-
// If isolated channels are disabled, then there are no .channel layers, we pack at level 1,
|
|
881
|
-
// otherwise we pack at level 2
|
|
882
|
-
this._storage = BlobAggregationStorage.wrap(
|
|
883
|
-
this.context.storage,
|
|
884
|
-
this.logger,
|
|
885
|
-
undefined /* allowPacking */,
|
|
886
|
-
this.disableIsolatedChannels ? 1 : 2,
|
|
887
|
-
);
|
|
888
|
-
}
|
|
889
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
890
|
-
return this._storage!;
|
|
849
|
+
return this.context.storage;
|
|
891
850
|
}
|
|
892
851
|
|
|
893
852
|
public get reSubmitFn(): (
|
|
@@ -1015,7 +974,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1015
974
|
}
|
|
1016
975
|
|
|
1017
976
|
private readonly createContainerMetadata: ICreateContainerMetadata;
|
|
1018
|
-
|
|
977
|
+
/**
|
|
978
|
+
* The summary number of the next summary that will be generated for this container. This is incremented every time
|
|
979
|
+
* a summary is generated.
|
|
980
|
+
*/
|
|
981
|
+
private nextSummaryNumber: number;
|
|
1019
982
|
private readonly opTracker: OpTracker;
|
|
1020
983
|
|
|
1021
984
|
private constructor(
|
|
@@ -1031,27 +994,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1031
994
|
existing: boolean,
|
|
1032
995
|
blobManagerSnapshot: IBlobManagerLoadInfo,
|
|
1033
996
|
private readonly requestHandler?: (request: IRequest, runtime: IContainerRuntime) => Promise<IResponse>,
|
|
1034
|
-
private _storage?: IDocumentStorageService,
|
|
1035
997
|
) {
|
|
1036
998
|
super();
|
|
1037
999
|
|
|
1038
1000
|
this.messageAtLastSummary = metadata?.message;
|
|
1039
1001
|
|
|
1040
|
-
// If this is an existing container, we get values from metadata.
|
|
1041
|
-
// otherwise, we initialize them.
|
|
1042
|
-
if (existing) {
|
|
1043
|
-
this.createContainerMetadata = {
|
|
1044
|
-
createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
|
|
1045
|
-
createContainerTimestamp: metadata?.createContainerTimestamp,
|
|
1046
|
-
};
|
|
1047
|
-
this.summaryCount = metadata?.summaryCount;
|
|
1048
|
-
} else {
|
|
1049
|
-
this.createContainerMetadata = {
|
|
1050
|
-
createContainerRuntimeVersion: pkgVersion,
|
|
1051
|
-
createContainerTimestamp: Date.now(),
|
|
1052
|
-
};
|
|
1053
|
-
}
|
|
1054
|
-
|
|
1055
1002
|
// Default to false (enabled).
|
|
1056
1003
|
this.disableIsolatedChannels = this.runtimeOptions.summaryOptions.disableIsolatedChannels ?? false;
|
|
1057
1004
|
|
|
@@ -1142,7 +1089,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1142
1089
|
this.handleContext,
|
|
1143
1090
|
blobManagerSnapshot,
|
|
1144
1091
|
() => this.storage,
|
|
1145
|
-
(blobId) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }),
|
|
1092
|
+
(blobId: string) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }),
|
|
1093
|
+
(blobPath: string) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
|
|
1146
1094
|
this,
|
|
1147
1095
|
this.logger,
|
|
1148
1096
|
);
|
|
@@ -1179,8 +1127,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1179
1127
|
|
|
1180
1128
|
if (this.summariesDisabled) {
|
|
1181
1129
|
this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
|
|
1182
|
-
}
|
|
1183
|
-
else {
|
|
1130
|
+
} else {
|
|
1184
1131
|
const orderedClientLogger = ChildLogger.create(this.logger, "OrderedClientElection");
|
|
1185
1132
|
const orderedClientCollection = new OrderedClientCollection(
|
|
1186
1133
|
orderedClientLogger,
|
|
@@ -1216,18 +1163,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1216
1163
|
this.summaryCollection,
|
|
1217
1164
|
async (runtime: IConnectableRuntime) => RunWhileConnectedCoordinator.create(runtime),
|
|
1218
1165
|
);
|
|
1219
|
-
}
|
|
1220
|
-
else if (SummarizerClientElection.clientDetailsPermitElection(this.context.clientDetails)) {
|
|
1166
|
+
} else if (SummarizerClientElection.clientDetailsPermitElection(this.context.clientDetails)) {
|
|
1221
1167
|
// Only create a SummaryManager and SummarizerClientElection
|
|
1222
1168
|
// if summaries are enabled and we are not the summarizer client.
|
|
1223
1169
|
const defaultAction = () => {
|
|
1224
1170
|
if (this.summaryCollection.opsSinceLastAck > maxOpsSinceLastSummary) {
|
|
1225
|
-
this.logger.sendErrorEvent({eventName: "SummaryStatus:Behind"});
|
|
1171
|
+
this.logger.sendErrorEvent({ eventName: "SummaryStatus:Behind" });
|
|
1226
1172
|
// unregister default to no log on every op after falling behind
|
|
1227
1173
|
// and register summary ack handler to re-register this handler
|
|
1228
1174
|
// after successful summary
|
|
1229
1175
|
this.summaryCollection.once(MessageType.SummaryAck, () => {
|
|
1230
|
-
this.logger.sendTelemetryEvent({eventName: "SummaryStatus:CaughtUp"});
|
|
1176
|
+
this.logger.sendTelemetryEvent({ eventName: "SummaryStatus:CaughtUp" });
|
|
1231
1177
|
// we've caught up, so re-register the default action to monitor for
|
|
1232
1178
|
// falling behind, and unregister ourself
|
|
1233
1179
|
this.summaryCollection.on("default", defaultAction);
|
|
@@ -1289,16 +1235,35 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1289
1235
|
|
|
1290
1236
|
// logging hardware telemetry
|
|
1291
1237
|
logger.sendTelemetryEvent({
|
|
1292
|
-
eventName:"DeviceSpec",
|
|
1238
|
+
eventName: "DeviceSpec",
|
|
1293
1239
|
...getDeviceSpec(),
|
|
1294
1240
|
});
|
|
1295
1241
|
|
|
1296
|
-
|
|
1242
|
+
let loadSummaryNumber: number;
|
|
1243
|
+
// Get the container creation metadata. For new container, we initialize these. For existing containers,
|
|
1244
|
+
// get the values from the metadata blob.
|
|
1245
|
+
if (existing) {
|
|
1246
|
+
this.createContainerMetadata = {
|
|
1247
|
+
createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
|
|
1248
|
+
createContainerTimestamp: metadata?.createContainerTimestamp,
|
|
1249
|
+
};
|
|
1250
|
+
// back-compat 0.59.3000 - Older document may either write summaryCount or not write it at all. If it does
|
|
1251
|
+
// not write it, initialize summaryNumber to 0.
|
|
1252
|
+
loadSummaryNumber = metadata?.summaryNumber ?? metadata?.summaryCount ?? 0;
|
|
1253
|
+
} else {
|
|
1254
|
+
this.createContainerMetadata = {
|
|
1255
|
+
createContainerRuntimeVersion: pkgVersion,
|
|
1256
|
+
createContainerTimestamp: Date.now(),
|
|
1257
|
+
};
|
|
1258
|
+
loadSummaryNumber = 0;
|
|
1259
|
+
}
|
|
1260
|
+
this.nextSummaryNumber = loadSummaryNumber + 1;
|
|
1261
|
+
|
|
1297
1262
|
this.logger.sendTelemetryEvent({
|
|
1298
1263
|
eventName: "ContainerLoadStats",
|
|
1299
1264
|
...this.createContainerMetadata,
|
|
1300
1265
|
...this.dataStores.containerLoadStats,
|
|
1301
|
-
|
|
1266
|
+
summaryNumber: loadSummaryNumber,
|
|
1302
1267
|
summaryFormatVersion: metadata?.summaryFormatVersion,
|
|
1303
1268
|
disableIsolatedChannels: metadata?.disableIsolatedChannels,
|
|
1304
1269
|
gcVersion: metadata?.gcFeature,
|
|
@@ -1334,7 +1299,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1334
1299
|
}
|
|
1335
1300
|
|
|
1336
1301
|
public get IFluidTokenProvider() {
|
|
1337
|
-
if (this.options
|
|
1302
|
+
if (this.options?.intelligence) {
|
|
1338
1303
|
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
1339
1304
|
return {
|
|
1340
1305
|
intelligence: this.options.intelligence,
|
|
@@ -1446,22 +1411,27 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1446
1411
|
return dataStoreChannel;
|
|
1447
1412
|
}
|
|
1448
1413
|
|
|
1449
|
-
|
|
1450
|
-
|
|
1414
|
+
/** Adds the container's metadata to the given summary tree. */
|
|
1415
|
+
private addMetadataToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
1416
|
+
const metadata: IContainerRuntimeMetadata = {
|
|
1451
1417
|
...this.createContainerMetadata,
|
|
1452
|
-
|
|
1418
|
+
// back-compat 0.59.3000: This is renamed to summaryNumber. Can be removed when 0.59.3000 saturates.
|
|
1419
|
+
summaryCount: this.nextSummaryNumber,
|
|
1420
|
+
// Increment the summary number for the next summary that will be generated.
|
|
1421
|
+
summaryNumber: this.nextSummaryNumber++,
|
|
1453
1422
|
summaryFormatVersion: 1,
|
|
1454
1423
|
disableIsolatedChannels: this.disableIsolatedChannels || undefined,
|
|
1455
|
-
|
|
1424
|
+
...this.garbageCollector.getMetadata(),
|
|
1456
1425
|
// The last message processed at the time of summary. If there are no new messages, use the message from the
|
|
1457
1426
|
// last summary.
|
|
1458
1427
|
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ?? this.messageAtLastSummary,
|
|
1459
|
-
sessionExpiryTimeoutMs: this.garbageCollector.sessionExpiryTimeoutMs,
|
|
1460
1428
|
};
|
|
1429
|
+
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
|
|
1461
1430
|
}
|
|
1462
1431
|
|
|
1463
1432
|
private addContainerStateToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
1464
|
-
|
|
1433
|
+
this.addMetadataToSummary(summaryTree);
|
|
1434
|
+
|
|
1465
1435
|
if (this.chunkMap.size > 0) {
|
|
1466
1436
|
const content = JSON.stringify([...this.chunkMap]);
|
|
1467
1437
|
addBlobToSummary(summaryTree, chunksBlobName, content);
|
|
@@ -1477,11 +1447,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1477
1447
|
addBlobToSummary(summaryTree, electedSummarizerBlobName, electedSummarizerContent);
|
|
1478
1448
|
}
|
|
1479
1449
|
|
|
1480
|
-
const
|
|
1450
|
+
const blobManagerSummary = this.blobManager.summarize();
|
|
1481
1451
|
// Some storage (like git) doesn't allow empty tree, so we can omit it.
|
|
1482
1452
|
// and the blob manager can handle the tree not existing when loading
|
|
1483
|
-
if (Object.keys(
|
|
1484
|
-
addTreeToSummary(summaryTree, blobsTreeName,
|
|
1453
|
+
if (Object.keys(blobManagerSummary.summary.tree).length > 0) {
|
|
1454
|
+
addTreeToSummary(summaryTree, blobsTreeName, blobManagerSummary);
|
|
1485
1455
|
}
|
|
1486
1456
|
|
|
1487
1457
|
if (this.garbageCollector.writeDataAtRoot) {
|
|
@@ -1880,8 +1850,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1880
1850
|
|
|
1881
1851
|
public createDetachedRootDataStore(
|
|
1882
1852
|
pkg: Readonly<string[]>,
|
|
1883
|
-
rootDataStoreId: string): IFluidDataStoreContextDetached
|
|
1884
|
-
{
|
|
1853
|
+
rootDataStoreId: string): IFluidDataStoreContextDetached {
|
|
1885
1854
|
return this.dataStores.createDetachedDataStoreCore(pkg, true, rootDataStoreId);
|
|
1886
1855
|
}
|
|
1887
1856
|
|
|
@@ -2064,17 +2033,17 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2064
2033
|
*/
|
|
2065
2034
|
public async summarize(options: {
|
|
2066
2035
|
/** True to generate the full tree with no handle reuse optimizations; defaults to false */
|
|
2067
|
-
fullTree?: boolean
|
|
2036
|
+
fullTree?: boolean;
|
|
2068
2037
|
/** True to track the state for this summary in the SummarizerNodes; defaults to true */
|
|
2069
|
-
trackState?: boolean
|
|
2038
|
+
trackState?: boolean;
|
|
2070
2039
|
/** Logger to use for correlated summary events */
|
|
2071
|
-
summaryLogger?: ITelemetryLogger
|
|
2040
|
+
summaryLogger?: ITelemetryLogger;
|
|
2072
2041
|
/** True to run garbage collection before summarizing; defaults to true */
|
|
2073
|
-
runGC?: boolean
|
|
2042
|
+
runGC?: boolean;
|
|
2074
2043
|
/** True to generate full GC data */
|
|
2075
|
-
fullGC?: boolean
|
|
2044
|
+
fullGC?: boolean;
|
|
2076
2045
|
/** True to run GC sweep phase after the mark phase */
|
|
2077
|
-
runSweep?: boolean
|
|
2046
|
+
runSweep?: boolean;
|
|
2078
2047
|
}): Promise<IRootSummaryTreeWithStats> {
|
|
2079
2048
|
this.verifyNotClosed();
|
|
2080
2049
|
|
|
@@ -2178,18 +2147,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2178
2147
|
}
|
|
2179
2148
|
|
|
2180
2149
|
/**
|
|
2181
|
-
* Returns the type of the GC node. Currently, there are nodes that belong to
|
|
2182
|
-
*
|
|
2150
|
+
* Returns the type of the GC node. Currently, there are nodes that belong to the root ("/"), data stores or
|
|
2151
|
+
* blob manager.
|
|
2183
2152
|
*/
|
|
2184
2153
|
public getNodeType(nodePath: string): GCNodeType {
|
|
2185
2154
|
if (this.isBlobPath(nodePath)) {
|
|
2186
2155
|
return GCNodeType.Blob;
|
|
2187
2156
|
}
|
|
2188
|
-
|
|
2189
|
-
return GCNodeType.DataStore;
|
|
2190
|
-
}
|
|
2191
|
-
// Root node ("/") and DDS nodes belong to "Other" node types.
|
|
2192
|
-
return GCNodeType.Other;
|
|
2157
|
+
return this.dataStores.getGCNodeType(nodePath) ?? GCNodeType.Other;
|
|
2193
2158
|
}
|
|
2194
2159
|
|
|
2195
2160
|
/**
|
|
@@ -2197,13 +2162,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2197
2162
|
* data store or an attachment blob.
|
|
2198
2163
|
*/
|
|
2199
2164
|
public getGCNodePackagePath(nodePath: string): readonly string[] | undefined {
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2165
|
+
switch (this.getNodeType(nodePath)) {
|
|
2166
|
+
case GCNodeType.Blob:
|
|
2167
|
+
return ["_blobs"];
|
|
2168
|
+
case GCNodeType.DataStore:
|
|
2169
|
+
case GCNodeType.SubDataStore:
|
|
2170
|
+
return this.dataStores.getDataStorePackagePath(nodePath);
|
|
2171
|
+
default:
|
|
2172
|
+
assert(false, 0x2de /* "Package path requested for unsupported node type." */);
|
|
2203
2173
|
}
|
|
2204
|
-
const dataStorePkgPath = this.dataStores.getDataStorePackagePath(nodePath);
|
|
2205
|
-
assert(dataStorePkgPath !== undefined, 0x2d6 /* "Package path requested for unknown node type." */);
|
|
2206
|
-
return dataStorePkgPath;
|
|
2207
2174
|
}
|
|
2208
2175
|
|
|
2209
2176
|
/**
|
|
@@ -2224,11 +2191,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2224
2191
|
public async collectGarbage(
|
|
2225
2192
|
options: {
|
|
2226
2193
|
/** Logger to use for logging GC events */
|
|
2227
|
-
logger?: ITelemetryLogger
|
|
2194
|
+
logger?: ITelemetryLogger;
|
|
2228
2195
|
/** True to run GC sweep phase after the mark phase */
|
|
2229
|
-
runSweep?: boolean
|
|
2196
|
+
runSweep?: boolean;
|
|
2230
2197
|
/** True to generate full GC data */
|
|
2231
|
-
fullGC?: boolean
|
|
2198
|
+
fullGC?: boolean;
|
|
2232
2199
|
},
|
|
2233
2200
|
): Promise<IGCStats> {
|
|
2234
2201
|
return this.garbageCollector.collectGarbage(options);
|
|
@@ -2254,14 +2221,25 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2254
2221
|
*/
|
|
2255
2222
|
public async submitSummary(options: ISubmitSummaryOptions): Promise<SubmitSummaryResult> {
|
|
2256
2223
|
const { fullTree, refreshLatestAck, summaryLogger } = options;
|
|
2224
|
+
// The summary number for this summary. This will be updated during the summary process, so get it now and
|
|
2225
|
+
// use it for all events logged during this summary.
|
|
2226
|
+
const summaryNumber = this.nextSummaryNumber;
|
|
2227
|
+
const summaryNumberLogger = ChildLogger.create(
|
|
2228
|
+
summaryLogger,
|
|
2229
|
+
undefined,
|
|
2230
|
+
{
|
|
2231
|
+
all: { summaryNumber },
|
|
2232
|
+
},
|
|
2233
|
+
);
|
|
2234
|
+
|
|
2257
2235
|
if (refreshLatestAck) {
|
|
2258
2236
|
const latestSummaryRefSeq = await this.refreshLatestSummaryAckFromServer(
|
|
2259
|
-
ChildLogger.create(
|
|
2237
|
+
ChildLogger.create(summaryNumberLogger, undefined, { all: { safeSummary: true } }));
|
|
2260
2238
|
|
|
2261
2239
|
if (latestSummaryRefSeq > this.deltaManager.lastSequenceNumber) {
|
|
2262
2240
|
// We need to catch up to the latest summary's reference sequence number before pausing.
|
|
2263
2241
|
await PerformanceEvent.timedExecAsync(
|
|
2264
|
-
|
|
2242
|
+
summaryNumberLogger,
|
|
2265
2243
|
{
|
|
2266
2244
|
eventName: "WaitingForSeq",
|
|
2267
2245
|
lastSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
@@ -2284,16 +2262,16 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2284
2262
|
// We should be here is we haven't processed be here. If we are of if the last message's sequence number
|
|
2285
2263
|
// doesn't match the last processed sequence number, log an error.
|
|
2286
2264
|
if (summaryRefSeqNum !== this.deltaManager.lastMessage?.sequenceNumber) {
|
|
2287
|
-
|
|
2265
|
+
summaryNumberLogger.sendErrorEvent({
|
|
2288
2266
|
eventName: "LastSequenceMismatch",
|
|
2289
2267
|
error: message,
|
|
2290
2268
|
});
|
|
2291
2269
|
}
|
|
2292
2270
|
|
|
2293
|
-
this.summarizerNode.startSummary(summaryRefSeqNum,
|
|
2271
|
+
this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger);
|
|
2294
2272
|
|
|
2295
2273
|
// Helper function to check whether we should still continue between each async step.
|
|
2296
|
-
const checkContinue = (): { continue: true; } | { continue: false; error: string } => {
|
|
2274
|
+
const checkContinue = (): { continue: true; } | { continue: false; error: string; } => {
|
|
2297
2275
|
// Do not check for loss of connectivity directly! Instead leave it up to
|
|
2298
2276
|
// RunWhileConnectedCoordinator to control policy in a single place.
|
|
2299
2277
|
// This will allow easier change of design if we chose to. For example, we may chose to allow
|
|
@@ -2331,13 +2309,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2331
2309
|
};
|
|
2332
2310
|
}
|
|
2333
2311
|
|
|
2334
|
-
// increment summary count
|
|
2335
|
-
if (this.summaryCount !== undefined) {
|
|
2336
|
-
this.summaryCount++;
|
|
2337
|
-
} else {
|
|
2338
|
-
this.summaryCount = 1;
|
|
2339
|
-
}
|
|
2340
|
-
|
|
2341
2312
|
const trace = Trace.start();
|
|
2342
2313
|
let summarizeResult: IRootSummaryTreeWithStats;
|
|
2343
2314
|
// If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
|
|
@@ -2347,7 +2318,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2347
2318
|
summarizeResult = await this.summarize({
|
|
2348
2319
|
fullTree: fullTree || forcedFullTree,
|
|
2349
2320
|
trackState: true,
|
|
2350
|
-
summaryLogger,
|
|
2321
|
+
summaryLogger: summaryNumberLogger,
|
|
2351
2322
|
runGC: this.garbageCollector.shouldRunGC,
|
|
2352
2323
|
});
|
|
2353
2324
|
} catch (error) {
|
|
@@ -2383,6 +2354,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2383
2354
|
gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
|
|
2384
2355
|
opsSizesSinceLastSummary: this.opTracker.opsSizeAccumulator,
|
|
2385
2356
|
nonSystemOpsSinceLastSummary: this.opTracker.nonSystemOpCount,
|
|
2357
|
+
summaryNumber,
|
|
2386
2358
|
...partialStats,
|
|
2387
2359
|
};
|
|
2388
2360
|
const generateSummaryData = {
|
|
@@ -2806,7 +2778,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2806
2778
|
getVersionDuration?: number | undefined;
|
|
2807
2779
|
getSnapshotDuration?: number | undefined;
|
|
2808
2780
|
}) => void; }) => {
|
|
2809
|
-
const stats: { getVersionDuration?: number; getSnapshotDuration?: number } = {};
|
|
2781
|
+
const stats: { getVersionDuration?: number; getSnapshotDuration?: number; } = {};
|
|
2810
2782
|
const trace = Trace.start();
|
|
2811
2783
|
|
|
2812
2784
|
const versions = await this.storage.getVersions(versionId, 1);
|
package/src/dataStoreContext.ts
CHANGED
|
@@ -246,7 +246,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
246
246
|
|
|
247
247
|
// The used state of this node as per the last GC run. This is used to update the used state of the channel
|
|
248
248
|
// if it realizes after GC is run.
|
|
249
|
-
private lastUsedState: { usedRoutes: string[]
|
|
249
|
+
private lastUsedState: { usedRoutes: string[]; gcTimestamp?: number; } | undefined;
|
|
250
250
|
|
|
251
251
|
public readonly id: string;
|
|
252
252
|
private readonly _containerRuntime: ContainerRuntime;
|
|
@@ -317,7 +317,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
317
317
|
}
|
|
318
318
|
|
|
319
319
|
private rejectDeferredRealize(reason: string, packageName?: string): never {
|
|
320
|
-
throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.PackageData }});
|
|
320
|
+
throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.PackageData } });
|
|
321
321
|
}
|
|
322
322
|
|
|
323
323
|
public async realize(): Promise<IFluidDataStoreChannel> {
|
|
@@ -326,9 +326,9 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
326
326
|
this.channelDeferred = new Deferred<IFluidDataStoreChannel>();
|
|
327
327
|
this.realizeCore(this.existing).catch((error) => {
|
|
328
328
|
const errorWrapped = DataProcessingError.wrapIfUnrecognized(error, "realizeFluidDataStoreContext");
|
|
329
|
-
errorWrapped.addTelemetryProperties({ fluidDataStoreId: { value: this.id, tag: "PackageData"} });
|
|
329
|
+
errorWrapped.addTelemetryProperties({ fluidDataStoreId: { value: this.id, tag: "PackageData" } });
|
|
330
330
|
this.channelDeferred?.reject(errorWrapped);
|
|
331
|
-
this.logger.sendErrorEvent({ eventName: "RealizeError"}, errorWrapped);
|
|
331
|
+
this.logger.sendErrorEvent({ eventName: "RealizeError" }, errorWrapped);
|
|
332
332
|
});
|
|
333
333
|
}
|
|
334
334
|
return this.channelDeferred.promise;
|
|
@@ -641,8 +641,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
641
641
|
throw new Error("Runtime already bound");
|
|
642
642
|
}
|
|
643
643
|
|
|
644
|
-
try
|
|
645
|
-
{
|
|
644
|
+
try {
|
|
646
645
|
assert(!this.detachedRuntimeCreation, 0x148 /* "Detached runtime creation on runtime bind" */);
|
|
647
646
|
assert(this.channelDeferred !== undefined, 0x149 /* "Undefined channel deferral" */);
|
|
648
647
|
assert(this.pkg !== undefined, 0x14a /* "Undefined package path" */);
|
|
@@ -680,7 +679,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
680
679
|
} catch (error) {
|
|
681
680
|
this.channelDeferred?.reject(error);
|
|
682
681
|
this.logger.sendErrorEvent(
|
|
683
|
-
{ eventName: "BindRuntimeError", fluidDataStoreId: { value: this.id, tag: "PackageData"} },
|
|
682
|
+
{ eventName: "BindRuntimeError", fluidDataStoreId: { value: this.id, tag: "PackageData" } },
|
|
684
683
|
error);
|
|
685
684
|
}
|
|
686
685
|
}
|
|
@@ -993,8 +992,7 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase {
|
|
|
993
992
|
*/
|
|
994
993
|
export class LocalDetachedFluidDataStoreContext
|
|
995
994
|
extends LocalFluidDataStoreContextBase
|
|
996
|
-
implements IFluidDataStoreContextDetached
|
|
997
|
-
{
|
|
995
|
+
implements IFluidDataStoreContextDetached {
|
|
998
996
|
constructor(props: ILocalFluidDataStoreContextProps) {
|
|
999
997
|
super(props);
|
|
1000
998
|
this.detachedRuntimeCreation = true;
|
|
@@ -1002,8 +1000,7 @@ export class LocalDetachedFluidDataStoreContext
|
|
|
1002
1000
|
|
|
1003
1001
|
public async attachRuntime(
|
|
1004
1002
|
registry: IProvideFluidDataStoreFactory,
|
|
1005
|
-
dataStoreChannel: IFluidDataStoreChannel)
|
|
1006
|
-
{
|
|
1003
|
+
dataStoreChannel: IFluidDataStoreChannel) {
|
|
1007
1004
|
assert(this.detachedRuntimeCreation, 0x154 /* "runtime creation is already attached" */);
|
|
1008
1005
|
assert(this.channelDeferred === undefined, 0x155 /* "channel deferral is already set" */);
|
|
1009
1006
|
|
package/src/dataStoreContexts.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { assert, Deferred, Lazy } from "@fluidframework/common-utils";
|
|
|
8
8
|
import { ChildLogger } from "@fluidframework/telemetry-utils";
|
|
9
9
|
import { FluidDataStoreContext, LocalFluidDataStoreContext } from "./dataStoreContext";
|
|
10
10
|
|
|
11
|
-
export class DataStoreContexts implements Iterable<[string,FluidDataStoreContext]>, IDisposable {
|
|
11
|
+
export class DataStoreContexts implements Iterable<[string, FluidDataStoreContext]>, IDisposable {
|
|
12
12
|
private readonly notBoundContexts = new Set<string>();
|
|
13
13
|
|
|
14
14
|
/** Attached and loaded context proxies */
|
|
@@ -23,7 +23,7 @@ import { FluidDataStoreContext, LocalFluidDataStoreContext } from "./dataStoreCo
|
|
|
23
23
|
*/
|
|
24
24
|
private readonly deferredContexts = new Map<string, Deferred<FluidDataStoreContext>>();
|
|
25
25
|
|
|
26
|
-
private readonly disposeOnce = new Lazy<void>(()=>{
|
|
26
|
+
private readonly disposeOnce = new Lazy<void>(() => {
|
|
27
27
|
// close/stop all store contexts
|
|
28
28
|
for (const [fluidDataStoreId, contextD] of this.deferredContexts) {
|
|
29
29
|
contextD.promise.then((context) => {
|
|
@@ -52,7 +52,7 @@ import { FluidDataStoreContext, LocalFluidDataStoreContext } from "./dataStoreCo
|
|
|
52
52
|
return this._contexts.size;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
public get disposed() { return this.disposeOnce.evaluated;}
|
|
55
|
+
public get disposed() { return this.disposeOnce.evaluated; }
|
|
56
56
|
public readonly dispose = () => this.disposeOnce.value;
|
|
57
57
|
|
|
58
58
|
public notBoundLength() {
|
|
@@ -106,8 +106,8 @@ import { FluidDataStoreContext, LocalFluidDataStoreContext } from "./dataStoreCo
|
|
|
106
106
|
/**
|
|
107
107
|
* Get the context with the given id, once it exists locally and is attached.
|
|
108
108
|
* e.g. If created locally, it must be bound, or if created remotely then it's fine as soon as it's sync'd in.
|
|
109
|
-
* @param id The id of the context to get
|
|
110
|
-
* @param wait If false, return undefined if the context isn't present and ready now. Otherwise, wait for it.
|
|
109
|
+
* @param id - The id of the context to get
|
|
110
|
+
* @param wait - If false, return undefined if the context isn't present and ready now. Otherwise, wait for it.
|
|
111
111
|
*/
|
|
112
112
|
public async getBoundOrRemoted(id: string, wait: boolean): Promise<FluidDataStoreContext | undefined> {
|
|
113
113
|
const deferredContext = this.ensureDeferred(id);
|