@fluidframework/container-runtime 0.56.5 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +9 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +6 -6
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +67 -26
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +146 -87
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +62 -0
- package/dist/dataStore.d.ts.map +1 -0
- package/dist/dataStore.js +135 -0
- package/dist/dataStore.js.map +1 -0
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +9 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +14 -19
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +66 -27
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +272 -97
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +1 -0
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +23 -15
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +4 -6
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryGenerator.d.ts +2 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +46 -29
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +9 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +6 -6
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +67 -26
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +147 -88
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +62 -0
- package/lib/dataStore.d.ts.map +1 -0
- package/lib/dataStore.js +130 -0
- package/lib/dataStore.js.map +1 -0
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +9 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +13 -18
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +66 -27
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +274 -99
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +3 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +1 -0
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +23 -15
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +4 -6
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryGenerator.d.ts +2 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +46 -29
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +13 -13
- package/src/blobManager.ts +12 -1
- package/src/connectionTelemetry.ts +7 -6
- package/src/containerRuntime.ts +244 -113
- package/src/dataStore.ts +187 -0
- package/src/dataStoreContext.ts +1 -1
- package/src/dataStores.ts +23 -38
- package/src/garbageCollection.ts +385 -150
- package/src/index.ts +3 -1
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +25 -16
- package/src/summarizerTypes.ts +4 -8
- package/src/summaryGenerator.ts +71 -23
package/src/containerRuntime.ts
CHANGED
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
TaggedLoggerAdapter,
|
|
47
47
|
MonitoringContext,
|
|
48
48
|
loggerToMonitoringContext,
|
|
49
|
+
TelemetryDataTag,
|
|
49
50
|
} from "@fluidframework/telemetry-utils";
|
|
50
51
|
import { DriverHeader, IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
51
52
|
import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
|
|
@@ -64,7 +65,6 @@ import {
|
|
|
64
65
|
ISummaryConfiguration,
|
|
65
66
|
ISummaryContent,
|
|
66
67
|
ISummaryTree,
|
|
67
|
-
ITree,
|
|
68
68
|
MessageType,
|
|
69
69
|
SummaryType,
|
|
70
70
|
} from "@fluidframework/protocol-definitions";
|
|
@@ -99,7 +99,6 @@ import {
|
|
|
99
99
|
requestFluidObject,
|
|
100
100
|
responseToException,
|
|
101
101
|
seqFromTree,
|
|
102
|
-
convertSummaryTreeToITree,
|
|
103
102
|
} from "@fluidframework/runtime-utils";
|
|
104
103
|
import { v4 as uuid } from "uuid";
|
|
105
104
|
import { ContainerFluidHandleContext } from "./containerHandleContext";
|
|
@@ -145,8 +144,14 @@ import {
|
|
|
145
144
|
IGarbageCollectionRuntime,
|
|
146
145
|
IGarbageCollector,
|
|
147
146
|
IGCStats,
|
|
148
|
-
IUsedStateStats,
|
|
149
147
|
} from "./garbageCollection";
|
|
148
|
+
import {
|
|
149
|
+
AliasResult,
|
|
150
|
+
channelToDataStore,
|
|
151
|
+
IDataStore,
|
|
152
|
+
IDataStoreAliasMessage,
|
|
153
|
+
isDataStoreAliasMessage,
|
|
154
|
+
} from "./dataStore";
|
|
150
155
|
|
|
151
156
|
export enum ContainerMessageType {
|
|
152
157
|
// An op to be delivered to store
|
|
@@ -283,12 +288,41 @@ export interface IContainerRuntimeOptions {
|
|
|
283
288
|
* 3. "bypass" will skip the check entirely. This is not recommended.
|
|
284
289
|
*/
|
|
285
290
|
loadSequenceNumberVerification?: "close" | "log" | "bypass";
|
|
291
|
+
/**
|
|
292
|
+
* Should the runtime use data store aliasing for creating root datastores.
|
|
293
|
+
* In case of aliasing conflicts, the runtime will raise an exception which does
|
|
294
|
+
* not effect the status of the container.
|
|
295
|
+
*/
|
|
296
|
+
useDataStoreAliasing?: boolean;
|
|
286
297
|
}
|
|
287
298
|
|
|
288
299
|
type IRuntimeMessageMetadata = undefined | {
|
|
289
300
|
batch?: boolean;
|
|
290
301
|
};
|
|
291
302
|
|
|
303
|
+
/**
|
|
304
|
+
* The summary tree returned by the root node. It adds state relevant to the root of the tree.
|
|
305
|
+
*/
|
|
306
|
+
export interface IRootSummaryTreeWithStats extends ISummaryTreeWithStats {
|
|
307
|
+
/** The garbage collection stats if GC ran, undefined otherwise. */
|
|
308
|
+
gcStats?: IGCStats;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Accepted header keys for requests coming to the runtime.
|
|
313
|
+
*/
|
|
314
|
+
export enum RuntimeHeaders {
|
|
315
|
+
/** True to wait for a data store to be created and loaded before returning it. */
|
|
316
|
+
wait = "wait",
|
|
317
|
+
/**
|
|
318
|
+
* True if the request is from an external app. Used for GC to handle scenarios where a data store
|
|
319
|
+
* is deleted and requested via an external app.
|
|
320
|
+
*/
|
|
321
|
+
externalRequest = "externalRequest",
|
|
322
|
+
/** True if the request is coming from an IFluidHandle. */
|
|
323
|
+
viaHandle = "viaHandle",
|
|
324
|
+
}
|
|
325
|
+
|
|
292
326
|
/**
|
|
293
327
|
* @deprecated
|
|
294
328
|
* Untagged logger is unsupported going forward. There are old loaders with old ContainerContexts that only
|
|
@@ -297,10 +331,11 @@ type IRuntimeMessageMetadata = undefined | {
|
|
|
297
331
|
*/
|
|
298
332
|
interface OldContainerContextWithLogger extends IContainerContext {
|
|
299
333
|
logger: ITelemetryBaseLogger;
|
|
300
|
-
}
|
|
334
|
+
}
|
|
301
335
|
|
|
302
336
|
// Local storage key to set the default flush mode to TurnBased
|
|
303
337
|
const turnBasedFlushModeKey = "Fluid.ContainerRuntime.FlushModeTurnBased";
|
|
338
|
+
const useDataStoreAliasingKey = "Fluid.ContainerRuntime.UseDataStoreAliasing";
|
|
304
339
|
|
|
305
340
|
export enum RuntimeMessage {
|
|
306
341
|
FluidDataStoreOp = "component",
|
|
@@ -651,7 +686,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
651
686
|
): Promise<ContainerRuntime> {
|
|
652
687
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
653
688
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
654
|
-
const passLogger = context.taggedLogger
|
|
689
|
+
const passLogger = context.taggedLogger ?? new TaggedLoggerAdapter((context as
|
|
655
690
|
OldContainerContextWithLogger).logger);
|
|
656
691
|
const logger = ChildLogger.create(passLogger, undefined, {
|
|
657
692
|
all: {
|
|
@@ -663,6 +698,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
663
698
|
summaryOptions = {},
|
|
664
699
|
gcOptions = {},
|
|
665
700
|
loadSequenceNumberVerification = "close",
|
|
701
|
+
useDataStoreAliasing = false,
|
|
666
702
|
} = runtimeOptions;
|
|
667
703
|
|
|
668
704
|
// We pack at data store level only. If isolated channels are disabled,
|
|
@@ -756,6 +792,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
756
792
|
summaryOptions,
|
|
757
793
|
gcOptions,
|
|
758
794
|
loadSequenceNumberVerification,
|
|
795
|
+
useDataStoreAliasing,
|
|
759
796
|
},
|
|
760
797
|
containerScope,
|
|
761
798
|
logger,
|
|
@@ -853,6 +890,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
853
890
|
private readonly summaryCollection: SummaryCollection;
|
|
854
891
|
|
|
855
892
|
private readonly summarizerNode: IRootSummarizerNodeWithGC;
|
|
893
|
+
private readonly _aliasingEnabled: boolean;
|
|
856
894
|
|
|
857
895
|
private _orderSequentiallyCalls: number = 0;
|
|
858
896
|
private _flushMode: FlushMode;
|
|
@@ -873,23 +911,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
873
911
|
}
|
|
874
912
|
|
|
875
913
|
private get summaryConfiguration() {
|
|
876
|
-
return
|
|
914
|
+
return {
|
|
877
915
|
// the defaults
|
|
878
916
|
... DefaultSummaryConfiguration,
|
|
879
917
|
// the server provided values
|
|
880
918
|
... this.context?.serviceConfiguration?.summary,
|
|
881
919
|
// the runtime configuration overrides
|
|
882
920
|
... this.runtimeOptions.summaryOptions?.summaryConfigOverrides,
|
|
883
|
-
|
|
921
|
+
};
|
|
884
922
|
}
|
|
885
923
|
|
|
886
924
|
private _disposed = false;
|
|
887
925
|
public get disposed() { return this._disposed; }
|
|
888
926
|
|
|
889
|
-
private dirtyContainer
|
|
927
|
+
private dirtyContainer: boolean;
|
|
890
928
|
private emitDirtyDocumentEvent = true;
|
|
891
929
|
|
|
892
|
-
private summarizerWarning = (warning: ContainerWarning) =>
|
|
930
|
+
private readonly summarizerWarning = (warning: ContainerWarning) =>
|
|
893
931
|
this.mc.logger.sendTelemetryEvent({ eventName: "summarizerWarning" }, warning);
|
|
894
932
|
/**
|
|
895
933
|
* Summarizer is responsible for coordinating when to send generate and send summaries.
|
|
@@ -914,8 +952,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
914
952
|
* and is the single source of truth for this container.
|
|
915
953
|
*/
|
|
916
954
|
public readonly disableIsolatedChannels: boolean;
|
|
917
|
-
/** The message
|
|
918
|
-
private
|
|
955
|
+
/** The last message processed at the time of the last summary. */
|
|
956
|
+
private messageAtLastSummary: ISummaryMetadataMessage | undefined;
|
|
919
957
|
|
|
920
958
|
private get summarizer(): Summarizer {
|
|
921
959
|
assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
|
|
@@ -946,7 +984,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
946
984
|
private _storage?: IDocumentStorageService,
|
|
947
985
|
) {
|
|
948
986
|
super();
|
|
949
|
-
|
|
987
|
+
|
|
988
|
+
this.messageAtLastSummary = metadata?.message;
|
|
950
989
|
|
|
951
990
|
// If this is an existing container, we get values from metadata.
|
|
952
991
|
// otherwise, we initialize them.
|
|
@@ -978,24 +1017,21 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
978
1017
|
this.mc.config.getBoolean(turnBasedFlushModeKey) ?? false
|
|
979
1018
|
? FlushMode.TurnBased : FlushMode.Immediate;
|
|
980
1019
|
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
* we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
986
|
-
* of this client's connection.
|
|
987
|
-
*/
|
|
988
|
-
const getCurrentTimestamp = () => {
|
|
989
|
-
const client = this.clientId !== undefined ? this.getAudience().getMember(this.clientId) : undefined;
|
|
990
|
-
const timestamp = client?.timestamp;
|
|
991
|
-
return this.deltaManager.lastMessage?.timestamp ?? timestamp ?? Date.now();
|
|
992
|
-
};
|
|
1020
|
+
this._aliasingEnabled =
|
|
1021
|
+
(this.mc.config.getBoolean(useDataStoreAliasingKey) ?? false) ||
|
|
1022
|
+
(runtimeOptions.useDataStoreAliasing ?? false);
|
|
1023
|
+
|
|
993
1024
|
this.garbageCollector = GarbageCollector.create(
|
|
994
1025
|
this,
|
|
995
1026
|
this.runtimeOptions.gcOptions,
|
|
996
1027
|
(unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
|
|
997
|
-
|
|
998
|
-
|
|
1028
|
+
(nodePath: string) => this.dataStores.getNodePackagePath(nodePath),
|
|
1029
|
+
/**
|
|
1030
|
+
* Returns the timestamp of the last message seen by this client. This is used by garbage collector as
|
|
1031
|
+
* the current reference timestamp for tracking unreferenced objects.
|
|
1032
|
+
*/
|
|
1033
|
+
() => this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp,
|
|
1034
|
+
() => this.messageAtLastSummary?.timestamp,
|
|
999
1035
|
context.baseSnapshot,
|
|
1000
1036
|
async <T>(id: string) => readAndParse<T>(this.storage, id),
|
|
1001
1037
|
this.mc.logger,
|
|
@@ -1047,7 +1083,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1047
1083
|
(id: string) => this.summarizerNode.deleteChild(id),
|
|
1048
1084
|
this.mc.logger,
|
|
1049
1085
|
async () => this.garbageCollector.getDataStoreBaseGCDetails(),
|
|
1050
|
-
(
|
|
1086
|
+
(path: string, timestampMs: number, packagePath?: readonly string[]) => this.garbageCollector.nodeUpdated(
|
|
1087
|
+
path,
|
|
1088
|
+
"Changed",
|
|
1089
|
+
timestampMs,
|
|
1090
|
+
packagePath,
|
|
1091
|
+
),
|
|
1051
1092
|
new Map<string, string>(dataStoreAliasMap),
|
|
1052
1093
|
this.garbageCollector.writeDataAtRoot,
|
|
1053
1094
|
);
|
|
@@ -1080,6 +1121,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1080
1121
|
|
|
1081
1122
|
this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
|
|
1082
1123
|
|
|
1124
|
+
const { attachState, pendingLocalState } = this.context;
|
|
1125
|
+
this.dirtyContainer = attachState !== AttachState.Attached
|
|
1126
|
+
|| (pendingLocalState as IPendingLocalState)?.pendingStates.length > 0;
|
|
1127
|
+
this.context.updateDirtyContainerState(this.dirtyContainer);
|
|
1128
|
+
|
|
1083
1129
|
// Map the deprecated generateSummaries flag to disableSummaries.
|
|
1084
1130
|
if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
|
|
1085
1131
|
this.runtimeOptions.summaryOptions.disableSummaries = true;
|
|
@@ -1305,18 +1351,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1305
1351
|
return create404Response(request);
|
|
1306
1352
|
}
|
|
1307
1353
|
} else if (requestParser.pathParts.length > 0) {
|
|
1308
|
-
|
|
1309
|
-
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
1310
|
-
* an error if the data store being requested is marked as unreferenced as per the data store's initial
|
|
1311
|
-
* summary.
|
|
1312
|
-
*
|
|
1313
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
1314
|
-
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
1315
|
-
*/
|
|
1316
|
-
const wait = typeof request.headers?.wait === "boolean" ? request.headers.wait : undefined;
|
|
1317
|
-
const dataStore = request.headers?.externalRequest && this.garbageCollector.shouldRunGC
|
|
1318
|
-
? await this.getDataStoreIfInitiallyReferenced(id, wait)
|
|
1319
|
-
: await this.getDataStore(id, wait);
|
|
1354
|
+
const dataStore = await this.getDataStoreFromRequest(id, request);
|
|
1320
1355
|
const subRequest = requestParser.createSubRequest(1);
|
|
1321
1356
|
// We always expect createSubRequest to include a leading slash, but asserting here to protect against
|
|
1322
1357
|
// unintentionally modifying the url if that changes.
|
|
@@ -1331,6 +1366,40 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1331
1366
|
}
|
|
1332
1367
|
}
|
|
1333
1368
|
|
|
1369
|
+
private async getDataStoreFromRequest(id: string, request: IRequest): Promise<IFluidRouter> {
|
|
1370
|
+
const wait = typeof request.headers?.[RuntimeHeaders.wait] === "boolean"
|
|
1371
|
+
? request.headers?.[RuntimeHeaders.wait]
|
|
1372
|
+
: true;
|
|
1373
|
+
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
1377
|
+
* an error if the data store being requested is marked as unreferenced as per the data store's base
|
|
1378
|
+
* GC data.
|
|
1379
|
+
*
|
|
1380
|
+
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
1381
|
+
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
1382
|
+
*/
|
|
1383
|
+
if (request.headers?.[RuntimeHeaders.externalRequest] && this.garbageCollector.shouldRunGC) {
|
|
1384
|
+
// The data store is referenced if used routes in the base summary has a route to self.
|
|
1385
|
+
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
1386
|
+
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
1387
|
+
if (!(usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/"))) {
|
|
1388
|
+
throw responseToException(create404Response(request), request);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
const dataStoreChannel = await dataStoreContext.realize();
|
|
1393
|
+
this.garbageCollector.nodeUpdated(
|
|
1394
|
+
`/${id}`,
|
|
1395
|
+
"Loaded",
|
|
1396
|
+
undefined /* timestampMs */,
|
|
1397
|
+
dataStoreContext.packagePath,
|
|
1398
|
+
request?.headers,
|
|
1399
|
+
);
|
|
1400
|
+
return dataStoreChannel;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1334
1403
|
private formMetadata(): IContainerRuntimeMetadata {
|
|
1335
1404
|
return {
|
|
1336
1405
|
...this.createContainerMetadata,
|
|
@@ -1338,50 +1407,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1338
1407
|
summaryFormatVersion: 1,
|
|
1339
1408
|
disableIsolatedChannels: this.disableIsolatedChannels || undefined,
|
|
1340
1409
|
gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
|
|
1341
|
-
// The last message processed at the time of summary. If there are no messages,
|
|
1342
|
-
//
|
|
1343
|
-
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ?? this.
|
|
1410
|
+
// The last message processed at the time of summary. If there are no new messages, use the message from the
|
|
1411
|
+
// last summary.
|
|
1412
|
+
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ?? this.messageAtLastSummary,
|
|
1344
1413
|
sessionExpiryTimeoutMs: this.garbageCollector.sessionExpiryTimeoutMs,
|
|
1345
1414
|
};
|
|
1346
1415
|
}
|
|
1347
1416
|
|
|
1348
|
-
/**
|
|
1349
|
-
* Retrieves the runtime for a data store if it's referenced as per the initially summary that it is loaded with.
|
|
1350
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted and marked
|
|
1351
|
-
* as unreferenced by GC.
|
|
1352
|
-
* @param id - Id supplied during creating the data store.
|
|
1353
|
-
* @param wait - True if you want to wait for it.
|
|
1354
|
-
* @returns the data store runtime if the data store exists and is initially referenced; undefined otherwise.
|
|
1355
|
-
*/
|
|
1356
|
-
private async getDataStoreIfInitiallyReferenced(id: string, wait = true): Promise<IFluidRouter> {
|
|
1357
|
-
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
1358
|
-
// The data store is referenced if used routes in the initial summary has a route to self.
|
|
1359
|
-
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
1360
|
-
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
1361
|
-
if (usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/")) {
|
|
1362
|
-
return dataStoreContext.realize();
|
|
1363
|
-
}
|
|
1364
|
-
|
|
1365
|
-
// The data store is unreferenced. Throw a 404 response exception.
|
|
1366
|
-
const request = { url: id };
|
|
1367
|
-
throw responseToException(create404Response(request), request);
|
|
1368
|
-
}
|
|
1369
|
-
|
|
1370
|
-
/**
|
|
1371
|
-
* Notifies this object to take the snapshot of the container.
|
|
1372
|
-
* @deprecated - Use summarize to get summary of the container runtime.
|
|
1373
|
-
*/
|
|
1374
|
-
public async snapshot(): Promise<ITree> {
|
|
1375
|
-
const summaryResult = await this.summarize({
|
|
1376
|
-
summaryLogger: this.logger,
|
|
1377
|
-
fullTree: true,
|
|
1378
|
-
trackState: false,
|
|
1379
|
-
runGC: this.garbageCollector.shouldRunGC,
|
|
1380
|
-
fullGC: true,
|
|
1381
|
-
});
|
|
1382
|
-
return convertSummaryTreeToITree(summaryResult.summary);
|
|
1383
|
-
}
|
|
1384
|
-
|
|
1385
1417
|
private addContainerStateToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
1386
1418
|
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
1387
1419
|
if (this.chunkMap.size > 0) {
|
|
@@ -1595,10 +1627,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1595
1627
|
return context.realize();
|
|
1596
1628
|
}
|
|
1597
1629
|
|
|
1598
|
-
protected async getDataStore(id: string, wait = true): Promise<IFluidRouter> {
|
|
1599
|
-
return (await this.dataStores.getDataStore(id, wait)).realize();
|
|
1600
|
-
}
|
|
1601
|
-
|
|
1602
1630
|
public setFlushMode(mode: FlushMode): void {
|
|
1603
1631
|
if (mode === this._flushMode) {
|
|
1604
1632
|
return;
|
|
@@ -1676,16 +1704,70 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1676
1704
|
}
|
|
1677
1705
|
}
|
|
1678
1706
|
|
|
1679
|
-
public async createDataStore(pkg: string | string[]): Promise<
|
|
1680
|
-
|
|
1707
|
+
public async createDataStore(pkg: string | string[]): Promise<IDataStore> {
|
|
1708
|
+
const internalId = uuid();
|
|
1709
|
+
return channelToDataStore(
|
|
1710
|
+
await this._createDataStore(pkg, false /* isRoot */, internalId),
|
|
1711
|
+
internalId,
|
|
1712
|
+
this,
|
|
1713
|
+
this.dataStores,
|
|
1714
|
+
this.mc.logger);
|
|
1681
1715
|
}
|
|
1682
1716
|
|
|
1683
|
-
|
|
1717
|
+
/**
|
|
1718
|
+
* Creates a root datastore directly with a user generated id and attaches it to storage.
|
|
1719
|
+
* It is vulnerable to name collisions and should not be used.
|
|
1720
|
+
*
|
|
1721
|
+
* This method will be removed. See #6465.
|
|
1722
|
+
*/
|
|
1723
|
+
private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1684
1724
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
1685
1725
|
fluidDataStore.bindToContext();
|
|
1686
1726
|
return fluidDataStore;
|
|
1687
1727
|
}
|
|
1688
1728
|
|
|
1729
|
+
public async createRootDataStore(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1730
|
+
return this._aliasingEnabled === true ?
|
|
1731
|
+
this.createAndAliasDataStore(pkg, rootDataStoreId) :
|
|
1732
|
+
this.createRootDataStoreLegacy(pkg, rootDataStoreId);
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1735
|
+
/**
|
|
1736
|
+
* Creates a data store then attempts to alias it.
|
|
1737
|
+
* If aliasing fails, it will raise an exception.
|
|
1738
|
+
*
|
|
1739
|
+
* This method will be removed. See #6465.
|
|
1740
|
+
*
|
|
1741
|
+
* @param pkg - Package name of the data store
|
|
1742
|
+
* @param alias - Alias to be assigned to the data store
|
|
1743
|
+
* @param props - Properties for the data store
|
|
1744
|
+
* @returns - An aliased data store which can can be found / loaded by alias.
|
|
1745
|
+
*/
|
|
1746
|
+
private async createAndAliasDataStore(pkg: string | string[], alias: string, props?: any): Promise<IFluidRouter> {
|
|
1747
|
+
const internalId = uuid();
|
|
1748
|
+
const dataStore = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
|
|
1749
|
+
const aliasedDataStore = channelToDataStore(dataStore, internalId, this,this.dataStores, this.mc.logger);
|
|
1750
|
+
const result = await aliasedDataStore.trySetAlias(alias);
|
|
1751
|
+
if (result !== AliasResult.Success) {
|
|
1752
|
+
throw new GenericError(
|
|
1753
|
+
"dataStoreAliasFailure",
|
|
1754
|
+
undefined /* error */,
|
|
1755
|
+
{
|
|
1756
|
+
alias: {
|
|
1757
|
+
value: alias,
|
|
1758
|
+
tag: TelemetryDataTag.UserData,
|
|
1759
|
+
},
|
|
1760
|
+
internalId: {
|
|
1761
|
+
value: internalId,
|
|
1762
|
+
tag: TelemetryDataTag.PackageData,
|
|
1763
|
+
},
|
|
1764
|
+
aliasResult: result,
|
|
1765
|
+
});
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
return aliasedDataStore;
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1689
1771
|
public createDetachedRootDataStore(
|
|
1690
1772
|
pkg: Readonly<string[]>,
|
|
1691
1773
|
rootDataStoreId: string): IFluidDataStoreContextDetached
|
|
@@ -1697,7 +1779,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1697
1779
|
return this.dataStores.createDetachedDataStoreCore(pkg, false);
|
|
1698
1780
|
}
|
|
1699
1781
|
|
|
1700
|
-
|
|
1782
|
+
/**
|
|
1783
|
+
* Creates a possibly root datastore directly with a possibly user generated id and attaches it to storage.
|
|
1784
|
+
* It is vulnerable to name collisions if both aforementioned conditions are true, and should not be used.
|
|
1785
|
+
*
|
|
1786
|
+
* This method will be removed. See #6465.
|
|
1787
|
+
*/
|
|
1788
|
+
private async _createDataStoreWithPropsLegacy(
|
|
1701
1789
|
pkg: string | string[],
|
|
1702
1790
|
props?: any,
|
|
1703
1791
|
id = uuid(),
|
|
@@ -1711,12 +1799,26 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1711
1799
|
return fluidDataStore;
|
|
1712
1800
|
}
|
|
1713
1801
|
|
|
1802
|
+
public async _createDataStoreWithProps(
|
|
1803
|
+
pkg: string | string[],
|
|
1804
|
+
props?: any,
|
|
1805
|
+
id = uuid(),
|
|
1806
|
+
isRoot = false,
|
|
1807
|
+
): Promise<IFluidRouter> {
|
|
1808
|
+
return this._aliasingEnabled === true && isRoot ?
|
|
1809
|
+
this.createAndAliasDataStore(pkg, id, props) :
|
|
1810
|
+
this._createDataStoreWithPropsLegacy(pkg, props, id, isRoot);
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1714
1813
|
private async _createDataStore(
|
|
1715
1814
|
pkg: string | string[],
|
|
1716
1815
|
isRoot: boolean,
|
|
1717
1816
|
id = uuid(),
|
|
1817
|
+
props?: any,
|
|
1718
1818
|
): Promise<IFluidDataStoreChannel> {
|
|
1719
|
-
return this.dataStores
|
|
1819
|
+
return this.dataStores
|
|
1820
|
+
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props)
|
|
1821
|
+
.realize();
|
|
1720
1822
|
}
|
|
1721
1823
|
|
|
1722
1824
|
private canSendOps() {
|
|
@@ -1782,6 +1884,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1782
1884
|
0x12e /* "Container Context should already be in attached state" */);
|
|
1783
1885
|
this.emit("attached");
|
|
1784
1886
|
}
|
|
1887
|
+
|
|
1888
|
+
if (attachState === AttachState.Attached && !this.pendingStateManager.hasPendingMessages()) {
|
|
1889
|
+
this.updateDocumentDirtyState(false);
|
|
1890
|
+
}
|
|
1785
1891
|
this.dataStores.setAttachState(attachState);
|
|
1786
1892
|
}
|
|
1787
1893
|
|
|
@@ -1837,32 +1943,40 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1837
1943
|
* Returns a summary of the runtime at the current sequence number.
|
|
1838
1944
|
*/
|
|
1839
1945
|
public async summarize(options: {
|
|
1840
|
-
/** Logger to use for correlated summary events */
|
|
1841
|
-
summaryLogger: ITelemetryLogger,
|
|
1842
1946
|
/** True to generate the full tree with no handle reuse optimizations; defaults to false */
|
|
1843
1947
|
fullTree?: boolean,
|
|
1844
1948
|
/** True to track the state for this summary in the SummarizerNodes; defaults to true */
|
|
1845
1949
|
trackState?: boolean,
|
|
1950
|
+
/** Logger to use for correlated summary events */
|
|
1951
|
+
summaryLogger?: ITelemetryLogger,
|
|
1846
1952
|
/** True to run garbage collection before summarizing; defaults to true */
|
|
1847
1953
|
runGC?: boolean,
|
|
1848
1954
|
/** True to generate full GC data */
|
|
1849
1955
|
fullGC?: boolean,
|
|
1850
1956
|
/** True to run GC sweep phase after the mark phase */
|
|
1851
1957
|
runSweep?: boolean,
|
|
1852
|
-
}): Promise<
|
|
1958
|
+
}): Promise<IRootSummaryTreeWithStats> {
|
|
1853
1959
|
this.verifyNotClosed();
|
|
1854
1960
|
|
|
1855
|
-
const {
|
|
1856
|
-
|
|
1961
|
+
const {
|
|
1962
|
+
fullTree = false,
|
|
1963
|
+
trackState = true,
|
|
1964
|
+
summaryLogger = this.logger,
|
|
1965
|
+
runGC = this.garbageCollector.shouldRunGC,
|
|
1966
|
+
runSweep,
|
|
1967
|
+
fullGC,
|
|
1968
|
+
} = options;
|
|
1969
|
+
|
|
1970
|
+
let gcStats: IGCStats | undefined;
|
|
1857
1971
|
if (runGC) {
|
|
1858
|
-
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1972
|
+
gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1859
1973
|
}
|
|
1860
1974
|
|
|
1861
1975
|
const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
|
|
1862
1976
|
assert(summarizeResult.summary.type === SummaryType.Tree,
|
|
1863
1977
|
0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1864
1978
|
|
|
1865
|
-
return summarizeResult as
|
|
1979
|
+
return { ...summarizeResult, gcStats } as IRootSummaryTreeWithStats;
|
|
1866
1980
|
}
|
|
1867
1981
|
|
|
1868
1982
|
/**
|
|
@@ -1890,9 +2004,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1890
2004
|
* @param usedRoutes - The routes that are used in all nodes in this Container.
|
|
1891
2005
|
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
|
|
1892
2006
|
* unreferenced as part of this GC run, this should be used to update the time when it happens.
|
|
1893
|
-
* @returns the statistics of the used state of the data stores.
|
|
1894
2007
|
*/
|
|
1895
|
-
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number)
|
|
2008
|
+
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
|
|
1896
2009
|
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1897
2010
|
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1898
2011
|
// always referenced, so the used routes is only self-route (empty string).
|
|
@@ -2008,15 +2121,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2008
2121
|
}
|
|
2009
2122
|
|
|
2010
2123
|
const trace = Trace.start();
|
|
2011
|
-
let summarizeResult:
|
|
2124
|
+
let summarizeResult: IRootSummaryTreeWithStats;
|
|
2012
2125
|
// If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
|
|
2013
2126
|
// state of all the nodes.
|
|
2014
2127
|
const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
|
|
2015
2128
|
try {
|
|
2016
2129
|
summarizeResult = await this.summarize({
|
|
2017
|
-
summaryLogger,
|
|
2018
2130
|
fullTree: fullTree || forcedFullTree,
|
|
2019
2131
|
trackState: true,
|
|
2132
|
+
summaryLogger,
|
|
2020
2133
|
runGC: this.garbageCollector.shouldRunGC,
|
|
2021
2134
|
});
|
|
2022
2135
|
} catch (error) {
|
|
@@ -2024,6 +2137,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2024
2137
|
}
|
|
2025
2138
|
const { summary: summaryTree, stats: partialStats } = summarizeResult;
|
|
2026
2139
|
|
|
2140
|
+
// Now that we have generated the summary, update the message at last summary to the last message processed.
|
|
2141
|
+
this.messageAtLastSummary = this.deltaManager.lastMessage;
|
|
2142
|
+
|
|
2027
2143
|
// Counting dataStores and handles
|
|
2028
2144
|
// Because handles are unchanged dataStores in the current logic,
|
|
2029
2145
|
// summarized dataStore count is total dataStore count minus handle count
|
|
@@ -2036,6 +2152,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2036
2152
|
const summaryStats: IGeneratedSummaryStats = {
|
|
2037
2153
|
dataStoreCount: this.dataStores.size,
|
|
2038
2154
|
summarizedDataStoreCount: this.dataStores.size - handleCount,
|
|
2155
|
+
gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
|
|
2039
2156
|
...partialStats,
|
|
2040
2157
|
};
|
|
2041
2158
|
const generateSummaryData = {
|
|
@@ -2176,6 +2293,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2176
2293
|
this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
|
|
2177
2294
|
}
|
|
2178
2295
|
|
|
2296
|
+
public submitDataStoreAliasOp(contents: any, localOpMetadata: unknown): void {
|
|
2297
|
+
const aliasMessage = contents as IDataStoreAliasMessage;
|
|
2298
|
+
if (!isDataStoreAliasMessage(aliasMessage)) {
|
|
2299
|
+
throw new UsageError("malformedDataStoreAliasMessage");
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2179
2305
|
public async uploadBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {
|
|
2180
2306
|
this.verifyNotClosed();
|
|
2181
2307
|
return this.blobManager.createBlob(blob);
|
|
@@ -2404,20 +2530,25 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2404
2530
|
|
|
2405
2531
|
private async fetchSnapshotFromStorage(
|
|
2406
2532
|
versionId: string | null, logger: ITelemetryLogger, event: ITelemetryGenericEvent) {
|
|
2407
|
-
return PerformanceEvent.timedExecAsync(
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2533
|
+
return PerformanceEvent.timedExecAsync(
|
|
2534
|
+
logger, event, async (perfEvent: {
|
|
2535
|
+
end: (arg0: {
|
|
2536
|
+
getVersionDuration?: number | undefined;
|
|
2537
|
+
getSnapshotDuration?: number | undefined;
|
|
2538
|
+
}) => void; }) => {
|
|
2539
|
+
const stats: { getVersionDuration?: number; getSnapshotDuration?: number } = {};
|
|
2540
|
+
const trace = Trace.start();
|
|
2541
|
+
|
|
2542
|
+
const versions = await this.storage.getVersions(versionId, 1);
|
|
2543
|
+
assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
|
|
2544
|
+
stats.getVersionDuration = trace.trace().duration;
|
|
2545
|
+
|
|
2546
|
+
const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
|
|
2547
|
+
assert(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
|
|
2548
|
+
stats.getSnapshotDuration = trace.trace().duration;
|
|
2549
|
+
|
|
2550
|
+
perfEvent.end(stats);
|
|
2551
|
+
return maybeSnapshot;
|
|
2421
2552
|
});
|
|
2422
2553
|
}
|
|
2423
2554
|
|