@fluidframework/container-runtime 0.56.7 → 0.57.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +9 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +6 -6
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +68 -28
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +148 -89
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +27 -0
- package/dist/dataStore.d.ts.map +1 -0
- package/dist/dataStore.js +113 -0
- package/dist/dataStore.js.map +1 -0
- package/dist/dataStoreContext.d.ts +1 -7
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +10 -6
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +9 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +14 -19
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +66 -27
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +272 -97
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +1 -0
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +23 -15
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +4 -6
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryGenerator.d.ts +2 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +46 -29
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +9 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +6 -6
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +68 -28
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +149 -90
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +27 -0
- package/lib/dataStore.d.ts.map +1 -0
- package/lib/dataStore.js +108 -0
- package/lib/dataStore.js.map +1 -0
- package/lib/dataStoreContext.d.ts +1 -7
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +10 -6
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +9 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +13 -18
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +66 -27
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +274 -99
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +1 -0
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +23 -15
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +4 -6
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryGenerator.d.ts +2 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +46 -29
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +13 -13
- package/src/blobManager.ts +12 -1
- package/src/connectionTelemetry.ts +7 -6
- package/src/containerRuntime.ts +244 -115
- package/src/dataStore.ts +151 -0
- package/src/dataStoreContext.ts +11 -14
- package/src/dataStores.ts +23 -38
- package/src/garbageCollection.ts +385 -150
- package/src/index.ts +2 -1
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +25 -16
- package/src/summarizerTypes.ts +4 -8
- package/src/summaryGenerator.ts +71 -23
package/src/containerRuntime.ts
CHANGED
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
TaggedLoggerAdapter,
|
|
48
48
|
MonitoringContext,
|
|
49
49
|
loggerToMonitoringContext,
|
|
50
|
+
TelemetryDataTag,
|
|
50
51
|
} from "@fluidframework/telemetry-utils";
|
|
51
52
|
import { DriverHeader, IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
52
53
|
import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
|
|
@@ -65,7 +66,6 @@ import {
|
|
|
65
66
|
ISummaryConfiguration,
|
|
66
67
|
ISummaryContent,
|
|
67
68
|
ISummaryTree,
|
|
68
|
-
ITree,
|
|
69
69
|
MessageType,
|
|
70
70
|
SummaryType,
|
|
71
71
|
} from "@fluidframework/protocol-definitions";
|
|
@@ -87,6 +87,7 @@ import {
|
|
|
87
87
|
SummarizeInternalFn,
|
|
88
88
|
channelsTreeName,
|
|
89
89
|
IAttachMessage,
|
|
90
|
+
IDataStore,
|
|
90
91
|
} from "@fluidframework/runtime-definitions";
|
|
91
92
|
import {
|
|
92
93
|
addBlobToSummary,
|
|
@@ -100,7 +101,6 @@ import {
|
|
|
100
101
|
requestFluidObject,
|
|
101
102
|
responseToException,
|
|
102
103
|
seqFromTree,
|
|
103
|
-
convertSummaryTreeToITree,
|
|
104
104
|
} from "@fluidframework/runtime-utils";
|
|
105
105
|
import { v4 as uuid } from "uuid";
|
|
106
106
|
import { ContainerFluidHandleContext } from "./containerHandleContext";
|
|
@@ -146,8 +146,12 @@ import {
|
|
|
146
146
|
IGarbageCollectionRuntime,
|
|
147
147
|
IGarbageCollector,
|
|
148
148
|
IGCStats,
|
|
149
|
-
IUsedStateStats,
|
|
150
149
|
} from "./garbageCollection";
|
|
150
|
+
import {
|
|
151
|
+
channelToDataStore,
|
|
152
|
+
IDataStoreAliasMessage,
|
|
153
|
+
isDataStoreAliasMessage,
|
|
154
|
+
} from "./dataStore";
|
|
151
155
|
|
|
152
156
|
export enum ContainerMessageType {
|
|
153
157
|
// An op to be delivered to store
|
|
@@ -284,12 +288,41 @@ export interface IContainerRuntimeOptions {
|
|
|
284
288
|
* 3. "bypass" will skip the check entirely. This is not recommended.
|
|
285
289
|
*/
|
|
286
290
|
loadSequenceNumberVerification?: "close" | "log" | "bypass";
|
|
291
|
+
/**
|
|
292
|
+
* Should the runtime use data store aliasing for creating root datastores.
|
|
293
|
+
* In case of aliasing conflicts, the runtime will raise an exception which does
|
|
294
|
+
* not effect the status of the container.
|
|
295
|
+
*/
|
|
296
|
+
useDataStoreAliasing?: boolean;
|
|
287
297
|
}
|
|
288
298
|
|
|
289
299
|
type IRuntimeMessageMetadata = undefined | {
|
|
290
300
|
batch?: boolean;
|
|
291
301
|
};
|
|
292
302
|
|
|
303
|
+
/**
|
|
304
|
+
* The summary tree returned by the root node. It adds state relevant to the root of the tree.
|
|
305
|
+
*/
|
|
306
|
+
export interface IRootSummaryTreeWithStats extends ISummaryTreeWithStats {
|
|
307
|
+
/** The garbage collection stats if GC ran, undefined otherwise. */
|
|
308
|
+
gcStats?: IGCStats;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Accepted header keys for requests coming to the runtime.
|
|
313
|
+
*/
|
|
314
|
+
export enum RuntimeHeaders {
|
|
315
|
+
/** True to wait for a data store to be created and loaded before returning it. */
|
|
316
|
+
wait = "wait",
|
|
317
|
+
/**
|
|
318
|
+
* True if the request is from an external app. Used for GC to handle scenarios where a data store
|
|
319
|
+
* is deleted and requested via an external app.
|
|
320
|
+
*/
|
|
321
|
+
externalRequest = "externalRequest",
|
|
322
|
+
/** True if the request is coming from an IFluidHandle. */
|
|
323
|
+
viaHandle = "viaHandle",
|
|
324
|
+
}
|
|
325
|
+
|
|
293
326
|
/**
|
|
294
327
|
* @deprecated
|
|
295
328
|
* Untagged logger is unsupported going forward. There are old loaders with old ContainerContexts that only
|
|
@@ -298,10 +331,11 @@ type IRuntimeMessageMetadata = undefined | {
|
|
|
298
331
|
*/
|
|
299
332
|
interface OldContainerContextWithLogger extends IContainerContext {
|
|
300
333
|
logger: ITelemetryBaseLogger;
|
|
301
|
-
}
|
|
334
|
+
}
|
|
302
335
|
|
|
303
336
|
// Local storage key to set the default flush mode to TurnBased
|
|
304
337
|
const turnBasedFlushModeKey = "Fluid.ContainerRuntime.FlushModeTurnBased";
|
|
338
|
+
const useDataStoreAliasingKey = "Fluid.ContainerRuntime.UseDataStoreAliasing";
|
|
305
339
|
const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
|
|
306
340
|
|
|
307
341
|
export enum RuntimeMessage {
|
|
@@ -653,7 +687,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
653
687
|
): Promise<ContainerRuntime> {
|
|
654
688
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
655
689
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
656
|
-
const passLogger = context.taggedLogger
|
|
690
|
+
const passLogger = context.taggedLogger ?? new TaggedLoggerAdapter((context as
|
|
657
691
|
OldContainerContextWithLogger).logger);
|
|
658
692
|
const logger = ChildLogger.create(passLogger, undefined, {
|
|
659
693
|
all: {
|
|
@@ -665,6 +699,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
665
699
|
summaryOptions = {},
|
|
666
700
|
gcOptions = {},
|
|
667
701
|
loadSequenceNumberVerification = "close",
|
|
702
|
+
useDataStoreAliasing = false,
|
|
668
703
|
} = runtimeOptions;
|
|
669
704
|
|
|
670
705
|
// We pack at data store level only. If isolated channels are disabled,
|
|
@@ -758,6 +793,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
758
793
|
summaryOptions,
|
|
759
794
|
gcOptions,
|
|
760
795
|
loadSequenceNumberVerification,
|
|
796
|
+
useDataStoreAliasing,
|
|
761
797
|
},
|
|
762
798
|
containerScope,
|
|
763
799
|
logger,
|
|
@@ -855,6 +891,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
855
891
|
private readonly summaryCollection: SummaryCollection;
|
|
856
892
|
|
|
857
893
|
private readonly summarizerNode: IRootSummarizerNodeWithGC;
|
|
894
|
+
private readonly _aliasingEnabled: boolean;
|
|
858
895
|
|
|
859
896
|
private readonly maxConsecutiveReconnects: number;
|
|
860
897
|
private readonly defaultMaxConsecutiveReconnects = 15;
|
|
@@ -880,23 +917,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
880
917
|
}
|
|
881
918
|
|
|
882
919
|
private get summaryConfiguration() {
|
|
883
|
-
return
|
|
920
|
+
return {
|
|
884
921
|
// the defaults
|
|
885
922
|
... DefaultSummaryConfiguration,
|
|
886
923
|
// the server provided values
|
|
887
924
|
... this.context?.serviceConfiguration?.summary,
|
|
888
925
|
// the runtime configuration overrides
|
|
889
926
|
... this.runtimeOptions.summaryOptions?.summaryConfigOverrides,
|
|
890
|
-
|
|
927
|
+
};
|
|
891
928
|
}
|
|
892
929
|
|
|
893
930
|
private _disposed = false;
|
|
894
931
|
public get disposed() { return this._disposed; }
|
|
895
932
|
|
|
896
|
-
private dirtyContainer
|
|
933
|
+
private dirtyContainer: boolean;
|
|
897
934
|
private emitDirtyDocumentEvent = true;
|
|
898
935
|
|
|
899
|
-
private summarizerWarning = (warning: ContainerWarning) =>
|
|
936
|
+
private readonly summarizerWarning = (warning: ContainerWarning) =>
|
|
900
937
|
this.mc.logger.sendTelemetryEvent({ eventName: "summarizerWarning" }, warning);
|
|
901
938
|
/**
|
|
902
939
|
* Summarizer is responsible for coordinating when to send generate and send summaries.
|
|
@@ -921,8 +958,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
921
958
|
* and is the single source of truth for this container.
|
|
922
959
|
*/
|
|
923
960
|
public readonly disableIsolatedChannels: boolean;
|
|
924
|
-
/** The message
|
|
925
|
-
private
|
|
961
|
+
/** The last message processed at the time of the last summary. */
|
|
962
|
+
private messageAtLastSummary: ISummaryMetadataMessage | undefined;
|
|
926
963
|
|
|
927
964
|
private get summarizer(): Summarizer {
|
|
928
965
|
assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
|
|
@@ -953,7 +990,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
953
990
|
private _storage?: IDocumentStorageService,
|
|
954
991
|
) {
|
|
955
992
|
super();
|
|
956
|
-
|
|
993
|
+
|
|
994
|
+
this.messageAtLastSummary = metadata?.message;
|
|
957
995
|
|
|
958
996
|
// If this is an existing container, we get values from metadata.
|
|
959
997
|
// otherwise, we initialize them.
|
|
@@ -985,18 +1023,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
985
1023
|
this.mc.config.getBoolean(turnBasedFlushModeKey) ?? false
|
|
986
1024
|
? FlushMode.TurnBased : FlushMode.Immediate;
|
|
987
1025
|
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
* We use the timestamp of the last op for current timestamp. However, there can be cases where
|
|
992
|
-
* we don't have an op (on demand summaries for instance). In those cases, we will use the timestamp
|
|
993
|
-
* of this client's connection.
|
|
994
|
-
*/
|
|
995
|
-
const getCurrentTimestamp = () => {
|
|
996
|
-
const client = this.clientId !== undefined ? this.getAudience().getMember(this.clientId) : undefined;
|
|
997
|
-
const timestamp = client?.timestamp;
|
|
998
|
-
return this.deltaManager.lastMessage?.timestamp ?? timestamp ?? Date.now();
|
|
999
|
-
};
|
|
1026
|
+
this._aliasingEnabled =
|
|
1027
|
+
(this.mc.config.getBoolean(useDataStoreAliasingKey) ?? false) ||
|
|
1028
|
+
(runtimeOptions.useDataStoreAliasing ?? false);
|
|
1000
1029
|
|
|
1001
1030
|
this.maxConsecutiveReconnects =
|
|
1002
1031
|
this.mc.config.getNumber(maxConsecutiveReconnectsKey) ?? this.defaultMaxConsecutiveReconnects;
|
|
@@ -1005,8 +1034,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1005
1034
|
this,
|
|
1006
1035
|
this.runtimeOptions.gcOptions,
|
|
1007
1036
|
(unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
|
|
1008
|
-
|
|
1009
|
-
|
|
1037
|
+
(nodePath: string) => this.dataStores.getNodePackagePath(nodePath),
|
|
1038
|
+
/**
|
|
1039
|
+
* Returns the timestamp of the last message seen by this client. This is used by garbage collector as
|
|
1040
|
+
* the current reference timestamp for tracking unreferenced objects.
|
|
1041
|
+
*/
|
|
1042
|
+
() => this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp,
|
|
1043
|
+
() => this.messageAtLastSummary?.timestamp,
|
|
1010
1044
|
context.baseSnapshot,
|
|
1011
1045
|
async <T>(id: string) => readAndParse<T>(this.storage, id),
|
|
1012
1046
|
this.mc.logger,
|
|
@@ -1058,7 +1092,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1058
1092
|
(id: string) => this.summarizerNode.deleteChild(id),
|
|
1059
1093
|
this.mc.logger,
|
|
1060
1094
|
async () => this.garbageCollector.getDataStoreBaseGCDetails(),
|
|
1061
|
-
(
|
|
1095
|
+
(path: string, timestampMs: number, packagePath?: readonly string[]) => this.garbageCollector.nodeUpdated(
|
|
1096
|
+
path,
|
|
1097
|
+
"Changed",
|
|
1098
|
+
timestampMs,
|
|
1099
|
+
packagePath,
|
|
1100
|
+
),
|
|
1062
1101
|
new Map<string, string>(dataStoreAliasMap),
|
|
1063
1102
|
this.garbageCollector.writeDataAtRoot,
|
|
1064
1103
|
);
|
|
@@ -1091,6 +1130,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1091
1130
|
|
|
1092
1131
|
this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
|
|
1093
1132
|
|
|
1133
|
+
const { attachState, pendingLocalState } = this.context;
|
|
1134
|
+
this.dirtyContainer = attachState !== AttachState.Attached
|
|
1135
|
+
|| (pendingLocalState as IPendingLocalState)?.pendingStates.length > 0;
|
|
1136
|
+
this.context.updateDirtyContainerState(this.dirtyContainer);
|
|
1137
|
+
|
|
1094
1138
|
// Map the deprecated generateSummaries flag to disableSummaries.
|
|
1095
1139
|
if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
|
|
1096
1140
|
this.runtimeOptions.summaryOptions.disableSummaries = true;
|
|
@@ -1316,18 +1360,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1316
1360
|
return create404Response(request);
|
|
1317
1361
|
}
|
|
1318
1362
|
} else if (requestParser.pathParts.length > 0) {
|
|
1319
|
-
|
|
1320
|
-
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
1321
|
-
* an error if the data store being requested is marked as unreferenced as per the data store's initial
|
|
1322
|
-
* summary.
|
|
1323
|
-
*
|
|
1324
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
1325
|
-
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
1326
|
-
*/
|
|
1327
|
-
const wait = typeof request.headers?.wait === "boolean" ? request.headers.wait : undefined;
|
|
1328
|
-
const dataStore = request.headers?.externalRequest && this.garbageCollector.shouldRunGC
|
|
1329
|
-
? await this.getDataStoreIfInitiallyReferenced(id, wait)
|
|
1330
|
-
: await this.getDataStore(id, wait);
|
|
1363
|
+
const dataStore = await this.getDataStoreFromRequest(id, request);
|
|
1331
1364
|
const subRequest = requestParser.createSubRequest(1);
|
|
1332
1365
|
// We always expect createSubRequest to include a leading slash, but asserting here to protect against
|
|
1333
1366
|
// unintentionally modifying the url if that changes.
|
|
@@ -1342,6 +1375,40 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1342
1375
|
}
|
|
1343
1376
|
}
|
|
1344
1377
|
|
|
1378
|
+
private async getDataStoreFromRequest(id: string, request: IRequest): Promise<IFluidRouter> {
|
|
1379
|
+
const wait = typeof request.headers?.[RuntimeHeaders.wait] === "boolean"
|
|
1380
|
+
? request.headers?.[RuntimeHeaders.wait]
|
|
1381
|
+
: true;
|
|
1382
|
+
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
1383
|
+
|
|
1384
|
+
/**
|
|
1385
|
+
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
1386
|
+
* an error if the data store being requested is marked as unreferenced as per the data store's base
|
|
1387
|
+
* GC data.
|
|
1388
|
+
*
|
|
1389
|
+
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
1390
|
+
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
1391
|
+
*/
|
|
1392
|
+
if (request.headers?.[RuntimeHeaders.externalRequest] && this.garbageCollector.shouldRunGC) {
|
|
1393
|
+
// The data store is referenced if used routes in the base summary has a route to self.
|
|
1394
|
+
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
1395
|
+
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
1396
|
+
if (!(usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/"))) {
|
|
1397
|
+
throw responseToException(create404Response(request), request);
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
const dataStoreChannel = await dataStoreContext.realize();
|
|
1402
|
+
this.garbageCollector.nodeUpdated(
|
|
1403
|
+
`/${id}`,
|
|
1404
|
+
"Loaded",
|
|
1405
|
+
undefined /* timestampMs */,
|
|
1406
|
+
dataStoreContext.packagePath,
|
|
1407
|
+
request?.headers,
|
|
1408
|
+
);
|
|
1409
|
+
return dataStoreChannel;
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1345
1412
|
private formMetadata(): IContainerRuntimeMetadata {
|
|
1346
1413
|
return {
|
|
1347
1414
|
...this.createContainerMetadata,
|
|
@@ -1349,50 +1416,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1349
1416
|
summaryFormatVersion: 1,
|
|
1350
1417
|
disableIsolatedChannels: this.disableIsolatedChannels || undefined,
|
|
1351
1418
|
gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
|
|
1352
|
-
// The last message processed at the time of summary. If there are no messages,
|
|
1353
|
-
//
|
|
1354
|
-
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ?? this.
|
|
1419
|
+
// The last message processed at the time of summary. If there are no new messages, use the message from the
|
|
1420
|
+
// last summary.
|
|
1421
|
+
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ?? this.messageAtLastSummary,
|
|
1355
1422
|
sessionExpiryTimeoutMs: this.garbageCollector.sessionExpiryTimeoutMs,
|
|
1356
1423
|
};
|
|
1357
1424
|
}
|
|
1358
1425
|
|
|
1359
|
-
/**
|
|
1360
|
-
* Retrieves the runtime for a data store if it's referenced as per the initially summary that it is loaded with.
|
|
1361
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted and marked
|
|
1362
|
-
* as unreferenced by GC.
|
|
1363
|
-
* @param id - Id supplied during creating the data store.
|
|
1364
|
-
* @param wait - True if you want to wait for it.
|
|
1365
|
-
* @returns the data store runtime if the data store exists and is initially referenced; undefined otherwise.
|
|
1366
|
-
*/
|
|
1367
|
-
private async getDataStoreIfInitiallyReferenced(id: string, wait = true): Promise<IFluidRouter> {
|
|
1368
|
-
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
1369
|
-
// The data store is referenced if used routes in the initial summary has a route to self.
|
|
1370
|
-
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
1371
|
-
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
1372
|
-
if (usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/")) {
|
|
1373
|
-
return dataStoreContext.realize();
|
|
1374
|
-
}
|
|
1375
|
-
|
|
1376
|
-
// The data store is unreferenced. Throw a 404 response exception.
|
|
1377
|
-
const request = { url: id };
|
|
1378
|
-
throw responseToException(create404Response(request), request);
|
|
1379
|
-
}
|
|
1380
|
-
|
|
1381
|
-
/**
|
|
1382
|
-
* Notifies this object to take the snapshot of the container.
|
|
1383
|
-
* @deprecated - Use summarize to get summary of the container runtime.
|
|
1384
|
-
*/
|
|
1385
|
-
public async snapshot(): Promise<ITree> {
|
|
1386
|
-
const summaryResult = await this.summarize({
|
|
1387
|
-
summaryLogger: this.logger,
|
|
1388
|
-
fullTree: true,
|
|
1389
|
-
trackState: false,
|
|
1390
|
-
runGC: this.garbageCollector.shouldRunGC,
|
|
1391
|
-
fullGC: true,
|
|
1392
|
-
});
|
|
1393
|
-
return convertSummaryTreeToITree(summaryResult.summary);
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
1426
|
private addContainerStateToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
1397
1427
|
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
1398
1428
|
if (this.chunkMap.size > 0) {
|
|
@@ -1657,10 +1687,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1657
1687
|
return context.realize();
|
|
1658
1688
|
}
|
|
1659
1689
|
|
|
1660
|
-
protected async getDataStore(id: string, wait = true): Promise<IFluidRouter> {
|
|
1661
|
-
return (await this.dataStores.getDataStore(id, wait)).realize();
|
|
1662
|
-
}
|
|
1663
|
-
|
|
1664
1690
|
public setFlushMode(mode: FlushMode): void {
|
|
1665
1691
|
if (mode === this._flushMode) {
|
|
1666
1692
|
return;
|
|
@@ -1738,16 +1764,70 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1738
1764
|
}
|
|
1739
1765
|
}
|
|
1740
1766
|
|
|
1741
|
-
public async createDataStore(pkg: string | string[]): Promise<
|
|
1742
|
-
|
|
1767
|
+
public async createDataStore(pkg: string | string[]): Promise<IDataStore> {
|
|
1768
|
+
const internalId = uuid();
|
|
1769
|
+
return channelToDataStore(
|
|
1770
|
+
await this._createDataStore(pkg, false /* isRoot */, internalId),
|
|
1771
|
+
internalId,
|
|
1772
|
+
this,
|
|
1773
|
+
this.dataStores,
|
|
1774
|
+
this.mc.logger);
|
|
1743
1775
|
}
|
|
1744
1776
|
|
|
1745
|
-
|
|
1777
|
+
/**
|
|
1778
|
+
* Creates a root datastore directly with a user generated id and attaches it to storage.
|
|
1779
|
+
* It is vulnerable to name collisions and should not be used.
|
|
1780
|
+
*
|
|
1781
|
+
* This method will be removed. See #6465.
|
|
1782
|
+
*/
|
|
1783
|
+
private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1746
1784
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
1747
1785
|
fluidDataStore.bindToContext();
|
|
1748
1786
|
return fluidDataStore;
|
|
1749
1787
|
}
|
|
1750
1788
|
|
|
1789
|
+
public async createRootDataStore(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1790
|
+
return this._aliasingEnabled === true ?
|
|
1791
|
+
this.createAndAliasDataStore(pkg, rootDataStoreId) :
|
|
1792
|
+
this.createRootDataStoreLegacy(pkg, rootDataStoreId);
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
/**
|
|
1796
|
+
* Creates a data store then attempts to alias it.
|
|
1797
|
+
* If aliasing fails, it will raise an exception.
|
|
1798
|
+
*
|
|
1799
|
+
* This method will be removed. See #6465.
|
|
1800
|
+
*
|
|
1801
|
+
* @param pkg - Package name of the data store
|
|
1802
|
+
* @param alias - Alias to be assigned to the data store
|
|
1803
|
+
* @param props - Properties for the data store
|
|
1804
|
+
* @returns - An aliased data store which can can be found / loaded by alias.
|
|
1805
|
+
*/
|
|
1806
|
+
private async createAndAliasDataStore(pkg: string | string[], alias: string, props?: any): Promise<IDataStore> {
|
|
1807
|
+
const internalId = uuid();
|
|
1808
|
+
const dataStore = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
|
|
1809
|
+
const aliasedDataStore = channelToDataStore(dataStore, internalId, this, this.dataStores, this.mc.logger);
|
|
1810
|
+
const result = await aliasedDataStore.trySetAlias(alias);
|
|
1811
|
+
if (result !== "Success") {
|
|
1812
|
+
throw new GenericError(
|
|
1813
|
+
"dataStoreAliasFailure",
|
|
1814
|
+
undefined /* error */,
|
|
1815
|
+
{
|
|
1816
|
+
alias: {
|
|
1817
|
+
value: alias,
|
|
1818
|
+
tag: TelemetryDataTag.UserData,
|
|
1819
|
+
},
|
|
1820
|
+
internalId: {
|
|
1821
|
+
value: internalId,
|
|
1822
|
+
tag: TelemetryDataTag.PackageData,
|
|
1823
|
+
},
|
|
1824
|
+
aliasResult: result,
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
return aliasedDataStore;
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1751
1831
|
public createDetachedRootDataStore(
|
|
1752
1832
|
pkg: Readonly<string[]>,
|
|
1753
1833
|
rootDataStoreId: string): IFluidDataStoreContextDetached
|
|
@@ -1759,26 +1839,46 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1759
1839
|
return this.dataStores.createDetachedDataStoreCore(pkg, false);
|
|
1760
1840
|
}
|
|
1761
1841
|
|
|
1762
|
-
|
|
1842
|
+
/**
|
|
1843
|
+
* Creates a possibly root datastore directly with a possibly user generated id and attaches it to storage.
|
|
1844
|
+
* It is vulnerable to name collisions if both aforementioned conditions are true, and should not be used.
|
|
1845
|
+
*
|
|
1846
|
+
* This method will be removed. See #6465.
|
|
1847
|
+
*/
|
|
1848
|
+
private async _createDataStoreWithPropsLegacy(
|
|
1763
1849
|
pkg: string | string[],
|
|
1764
1850
|
props?: any,
|
|
1765
1851
|
id = uuid(),
|
|
1766
1852
|
isRoot = false,
|
|
1767
|
-
): Promise<
|
|
1853
|
+
): Promise<IDataStore> {
|
|
1768
1854
|
const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
|
|
1769
1855
|
Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
|
|
1770
1856
|
if (isRoot) {
|
|
1771
1857
|
fluidDataStore.bindToContext();
|
|
1772
1858
|
}
|
|
1773
|
-
return fluidDataStore;
|
|
1859
|
+
return channelToDataStore(fluidDataStore, id, this, this.dataStores, this.mc.logger);
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
public async _createDataStoreWithProps(
|
|
1863
|
+
pkg: string | string[],
|
|
1864
|
+
props?: any,
|
|
1865
|
+
id = uuid(),
|
|
1866
|
+
isRoot = false,
|
|
1867
|
+
): Promise<IDataStore> {
|
|
1868
|
+
return this._aliasingEnabled === true && isRoot ?
|
|
1869
|
+
this.createAndAliasDataStore(pkg, id, props) :
|
|
1870
|
+
this._createDataStoreWithPropsLegacy(pkg, props, id, isRoot);
|
|
1774
1871
|
}
|
|
1775
1872
|
|
|
1776
1873
|
private async _createDataStore(
|
|
1777
1874
|
pkg: string | string[],
|
|
1778
1875
|
isRoot: boolean,
|
|
1779
1876
|
id = uuid(),
|
|
1877
|
+
props?: any,
|
|
1780
1878
|
): Promise<IFluidDataStoreChannel> {
|
|
1781
|
-
return this.dataStores
|
|
1879
|
+
return this.dataStores
|
|
1880
|
+
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props)
|
|
1881
|
+
.realize();
|
|
1782
1882
|
}
|
|
1783
1883
|
|
|
1784
1884
|
private canSendOps() {
|
|
@@ -1844,6 +1944,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1844
1944
|
0x12e /* "Container Context should already be in attached state" */);
|
|
1845
1945
|
this.emit("attached");
|
|
1846
1946
|
}
|
|
1947
|
+
|
|
1948
|
+
if (attachState === AttachState.Attached && !this.pendingStateManager.hasPendingMessages()) {
|
|
1949
|
+
this.updateDocumentDirtyState(false);
|
|
1950
|
+
}
|
|
1847
1951
|
this.dataStores.setAttachState(attachState);
|
|
1848
1952
|
}
|
|
1849
1953
|
|
|
@@ -1899,32 +2003,40 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1899
2003
|
* Returns a summary of the runtime at the current sequence number.
|
|
1900
2004
|
*/
|
|
1901
2005
|
public async summarize(options: {
|
|
1902
|
-
/** Logger to use for correlated summary events */
|
|
1903
|
-
summaryLogger: ITelemetryLogger,
|
|
1904
2006
|
/** True to generate the full tree with no handle reuse optimizations; defaults to false */
|
|
1905
2007
|
fullTree?: boolean,
|
|
1906
2008
|
/** True to track the state for this summary in the SummarizerNodes; defaults to true */
|
|
1907
2009
|
trackState?: boolean,
|
|
2010
|
+
/** Logger to use for correlated summary events */
|
|
2011
|
+
summaryLogger?: ITelemetryLogger,
|
|
1908
2012
|
/** True to run garbage collection before summarizing; defaults to true */
|
|
1909
2013
|
runGC?: boolean,
|
|
1910
2014
|
/** True to generate full GC data */
|
|
1911
2015
|
fullGC?: boolean,
|
|
1912
2016
|
/** True to run GC sweep phase after the mark phase */
|
|
1913
2017
|
runSweep?: boolean,
|
|
1914
|
-
}): Promise<
|
|
2018
|
+
}): Promise<IRootSummaryTreeWithStats> {
|
|
1915
2019
|
this.verifyNotClosed();
|
|
1916
2020
|
|
|
1917
|
-
const {
|
|
1918
|
-
|
|
2021
|
+
const {
|
|
2022
|
+
fullTree = false,
|
|
2023
|
+
trackState = true,
|
|
2024
|
+
summaryLogger = this.logger,
|
|
2025
|
+
runGC = this.garbageCollector.shouldRunGC,
|
|
2026
|
+
runSweep,
|
|
2027
|
+
fullGC,
|
|
2028
|
+
} = options;
|
|
2029
|
+
|
|
2030
|
+
let gcStats: IGCStats | undefined;
|
|
1919
2031
|
if (runGC) {
|
|
1920
|
-
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
2032
|
+
gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1921
2033
|
}
|
|
1922
2034
|
|
|
1923
2035
|
const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
|
|
1924
2036
|
assert(summarizeResult.summary.type === SummaryType.Tree,
|
|
1925
2037
|
0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1926
2038
|
|
|
1927
|
-
return summarizeResult as
|
|
2039
|
+
return { ...summarizeResult, gcStats } as IRootSummaryTreeWithStats;
|
|
1928
2040
|
}
|
|
1929
2041
|
|
|
1930
2042
|
/**
|
|
@@ -1952,9 +2064,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1952
2064
|
* @param usedRoutes - The routes that are used in all nodes in this Container.
|
|
1953
2065
|
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
|
|
1954
2066
|
* unreferenced as part of this GC run, this should be used to update the time when it happens.
|
|
1955
|
-
* @returns the statistics of the used state of the data stores.
|
|
1956
2067
|
*/
|
|
1957
|
-
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number)
|
|
2068
|
+
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
|
|
1958
2069
|
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1959
2070
|
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1960
2071
|
// always referenced, so the used routes is only self-route (empty string).
|
|
@@ -2070,15 +2181,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2070
2181
|
}
|
|
2071
2182
|
|
|
2072
2183
|
const trace = Trace.start();
|
|
2073
|
-
let summarizeResult:
|
|
2184
|
+
let summarizeResult: IRootSummaryTreeWithStats;
|
|
2074
2185
|
// If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
|
|
2075
2186
|
// state of all the nodes.
|
|
2076
2187
|
const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
|
|
2077
2188
|
try {
|
|
2078
2189
|
summarizeResult = await this.summarize({
|
|
2079
|
-
summaryLogger,
|
|
2080
2190
|
fullTree: fullTree || forcedFullTree,
|
|
2081
2191
|
trackState: true,
|
|
2192
|
+
summaryLogger,
|
|
2082
2193
|
runGC: this.garbageCollector.shouldRunGC,
|
|
2083
2194
|
});
|
|
2084
2195
|
} catch (error) {
|
|
@@ -2086,6 +2197,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2086
2197
|
}
|
|
2087
2198
|
const { summary: summaryTree, stats: partialStats } = summarizeResult;
|
|
2088
2199
|
|
|
2200
|
+
// Now that we have generated the summary, update the message at last summary to the last message processed.
|
|
2201
|
+
this.messageAtLastSummary = this.deltaManager.lastMessage;
|
|
2202
|
+
|
|
2089
2203
|
// Counting dataStores and handles
|
|
2090
2204
|
// Because handles are unchanged dataStores in the current logic,
|
|
2091
2205
|
// summarized dataStore count is total dataStore count minus handle count
|
|
@@ -2098,6 +2212,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2098
2212
|
const summaryStats: IGeneratedSummaryStats = {
|
|
2099
2213
|
dataStoreCount: this.dataStores.size,
|
|
2100
2214
|
summarizedDataStoreCount: this.dataStores.size - handleCount,
|
|
2215
|
+
gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
|
|
2101
2216
|
...partialStats,
|
|
2102
2217
|
};
|
|
2103
2218
|
const generateSummaryData = {
|
|
@@ -2238,6 +2353,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2238
2353
|
this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
|
|
2239
2354
|
}
|
|
2240
2355
|
|
|
2356
|
+
public submitDataStoreAliasOp(contents: any, localOpMetadata: unknown): void {
|
|
2357
|
+
const aliasMessage = contents as IDataStoreAliasMessage;
|
|
2358
|
+
if (!isDataStoreAliasMessage(aliasMessage)) {
|
|
2359
|
+
throw new UsageError("malformedDataStoreAliasMessage");
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2362
|
+
this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2241
2365
|
public async uploadBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {
|
|
2242
2366
|
this.verifyNotClosed();
|
|
2243
2367
|
return this.blobManager.createBlob(blob);
|
|
@@ -2466,20 +2590,25 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2466
2590
|
|
|
2467
2591
|
private async fetchSnapshotFromStorage(
|
|
2468
2592
|
versionId: string | null, logger: ITelemetryLogger, event: ITelemetryGenericEvent) {
|
|
2469
|
-
return PerformanceEvent.timedExecAsync(
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2593
|
+
return PerformanceEvent.timedExecAsync(
|
|
2594
|
+
logger, event, async (perfEvent: {
|
|
2595
|
+
end: (arg0: {
|
|
2596
|
+
getVersionDuration?: number | undefined;
|
|
2597
|
+
getSnapshotDuration?: number | undefined;
|
|
2598
|
+
}) => void; }) => {
|
|
2599
|
+
const stats: { getVersionDuration?: number; getSnapshotDuration?: number } = {};
|
|
2600
|
+
const trace = Trace.start();
|
|
2601
|
+
|
|
2602
|
+
const versions = await this.storage.getVersions(versionId, 1);
|
|
2603
|
+
assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
|
|
2604
|
+
stats.getVersionDuration = trace.trace().duration;
|
|
2605
|
+
|
|
2606
|
+
const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
|
|
2607
|
+
assert(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
|
|
2608
|
+
stats.getSnapshotDuration = trace.trace().duration;
|
|
2609
|
+
|
|
2610
|
+
perfEvent.end(stats);
|
|
2611
|
+
return maybeSnapshot;
|
|
2483
2612
|
});
|
|
2484
2613
|
}
|
|
2485
2614
|
|