@fluidframework/container-runtime 0.56.0 → 0.57.0-51086
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +9 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +6 -6
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +65 -25
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +149 -79
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +62 -0
- package/dist/dataStore.d.ts.map +1 -0
- package/dist/dataStore.js +135 -0
- package/dist/dataStore.js.map +1 -0
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +9 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +14 -19
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +47 -21
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +195 -61
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +1 -0
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +23 -15
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +6 -6
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryGenerator.d.ts +2 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +46 -28
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +9 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +6 -6
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +65 -25
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +150 -80
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +62 -0
- package/lib/dataStore.d.ts.map +1 -0
- package/lib/dataStore.js +130 -0
- package/lib/dataStore.js.map +1 -0
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +9 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +13 -18
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +47 -21
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +197 -63
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +3 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +1 -0
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +23 -15
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +6 -6
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryGenerator.d.ts +2 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +46 -28
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +13 -13
- package/src/blobManager.ts +12 -1
- package/src/connectionTelemetry.ts +7 -6
- package/src/containerRuntime.ts +231 -103
- package/src/dataStore.ts +187 -0
- package/src/dataStoreContext.ts +1 -1
- package/src/dataStores.ts +18 -38
- package/src/garbageCollection.ts +283 -105
- package/src/index.ts +3 -1
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +25 -16
- package/src/summarizerTypes.ts +6 -8
- package/src/summaryGenerator.ts +72 -23
package/src/containerRuntime.ts
CHANGED
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
TaggedLoggerAdapter,
|
|
47
47
|
MonitoringContext,
|
|
48
48
|
loggerToMonitoringContext,
|
|
49
|
+
TelemetryDataTag,
|
|
49
50
|
} from "@fluidframework/telemetry-utils";
|
|
50
51
|
import { DriverHeader, IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
51
52
|
import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
|
|
@@ -64,7 +65,6 @@ import {
|
|
|
64
65
|
ISummaryConfiguration,
|
|
65
66
|
ISummaryContent,
|
|
66
67
|
ISummaryTree,
|
|
67
|
-
ITree,
|
|
68
68
|
MessageType,
|
|
69
69
|
SummaryType,
|
|
70
70
|
} from "@fluidframework/protocol-definitions";
|
|
@@ -99,7 +99,6 @@ import {
|
|
|
99
99
|
requestFluidObject,
|
|
100
100
|
responseToException,
|
|
101
101
|
seqFromTree,
|
|
102
|
-
convertSummaryTreeToITree,
|
|
103
102
|
} from "@fluidframework/runtime-utils";
|
|
104
103
|
import { v4 as uuid } from "uuid";
|
|
105
104
|
import { ContainerFluidHandleContext } from "./containerHandleContext";
|
|
@@ -145,8 +144,14 @@ import {
|
|
|
145
144
|
IGarbageCollectionRuntime,
|
|
146
145
|
IGarbageCollector,
|
|
147
146
|
IGCStats,
|
|
148
|
-
IUsedStateStats,
|
|
149
147
|
} from "./garbageCollection";
|
|
148
|
+
import {
|
|
149
|
+
AliasResult,
|
|
150
|
+
channelToDataStore,
|
|
151
|
+
IDataStore,
|
|
152
|
+
IDataStoreAliasMessage,
|
|
153
|
+
isDataStoreAliasMessage,
|
|
154
|
+
} from "./dataStore";
|
|
150
155
|
|
|
151
156
|
export enum ContainerMessageType {
|
|
152
157
|
// An op to be delivered to store
|
|
@@ -283,12 +288,41 @@ export interface IContainerRuntimeOptions {
|
|
|
283
288
|
* 3. "bypass" will skip the check entirely. This is not recommended.
|
|
284
289
|
*/
|
|
285
290
|
loadSequenceNumberVerification?: "close" | "log" | "bypass";
|
|
291
|
+
/**
|
|
292
|
+
* Should the runtime use data store aliasing for creating root datastores.
|
|
293
|
+
* In case of aliasing conflicts, the runtime will raise an exception which does
|
|
294
|
+
* not effect the status of the container.
|
|
295
|
+
*/
|
|
296
|
+
useDataStoreAliasing?: boolean;
|
|
286
297
|
}
|
|
287
298
|
|
|
288
299
|
type IRuntimeMessageMetadata = undefined | {
|
|
289
300
|
batch?: boolean;
|
|
290
301
|
};
|
|
291
302
|
|
|
303
|
+
/**
|
|
304
|
+
* The summary tree returned by the root node. It adds state relevant to the root of the tree.
|
|
305
|
+
*/
|
|
306
|
+
export interface IRootSummaryTreeWithStats extends ISummaryTreeWithStats {
|
|
307
|
+
/** The garbage collection stats if GC ran, undefined otherwise. */
|
|
308
|
+
gcStats?: IGCStats;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Accepted header keys for requests coming to the runtime.
|
|
313
|
+
*/
|
|
314
|
+
export enum RuntimeHeaders {
|
|
315
|
+
/** True to wait for a data store to be created and loaded before returning it. */
|
|
316
|
+
wait = "wait",
|
|
317
|
+
/**
|
|
318
|
+
* True if the request is from an external app. Used for GC to handle scenarios where a data store
|
|
319
|
+
* is deleted and requested via an external app.
|
|
320
|
+
*/
|
|
321
|
+
externalRequest = "externalRequest",
|
|
322
|
+
/** True if the request is coming from an IFluidHandle. */
|
|
323
|
+
viaHandle = "viaHandle",
|
|
324
|
+
}
|
|
325
|
+
|
|
292
326
|
/**
|
|
293
327
|
* @deprecated
|
|
294
328
|
* Untagged logger is unsupported going forward. There are old loaders with old ContainerContexts that only
|
|
@@ -297,10 +331,11 @@ type IRuntimeMessageMetadata = undefined | {
|
|
|
297
331
|
*/
|
|
298
332
|
interface OldContainerContextWithLogger extends IContainerContext {
|
|
299
333
|
logger: ITelemetryBaseLogger;
|
|
300
|
-
}
|
|
334
|
+
}
|
|
301
335
|
|
|
302
336
|
// Local storage key to set the default flush mode to TurnBased
|
|
303
337
|
const turnBasedFlushModeKey = "Fluid.ContainerRuntime.FlushModeTurnBased";
|
|
338
|
+
const useDataStoreAliasingKey = "Fluid.ContainerRuntime.UseDataStoreAliasing";
|
|
304
339
|
|
|
305
340
|
export enum RuntimeMessage {
|
|
306
341
|
FluidDataStoreOp = "component",
|
|
@@ -391,6 +426,11 @@ class ScheduleManagerCore {
|
|
|
391
426
|
for (const pending of allPending) {
|
|
392
427
|
this.trackPending(pending);
|
|
393
428
|
}
|
|
429
|
+
|
|
430
|
+
// We are intentionally directly listening to the "op" to inspect system ops as well.
|
|
431
|
+
// If we do not observe system ops, we are likely to hit 0x296 assert when system ops
|
|
432
|
+
// precedes start of incomplete batch.
|
|
433
|
+
this.deltaManager.on("op", (message) => this.afterOpProcessing(message.sequenceNumber));
|
|
394
434
|
}
|
|
395
435
|
|
|
396
436
|
/**
|
|
@@ -536,8 +576,6 @@ export class ScheduleManager {
|
|
|
536
576
|
private batchClientId: string | undefined;
|
|
537
577
|
private hitError = false;
|
|
538
578
|
|
|
539
|
-
private readonly scheduler: ScheduleManagerCore;
|
|
540
|
-
|
|
541
579
|
constructor(
|
|
542
580
|
private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
|
|
543
581
|
private readonly emitter: EventEmitter,
|
|
@@ -547,7 +585,7 @@ export class ScheduleManager {
|
|
|
547
585
|
this.deltaManager,
|
|
548
586
|
ChildLogger.create(this.logger, "DeltaScheduler"),
|
|
549
587
|
);
|
|
550
|
-
|
|
588
|
+
void new ScheduleManagerCore(deltaManager, logger);
|
|
551
589
|
}
|
|
552
590
|
|
|
553
591
|
public beforeOpProcessing(message: ISequencedDocumentMessage) {
|
|
@@ -572,10 +610,6 @@ export class ScheduleManager {
|
|
|
572
610
|
// If this is no longer true, we need to revisit what we do where we set this.hitError.
|
|
573
611
|
assert(!this.hitError, 0x2a3 /* "container should be closed on any error" */);
|
|
574
612
|
|
|
575
|
-
// Let the scheduler know how far we progressed, to decide if op processing
|
|
576
|
-
// should be paused or not.
|
|
577
|
-
this.scheduler.afterOpProcessing(message.sequenceNumber);
|
|
578
|
-
|
|
579
613
|
if (error) {
|
|
580
614
|
// We assume here that loader will close container and stop processing all future ops.
|
|
581
615
|
// This is implicit dependency. If this flow changes, this code might no longer be correct.
|
|
@@ -652,7 +686,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
652
686
|
): Promise<ContainerRuntime> {
|
|
653
687
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
654
688
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
655
|
-
const passLogger = context.taggedLogger
|
|
689
|
+
const passLogger = context.taggedLogger ?? new TaggedLoggerAdapter((context as
|
|
656
690
|
OldContainerContextWithLogger).logger);
|
|
657
691
|
const logger = ChildLogger.create(passLogger, undefined, {
|
|
658
692
|
all: {
|
|
@@ -664,6 +698,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
664
698
|
summaryOptions = {},
|
|
665
699
|
gcOptions = {},
|
|
666
700
|
loadSequenceNumberVerification = "close",
|
|
701
|
+
useDataStoreAliasing = false,
|
|
667
702
|
} = runtimeOptions;
|
|
668
703
|
|
|
669
704
|
// We pack at data store level only. If isolated channels are disabled,
|
|
@@ -757,6 +792,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
757
792
|
summaryOptions,
|
|
758
793
|
gcOptions,
|
|
759
794
|
loadSequenceNumberVerification,
|
|
795
|
+
useDataStoreAliasing,
|
|
760
796
|
},
|
|
761
797
|
containerScope,
|
|
762
798
|
logger,
|
|
@@ -854,6 +890,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
854
890
|
private readonly summaryCollection: SummaryCollection;
|
|
855
891
|
|
|
856
892
|
private readonly summarizerNode: IRootSummarizerNodeWithGC;
|
|
893
|
+
private readonly _aliasingEnabled: boolean;
|
|
857
894
|
|
|
858
895
|
private _orderSequentiallyCalls: number = 0;
|
|
859
896
|
private _flushMode: FlushMode;
|
|
@@ -874,23 +911,23 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
874
911
|
}
|
|
875
912
|
|
|
876
913
|
private get summaryConfiguration() {
|
|
877
|
-
return
|
|
914
|
+
return {
|
|
878
915
|
// the defaults
|
|
879
916
|
... DefaultSummaryConfiguration,
|
|
880
917
|
// the server provided values
|
|
881
918
|
... this.context?.serviceConfiguration?.summary,
|
|
882
919
|
// the runtime configuration overrides
|
|
883
920
|
... this.runtimeOptions.summaryOptions?.summaryConfigOverrides,
|
|
884
|
-
|
|
921
|
+
};
|
|
885
922
|
}
|
|
886
923
|
|
|
887
924
|
private _disposed = false;
|
|
888
925
|
public get disposed() { return this._disposed; }
|
|
889
926
|
|
|
890
|
-
private dirtyContainer
|
|
927
|
+
private dirtyContainer: boolean;
|
|
891
928
|
private emitDirtyDocumentEvent = true;
|
|
892
929
|
|
|
893
|
-
private summarizerWarning = (warning: ContainerWarning) =>
|
|
930
|
+
private readonly summarizerWarning = (warning: ContainerWarning) =>
|
|
894
931
|
this.mc.logger.sendTelemetryEvent({ eventName: "summarizerWarning" }, warning);
|
|
895
932
|
/**
|
|
896
933
|
* Summarizer is responsible for coordinating when to send generate and send summaries.
|
|
@@ -979,6 +1016,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
979
1016
|
this.mc.config.getBoolean(turnBasedFlushModeKey) ?? false
|
|
980
1017
|
? FlushMode.TurnBased : FlushMode.Immediate;
|
|
981
1018
|
|
|
1019
|
+
this._aliasingEnabled =
|
|
1020
|
+
(this.mc.config.getBoolean(useDataStoreAliasingKey) ?? false) ||
|
|
1021
|
+
(runtimeOptions.useDataStoreAliasing ?? false);
|
|
1022
|
+
|
|
982
1023
|
/**
|
|
983
1024
|
* Function that return the current server timestamp. This is used by the garbage collector to set the
|
|
984
1025
|
* time when a node becomes unreferenced.
|
|
@@ -995,6 +1036,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
995
1036
|
this,
|
|
996
1037
|
this.runtimeOptions.gcOptions,
|
|
997
1038
|
(unusedRoutes: string[]) => this.dataStores.deleteUnusedRoutes(unusedRoutes),
|
|
1039
|
+
(nodePath: string) => this.dataStores.getNodePackagePath(nodePath),
|
|
998
1040
|
getCurrentTimestamp,
|
|
999
1041
|
this.closeFn,
|
|
1000
1042
|
context.baseSnapshot,
|
|
@@ -1048,7 +1090,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1048
1090
|
(id: string) => this.summarizerNode.deleteChild(id),
|
|
1049
1091
|
this.mc.logger,
|
|
1050
1092
|
async () => this.garbageCollector.getDataStoreBaseGCDetails(),
|
|
1051
|
-
(
|
|
1093
|
+
(dataStorePath: string, packagePath?: readonly string[]) =>
|
|
1094
|
+
this.garbageCollector.nodeUpdated(dataStorePath, "Changed", packagePath),
|
|
1052
1095
|
new Map<string, string>(dataStoreAliasMap),
|
|
1053
1096
|
this.garbageCollector.writeDataAtRoot,
|
|
1054
1097
|
);
|
|
@@ -1081,6 +1124,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1081
1124
|
|
|
1082
1125
|
this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
|
|
1083
1126
|
|
|
1127
|
+
const { attachState, pendingLocalState } = this.context;
|
|
1128
|
+
this.dirtyContainer = attachState !== AttachState.Attached
|
|
1129
|
+
|| (pendingLocalState as IPendingLocalState)?.pendingStates.length > 0;
|
|
1130
|
+
this.context.updateDirtyContainerState(this.dirtyContainer);
|
|
1131
|
+
|
|
1084
1132
|
// Map the deprecated generateSummaries flag to disableSummaries.
|
|
1085
1133
|
if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
|
|
1086
1134
|
this.runtimeOptions.summaryOptions.disableSummaries = true;
|
|
@@ -1306,18 +1354,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1306
1354
|
return create404Response(request);
|
|
1307
1355
|
}
|
|
1308
1356
|
} else if (requestParser.pathParts.length > 0) {
|
|
1309
|
-
|
|
1310
|
-
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
1311
|
-
* an error if the data store being requested is marked as unreferenced as per the data store's initial
|
|
1312
|
-
* summary.
|
|
1313
|
-
*
|
|
1314
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
1315
|
-
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
1316
|
-
*/
|
|
1317
|
-
const wait = typeof request.headers?.wait === "boolean" ? request.headers.wait : undefined;
|
|
1318
|
-
const dataStore = request.headers?.externalRequest && this.garbageCollector.shouldRunGC
|
|
1319
|
-
? await this.getDataStoreIfInitiallyReferenced(id, wait)
|
|
1320
|
-
: await this.getDataStore(id, wait);
|
|
1357
|
+
const dataStore = await this.getDataStoreFromRequest(id, request);
|
|
1321
1358
|
const subRequest = requestParser.createSubRequest(1);
|
|
1322
1359
|
// We always expect createSubRequest to include a leading slash, but asserting here to protect against
|
|
1323
1360
|
// unintentionally modifying the url if that changes.
|
|
@@ -1332,6 +1369,36 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1332
1369
|
}
|
|
1333
1370
|
}
|
|
1334
1371
|
|
|
1372
|
+
private async getDataStoreFromRequest(id: string, request: IRequest): Promise<IFluidRouter> {
|
|
1373
|
+
const wait = typeof request.headers?.[RuntimeHeaders.wait] === "boolean"
|
|
1374
|
+
? request.headers?.[RuntimeHeaders.wait]
|
|
1375
|
+
: true;
|
|
1376
|
+
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
1377
|
+
|
|
1378
|
+
/**
|
|
1379
|
+
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
1380
|
+
* an error if the data store being requested is marked as unreferenced as per the data store's base
|
|
1381
|
+
* GC data.
|
|
1382
|
+
*
|
|
1383
|
+
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
1384
|
+
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
1385
|
+
*/
|
|
1386
|
+
if (request.headers?.[RuntimeHeaders.externalRequest] && this.garbageCollector.shouldRunGC) {
|
|
1387
|
+
// The data store is referenced if used routes in the base summary has a route to self.
|
|
1388
|
+
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
1389
|
+
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
1390
|
+
if (!(usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/"))) {
|
|
1391
|
+
throw responseToException(create404Response(request), request);
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
const dataStoreChannel = await dataStoreContext.realize();
|
|
1396
|
+
// Let the garbage collector know that a data store was requested / loaded. Realize the data store first so
|
|
1397
|
+
// that the package path is available.
|
|
1398
|
+
this.garbageCollector.nodeUpdated(`/${id}`, "Loaded", dataStoreContext.packagePath, request?.headers);
|
|
1399
|
+
return dataStoreChannel;
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1335
1402
|
private formMetadata(): IContainerRuntimeMetadata {
|
|
1336
1403
|
return {
|
|
1337
1404
|
...this.createContainerMetadata,
|
|
@@ -1346,43 +1413,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1346
1413
|
};
|
|
1347
1414
|
}
|
|
1348
1415
|
|
|
1349
|
-
/**
|
|
1350
|
-
* Retrieves the runtime for a data store if it's referenced as per the initially summary that it is loaded with.
|
|
1351
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted and marked
|
|
1352
|
-
* as unreferenced by GC.
|
|
1353
|
-
* @param id - Id supplied during creating the data store.
|
|
1354
|
-
* @param wait - True if you want to wait for it.
|
|
1355
|
-
* @returns the data store runtime if the data store exists and is initially referenced; undefined otherwise.
|
|
1356
|
-
*/
|
|
1357
|
-
private async getDataStoreIfInitiallyReferenced(id: string, wait = true): Promise<IFluidRouter> {
|
|
1358
|
-
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
1359
|
-
// The data store is referenced if used routes in the initial summary has a route to self.
|
|
1360
|
-
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
1361
|
-
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
1362
|
-
if (usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/")) {
|
|
1363
|
-
return dataStoreContext.realize();
|
|
1364
|
-
}
|
|
1365
|
-
|
|
1366
|
-
// The data store is unreferenced. Throw a 404 response exception.
|
|
1367
|
-
const request = { url: id };
|
|
1368
|
-
throw responseToException(create404Response(request), request);
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
/**
|
|
1372
|
-
* Notifies this object to take the snapshot of the container.
|
|
1373
|
-
* @deprecated - Use summarize to get summary of the container runtime.
|
|
1374
|
-
*/
|
|
1375
|
-
public async snapshot(): Promise<ITree> {
|
|
1376
|
-
const summaryResult = await this.summarize({
|
|
1377
|
-
summaryLogger: this.logger,
|
|
1378
|
-
fullTree: true,
|
|
1379
|
-
trackState: false,
|
|
1380
|
-
runGC: this.garbageCollector.shouldRunGC,
|
|
1381
|
-
fullGC: true,
|
|
1382
|
-
});
|
|
1383
|
-
return convertSummaryTreeToITree(summaryResult.summary);
|
|
1384
|
-
}
|
|
1385
|
-
|
|
1386
1416
|
private addContainerStateToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
1387
1417
|
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
1388
1418
|
if (this.chunkMap.size > 0) {
|
|
@@ -1596,10 +1626,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1596
1626
|
return context.realize();
|
|
1597
1627
|
}
|
|
1598
1628
|
|
|
1599
|
-
protected async getDataStore(id: string, wait = true): Promise<IFluidRouter> {
|
|
1600
|
-
return (await this.dataStores.getDataStore(id, wait)).realize();
|
|
1601
|
-
}
|
|
1602
|
-
|
|
1603
1629
|
public setFlushMode(mode: FlushMode): void {
|
|
1604
1630
|
if (mode === this._flushMode) {
|
|
1605
1631
|
return;
|
|
@@ -1677,16 +1703,70 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1677
1703
|
}
|
|
1678
1704
|
}
|
|
1679
1705
|
|
|
1680
|
-
public async createDataStore(pkg: string | string[]): Promise<
|
|
1681
|
-
|
|
1706
|
+
public async createDataStore(pkg: string | string[]): Promise<IDataStore> {
|
|
1707
|
+
const internalId = uuid();
|
|
1708
|
+
return channelToDataStore(
|
|
1709
|
+
await this._createDataStore(pkg, false /* isRoot */, internalId),
|
|
1710
|
+
internalId,
|
|
1711
|
+
this,
|
|
1712
|
+
this.dataStores,
|
|
1713
|
+
this.mc.logger);
|
|
1682
1714
|
}
|
|
1683
1715
|
|
|
1684
|
-
|
|
1716
|
+
/**
|
|
1717
|
+
* Creates a root datastore directly with a user generated id and attaches it to storage.
|
|
1718
|
+
* It is vulnerable to name collisions and should not be used.
|
|
1719
|
+
*
|
|
1720
|
+
* This method will be removed. See #6465.
|
|
1721
|
+
*/
|
|
1722
|
+
private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1685
1723
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
1686
1724
|
fluidDataStore.bindToContext();
|
|
1687
1725
|
return fluidDataStore;
|
|
1688
1726
|
}
|
|
1689
1727
|
|
|
1728
|
+
public async createRootDataStore(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
1729
|
+
return this._aliasingEnabled === true ?
|
|
1730
|
+
this.createAndAliasDataStore(pkg, rootDataStoreId) :
|
|
1731
|
+
this.createRootDataStoreLegacy(pkg, rootDataStoreId);
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
/**
|
|
1735
|
+
* Creates a data store then attempts to alias it.
|
|
1736
|
+
* If aliasing fails, it will raise an exception.
|
|
1737
|
+
*
|
|
1738
|
+
* This method will be removed. See #6465.
|
|
1739
|
+
*
|
|
1740
|
+
* @param pkg - Package name of the data store
|
|
1741
|
+
* @param alias - Alias to be assigned to the data store
|
|
1742
|
+
* @param props - Properties for the data store
|
|
1743
|
+
* @returns - An aliased data store which can can be found / loaded by alias.
|
|
1744
|
+
*/
|
|
1745
|
+
private async createAndAliasDataStore(pkg: string | string[], alias: string, props?: any): Promise<IFluidRouter> {
|
|
1746
|
+
const internalId = uuid();
|
|
1747
|
+
const dataStore = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
|
|
1748
|
+
const aliasedDataStore = channelToDataStore(dataStore, internalId, this,this.dataStores, this.mc.logger);
|
|
1749
|
+
const result = await aliasedDataStore.trySetAlias(alias);
|
|
1750
|
+
if (result !== AliasResult.Success) {
|
|
1751
|
+
throw new GenericError(
|
|
1752
|
+
"dataStoreAliasFailure",
|
|
1753
|
+
undefined /* error */,
|
|
1754
|
+
{
|
|
1755
|
+
alias: {
|
|
1756
|
+
value: alias,
|
|
1757
|
+
tag: TelemetryDataTag.UserData,
|
|
1758
|
+
},
|
|
1759
|
+
internalId: {
|
|
1760
|
+
value: internalId,
|
|
1761
|
+
tag: TelemetryDataTag.PackageData,
|
|
1762
|
+
},
|
|
1763
|
+
aliasResult: result,
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
return aliasedDataStore;
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1690
1770
|
public createDetachedRootDataStore(
|
|
1691
1771
|
pkg: Readonly<string[]>,
|
|
1692
1772
|
rootDataStoreId: string): IFluidDataStoreContextDetached
|
|
@@ -1698,7 +1778,13 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1698
1778
|
return this.dataStores.createDetachedDataStoreCore(pkg, false);
|
|
1699
1779
|
}
|
|
1700
1780
|
|
|
1701
|
-
|
|
1781
|
+
/**
|
|
1782
|
+
* Creates a possibly root datastore directly with a possibly user generated id and attaches it to storage.
|
|
1783
|
+
* It is vulnerable to name collisions if both aforementioned conditions are true, and should not be used.
|
|
1784
|
+
*
|
|
1785
|
+
* This method will be removed. See #6465.
|
|
1786
|
+
*/
|
|
1787
|
+
private async _createDataStoreWithPropsLegacy(
|
|
1702
1788
|
pkg: string | string[],
|
|
1703
1789
|
props?: any,
|
|
1704
1790
|
id = uuid(),
|
|
@@ -1712,12 +1798,26 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1712
1798
|
return fluidDataStore;
|
|
1713
1799
|
}
|
|
1714
1800
|
|
|
1801
|
+
public async _createDataStoreWithProps(
|
|
1802
|
+
pkg: string | string[],
|
|
1803
|
+
props?: any,
|
|
1804
|
+
id = uuid(),
|
|
1805
|
+
isRoot = false,
|
|
1806
|
+
): Promise<IFluidRouter> {
|
|
1807
|
+
return this._aliasingEnabled === true && isRoot ?
|
|
1808
|
+
this.createAndAliasDataStore(pkg, id, props) :
|
|
1809
|
+
this._createDataStoreWithPropsLegacy(pkg, props, id, isRoot);
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1715
1812
|
private async _createDataStore(
|
|
1716
1813
|
pkg: string | string[],
|
|
1717
1814
|
isRoot: boolean,
|
|
1718
1815
|
id = uuid(),
|
|
1816
|
+
props?: any,
|
|
1719
1817
|
): Promise<IFluidDataStoreChannel> {
|
|
1720
|
-
return this.dataStores
|
|
1818
|
+
return this.dataStores
|
|
1819
|
+
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props)
|
|
1820
|
+
.realize();
|
|
1721
1821
|
}
|
|
1722
1822
|
|
|
1723
1823
|
private canSendOps() {
|
|
@@ -1783,6 +1883,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1783
1883
|
0x12e /* "Container Context should already be in attached state" */);
|
|
1784
1884
|
this.emit("attached");
|
|
1785
1885
|
}
|
|
1886
|
+
|
|
1887
|
+
if (attachState === AttachState.Attached && !this.pendingStateManager.hasPendingMessages()) {
|
|
1888
|
+
this.updateDocumentDirtyState(false);
|
|
1889
|
+
}
|
|
1786
1890
|
this.dataStores.setAttachState(attachState);
|
|
1787
1891
|
}
|
|
1788
1892
|
|
|
@@ -1838,32 +1942,40 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1838
1942
|
* Returns a summary of the runtime at the current sequence number.
|
|
1839
1943
|
*/
|
|
1840
1944
|
public async summarize(options: {
|
|
1841
|
-
/** Logger to use for correlated summary events */
|
|
1842
|
-
summaryLogger: ITelemetryLogger,
|
|
1843
1945
|
/** True to generate the full tree with no handle reuse optimizations; defaults to false */
|
|
1844
1946
|
fullTree?: boolean,
|
|
1845
1947
|
/** True to track the state for this summary in the SummarizerNodes; defaults to true */
|
|
1846
1948
|
trackState?: boolean,
|
|
1949
|
+
/** Logger to use for correlated summary events */
|
|
1950
|
+
summaryLogger?: ITelemetryLogger,
|
|
1847
1951
|
/** True to run garbage collection before summarizing; defaults to true */
|
|
1848
1952
|
runGC?: boolean,
|
|
1849
1953
|
/** True to generate full GC data */
|
|
1850
1954
|
fullGC?: boolean,
|
|
1851
1955
|
/** True to run GC sweep phase after the mark phase */
|
|
1852
1956
|
runSweep?: boolean,
|
|
1853
|
-
}): Promise<
|
|
1957
|
+
}): Promise<IRootSummaryTreeWithStats> {
|
|
1854
1958
|
this.verifyNotClosed();
|
|
1855
1959
|
|
|
1856
|
-
const {
|
|
1857
|
-
|
|
1960
|
+
const {
|
|
1961
|
+
fullTree = false,
|
|
1962
|
+
trackState = true,
|
|
1963
|
+
summaryLogger = this.logger,
|
|
1964
|
+
runGC = this.garbageCollector.shouldRunGC,
|
|
1965
|
+
runSweep,
|
|
1966
|
+
fullGC,
|
|
1967
|
+
} = options;
|
|
1968
|
+
|
|
1969
|
+
let gcStats: IGCStats | undefined;
|
|
1858
1970
|
if (runGC) {
|
|
1859
|
-
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1971
|
+
gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1860
1972
|
}
|
|
1861
1973
|
|
|
1862
1974
|
const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
|
|
1863
1975
|
assert(summarizeResult.summary.type === SummaryType.Tree,
|
|
1864
1976
|
0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1865
1977
|
|
|
1866
|
-
return summarizeResult as
|
|
1978
|
+
return { ...summarizeResult, gcStats } as IRootSummaryTreeWithStats;
|
|
1867
1979
|
}
|
|
1868
1980
|
|
|
1869
1981
|
/**
|
|
@@ -1891,9 +2003,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1891
2003
|
* @param usedRoutes - The routes that are used in all nodes in this Container.
|
|
1892
2004
|
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
|
|
1893
2005
|
* unreferenced as part of this GC run, this should be used to update the time when it happens.
|
|
1894
|
-
* @returns the statistics of the used state of the data stores.
|
|
1895
2006
|
*/
|
|
1896
|
-
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number)
|
|
2007
|
+
public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
|
|
1897
2008
|
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
1898
2009
|
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1899
2010
|
// always referenced, so the used routes is only self-route (empty string).
|
|
@@ -2009,14 +2120,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2009
2120
|
}
|
|
2010
2121
|
|
|
2011
2122
|
const trace = Trace.start();
|
|
2012
|
-
let summarizeResult:
|
|
2123
|
+
let summarizeResult: IRootSummaryTreeWithStats;
|
|
2124
|
+
// If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
|
|
2125
|
+
// state of all the nodes.
|
|
2126
|
+
const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
|
|
2013
2127
|
try {
|
|
2014
2128
|
summarizeResult = await this.summarize({
|
|
2015
|
-
|
|
2016
|
-
// If the GC state needs to be reset, we need to regenerate the summary and update the unreferenced
|
|
2017
|
-
// state of all the nodes.
|
|
2018
|
-
fullTree: fullTree || this.garbageCollector.summaryStateNeedsReset,
|
|
2129
|
+
fullTree: fullTree || forcedFullTree,
|
|
2019
2130
|
trackState: true,
|
|
2131
|
+
summaryLogger,
|
|
2020
2132
|
runGC: this.garbageCollector.shouldRunGC,
|
|
2021
2133
|
});
|
|
2022
2134
|
} catch (error) {
|
|
@@ -2036,6 +2148,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2036
2148
|
const summaryStats: IGeneratedSummaryStats = {
|
|
2037
2149
|
dataStoreCount: this.dataStores.size,
|
|
2038
2150
|
summarizedDataStoreCount: this.dataStores.size - handleCount,
|
|
2151
|
+
gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
|
|
2039
2152
|
...partialStats,
|
|
2040
2153
|
};
|
|
2041
2154
|
const generateSummaryData = {
|
|
@@ -2043,6 +2156,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2043
2156
|
summaryTree,
|
|
2044
2157
|
summaryStats,
|
|
2045
2158
|
generateDuration: trace.trace().duration,
|
|
2159
|
+
forcedFullTree,
|
|
2046
2160
|
} as const;
|
|
2047
2161
|
|
|
2048
2162
|
continueResult = checkContinue();
|
|
@@ -2175,6 +2289,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2175
2289
|
this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
|
|
2176
2290
|
}
|
|
2177
2291
|
|
|
2292
|
+
public submitDataStoreAliasOp(contents: any, localOpMetadata: unknown): void {
|
|
2293
|
+
const aliasMessage = contents as IDataStoreAliasMessage;
|
|
2294
|
+
if (!isDataStoreAliasMessage(aliasMessage)) {
|
|
2295
|
+
throw new UsageError("malformedDataStoreAliasMessage");
|
|
2296
|
+
}
|
|
2297
|
+
|
|
2298
|
+
this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2178
2301
|
public async uploadBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {
|
|
2179
2302
|
this.verifyNotClosed();
|
|
2180
2303
|
return this.blobManager.createBlob(blob);
|
|
@@ -2403,20 +2526,25 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2403
2526
|
|
|
2404
2527
|
private async fetchSnapshotFromStorage(
|
|
2405
2528
|
versionId: string | null, logger: ITelemetryLogger, event: ITelemetryGenericEvent) {
|
|
2406
|
-
return PerformanceEvent.timedExecAsync(
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2529
|
+
return PerformanceEvent.timedExecAsync(
|
|
2530
|
+
logger, event, async (perfEvent: {
|
|
2531
|
+
end: (arg0: {
|
|
2532
|
+
getVersionDuration?: number | undefined;
|
|
2533
|
+
getSnapshotDuration?: number | undefined;
|
|
2534
|
+
}) => void; }) => {
|
|
2535
|
+
const stats: { getVersionDuration?: number; getSnapshotDuration?: number } = {};
|
|
2536
|
+
const trace = Trace.start();
|
|
2537
|
+
|
|
2538
|
+
const versions = await this.storage.getVersions(versionId, 1);
|
|
2539
|
+
assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
|
|
2540
|
+
stats.getVersionDuration = trace.trace().duration;
|
|
2541
|
+
|
|
2542
|
+
const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
|
|
2543
|
+
assert(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
|
|
2544
|
+
stats.getSnapshotDuration = trace.trace().duration;
|
|
2545
|
+
|
|
2546
|
+
perfEvent.end(stats);
|
|
2547
|
+
return maybeSnapshot;
|
|
2420
2548
|
});
|
|
2421
2549
|
}
|
|
2422
2550
|
|