@fluidframework/container-runtime 0.56.0 → 0.57.0-51086
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +9 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +6 -6
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +65 -25
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +149 -79
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +62 -0
- package/dist/dataStore.d.ts.map +1 -0
- package/dist/dataStore.js +135 -0
- package/dist/dataStore.js.map +1 -0
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +9 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +14 -19
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +47 -21
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +195 -61
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +1 -0
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +23 -15
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +6 -6
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryGenerator.d.ts +2 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +46 -28
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +9 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +6 -6
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +65 -25
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +150 -80
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +62 -0
- package/lib/dataStore.d.ts.map +1 -0
- package/lib/dataStore.js +130 -0
- package/lib/dataStore.js.map +1 -0
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +9 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +13 -18
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +47 -21
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +197 -63
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +3 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +1 -0
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +23 -15
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +6 -6
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryGenerator.d.ts +2 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +46 -28
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +13 -13
- package/src/blobManager.ts +12 -1
- package/src/connectionTelemetry.ts +7 -6
- package/src/containerRuntime.ts +231 -103
- package/src/dataStore.ts +187 -0
- package/src/dataStoreContext.ts +1 -1
- package/src/dataStores.ts +18 -38
- package/src/garbageCollection.ts +283 -105
- package/src/index.ts +3 -1
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +25 -16
- package/src/summarizerTypes.ts +6 -8
- package/src/summaryGenerator.ts +72 -23
package/lib/containerRuntime.js
CHANGED
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
|
|
6
6
|
import { assert, Trace, TypedEventEmitter, unreachableCase, performance, } from "@fluidframework/common-utils";
|
|
7
|
-
import { ChildLogger, raiseConnectedEvent, PerformanceEvent, normalizeError, TaggedLoggerAdapter, loggerToMonitoringContext, } from "@fluidframework/telemetry-utils";
|
|
7
|
+
import { ChildLogger, raiseConnectedEvent, PerformanceEvent, normalizeError, TaggedLoggerAdapter, loggerToMonitoringContext, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
|
|
8
8
|
import { DriverHeader } from "@fluidframework/driver-definitions";
|
|
9
9
|
import { readAndParse, BlobAggregationStorage } from "@fluidframework/driver-utils";
|
|
10
10
|
import { DataCorruptionError, GenericError, UsageError, extractSafePropertiesFromMessage, } from "@fluidframework/container-utils";
|
|
11
11
|
import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
|
|
12
12
|
import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
|
|
13
|
-
import { addBlobToSummary, addTreeToSummary, convertToSummaryTree, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree,
|
|
13
|
+
import { addBlobToSummary, addTreeToSummary, convertToSummaryTree, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree, } from "@fluidframework/runtime-utils";
|
|
14
14
|
import { v4 as uuid } from "uuid";
|
|
15
15
|
import { ContainerFluidHandleContext } from "./containerHandleContext";
|
|
16
16
|
import { FluidDataStoreRegistry } from "./dataStoreRegistry";
|
|
@@ -29,6 +29,7 @@ import { SummarizerClientElection, summarizerClientType } from "./summarizerClie
|
|
|
29
29
|
import { formExponentialFn, Throttler } from "./throttler";
|
|
30
30
|
import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
|
|
31
31
|
import { GarbageCollector, gcTreeKey, } from "./garbageCollection";
|
|
32
|
+
import { AliasResult, channelToDataStore, isDataStoreAliasMessage, } from "./dataStore";
|
|
32
33
|
export var ContainerMessageType;
|
|
33
34
|
(function (ContainerMessageType) {
|
|
34
35
|
// An op to be delivered to store
|
|
@@ -56,9 +57,24 @@ const DefaultSummaryConfiguration = {
|
|
|
56
57
|
// the min of the two will be chosen
|
|
57
58
|
maxAckWaitTime: 120000,
|
|
58
59
|
};
|
|
59
|
-
|
|
60
|
+
/**
|
|
61
|
+
* Accepted header keys for requests coming to the runtime.
|
|
62
|
+
*/
|
|
63
|
+
export var RuntimeHeaders;
|
|
64
|
+
(function (RuntimeHeaders) {
|
|
65
|
+
/** True to wait for a data store to be created and loaded before returning it. */
|
|
66
|
+
RuntimeHeaders["wait"] = "wait";
|
|
67
|
+
/**
|
|
68
|
+
* True if the request is from an external app. Used for GC to handle scenarios where a data store
|
|
69
|
+
* is deleted and requested via an external app.
|
|
70
|
+
*/
|
|
71
|
+
RuntimeHeaders["externalRequest"] = "externalRequest";
|
|
72
|
+
/** True if the request is coming from an IFluidHandle. */
|
|
73
|
+
RuntimeHeaders["viaHandle"] = "viaHandle";
|
|
74
|
+
})(RuntimeHeaders || (RuntimeHeaders = {}));
|
|
60
75
|
// Local storage key to set the default flush mode to TurnBased
|
|
61
76
|
const turnBasedFlushModeKey = "Fluid.ContainerRuntime.FlushModeTurnBased";
|
|
77
|
+
const useDataStoreAliasingKey = "Fluid.ContainerRuntime.UseDataStoreAliasing";
|
|
62
78
|
export var RuntimeMessage;
|
|
63
79
|
(function (RuntimeMessage) {
|
|
64
80
|
RuntimeMessage["FluidDataStoreOp"] = "component";
|
|
@@ -136,6 +152,10 @@ class ScheduleManagerCore {
|
|
|
136
152
|
for (const pending of allPending) {
|
|
137
153
|
this.trackPending(pending);
|
|
138
154
|
}
|
|
155
|
+
// We are intentionally directly listening to the "op" to inspect system ops as well.
|
|
156
|
+
// If we do not observe system ops, we are likely to hit 0x296 assert when system ops
|
|
157
|
+
// precedes start of incomplete batch.
|
|
158
|
+
this.deltaManager.on("op", (message) => this.afterOpProcessing(message.sequenceNumber));
|
|
139
159
|
}
|
|
140
160
|
/**
|
|
141
161
|
* The only public function in this class - called when we processed an op,
|
|
@@ -260,7 +280,7 @@ export class ScheduleManager {
|
|
|
260
280
|
this.logger = logger;
|
|
261
281
|
this.hitError = false;
|
|
262
282
|
this.deltaScheduler = new DeltaScheduler(this.deltaManager, ChildLogger.create(this.logger, "DeltaScheduler"));
|
|
263
|
-
|
|
283
|
+
void new ScheduleManagerCore(deltaManager, logger);
|
|
264
284
|
}
|
|
265
285
|
beforeOpProcessing(message) {
|
|
266
286
|
var _a;
|
|
@@ -282,9 +302,6 @@ export class ScheduleManager {
|
|
|
282
302
|
var _a;
|
|
283
303
|
// If this is no longer true, we need to revisit what we do where we set this.hitError.
|
|
284
304
|
assert(!this.hitError, 0x2a3 /* "container should be closed on any error" */);
|
|
285
|
-
// Let the scheduler know how far we progressed, to decide if op processing
|
|
286
|
-
// should be paused or not.
|
|
287
|
-
this.scheduler.afterOpProcessing(message.sequenceNumber);
|
|
288
305
|
if (error) {
|
|
289
306
|
// We assume here that loader will close container and stop processing all future ops.
|
|
290
307
|
// This is implicit dependency. If this flow changes, this code might no longer be correct.
|
|
@@ -331,7 +348,7 @@ export function getDeviceSpec() {
|
|
|
331
348
|
*/
|
|
332
349
|
export class ContainerRuntime extends TypedEventEmitter {
|
|
333
350
|
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, requestHandler, _storage) {
|
|
334
|
-
var _a, _b, _c, _d, _e;
|
|
351
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
335
352
|
super();
|
|
336
353
|
this.context = context;
|
|
337
354
|
this.registry = registry;
|
|
@@ -345,7 +362,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
345
362
|
this.flushTrigger = false;
|
|
346
363
|
this.paused = false;
|
|
347
364
|
this._disposed = false;
|
|
348
|
-
this.dirtyContainer = false;
|
|
349
365
|
this.emitDirtyDocumentEvent = true;
|
|
350
366
|
this.summarizerWarning = (warning) => this.mc.logger.sendTelemetryEvent({ eventName: "summarizerWarning" }, warning);
|
|
351
367
|
/**
|
|
@@ -421,6 +437,9 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
421
437
|
this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
|
|
422
438
|
this._flushMode =
|
|
423
439
|
((_b = this.mc.config.getBoolean(turnBasedFlushModeKey)) !== null && _b !== void 0 ? _b : false) ? FlushMode.TurnBased : FlushMode.Immediate;
|
|
440
|
+
this._aliasingEnabled =
|
|
441
|
+
((_c = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _c !== void 0 ? _c : false) ||
|
|
442
|
+
((_d = runtimeOptions.useDataStoreAliasing) !== null && _d !== void 0 ? _d : false);
|
|
424
443
|
/**
|
|
425
444
|
* Function that return the current server timestamp. This is used by the garbage collector to set the
|
|
426
445
|
* time when a node becomes unreferenced.
|
|
@@ -434,7 +453,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
434
453
|
const timestamp = client === null || client === void 0 ? void 0 : client.timestamp;
|
|
435
454
|
return (_c = (_b = (_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp) !== null && _b !== void 0 ? _b : timestamp) !== null && _c !== void 0 ? _c : Date.now();
|
|
436
455
|
};
|
|
437
|
-
this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), getCurrentTimestamp, this.closeFn, context.baseSnapshot, async (id) => readAndParse(this.storage, id), this.mc.logger, existing, metadata);
|
|
456
|
+
this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), (nodePath) => this.dataStores.getNodePackagePath(nodePath), getCurrentTimestamp, this.closeFn, context.baseSnapshot, async (id) => readAndParse(this.storage, id), this.mc.logger, existing, metadata);
|
|
438
457
|
const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
|
|
439
458
|
this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"),
|
|
440
459
|
// Summarize function to call when summarize is called. Summarizer node always tracks summary state.
|
|
@@ -455,7 +474,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
455
474
|
if (this.context.baseSnapshot) {
|
|
456
475
|
this.summarizerNode.loadBaseSummaryWithoutDifferential(this.context.baseSnapshot);
|
|
457
476
|
}
|
|
458
|
-
this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getDataStoreBaseGCDetails(), (
|
|
477
|
+
this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getDataStoreBaseGCDetails(), (dataStorePath, packagePath) => this.garbageCollector.nodeUpdated(dataStorePath, "Changed", packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
|
|
459
478
|
this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }), this, this.logger);
|
|
460
479
|
this.scheduleManager = new ScheduleManager(context.deltaManager, this, ChildLogger.create(this.logger, "ScheduleManager"));
|
|
461
480
|
this.deltaSender = this.deltaManager;
|
|
@@ -464,6 +483,10 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
464
483
|
this.clearPartialChunks(clientId);
|
|
465
484
|
});
|
|
466
485
|
this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
|
|
486
|
+
const { attachState, pendingLocalState } = this.context;
|
|
487
|
+
this.dirtyContainer = attachState !== AttachState.Attached
|
|
488
|
+
|| ((_e = pendingLocalState) === null || _e === void 0 ? void 0 : _e.pendingStates.length) > 0;
|
|
489
|
+
this.context.updateDirtyContainerState(this.dirtyContainer);
|
|
467
490
|
// Map the deprecated generateSummaries flag to disableSummaries.
|
|
468
491
|
if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
|
|
469
492
|
this.runtimeOptions.summaryOptions.disableSummaries = true;
|
|
@@ -475,8 +498,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
475
498
|
const orderedClientLogger = ChildLogger.create(this.logger, "OrderedClientElection");
|
|
476
499
|
const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
|
|
477
500
|
const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
|
|
478
|
-
const summarizerClientElectionEnabled = (
|
|
479
|
-
const maxOpsSinceLastSummary = (
|
|
501
|
+
const summarizerClientElectionEnabled = (_f = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _f !== void 0 ? _f : ((_g = this.runtimeOptions.summaryOptions) === null || _g === void 0 ? void 0 : _g.summarizerClientElection) === true;
|
|
502
|
+
const maxOpsSinceLastSummary = (_h = this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary) !== null && _h !== void 0 ? _h : 7000;
|
|
480
503
|
this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, maxOpsSinceLastSummary, summarizerClientElectionEnabled);
|
|
481
504
|
if (this.context.clientDetails.type === summarizerClientType) {
|
|
482
505
|
this._summarizer = new Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime));
|
|
@@ -560,7 +583,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
560
583
|
runtimeVersion: pkgVersion,
|
|
561
584
|
},
|
|
562
585
|
});
|
|
563
|
-
const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", } = runtimeOptions;
|
|
586
|
+
const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", useDataStoreAliasing = false, } = runtimeOptions;
|
|
564
587
|
// We pack at data store level only. If isolated channels are disabled,
|
|
565
588
|
// then there are no .channel layers, we pack at level 1, otherwise we pack at level 2
|
|
566
589
|
const packingLevel = summaryOptions.disableIsolatedChannels ? 1 : 2;
|
|
@@ -628,6 +651,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
628
651
|
summaryOptions,
|
|
629
652
|
gcOptions,
|
|
630
653
|
loadSequenceNumberVerification,
|
|
654
|
+
useDataStoreAliasing,
|
|
631
655
|
}, containerScope, logger, loadExisting, blobManagerSnapshot, requestHandler, storage);
|
|
632
656
|
return runtime;
|
|
633
657
|
}
|
|
@@ -767,7 +791,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
767
791
|
* @param request - Request made to the handler.
|
|
768
792
|
*/
|
|
769
793
|
async resolveHandle(request) {
|
|
770
|
-
var _a, _b;
|
|
771
794
|
try {
|
|
772
795
|
const requestParser = RequestParser.create(request);
|
|
773
796
|
const id = requestParser.pathParts[0];
|
|
@@ -788,18 +811,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
788
811
|
}
|
|
789
812
|
}
|
|
790
813
|
else if (requestParser.pathParts.length > 0) {
|
|
791
|
-
|
|
792
|
-
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
793
|
-
* an error if the data store being requested is marked as unreferenced as per the data store's initial
|
|
794
|
-
* summary.
|
|
795
|
-
*
|
|
796
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
797
|
-
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
798
|
-
*/
|
|
799
|
-
const wait = typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a.wait) === "boolean" ? request.headers.wait : undefined;
|
|
800
|
-
const dataStore = ((_b = request.headers) === null || _b === void 0 ? void 0 : _b.externalRequest) && this.garbageCollector.shouldRunGC
|
|
801
|
-
? await this.getDataStoreIfInitiallyReferenced(id, wait)
|
|
802
|
-
: await this.getDataStore(id, wait);
|
|
814
|
+
const dataStore = await this.getDataStoreFromRequest(id, request);
|
|
803
815
|
const subRequest = requestParser.createSubRequest(1);
|
|
804
816
|
// We always expect createSubRequest to include a leading slash, but asserting here to protect against
|
|
805
817
|
// unintentionally modifying the url if that changes.
|
|
@@ -812,6 +824,33 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
812
824
|
return exceptionToResponse(error);
|
|
813
825
|
}
|
|
814
826
|
}
|
|
827
|
+
async getDataStoreFromRequest(id, request) {
|
|
828
|
+
var _a, _b, _c;
|
|
829
|
+
const wait = typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a[RuntimeHeaders.wait]) === "boolean"
|
|
830
|
+
? (_b = request.headers) === null || _b === void 0 ? void 0 : _b[RuntimeHeaders.wait] : true;
|
|
831
|
+
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
832
|
+
/**
|
|
833
|
+
* If GC should run and this an external app request with "externalRequest" header, we need to return
|
|
834
|
+
* an error if the data store being requested is marked as unreferenced as per the data store's base
|
|
835
|
+
* GC data.
|
|
836
|
+
*
|
|
837
|
+
* This is a workaround to handle scenarios where a data store shared with an external app is deleted
|
|
838
|
+
* and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
|
|
839
|
+
*/
|
|
840
|
+
if (((_c = request.headers) === null || _c === void 0 ? void 0 : _c[RuntimeHeaders.externalRequest]) && this.garbageCollector.shouldRunGC) {
|
|
841
|
+
// The data store is referenced if used routes in the base summary has a route to self.
|
|
842
|
+
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
843
|
+
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
844
|
+
if (!(usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/"))) {
|
|
845
|
+
throw responseToException(create404Response(request), request);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
const dataStoreChannel = await dataStoreContext.realize();
|
|
849
|
+
// Let the garbage collector know that a data store was requested / loaded. Realize the data store first so
|
|
850
|
+
// that the package path is available.
|
|
851
|
+
this.garbageCollector.nodeUpdated(`/${id}`, "Loaded", dataStoreContext.packagePath, request === null || request === void 0 ? void 0 : request.headers);
|
|
852
|
+
return dataStoreChannel;
|
|
853
|
+
}
|
|
815
854
|
formMetadata() {
|
|
816
855
|
var _a;
|
|
817
856
|
return Object.assign(Object.assign({}, this.createContainerMetadata), { summaryCount: this.summaryCount, summaryFormatVersion: 1, disableIsolatedChannels: this.disableIsolatedChannels || undefined, gcFeature: this.garbageCollector.gcSummaryFeatureVersion,
|
|
@@ -819,40 +858,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
819
858
|
// the base summary we loaded from. So, use the message from its metadata blob.
|
|
820
859
|
message: (_a = extractSummaryMetadataMessage(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.baseSummaryMessage, sessionExpiryTimeoutMs: this.garbageCollector.sessionExpiryTimeoutMs });
|
|
821
860
|
}
|
|
822
|
-
/**
|
|
823
|
-
* Retrieves the runtime for a data store if it's referenced as per the initially summary that it is loaded with.
|
|
824
|
-
* This is a workaround to handle scenarios where a data store shared with an external app is deleted and marked
|
|
825
|
-
* as unreferenced by GC.
|
|
826
|
-
* @param id - Id supplied during creating the data store.
|
|
827
|
-
* @param wait - True if you want to wait for it.
|
|
828
|
-
* @returns the data store runtime if the data store exists and is initially referenced; undefined otherwise.
|
|
829
|
-
*/
|
|
830
|
-
async getDataStoreIfInitiallyReferenced(id, wait = true) {
|
|
831
|
-
const dataStoreContext = await this.dataStores.getDataStore(id, wait);
|
|
832
|
-
// The data store is referenced if used routes in the initial summary has a route to self.
|
|
833
|
-
// Older documents may not have used routes in the summary. They are considered referenced.
|
|
834
|
-
const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
|
|
835
|
-
if (usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/")) {
|
|
836
|
-
return dataStoreContext.realize();
|
|
837
|
-
}
|
|
838
|
-
// The data store is unreferenced. Throw a 404 response exception.
|
|
839
|
-
const request = { url: id };
|
|
840
|
-
throw responseToException(create404Response(request), request);
|
|
841
|
-
}
|
|
842
|
-
/**
|
|
843
|
-
* Notifies this object to take the snapshot of the container.
|
|
844
|
-
* @deprecated - Use summarize to get summary of the container runtime.
|
|
845
|
-
*/
|
|
846
|
-
async snapshot() {
|
|
847
|
-
const summaryResult = await this.summarize({
|
|
848
|
-
summaryLogger: this.logger,
|
|
849
|
-
fullTree: true,
|
|
850
|
-
trackState: false,
|
|
851
|
-
runGC: this.garbageCollector.shouldRunGC,
|
|
852
|
-
fullGC: true,
|
|
853
|
-
});
|
|
854
|
-
return convertSummaryTreeToITree(summaryResult.summary);
|
|
855
|
-
}
|
|
856
861
|
addContainerStateToSummary(summaryTree) {
|
|
857
862
|
var _a;
|
|
858
863
|
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata()));
|
|
@@ -1014,9 +1019,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1014
1019
|
assert(await context.isRoot(), 0x12b /* "did not get root data store" */);
|
|
1015
1020
|
return context.realize();
|
|
1016
1021
|
}
|
|
1017
|
-
async getDataStore(id, wait = true) {
|
|
1018
|
-
return (await this.dataStores.getDataStore(id, wait)).realize();
|
|
1019
|
-
}
|
|
1020
1022
|
setFlushMode(mode) {
|
|
1021
1023
|
if (mode === this._flushMode) {
|
|
1022
1024
|
return;
|
|
@@ -1081,28 +1083,84 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1081
1083
|
}
|
|
1082
1084
|
}
|
|
1083
1085
|
async createDataStore(pkg) {
|
|
1084
|
-
|
|
1086
|
+
const internalId = uuid();
|
|
1087
|
+
return channelToDataStore(await this._createDataStore(pkg, false /* isRoot */, internalId), internalId, this, this.dataStores, this.mc.logger);
|
|
1085
1088
|
}
|
|
1086
|
-
|
|
1089
|
+
/**
|
|
1090
|
+
* Creates a root datastore directly with a user generated id and attaches it to storage.
|
|
1091
|
+
* It is vulnerable to name collisions and should not be used.
|
|
1092
|
+
*
|
|
1093
|
+
* This method will be removed. See #6465.
|
|
1094
|
+
*/
|
|
1095
|
+
async createRootDataStoreLegacy(pkg, rootDataStoreId) {
|
|
1087
1096
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
1088
1097
|
fluidDataStore.bindToContext();
|
|
1089
1098
|
return fluidDataStore;
|
|
1090
1099
|
}
|
|
1100
|
+
async createRootDataStore(pkg, rootDataStoreId) {
|
|
1101
|
+
return this._aliasingEnabled === true ?
|
|
1102
|
+
this.createAndAliasDataStore(pkg, rootDataStoreId) :
|
|
1103
|
+
this.createRootDataStoreLegacy(pkg, rootDataStoreId);
|
|
1104
|
+
}
|
|
1105
|
+
/**
|
|
1106
|
+
* Creates a data store then attempts to alias it.
|
|
1107
|
+
* If aliasing fails, it will raise an exception.
|
|
1108
|
+
*
|
|
1109
|
+
* This method will be removed. See #6465.
|
|
1110
|
+
*
|
|
1111
|
+
* @param pkg - Package name of the data store
|
|
1112
|
+
* @param alias - Alias to be assigned to the data store
|
|
1113
|
+
* @param props - Properties for the data store
|
|
1114
|
+
* @returns - An aliased data store which can can be found / loaded by alias.
|
|
1115
|
+
*/
|
|
1116
|
+
async createAndAliasDataStore(pkg, alias, props) {
|
|
1117
|
+
const internalId = uuid();
|
|
1118
|
+
const dataStore = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
|
|
1119
|
+
const aliasedDataStore = channelToDataStore(dataStore, internalId, this, this.dataStores, this.mc.logger);
|
|
1120
|
+
const result = await aliasedDataStore.trySetAlias(alias);
|
|
1121
|
+
if (result !== AliasResult.Success) {
|
|
1122
|
+
throw new GenericError("dataStoreAliasFailure", undefined /* error */, {
|
|
1123
|
+
alias: {
|
|
1124
|
+
value: alias,
|
|
1125
|
+
tag: TelemetryDataTag.UserData,
|
|
1126
|
+
},
|
|
1127
|
+
internalId: {
|
|
1128
|
+
value: internalId,
|
|
1129
|
+
tag: TelemetryDataTag.PackageData,
|
|
1130
|
+
},
|
|
1131
|
+
aliasResult: result,
|
|
1132
|
+
});
|
|
1133
|
+
}
|
|
1134
|
+
return aliasedDataStore;
|
|
1135
|
+
}
|
|
1091
1136
|
createDetachedRootDataStore(pkg, rootDataStoreId) {
|
|
1092
1137
|
return this.dataStores.createDetachedDataStoreCore(pkg, true, rootDataStoreId);
|
|
1093
1138
|
}
|
|
1094
1139
|
createDetachedDataStore(pkg) {
|
|
1095
1140
|
return this.dataStores.createDetachedDataStoreCore(pkg, false);
|
|
1096
1141
|
}
|
|
1097
|
-
|
|
1142
|
+
/**
|
|
1143
|
+
* Creates a possibly root datastore directly with a possibly user generated id and attaches it to storage.
|
|
1144
|
+
* It is vulnerable to name collisions if both aforementioned conditions are true, and should not be used.
|
|
1145
|
+
*
|
|
1146
|
+
* This method will be removed. See #6465.
|
|
1147
|
+
*/
|
|
1148
|
+
async _createDataStoreWithPropsLegacy(pkg, props, id = uuid(), isRoot = false) {
|
|
1098
1149
|
const fluidDataStore = await this.dataStores._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
|
|
1099
1150
|
if (isRoot) {
|
|
1100
1151
|
fluidDataStore.bindToContext();
|
|
1101
1152
|
}
|
|
1102
1153
|
return fluidDataStore;
|
|
1103
1154
|
}
|
|
1104
|
-
async
|
|
1105
|
-
return this.
|
|
1155
|
+
async _createDataStoreWithProps(pkg, props, id = uuid(), isRoot = false) {
|
|
1156
|
+
return this._aliasingEnabled === true && isRoot ?
|
|
1157
|
+
this.createAndAliasDataStore(pkg, id, props) :
|
|
1158
|
+
this._createDataStoreWithPropsLegacy(pkg, props, id, isRoot);
|
|
1159
|
+
}
|
|
1160
|
+
async _createDataStore(pkg, isRoot, id = uuid(), props) {
|
|
1161
|
+
return this.dataStores
|
|
1162
|
+
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props)
|
|
1163
|
+
.realize();
|
|
1106
1164
|
}
|
|
1107
1165
|
canSendOps() {
|
|
1108
1166
|
return this.connected && !this.deltaManager.readOnlyInfo.readonly;
|
|
@@ -1160,6 +1218,9 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1160
1218
|
assert(this.attachState === AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
|
|
1161
1219
|
this.emit("attached");
|
|
1162
1220
|
}
|
|
1221
|
+
if (attachState === AttachState.Attached && !this.pendingStateManager.hasPendingMessages()) {
|
|
1222
|
+
this.updateDocumentDirtyState(false);
|
|
1223
|
+
}
|
|
1163
1224
|
this.dataStores.setAttachState(attachState);
|
|
1164
1225
|
}
|
|
1165
1226
|
/**
|
|
@@ -1206,13 +1267,14 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1206
1267
|
*/
|
|
1207
1268
|
async summarize(options) {
|
|
1208
1269
|
this.verifyNotClosed();
|
|
1209
|
-
const {
|
|
1270
|
+
const { fullTree = false, trackState = true, summaryLogger = this.logger, runGC = this.garbageCollector.shouldRunGC, runSweep, fullGC, } = options;
|
|
1271
|
+
let gcStats;
|
|
1210
1272
|
if (runGC) {
|
|
1211
|
-
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1273
|
+
gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
|
|
1212
1274
|
}
|
|
1213
1275
|
const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState);
|
|
1214
1276
|
assert(summarizeResult.summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1215
|
-
return summarizeResult;
|
|
1277
|
+
return Object.assign(Object.assign({}, summarizeResult), { gcStats });
|
|
1216
1278
|
}
|
|
1217
1279
|
/**
|
|
1218
1280
|
* Implementation of IGarbageCollectionRuntime::updateStateBeforeGC.
|
|
@@ -1237,7 +1299,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1237
1299
|
* @param usedRoutes - The routes that are used in all nodes in this Container.
|
|
1238
1300
|
* @param gcTimestamp - The time when GC was run that generated these used routes. If any node node becomes
|
|
1239
1301
|
* unreferenced as part of this GC run, this should be used to update the time when it happens.
|
|
1240
|
-
* @returns the statistics of the used state of the data stores.
|
|
1241
1302
|
*/
|
|
1242
1303
|
updateUsedRoutes(usedRoutes, gcTimestamp) {
|
|
1243
1304
|
// Update our summarizer node's used routes. Updating used routes in summarizer node before
|
|
@@ -1271,7 +1332,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1271
1332
|
* @param options - options controlling how the summary is generated or submitted
|
|
1272
1333
|
*/
|
|
1273
1334
|
async submitSummary(options) {
|
|
1274
|
-
var _a;
|
|
1335
|
+
var _a, _b;
|
|
1275
1336
|
const { fullTree, refreshLatestAck, summaryLogger } = options;
|
|
1276
1337
|
if (refreshLatestAck) {
|
|
1277
1338
|
const latestSummaryRefSeq = await this.refreshLatestSummaryAckFromServer(ChildLogger.create(summaryLogger, undefined, { all: { safeSummary: true } }));
|
|
@@ -1330,13 +1391,14 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1330
1391
|
}
|
|
1331
1392
|
const trace = Trace.start();
|
|
1332
1393
|
let summarizeResult;
|
|
1394
|
+
// If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
|
|
1395
|
+
// state of all the nodes.
|
|
1396
|
+
const forcedFullTree = this.garbageCollector.summaryStateNeedsReset;
|
|
1333
1397
|
try {
|
|
1334
1398
|
summarizeResult = await this.summarize({
|
|
1335
|
-
|
|
1336
|
-
// If the GC state needs to be reset, we need to regenerate the summary and update the unreferenced
|
|
1337
|
-
// state of all the nodes.
|
|
1338
|
-
fullTree: fullTree || this.garbageCollector.summaryStateNeedsReset,
|
|
1399
|
+
fullTree: fullTree || forcedFullTree,
|
|
1339
1400
|
trackState: true,
|
|
1401
|
+
summaryLogger,
|
|
1340
1402
|
runGC: this.garbageCollector.shouldRunGC,
|
|
1341
1403
|
});
|
|
1342
1404
|
}
|
|
@@ -1350,12 +1412,13 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1350
1412
|
const dataStoreTree = this.disableIsolatedChannels ? summaryTree : summaryTree.tree[channelsTreeName];
|
|
1351
1413
|
assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
|
|
1352
1414
|
const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === SummaryType.Handle).length;
|
|
1353
|
-
const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount }, partialStats);
|
|
1415
|
+
const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount, gcStateUpdatedDataStoreCount: (_a = summarizeResult.gcStats) === null || _a === void 0 ? void 0 : _a.updatedDataStoreCount }, partialStats);
|
|
1354
1416
|
const generateSummaryData = {
|
|
1355
1417
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1356
1418
|
summaryTree,
|
|
1357
1419
|
summaryStats,
|
|
1358
1420
|
generateDuration: trace.trace().duration,
|
|
1421
|
+
forcedFullTree,
|
|
1359
1422
|
};
|
|
1360
1423
|
continueResult = checkContinue();
|
|
1361
1424
|
if (!continueResult.continue) {
|
|
@@ -1365,7 +1428,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1365
1428
|
const summaryContext = lastAck === undefined
|
|
1366
1429
|
? {
|
|
1367
1430
|
proposalHandle: undefined,
|
|
1368
|
-
ackHandle: (
|
|
1431
|
+
ackHandle: (_b = this.context.getLoadedFromVersion()) === null || _b === void 0 ? void 0 : _b.id,
|
|
1369
1432
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1370
1433
|
}
|
|
1371
1434
|
: {
|
|
@@ -1460,6 +1523,13 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1460
1523
|
};
|
|
1461
1524
|
this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
|
|
1462
1525
|
}
|
|
1526
|
+
submitDataStoreAliasOp(contents, localOpMetadata) {
|
|
1527
|
+
const aliasMessage = contents;
|
|
1528
|
+
if (!isDataStoreAliasMessage(aliasMessage)) {
|
|
1529
|
+
throw new UsageError("malformedDataStoreAliasMessage");
|
|
1530
|
+
}
|
|
1531
|
+
this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
|
|
1532
|
+
}
|
|
1463
1533
|
async uploadBlob(blob) {
|
|
1464
1534
|
this.verifyNotClosed();
|
|
1465
1535
|
return this.blobManager.createBlob(blob);
|