@fluidframework/container-runtime 2.0.0-internal.2.2.1 → 2.0.0-internal.2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +19 -8
- package/dist/batchTracker.d.ts +1 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +44 -33
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +130 -97
- package/dist/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +39 -8
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +117 -61
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +1 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +4 -3
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +9 -6
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +30 -24
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +41 -20
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +205 -151
- package/dist/garbageCollection.js.map +1 -1
- package/dist/garbageCollectionConstants.d.ts +6 -3
- package/dist/garbageCollectionConstants.d.ts.map +1 -1
- package/dist/garbageCollectionConstants.js +7 -7
- package/dist/garbageCollectionConstants.js.map +1 -1
- package/dist/garbageCollectionTombstoneUtils.d.ts +13 -0
- package/dist/garbageCollectionTombstoneUtils.d.ts.map +1 -0
- package/dist/garbageCollectionTombstoneUtils.js +28 -0
- package/dist/garbageCollectionTombstoneUtils.js.map +1 -0
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -5
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +13 -1
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +35 -1
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +25 -1
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +2 -2
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +24 -10
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +2 -1
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +30 -17
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +34 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +114 -5
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +5 -0
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +24 -14
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +17 -2
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +0 -1
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/scheduleManager.d.ts +0 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +9 -20
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summarizer.d.ts +0 -1
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +2 -1
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerTypes.d.ts +1 -0
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js +1 -2
- package/dist/summaryFormat.js.map +1 -1
- package/lib/batchTracker.d.ts +1 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +44 -33
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +131 -98
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +39 -8
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +115 -59
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +1 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +5 -4
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +9 -6
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +32 -26
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +41 -20
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +201 -147
- package/lib/garbageCollection.js.map +1 -1
- package/lib/garbageCollectionConstants.d.ts +6 -3
- package/lib/garbageCollectionConstants.d.ts.map +1 -1
- package/lib/garbageCollectionConstants.js +6 -6
- package/lib/garbageCollectionConstants.js.map +1 -1
- package/lib/garbageCollectionTombstoneUtils.d.ts +13 -0
- package/lib/garbageCollectionTombstoneUtils.d.ts.map +1 -0
- package/lib/garbageCollectionTombstoneUtils.js +24 -0
- package/lib/garbageCollectionTombstoneUtils.js.map +1 -0
- package/lib/index.d.ts +0 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +0 -1
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +13 -1
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +35 -1
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +25 -1
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +2 -2
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +24 -10
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +2 -1
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +30 -17
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +34 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +112 -4
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +5 -0
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +24 -14
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +17 -2
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +0 -1
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/scheduleManager.d.ts +0 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +9 -20
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summarizer.d.ts +0 -1
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +2 -1
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerTypes.d.ts +1 -0
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js +1 -2
- package/lib/summaryFormat.js.map +1 -1
- package/package.json +37 -19
- package/src/batchTracker.ts +1 -1
- package/src/blobManager.ts +146 -103
- package/src/containerRuntime.ts +166 -65
- package/src/dataStoreContext.ts +5 -5
- package/src/dataStores.ts +40 -30
- package/src/garbageCollection.ts +254 -183
- package/src/garbageCollectionConstants.ts +7 -6
- package/src/garbageCollectionTombstoneUtils.ts +31 -0
- package/src/index.ts +0 -5
- package/src/opLifecycle/batchManager.ts +59 -1
- package/src/opLifecycle/definitions.ts +27 -1
- package/src/opLifecycle/index.ts +2 -1
- package/src/opLifecycle/opCompressor.ts +29 -12
- package/src/opLifecycle/opDecompressor.ts +39 -18
- package/src/opLifecycle/opSplitter.ts +141 -7
- package/src/opLifecycle/outbox.ts +32 -16
- package/src/opLifecycle/remoteMessageProcessor.ts +19 -3
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +0 -1
- package/src/scheduleManager.ts +19 -30
- package/src/summarizer.ts +1 -1
- package/src/summarizerTypes.ts +1 -0
- package/src/summaryFormat.ts +1 -2
package/dist/containerRuntime.js
CHANGED
|
@@ -28,7 +28,6 @@ const summarizerClientElection_1 = require("./summarizerClientElection");
|
|
|
28
28
|
const throttler_1 = require("./throttler");
|
|
29
29
|
const runWhileConnectedCoordinator_1 = require("./runWhileConnectedCoordinator");
|
|
30
30
|
const garbageCollection_1 = require("./garbageCollection");
|
|
31
|
-
const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
|
|
32
31
|
const dataStore_1 = require("./dataStore");
|
|
33
32
|
const batchTracker_1 = require("./batchTracker");
|
|
34
33
|
const serializedSnapshotStorage_1 = require("./serializedSnapshotStorage");
|
|
@@ -218,12 +217,17 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
218
217
|
this.nextSummaryNumber = loadSummaryNumber + 1;
|
|
219
218
|
this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
|
|
220
219
|
this._connected = this.context.connected;
|
|
221
|
-
this.remoteMessageProcessor = new opLifecycle_1.RemoteMessageProcessor(new opLifecycle_1.OpSplitter(chunks), new opLifecycle_1.OpDecompressor());
|
|
222
|
-
this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
|
|
223
220
|
this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.logger, "ContainerRuntime"));
|
|
221
|
+
const opSplitter = new opLifecycle_1.OpSplitter(chunks, this.context.submitBatchFn, this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true ?
|
|
222
|
+
Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
|
|
223
|
+
this.remoteMessageProcessor = new opLifecycle_1.RemoteMessageProcessor(opSplitter, new opLifecycle_1.OpDecompressor());
|
|
224
|
+
this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
|
|
224
225
|
if (this.summaryConfiguration.state === "enabled") {
|
|
225
226
|
this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
|
|
226
227
|
}
|
|
228
|
+
this.enableOpReentryCheck = runtimeOptions.enableOpReentryCheck === true
|
|
229
|
+
// Allow for a break-glass config to override the options
|
|
230
|
+
&& this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
|
|
227
231
|
this.summariesDisabled = this.isSummariesDisabled();
|
|
228
232
|
this.heuristicsDisabled = this.isHeuristicsDisabled();
|
|
229
233
|
this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
|
|
@@ -272,16 +276,20 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
272
276
|
throwOnFailure: true,
|
|
273
277
|
// If GC should not run, let the summarizer node know so that it does not track GC state.
|
|
274
278
|
gcDisabled: !this.garbageCollector.shouldRunGC,
|
|
275
|
-
}
|
|
279
|
+
},
|
|
280
|
+
// Function to get GC data if needed. This will always be called by the root summarizer node to get GC data.
|
|
281
|
+
async (fullGC) => this.getGCDataInternal(fullGC),
|
|
282
|
+
// Function to get the GC details from the base snapshot we loaded from.
|
|
283
|
+
async () => this.garbageCollector.getBaseGCDetails());
|
|
276
284
|
if (baseSnapshot) {
|
|
277
285
|
this.summarizerNode.updateBaseSummaryState(baseSnapshot);
|
|
278
286
|
}
|
|
279
287
|
this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap));
|
|
280
|
-
this.blobManager = new blobManager_1.BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (
|
|
288
|
+
this.blobManager = new blobManager_1.BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
|
|
281
289
|
if (!this.disposed) {
|
|
282
|
-
this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
|
|
290
|
+
this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { localId, blobId });
|
|
283
291
|
}
|
|
284
|
-
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
|
|
292
|
+
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (fromPath, toPath) => this.garbageCollector.addedOutboundReference(fromPath, toPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
|
|
285
293
|
this.scheduleManager = new scheduleManager_1.ScheduleManager(context.deltaManager, this, () => this.clientId, telemetry_utils_1.ChildLogger.create(this.logger, "ScheduleManager"));
|
|
286
294
|
this.pendingStateManager = new pendingStateManager_1.PendingStateManager({
|
|
287
295
|
applyStashedOp: this.applyStashedOp.bind(this),
|
|
@@ -293,15 +301,23 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
293
301
|
rollback: this.rollback.bind(this),
|
|
294
302
|
orderSequentially: this.orderSequentially.bind(this),
|
|
295
303
|
}, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
|
|
304
|
+
const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true ?
|
|
305
|
+
{
|
|
306
|
+
minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
|
|
307
|
+
compressionAlgorithm: CompressionAlgorithms.lz4
|
|
308
|
+
} : runtimeOptions.compressionOptions;
|
|
296
309
|
this.outbox = new opLifecycle_1.Outbox({
|
|
297
310
|
shouldSend: () => this.canSendOps(),
|
|
298
311
|
pendingStateManager: this.pendingStateManager,
|
|
299
312
|
containerContext: this.context,
|
|
300
313
|
compressor: new opLifecycle_1.OpCompressor(this.mc.logger),
|
|
314
|
+
splitter: opSplitter,
|
|
301
315
|
config: {
|
|
302
|
-
compressionOptions
|
|
316
|
+
compressionOptions,
|
|
303
317
|
maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
|
|
318
|
+
enableOpReentryCheck: this.enableOpReentryCheck,
|
|
304
319
|
},
|
|
320
|
+
logger: this.mc.logger,
|
|
305
321
|
});
|
|
306
322
|
this.context.quorum.on("removeMember", (clientId) => {
|
|
307
323
|
this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
|
|
@@ -389,7 +405,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
389
405
|
* allows mixin classes to leverage this method to define their own async initializer.
|
|
390
406
|
*/
|
|
391
407
|
static async load(context, registryEntries, requestHandler, runtimeOptions = {}, containerScope = context.scope, existing, containerRuntimeCtor = ContainerRuntime) {
|
|
392
|
-
var _a, _b, _c;
|
|
408
|
+
var _a, _b, _c, _d;
|
|
393
409
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
394
410
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
395
411
|
const backCompatContext = context;
|
|
@@ -402,7 +418,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
402
418
|
const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = {
|
|
403
419
|
minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
|
|
404
420
|
compressionAlgorithm: CompressionAlgorithms.lz4
|
|
405
|
-
}, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, } = runtimeOptions;
|
|
421
|
+
}, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, chunkSizeInBytes = Number.POSITIVE_INFINITY, enableOpReentryCheck = false, } = runtimeOptions;
|
|
406
422
|
const pendingRuntimeState = context.pendingLocalState;
|
|
407
423
|
const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
|
|
408
424
|
const storage = !pendingRuntimeState ?
|
|
@@ -447,7 +463,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
447
463
|
logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
|
|
448
464
|
}
|
|
449
465
|
else {
|
|
466
|
+
// Call both close and dispose as close implementation will no longer dispose runtime in future (2.0.0-internal.3.0.0)
|
|
450
467
|
context.closeFn(error);
|
|
468
|
+
(_d = context.disposeFn) === null || _d === void 0 ? void 0 : _d.call(context, error);
|
|
451
469
|
}
|
|
452
470
|
}
|
|
453
471
|
}
|
|
@@ -459,6 +477,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
459
477
|
enableOfflineLoad,
|
|
460
478
|
compressionOptions,
|
|
461
479
|
maxBatchSizeInBytes,
|
|
480
|
+
chunkSizeInBytes,
|
|
481
|
+
enableOpReentryCheck,
|
|
462
482
|
}, containerScope, logger, loadExisting, blobManagerSnapshot, storage, requestHandler);
|
|
463
483
|
if (pendingRuntimeState) {
|
|
464
484
|
await runtime.processSavedOps(pendingRuntimeState);
|
|
@@ -488,8 +508,18 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
488
508
|
// eslint-disable-next-line @typescript-eslint/unbound-method
|
|
489
509
|
return this.reSubmit;
|
|
490
510
|
}
|
|
511
|
+
get disposeFn() {
|
|
512
|
+
var _a;
|
|
513
|
+
// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
|
|
514
|
+
return (_a = this.context.disposeFn) !== null && _a !== void 0 ? _a : this.context.closeFn;
|
|
515
|
+
}
|
|
491
516
|
get closeFn() {
|
|
492
|
-
|
|
517
|
+
// Also call disposeFn to retain functionality of runtime being disposed on close
|
|
518
|
+
return (error) => {
|
|
519
|
+
var _a, _b;
|
|
520
|
+
this.context.closeFn(error);
|
|
521
|
+
(_b = (_a = this.context).disposeFn) === null || _b === void 0 ? void 0 : _b.call(_a, error);
|
|
522
|
+
};
|
|
493
523
|
}
|
|
494
524
|
get flushMode() {
|
|
495
525
|
return this._flushMode;
|
|
@@ -742,7 +772,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
742
772
|
}
|
|
743
773
|
const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
|
|
744
774
|
if (gcSummary !== undefined) {
|
|
745
|
-
(0, runtime_utils_1.addSummarizeResultToSummary)(summaryTree,
|
|
775
|
+
(0, runtime_utils_1.addSummarizeResultToSummary)(summaryTree, runtime_definitions_1.gcTreeKey, gcSummary);
|
|
746
776
|
}
|
|
747
777
|
}
|
|
748
778
|
// Track how many times the container tries to reconnect with pending messages.
|
|
@@ -867,9 +897,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
867
897
|
if (reconnection) {
|
|
868
898
|
this.consecutiveReconnects++;
|
|
869
899
|
if (!this.shouldContinueReconnecting()) {
|
|
870
|
-
this.closeFn(container_utils_1.DataProcessingError.create(
|
|
871
|
-
// eslint-disable-next-line max-len
|
|
872
|
-
"Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
|
|
900
|
+
this.closeFn(container_utils_1.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
|
|
873
901
|
dataLoss: 1,
|
|
874
902
|
attempts: this.consecutiveReconnects,
|
|
875
903
|
pendingMessages: this.pendingStateManager.pendingMessagesCount,
|
|
@@ -903,7 +931,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
903
931
|
this.scheduleManager.beforeOpProcessing(message);
|
|
904
932
|
try {
|
|
905
933
|
let localOpMetadata;
|
|
906
|
-
if (local && runtimeMessage) {
|
|
934
|
+
if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
|
|
907
935
|
localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
|
|
908
936
|
}
|
|
909
937
|
// If there are no more pending messages after processing a local message,
|
|
@@ -1020,6 +1048,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1020
1048
|
}
|
|
1021
1049
|
orderSequentially(callback) {
|
|
1022
1050
|
let checkpoint;
|
|
1051
|
+
let result;
|
|
1023
1052
|
if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
|
|
1024
1053
|
// Note: we are not touching this.pendingAttachBatch here, for two reasons:
|
|
1025
1054
|
// 1. It would not help, as we flush attach ops as they become available.
|
|
@@ -1028,7 +1057,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1028
1057
|
}
|
|
1029
1058
|
try {
|
|
1030
1059
|
this._orderSequentiallyCalls++;
|
|
1031
|
-
callback();
|
|
1060
|
+
result = callback();
|
|
1032
1061
|
}
|
|
1033
1062
|
catch (error) {
|
|
1034
1063
|
if (checkpoint) {
|
|
@@ -1056,6 +1085,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1056
1085
|
if (this.flushMode === runtime_definitions_1.FlushMode.Immediate && this._orderSequentiallyCalls === 0) {
|
|
1057
1086
|
this.flush();
|
|
1058
1087
|
}
|
|
1088
|
+
return result;
|
|
1059
1089
|
}
|
|
1060
1090
|
async createDataStore(pkg) {
|
|
1061
1091
|
const internalId = (0, uuid_1.v4)();
|
|
@@ -1221,6 +1251,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1221
1251
|
async updateStateBeforeGC() {
|
|
1222
1252
|
return this.dataStores.updateStateBeforeGC();
|
|
1223
1253
|
}
|
|
1254
|
+
async getGCDataInternal(fullGC) {
|
|
1255
|
+
return this.dataStores.getGCData(fullGC);
|
|
1256
|
+
}
|
|
1224
1257
|
/**
|
|
1225
1258
|
* Implementation of IGarbageCollectionRuntime::getGCData.
|
|
1226
1259
|
* Generates and returns the GC data for this container.
|
|
@@ -1228,7 +1261,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1228
1261
|
*/
|
|
1229
1262
|
async getGCData(fullGC) {
|
|
1230
1263
|
const builder = new garbage_collector_1.GCDataBuilder();
|
|
1231
|
-
const dsGCData = await this.
|
|
1264
|
+
const dsGCData = await this.summarizerNode.getGCData(fullGC);
|
|
1232
1265
|
builder.addNodes(dsGCData.gcNodes);
|
|
1233
1266
|
const blobsGCData = this.blobManager.getGCData(fullGC);
|
|
1234
1267
|
builder.addNodes(blobsGCData.gcNodes);
|
|
@@ -1244,39 +1277,26 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1244
1277
|
// summarizing is required and asserted by the the summarizer node. We are the root and are
|
|
1245
1278
|
// always referenced, so the used routes is only self-route (empty string).
|
|
1246
1279
|
this.summarizerNode.updateUsedRoutes([""]);
|
|
1247
|
-
const
|
|
1248
|
-
|
|
1249
|
-
for (const route of usedRoutes) {
|
|
1250
|
-
if (this.isBlobPath(route)) {
|
|
1251
|
-
blobManagerUsedRoutes.push(route);
|
|
1252
|
-
}
|
|
1253
|
-
else {
|
|
1254
|
-
dataStoreUsedRoutes.push(route);
|
|
1255
|
-
}
|
|
1256
|
-
}
|
|
1257
|
-
this.blobManager.updateUsedRoutes(blobManagerUsedRoutes);
|
|
1258
|
-
this.dataStores.updateUsedRoutes(dataStoreUsedRoutes);
|
|
1280
|
+
const { dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(usedRoutes);
|
|
1281
|
+
this.dataStores.updateUsedRoutes(dataStoreRoutes);
|
|
1259
1282
|
}
|
|
1260
1283
|
/**
|
|
1261
|
-
* This is called to update objects whose routes are unused.
|
|
1262
|
-
*
|
|
1263
|
-
* @param unusedRoutes - The routes that are unused in all data stores and attachment blobs in this Container.
|
|
1264
|
-
* @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
|
|
1265
|
-
* are deleted.
|
|
1284
|
+
* This is called to update objects whose routes are unused.
|
|
1285
|
+
* @param unusedRoutes - Data store and attachment blob routes that are unused in this Container.
|
|
1266
1286
|
*/
|
|
1267
|
-
updateUnusedRoutes(unusedRoutes
|
|
1268
|
-
const
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
}
|
|
1278
|
-
this.blobManager.
|
|
1279
|
-
this.dataStores.
|
|
1287
|
+
updateUnusedRoutes(unusedRoutes) {
|
|
1288
|
+
const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
|
|
1289
|
+
this.blobManager.updateUnusedRoutes(blobManagerRoutes);
|
|
1290
|
+
this.dataStores.updateUnusedRoutes(dataStoreRoutes);
|
|
1291
|
+
}
|
|
1292
|
+
/**
|
|
1293
|
+
* This is called to update objects that are tombstones.
|
|
1294
|
+
* @param tombstonedRoutes - Data store and attachment blob routes that are tombstones in this Container.
|
|
1295
|
+
*/
|
|
1296
|
+
updateTombstonedRoutes(tombstonedRoutes) {
|
|
1297
|
+
const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(tombstonedRoutes);
|
|
1298
|
+
this.blobManager.updateTombstonedRoutes(blobManagerRoutes);
|
|
1299
|
+
this.dataStores.updateTombstonedRoutes(dataStoreRoutes);
|
|
1280
1300
|
}
|
|
1281
1301
|
/**
|
|
1282
1302
|
* Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
|
|
@@ -1305,7 +1325,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1305
1325
|
async getGCNodePackagePath(nodePath) {
|
|
1306
1326
|
switch (this.getNodeType(nodePath)) {
|
|
1307
1327
|
case garbageCollection_1.GCNodeType.Blob:
|
|
1308
|
-
return [
|
|
1328
|
+
return [blobManager_1.BlobManager.basePath];
|
|
1309
1329
|
case garbageCollection_1.GCNodeType.DataStore:
|
|
1310
1330
|
case garbageCollection_1.GCNodeType.SubDataStore:
|
|
1311
1331
|
return this.dataStores.getDataStorePackagePath(nodePath);
|
|
@@ -1323,6 +1343,25 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1323
1343
|
}
|
|
1324
1344
|
return true;
|
|
1325
1345
|
}
|
|
1346
|
+
/**
|
|
1347
|
+
* From a given list of routes, separate and return routes that belong to blob manager and data stores.
|
|
1348
|
+
* @param routes - A list of routes that can belong to data stores or blob manager.
|
|
1349
|
+
* @returns - Two route lists - One that contains routes for blob manager and another one that contains routes
|
|
1350
|
+
* for data stores.
|
|
1351
|
+
*/
|
|
1352
|
+
getDataStoreAndBlobManagerRoutes(routes) {
|
|
1353
|
+
const blobManagerRoutes = [];
|
|
1354
|
+
const dataStoreRoutes = [];
|
|
1355
|
+
for (const route of routes) {
|
|
1356
|
+
if (this.isBlobPath(route)) {
|
|
1357
|
+
blobManagerRoutes.push(route);
|
|
1358
|
+
}
|
|
1359
|
+
else {
|
|
1360
|
+
dataStoreRoutes.push(route);
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
return { blobManagerRoutes, dataStoreRoutes };
|
|
1364
|
+
}
|
|
1326
1365
|
/**
|
|
1327
1366
|
* Runs garbage collection and updates the reference / used state of the nodes in the container.
|
|
1328
1367
|
* @returns the statistics of the garbage collection run; undefined if GC did not run.
|
|
@@ -1394,7 +1433,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1394
1433
|
if (this.deltaManager.lastSequenceNumber !== summaryRefSeqNum) {
|
|
1395
1434
|
return {
|
|
1396
1435
|
continue: false,
|
|
1397
|
-
// eslint-disable-next-line max-len
|
|
1398
1436
|
error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
|
|
1399
1437
|
};
|
|
1400
1438
|
}
|
|
@@ -1402,7 +1440,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1402
1440
|
if (lastAck !== this.summaryCollection.latestAck) {
|
|
1403
1441
|
return {
|
|
1404
1442
|
continue: false,
|
|
1405
|
-
// eslint-disable-next-line max-len
|
|
1406
1443
|
error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
|
|
1407
1444
|
};
|
|
1408
1445
|
}
|
|
@@ -1447,8 +1484,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1447
1484
|
const dataStoreTree = summaryTree.tree[runtime_definitions_1.channelsTreeName];
|
|
1448
1485
|
(0, common_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
|
|
1449
1486
|
const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === protocol_definitions_1.SummaryType.Handle).length;
|
|
1450
|
-
const gcSummaryTreeStats = summaryTree.tree[
|
|
1451
|
-
? (0, runtime_utils_1.calculateStats)(summaryTree.tree[
|
|
1487
|
+
const gcSummaryTreeStats = summaryTree.tree[runtime_definitions_1.gcTreeKey]
|
|
1488
|
+
? (0, runtime_utils_1.calculateStats)(summaryTree.tree[runtime_definitions_1.gcTreeKey])
|
|
1452
1489
|
: undefined;
|
|
1453
1490
|
const summaryStats = Object.assign({ dataStoreCount: this.dataStores.size, summarizedDataStoreCount: this.dataStores.size - handleCount, gcStateUpdatedDataStoreCount: (_a = summarizeResult.gcStats) === null || _a === void 0 ? void 0 : _a.updatedDataStoreCount, gcBlobNodeCount: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.blobNodeCount, gcTotalBlobsSize: gcSummaryTreeStats === null || gcSummaryTreeStats === void 0 ? void 0 : gcSummaryTreeStats.totalBlobSize, summaryNumber }, partialStats);
|
|
1454
1491
|
const generateSummaryData = {
|
|
@@ -1617,7 +1654,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1617
1654
|
else if (!this.flushMicroTaskExists) {
|
|
1618
1655
|
this.flushMicroTaskExists = true;
|
|
1619
1656
|
// Queue a microtask to detect the end of the turn and force a flush.
|
|
1620
|
-
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
1621
1657
|
Promise.resolve().then(() => {
|
|
1622
1658
|
this.flushMicroTaskExists = false;
|
|
1623
1659
|
this.flush();
|
|
@@ -1710,13 +1746,32 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1710
1746
|
// It should only be done by the summarizerNode, if required.
|
|
1711
1747
|
// When fetching from storage we will always get the latest version and do not use the ackHandle.
|
|
1712
1748
|
const snapshotTreeFetcher = async () => {
|
|
1713
|
-
const fetchResult = await this.
|
|
1749
|
+
const fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
|
|
1714
1750
|
eventName: "RefreshLatestSummaryGetSnapshot",
|
|
1715
1751
|
ackHandle,
|
|
1716
1752
|
summaryRefSeq,
|
|
1717
1753
|
fetchLatest: true,
|
|
1718
1754
|
});
|
|
1719
1755
|
const latestSnapshotRefSeq = await (0, runtime_utils_1.seqFromTree)(fetchResult.snapshotTree, readAndParseBlob);
|
|
1756
|
+
/**
|
|
1757
|
+
* If the fetched snapshot is older than the one for which the ack was received, close the container.
|
|
1758
|
+
* This should never happen because an ack should be sent after the latest summary is updated in the server.
|
|
1759
|
+
* However, there are couple of scenarios where it's possible:
|
|
1760
|
+
* 1. A file was modified externally resulting in modifying the snapshot's sequence number. This can lead to
|
|
1761
|
+
* the document being unusable and we should not proceed.
|
|
1762
|
+
* 2. The server DB failed after the ack was sent which may delete the corresponding snapshot. Ideally, in
|
|
1763
|
+
* such cases, the file will be rolled back along with the ack and we will eventually reach a consistent
|
|
1764
|
+
* state.
|
|
1765
|
+
*/
|
|
1766
|
+
if (latestSnapshotRefSeq < summaryRefSeq) {
|
|
1767
|
+
const error = container_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
|
|
1768
|
+
ackHandle,
|
|
1769
|
+
summaryRefSeq,
|
|
1770
|
+
latestSnapshotRefSeq,
|
|
1771
|
+
});
|
|
1772
|
+
this.closeFn(error);
|
|
1773
|
+
throw error;
|
|
1774
|
+
}
|
|
1720
1775
|
summaryLogger.sendTelemetryEvent({
|
|
1721
1776
|
eventName: "LatestSummaryRetrieved",
|
|
1722
1777
|
ackHandle,
|
|
@@ -1730,7 +1785,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1730
1785
|
};
|
|
1731
1786
|
const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, snapshotTreeFetcher, readAndParseBlob, summaryLogger);
|
|
1732
1787
|
// Notify the garbage collector so it can update its latest summary state.
|
|
1733
|
-
await this.garbageCollector.
|
|
1788
|
+
await this.garbageCollector.refreshLatestSummary(result, proposalHandle, summaryRefSeq, readAndParseBlob);
|
|
1734
1789
|
}
|
|
1735
1790
|
/**
|
|
1736
1791
|
* Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
|
|
@@ -1739,22 +1794,22 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1739
1794
|
* @returns downloaded snapshot's reference sequence number
|
|
1740
1795
|
*/
|
|
1741
1796
|
async refreshLatestSummaryAckFromServer(summaryLogger) {
|
|
1742
|
-
const { snapshotTree, versionId } = await this.
|
|
1797
|
+
const { snapshotTree, versionId } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
|
|
1743
1798
|
eventName: "RefreshLatestSummaryGetSnapshot",
|
|
1744
1799
|
fetchLatest: true,
|
|
1745
|
-
}
|
|
1800
|
+
});
|
|
1746
1801
|
const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
|
|
1747
1802
|
const latestSnapshotRefSeq = await (0, runtime_utils_1.seqFromTree)(snapshotTree, readAndParseBlob);
|
|
1748
1803
|
const result = await this.summarizerNode.refreshLatestSummary(undefined, latestSnapshotRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
|
|
1749
1804
|
// Notify the garbage collector so it can update its latest summary state.
|
|
1750
|
-
await this.garbageCollector.
|
|
1805
|
+
await this.garbageCollector.refreshLatestSummary(result, undefined, latestSnapshotRefSeq, readAndParseBlob);
|
|
1751
1806
|
return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
|
|
1752
1807
|
}
|
|
1753
|
-
async
|
|
1808
|
+
async fetchLatestSnapshotFromStorage(logger, event) {
|
|
1754
1809
|
return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
1755
1810
|
const stats = {};
|
|
1756
1811
|
const trace = common_utils_1.Trace.start();
|
|
1757
|
-
const versions = await this.storage.getVersions(
|
|
1812
|
+
const versions = await this.storage.getVersions(null, 1, "refreshLatestSummaryAckFromServer", driver_definitions_1.FetchSource.noCache);
|
|
1758
1813
|
(0, common_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
|
|
1759
1814
|
stats.getVersionDuration = trace.trace().duration;
|
|
1760
1815
|
const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
|
|
@@ -1871,6 +1926,7 @@ exports.ContainerRuntime = ContainerRuntime;
|
|
|
1871
1926
|
const waitForSeq = async (deltaManager, targetSeq) => new Promise((resolve, reject) => {
|
|
1872
1927
|
// TODO: remove cast to any when actual event is determined
|
|
1873
1928
|
deltaManager.on("closed", reject);
|
|
1929
|
+
deltaManager.on("disposed", reject);
|
|
1874
1930
|
// If we already reached target sequence number, simply resolve the promise.
|
|
1875
1931
|
if (deltaManager.lastSequenceNumber >= targetSeq) {
|
|
1876
1932
|
resolve();
|