@fluidframework/container-runtime 2.40.0-336023 → 2.41.0-337492
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/api-report/container-runtime.legacy.alpha.api.md +4 -0
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts +31 -8
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +90 -17
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/channelCollection.d.ts +8 -2
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +29 -6
- package/dist/channelCollection.js.map +1 -1
- package/dist/compatUtils.d.ts +19 -10
- package/dist/compatUtils.d.ts.map +1 -1
- package/dist/compatUtils.js +39 -32
- package/dist/compatUtils.js.map +1 -1
- package/dist/containerRuntime.d.ts +29 -13
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +139 -149
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +12 -4
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +37 -18
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/legacy.d.ts +1 -0
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +20 -7
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +16 -20
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +22 -8
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +11 -16
- package/dist/pendingStateManager.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts +31 -8
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +91 -18
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/channelCollection.d.ts +8 -2
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +29 -6
- package/lib/channelCollection.js.map +1 -1
- package/lib/compatUtils.d.ts +19 -10
- package/lib/compatUtils.d.ts.map +1 -1
- package/lib/compatUtils.js +36 -29
- package/lib/compatUtils.js.map +1 -1
- package/lib/containerRuntime.d.ts +29 -13
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +60 -70
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +12 -4
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +38 -19
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/index.d.ts +1 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/legacy.d.ts +1 -0
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +20 -7
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +16 -20
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +22 -8
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +11 -16
- package/lib/pendingStateManager.js.map +1 -1
- package/package.json +18 -18
- package/src/blobManager/blobManager.ts +141 -33
- package/src/channelCollection.ts +42 -6
- package/src/compatUtils.ts +53 -30
- package/src/containerRuntime.ts +102 -81
- package/src/dataStoreContext.ts +44 -25
- package/src/index.ts +1 -0
- package/src/opLifecycle/index.ts +1 -0
- package/src/opLifecycle/outbox.ts +42 -33
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +37 -20
package/dist/containerRuntime.js
CHANGED
|
@@ -12,9 +12,10 @@ const internal_2 = require("@fluidframework/core-utils/internal");
|
|
|
12
12
|
const driver_definitions_1 = require("@fluidframework/driver-definitions");
|
|
13
13
|
const internal_3 = require("@fluidframework/driver-definitions/internal");
|
|
14
14
|
const internal_4 = require("@fluidframework/driver-utils/internal");
|
|
15
|
-
const internal_5 = require("@fluidframework/
|
|
16
|
-
const internal_6 = require("@fluidframework/runtime-
|
|
17
|
-
const internal_7 = require("@fluidframework/
|
|
15
|
+
const internal_5 = require("@fluidframework/id-compressor/internal");
|
|
16
|
+
const internal_6 = require("@fluidframework/runtime-definitions/internal");
|
|
17
|
+
const internal_7 = require("@fluidframework/runtime-utils/internal");
|
|
18
|
+
const internal_8 = require("@fluidframework/telemetry-utils/internal");
|
|
18
19
|
const uuid_1 = require("uuid");
|
|
19
20
|
const batchTracker_js_1 = require("./batchTracker.js");
|
|
20
21
|
const index_js_1 = require("./blobManager/index.js");
|
|
@@ -51,7 +52,7 @@ const throttler_js_1 = require("./throttler.js");
|
|
|
51
52
|
*
|
|
52
53
|
*/
|
|
53
54
|
function getUnknownMessageTypeError(unknownContainerRuntimeMessageType, codePath, sequencedMessage) {
|
|
54
|
-
return
|
|
55
|
+
return internal_8.DataProcessingError.create("Runtime message of unknown type", codePath, sequencedMessage, {
|
|
55
56
|
messageDetails: {
|
|
56
57
|
type: unknownContainerRuntimeMessageType,
|
|
57
58
|
},
|
|
@@ -85,6 +86,7 @@ exports.defaultRuntimeHeaderData = {
|
|
|
85
86
|
viaHandle: false,
|
|
86
87
|
allowTombstone: false,
|
|
87
88
|
};
|
|
89
|
+
const defaultStagingCommitOptions = { squash: false };
|
|
88
90
|
const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
|
|
89
91
|
// The actual limit is 1Mb (socket.io and Kafka limits)
|
|
90
92
|
// We can't estimate it fully, as we
|
|
@@ -215,17 +217,18 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
215
217
|
* - containerRuntimeCtor - Constructor to use to create the ContainerRuntime instance.
|
|
216
218
|
* This allows mixin classes to leverage this method to define their own async initializer.
|
|
217
219
|
* - provideEntryPoint - Promise that resolves to an object which will act as entryPoint for the Container.
|
|
220
|
+
* - minVersionForCollab - Minimum version of the FF runtime that this runtime supports collaboration with.
|
|
218
221
|
* This object should provide all the functionality that the Container is expected to provide to the loader layer.
|
|
219
222
|
*/
|
|
220
223
|
static async loadRuntime(params) {
|
|
221
|
-
const { context, registryEntries, existing, requestHandler, provideEntryPoint, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
|
|
224
|
+
const { context, registryEntries, existing, requestHandler, provideEntryPoint, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, minVersionForCollab = compatUtils_js_1.defaultMinVersionForCollab, } = params;
|
|
222
225
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
223
226
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
224
227
|
const backCompatContext = context;
|
|
225
228
|
const passLogger = backCompatContext.taggedLogger ??
|
|
226
229
|
// eslint-disable-next-line import/no-deprecated
|
|
227
|
-
new
|
|
228
|
-
const logger = (0,
|
|
230
|
+
new internal_8.TaggedLoggerAdapter(backCompatContext.logger);
|
|
231
|
+
const logger = (0, internal_8.createChildLogger)({
|
|
229
232
|
logger: passLogger,
|
|
230
233
|
properties: {
|
|
231
234
|
all: {
|
|
@@ -233,28 +236,26 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
233
236
|
},
|
|
234
237
|
},
|
|
235
238
|
});
|
|
236
|
-
const mc = (0,
|
|
239
|
+
const mc = (0, internal_8.loggerToMonitoringContext)(logger);
|
|
237
240
|
// Some options require a minimum version of the FF runtime to operate, so the default configs will be generated
|
|
238
|
-
// based on the
|
|
239
|
-
// For example, if
|
|
240
|
-
// 1.0.0 or later. If the
|
|
241
|
+
// based on the minVersionForCollab.
|
|
242
|
+
// For example, if minVersionForCollab is set to "1.0.0", the default configs will ensure compatibility with FF runtime
|
|
243
|
+
// 1.0.0 or later. If the minVersionForCollab is set to "2.10.0", the default values will be generated to ensure compatibility
|
|
241
244
|
// with FF runtime 2.10.0 or later.
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
if (!(0, compatUtils_js_1.isValidCompatVersion)(compatibilityVersion)) {
|
|
245
|
-
throw new internal_7.UsageError(`Invalid compatibility version: ${compatibilityVersion}. It must be an existing FF version (i.e. 2.22.1).`);
|
|
245
|
+
if (!(0, compatUtils_js_1.isValidMinVersionForCollab)(minVersionForCollab)) {
|
|
246
|
+
throw new internal_8.UsageError(`Invalid minVersionForCollab: ${minVersionForCollab}. It must be an existing FF version (i.e. 2.22.1).`);
|
|
246
247
|
}
|
|
247
|
-
const
|
|
248
|
+
const defaultsAffectingDocSchema = (0, compatUtils_js_1.getMinVersionForCollabDefaults)(minVersionForCollab);
|
|
248
249
|
// The following are the default values for the options that do not affect the DocumentSchema.
|
|
249
|
-
const
|
|
250
|
+
const defaultsNotAffectingDocSchema = {
|
|
250
251
|
summaryOptions: {},
|
|
251
252
|
loadSequenceNumberVerification: "close",
|
|
252
253
|
maxBatchSizeInBytes: defaultMaxBatchSizeInBytes,
|
|
253
254
|
chunkSizeInBytes: defaultChunkSizeInBytes,
|
|
254
255
|
};
|
|
255
256
|
const defaultConfigs = {
|
|
256
|
-
...
|
|
257
|
-
...
|
|
257
|
+
...defaultsAffectingDocSchema,
|
|
258
|
+
...defaultsNotAffectingDocSchema,
|
|
258
259
|
};
|
|
259
260
|
// Here we set each option to its corresponding default config value if it's not provided in runtimeOptions.
|
|
260
261
|
// Note: We cannot do a simple object merge of defaultConfigs/runtimeOptions because in most cases we don't want
|
|
@@ -307,7 +308,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
307
308
|
// Older runtimes do not understand new schema, and thus could corrupt document if they proceed, thus we are using
|
|
308
309
|
// this poison pill to prevent them from proceeding.
|
|
309
310
|
// "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
|
|
310
|
-
const error = new
|
|
311
|
+
const error = new internal_8.DataCorruptionError(
|
|
311
312
|
// pre-0.58 error message: SummaryMetadataMismatch
|
|
312
313
|
"Summary metadata mismatch", { runtimeVersion: packageVersion_js_1.pkgVersion, runtimeSequenceNumber, protocolSequenceNumber });
|
|
313
314
|
if (loadSequenceNumberVerification === "log") {
|
|
@@ -365,8 +366,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
365
366
|
else {
|
|
366
367
|
idCompressorMode = desiredIdCompressorMode;
|
|
367
368
|
}
|
|
368
|
-
const createIdCompressorFn =
|
|
369
|
-
const { createIdCompressor, deserializeIdCompressor, createSessionId } = await import("@fluidframework/id-compressor/internal");
|
|
369
|
+
const createIdCompressorFn = () => {
|
|
370
370
|
/**
|
|
371
371
|
* Because the IdCompressor emits so much telemetry, this function is used to sample
|
|
372
372
|
* approximately 5% of all clients. Only the given percentage of sessions will emit telemetry.
|
|
@@ -379,16 +379,16 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
379
379
|
},
|
|
380
380
|
};
|
|
381
381
|
})();
|
|
382
|
-
const compressorLogger = (0,
|
|
382
|
+
const compressorLogger = (0, internal_8.createSampledLogger)(logger, idCompressorEventSampler);
|
|
383
383
|
const pendingLocalState = context.pendingLocalState;
|
|
384
384
|
if (pendingLocalState?.pendingIdCompressorState !== undefined) {
|
|
385
|
-
return deserializeIdCompressor(pendingLocalState.pendingIdCompressorState, compressorLogger);
|
|
385
|
+
return (0, internal_5.deserializeIdCompressor)(pendingLocalState.pendingIdCompressorState, compressorLogger);
|
|
386
386
|
}
|
|
387
387
|
else if (serializedIdCompressor === undefined) {
|
|
388
|
-
return createIdCompressor(compressorLogger);
|
|
388
|
+
return (0, internal_5.createIdCompressor)(compressorLogger);
|
|
389
389
|
}
|
|
390
390
|
else {
|
|
391
|
-
return deserializeIdCompressor(serializedIdCompressor, createSessionId(), compressorLogger);
|
|
391
|
+
return (0, internal_5.deserializeIdCompressor)(serializedIdCompressor, (0, internal_5.createSessionId)(), compressorLogger);
|
|
392
392
|
}
|
|
393
393
|
};
|
|
394
394
|
const compressionLz4 = compressionOptions.minimumBatchSizeInBytes !== Number.POSITIVE_INFINITY &&
|
|
@@ -404,7 +404,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
404
404
|
runtime.onSchemaChange(schema);
|
|
405
405
|
});
|
|
406
406
|
if (compressionLz4 && !enableGroupedBatching) {
|
|
407
|
-
throw new
|
|
407
|
+
throw new internal_8.UsageError("If compression is enabled, op grouping must be enabled too");
|
|
408
408
|
}
|
|
409
409
|
const featureGatesForTelemetry = {};
|
|
410
410
|
// Make sure we've got all the options including internal ones
|
|
@@ -421,7 +421,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
421
421
|
explicitSchemaControl,
|
|
422
422
|
createBlobPayloadPending,
|
|
423
423
|
};
|
|
424
|
-
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], internalRuntimeOptions, containerScope, logger, existing, blobManagerLoadInfo, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, undefined, // summaryConfiguration
|
|
424
|
+
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], internalRuntimeOptions, containerScope, logger, existing, blobManagerLoadInfo, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, minVersionForCollab, requestHandler, undefined, // summaryConfiguration
|
|
425
425
|
recentBatchInfo);
|
|
426
426
|
runtime.blobManager.stashedBlobsUploadP.then(() => {
|
|
427
427
|
// make sure we didn't reconnect before the promise resolved
|
|
@@ -548,7 +548,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
548
548
|
/***/
|
|
549
549
|
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope,
|
|
550
550
|
// Create a custom ITelemetryBaseLogger to output telemetry events.
|
|
551
|
-
baseLogger, existing, blobManagerLoadInfo, _storage,
|
|
551
|
+
baseLogger, existing, blobManagerLoadInfo, _storage, createIdCompressorFn, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, minVersionForCollab, requestHandler,
|
|
552
552
|
// // eslint-disable-next-line unicorn/no-object-as-default-parameter
|
|
553
553
|
summaryConfiguration = {
|
|
554
554
|
// the defaults
|
|
@@ -564,8 +564,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
564
564
|
this.containerScope = containerScope;
|
|
565
565
|
this.baseLogger = baseLogger;
|
|
566
566
|
this._storage = _storage;
|
|
567
|
-
this.
|
|
567
|
+
this.createIdCompressorFn = createIdCompressorFn;
|
|
568
568
|
this.documentsSchemaController = documentsSchemaController;
|
|
569
|
+
this.minVersionForCollab = minVersionForCollab;
|
|
569
570
|
this.requestHandler = requestHandler;
|
|
570
571
|
this.summaryConfiguration = summaryConfiguration;
|
|
571
572
|
this.imminentClosure = false;
|
|
@@ -604,10 +605,12 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
604
605
|
this.outbox.flush();
|
|
605
606
|
const exitStagingMode = (discardOrCommit) => () => {
|
|
606
607
|
// Final flush of any last staged changes
|
|
607
|
-
this.outbox.flush(
|
|
608
|
+
this.outbox.flush();
|
|
608
609
|
this.stageControls = undefined;
|
|
609
610
|
discardOrCommit();
|
|
611
|
+
this.channelCollection.notifyStagingMode(false);
|
|
610
612
|
};
|
|
613
|
+
// eslint-disable-next-line import/no-deprecated
|
|
611
614
|
const stageControls = {
|
|
612
615
|
discardChanges: exitStagingMode(() => {
|
|
613
616
|
// Pop all staged batches from the PSM and roll them back in LIFO order
|
|
@@ -619,18 +622,19 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
619
622
|
this.updateDocumentDirtyState(this.pendingMessagesCount !== 0);
|
|
620
623
|
}
|
|
621
624
|
}),
|
|
622
|
-
commitChanges:
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
}),
|
|
625
|
+
commitChanges: (optionsParam) => {
|
|
626
|
+
const options = { ...defaultStagingCommitOptions, ...optionsParam };
|
|
627
|
+
return exitStagingMode(() => {
|
|
628
|
+
this.pendingStateManager.replayPendingStates({
|
|
629
|
+
onlyStagedBatches: true,
|
|
630
|
+
squash: options.squash ?? false,
|
|
631
|
+
});
|
|
632
|
+
})();
|
|
633
|
+
},
|
|
632
634
|
};
|
|
633
|
-
|
|
635
|
+
this.stageControls = stageControls;
|
|
636
|
+
this.channelCollection.notifyStagingMode(true);
|
|
637
|
+
return this.stageControls;
|
|
634
638
|
};
|
|
635
639
|
this.readAndParseBlob = async (id) => (0, internal_4.readAndParse)(this.storage, id);
|
|
636
640
|
const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, pendingLocalState, supportedFeatures, snapshotWithContents, } = context;
|
|
@@ -639,7 +643,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
639
643
|
// Validate that the Loader is compatible with this Runtime.
|
|
640
644
|
const maybeloaderCompatDetailsForRuntime = context;
|
|
641
645
|
(0, runtimeLayerCompatState_js_1.validateLoaderCompatibility)(maybeloaderCompatDetailsForRuntime.ILayerCompatDetails, this.disposeFn);
|
|
642
|
-
this.mc = (0,
|
|
646
|
+
this.mc = (0, internal_8.createChildMonitoringContext)({
|
|
643
647
|
logger: this.baseLogger,
|
|
644
648
|
namespace: "ContainerRuntime",
|
|
645
649
|
});
|
|
@@ -779,19 +783,19 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
779
783
|
const referenceSequenceNumbersSupported = maybeloaderCompatDetailsForRuntime.ILayerCompatDetails === undefined
|
|
780
784
|
? supportedFeatures?.get("referenceSequenceNumbers") === true
|
|
781
785
|
: true;
|
|
782
|
-
if (runtimeOptions.flushMode ===
|
|
786
|
+
if (runtimeOptions.flushMode === internal_6.FlushModeExperimental.Async &&
|
|
783
787
|
!referenceSequenceNumbersSupported) {
|
|
784
788
|
// The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
|
|
785
789
|
this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
|
|
786
|
-
this._flushMode =
|
|
790
|
+
this._flushMode = internal_6.FlushMode.TurnBased;
|
|
787
791
|
}
|
|
788
792
|
else {
|
|
789
793
|
this._flushMode = runtimeOptions.flushMode;
|
|
790
794
|
}
|
|
791
795
|
this.offlineEnabled =
|
|
792
796
|
this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad") ?? false;
|
|
793
|
-
if (this.offlineEnabled && this._flushMode !==
|
|
794
|
-
const error = new
|
|
797
|
+
if (this.offlineEnabled && this._flushMode !== internal_6.FlushMode.TurnBased) {
|
|
798
|
+
const error = new internal_8.UsageError("Offline mode is only supported in turn-based mode");
|
|
795
799
|
this.closeFn(error);
|
|
796
800
|
throw error;
|
|
797
801
|
}
|
|
@@ -809,7 +813,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
809
813
|
// This is a runtime enforcement of what's already explicit in the policy's type itself,
|
|
810
814
|
// which dictates the value is either undefined or exactly 5 days in ms.
|
|
811
815
|
// As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
|
|
812
|
-
throw new
|
|
816
|
+
throw new internal_8.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
|
|
813
817
|
}
|
|
814
818
|
}
|
|
815
819
|
this.garbageCollector = index_js_2.GarbageCollector.create({
|
|
@@ -834,7 +838,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
834
838
|
const summaryReferenceSequenceNumber = baseSnapshot === undefined || metadata?.disableIsolatedChannels === true
|
|
835
839
|
? undefined
|
|
836
840
|
: loadedFromSequenceNumber;
|
|
837
|
-
this.summarizerNode = (0, index_js_4.createRootSummarizerNodeWithGC)((0,
|
|
841
|
+
this.summarizerNode = (0, index_js_4.createRootSummarizerNodeWithGC)((0, internal_8.createChildLogger)({ logger: this.baseLogger, namespace: "SummarizerNode" }),
|
|
838
842
|
// Summarize function to call when summarize is called. Summarizer node always tracks summary state.
|
|
839
843
|
async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext),
|
|
840
844
|
// Latest change sequence number, no changes since summary applied yet
|
|
@@ -900,8 +904,8 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
900
904
|
stashedBlobs: pendingRuntimeState?.pendingAttachmentBlobs,
|
|
901
905
|
createBlobPayloadPending: this.sessionSchema.createBlobPayloadPending === true,
|
|
902
906
|
});
|
|
903
|
-
this.deltaScheduler = new deltaScheduler_js_1.DeltaScheduler(this.innerDeltaManager, this, (0,
|
|
904
|
-
this.inboundBatchAggregator = new inboundBatchAggregator_js_1.InboundBatchAggregator(this.innerDeltaManager, () => this.clientId, (0,
|
|
907
|
+
this.deltaScheduler = new deltaScheduler_js_1.DeltaScheduler(this.innerDeltaManager, this, (0, internal_8.createChildLogger)({ logger: this.baseLogger, namespace: "DeltaScheduler" }));
|
|
908
|
+
this.inboundBatchAggregator = new inboundBatchAggregator_js_1.InboundBatchAggregator(this.innerDeltaManager, () => this.clientId, (0, internal_8.createChildLogger)({ logger: this.baseLogger, namespace: "InboundBatchAggregator" }));
|
|
905
909
|
const legacySendBatchFn = (0, exports.makeLegacySendBatchFn)(submitFn, this.innerDeltaManager);
|
|
906
910
|
this.skipSafetyFlushDuringProcessStack =
|
|
907
911
|
// Keep the old flag name even though we renamed the class member (it shipped in 2.31.0)
|
|
@@ -994,6 +998,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
994
998
|
telemetryDocumentId: this.telemetryDocumentId,
|
|
995
999
|
groupedBatchingEnabled: this.groupedBatchingEnabled,
|
|
996
1000
|
initialSequenceNumber: this.deltaManager.initialSequenceNumber,
|
|
1001
|
+
minVersionForCollab: this.minVersionForCollab,
|
|
997
1002
|
});
|
|
998
1003
|
(0, connectionTelemetry_js_1.ReportOpPerfTelemetry)(this.clientId, this._deltaManager, this, this.baseLogger);
|
|
999
1004
|
(0, batchTracker_js_1.BindBatchTracker)(this, this.baseLogger);
|
|
@@ -1021,7 +1026,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1021
1026
|
// As it's implemented right now (with async initialization), this will only work for "off" -> "delayed" transitions.
|
|
1022
1027
|
// Anything else is too risky, and requires ability to initialize ID compressor synchronously!
|
|
1023
1028
|
if (schema.runtime.idCompressorMode !== undefined) {
|
|
1024
|
-
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
1025
1029
|
this.loadIdCompressor();
|
|
1026
1030
|
}
|
|
1027
1031
|
}
|
|
@@ -1046,7 +1050,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1046
1050
|
await this.initializeSummarizer(loader);
|
|
1047
1051
|
if (this.sessionSchema.idCompressorMode === "on" ||
|
|
1048
1052
|
(this.sessionSchema.idCompressorMode === "delayed" && this.connected)) {
|
|
1049
|
-
this._idCompressor =
|
|
1053
|
+
this._idCompressor = this.createIdCompressorFn();
|
|
1050
1054
|
// This is called from loadRuntime(), long before we process any ops, so there should be no ops accumulated yet.
|
|
1051
1055
|
(0, internal_2.assert)(this.pendingIdCompressorOps.length === 0, 0x8ec /* no pending ops */);
|
|
1052
1056
|
}
|
|
@@ -1068,7 +1072,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1068
1072
|
this.summaryConfiguration.initialSummarizerDelayMs,
|
|
1069
1073
|
};
|
|
1070
1074
|
const summaryCollection = new index_js_4.SummaryCollection(this.deltaManager, this.baseLogger);
|
|
1071
|
-
const orderedClientLogger = (0,
|
|
1075
|
+
const orderedClientLogger = (0, internal_8.createChildLogger)({
|
|
1072
1076
|
logger: this.baseLogger,
|
|
1073
1077
|
namespace: "OrderedClientElection",
|
|
1074
1078
|
});
|
|
@@ -1193,7 +1197,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1193
1197
|
// Snapshots should only move forward. If we observe an older snapshot than the one we loaded from, then likely
|
|
1194
1198
|
// the file has been overwritten or service lost data.
|
|
1195
1199
|
if (snapshotSeqNumber < this.deltaManager.initialSequenceNumber) {
|
|
1196
|
-
throw
|
|
1200
|
+
throw internal_8.DataProcessingError.create("Downloaded snapshot older than snapshot we loaded from", "getSnapshotForLoadingGroupId", undefined, {
|
|
1197
1201
|
loadingGroupIds: sortedLoadingGroupIds.join(","),
|
|
1198
1202
|
snapshotSeqNumber,
|
|
1199
1203
|
initialSequenceNumber: this.deltaManager.initialSequenceNumber,
|
|
@@ -1216,7 +1220,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1216
1220
|
targetSequenceNumber: snapshotSeqNumber, // This is so we reuse some columns in telemetry
|
|
1217
1221
|
sequenceNumber: this.deltaManager.lastSequenceNumber, // This is so we reuse some columns in telemetry
|
|
1218
1222
|
};
|
|
1219
|
-
const event =
|
|
1223
|
+
const event = internal_8.PerformanceEvent.start(this.mc.logger, {
|
|
1220
1224
|
...props,
|
|
1221
1225
|
});
|
|
1222
1226
|
// If the inbound deltas queue is paused or disconnected, we expect a reconnect and unpause
|
|
@@ -1247,7 +1251,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1247
1251
|
let childTree = snapshotTree;
|
|
1248
1252
|
for (const part of pathParts) {
|
|
1249
1253
|
if (hasIsolatedChannels) {
|
|
1250
|
-
childTree = childTree?.trees[
|
|
1254
|
+
childTree = childTree?.trees[internal_6.channelsTreeName];
|
|
1251
1255
|
}
|
|
1252
1256
|
childTree = childTree?.trees[part];
|
|
1253
1257
|
}
|
|
@@ -1261,7 +1265,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1261
1265
|
// @ts-expect-error expected to be used by LTS Loaders and Containers
|
|
1262
1266
|
async request(request) {
|
|
1263
1267
|
try {
|
|
1264
|
-
const parser =
|
|
1268
|
+
const parser = internal_7.RequestParser.create(request);
|
|
1265
1269
|
const id = parser.pathParts[0];
|
|
1266
1270
|
if (id === index_js_4.summarizerRequestUrl && parser.pathParts.length === 1) {
|
|
1267
1271
|
if (this._summarizer !== undefined) {
|
|
@@ -1271,16 +1275,16 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1271
1275
|
value: this._summarizer,
|
|
1272
1276
|
};
|
|
1273
1277
|
}
|
|
1274
|
-
return (0,
|
|
1278
|
+
return (0, internal_7.create404Response)(request);
|
|
1275
1279
|
}
|
|
1276
1280
|
if (this.requestHandler !== undefined) {
|
|
1277
1281
|
// eslint-disable-next-line @typescript-eslint/return-await -- Adding an await here causes test failures
|
|
1278
1282
|
return this.requestHandler(parser, this);
|
|
1279
1283
|
}
|
|
1280
|
-
return (0,
|
|
1284
|
+
return (0, internal_7.create404Response)(request);
|
|
1281
1285
|
}
|
|
1282
1286
|
catch (error) {
|
|
1283
|
-
return (0,
|
|
1287
|
+
return (0, internal_7.exceptionToResponse)(error);
|
|
1284
1288
|
}
|
|
1285
1289
|
}
|
|
1286
1290
|
/**
|
|
@@ -1289,7 +1293,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1289
1293
|
*/
|
|
1290
1294
|
async resolveHandle(request) {
|
|
1291
1295
|
try {
|
|
1292
|
-
const requestParser =
|
|
1296
|
+
const requestParser = internal_7.RequestParser.create(request);
|
|
1293
1297
|
const id = requestParser.pathParts[0];
|
|
1294
1298
|
if (id === "_channels") {
|
|
1295
1299
|
// eslint-disable-next-line @typescript-eslint/return-await -- Adding an await here causes test failures
|
|
@@ -1297,10 +1301,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1297
1301
|
}
|
|
1298
1302
|
if (id === index_js_1.blobManagerBasePath && requestParser.isLeaf(2)) {
|
|
1299
1303
|
const localId = requestParser.pathParts[1];
|
|
1300
|
-
const payloadPending = requestParser.headers?.[
|
|
1304
|
+
const payloadPending = requestParser.headers?.[internal_7.RuntimeHeaders.payloadPending] === true;
|
|
1301
1305
|
if (!this.blobManager.hasBlob(localId) &&
|
|
1302
|
-
requestParser.headers?.[
|
|
1303
|
-
return (0,
|
|
1306
|
+
requestParser.headers?.[internal_7.RuntimeHeaders.wait] === false) {
|
|
1307
|
+
return (0, internal_7.create404Response)(request);
|
|
1304
1308
|
}
|
|
1305
1309
|
const blob = await this.blobManager.getBlob(localId, payloadPending);
|
|
1306
1310
|
return {
|
|
@@ -1312,10 +1316,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1312
1316
|
else if (requestParser.pathParts.length > 0) {
|
|
1313
1317
|
return await this.channelCollection.request(request);
|
|
1314
1318
|
}
|
|
1315
|
-
return (0,
|
|
1319
|
+
return (0, internal_7.create404Response)(request);
|
|
1316
1320
|
}
|
|
1317
1321
|
catch (error) {
|
|
1318
|
-
return (0,
|
|
1322
|
+
return (0, internal_7.exceptionToResponse)(error);
|
|
1319
1323
|
}
|
|
1320
1324
|
}
|
|
1321
1325
|
/**
|
|
@@ -1356,39 +1360,39 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1356
1360
|
lastMessage: explicitSchemaControl ? message : undefined,
|
|
1357
1361
|
documentSchema,
|
|
1358
1362
|
};
|
|
1359
|
-
(0,
|
|
1363
|
+
(0, internal_7.addBlobToSummary)(summaryTree, index_js_4.metadataBlobName, JSON.stringify(metadata));
|
|
1360
1364
|
}
|
|
1361
1365
|
addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
|
|
1362
1366
|
this.addMetadataToSummary(summaryTree);
|
|
1363
1367
|
if (this._idCompressor) {
|
|
1364
1368
|
const idCompressorState = JSON.stringify(this._idCompressor.serialize(false));
|
|
1365
|
-
(0,
|
|
1369
|
+
(0, internal_7.addBlobToSummary)(summaryTree, index_js_4.idCompressorBlobName, idCompressorState);
|
|
1366
1370
|
}
|
|
1367
1371
|
if (this.remoteMessageProcessor.partialMessages.size > 0) {
|
|
1368
1372
|
const content = JSON.stringify([...this.remoteMessageProcessor.partialMessages]);
|
|
1369
|
-
(0,
|
|
1373
|
+
(0, internal_7.addBlobToSummary)(summaryTree, index_js_4.chunksBlobName, content);
|
|
1370
1374
|
}
|
|
1371
1375
|
const recentBatchInfo = this.duplicateBatchDetector?.getRecentBatchInfoForSummary(telemetryContext);
|
|
1372
1376
|
if (recentBatchInfo !== undefined) {
|
|
1373
|
-
(0,
|
|
1377
|
+
(0, internal_7.addBlobToSummary)(summaryTree, index_js_4.recentBatchInfoBlobName, JSON.stringify(recentBatchInfo));
|
|
1374
1378
|
}
|
|
1375
1379
|
const dataStoreAliases = this.channelCollection.aliases;
|
|
1376
1380
|
if (dataStoreAliases.size > 0) {
|
|
1377
|
-
(0,
|
|
1381
|
+
(0, internal_7.addBlobToSummary)(summaryTree, index_js_4.aliasBlobName, JSON.stringify([...dataStoreAliases]));
|
|
1378
1382
|
}
|
|
1379
1383
|
if (this.summarizerClientElection) {
|
|
1380
1384
|
const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
|
|
1381
|
-
(0,
|
|
1385
|
+
(0, internal_7.addBlobToSummary)(summaryTree, index_js_4.electedSummarizerBlobName, electedSummarizerContent);
|
|
1382
1386
|
}
|
|
1383
1387
|
const blobManagerSummary = this.blobManager.summarize();
|
|
1384
1388
|
// Some storage (like git) doesn't allow empty tree, so we can omit it.
|
|
1385
1389
|
// and the blob manager can handle the tree not existing when loading
|
|
1386
1390
|
if (Object.keys(blobManagerSummary.summary.tree).length > 0) {
|
|
1387
|
-
(0,
|
|
1391
|
+
(0, internal_7.addSummarizeResultToSummary)(summaryTree, index_js_1.blobsTreeName, blobManagerSummary);
|
|
1388
1392
|
}
|
|
1389
1393
|
const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
|
|
1390
1394
|
if (gcSummary !== undefined) {
|
|
1391
|
-
(0,
|
|
1395
|
+
(0, internal_7.addSummarizeResultToSummary)(summaryTree, internal_6.gcTreeKey, gcSummary);
|
|
1392
1396
|
}
|
|
1393
1397
|
}
|
|
1394
1398
|
// Track how many times the container tries to reconnect with pending messages.
|
|
@@ -1493,7 +1497,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1493
1497
|
}
|
|
1494
1498
|
case messageTypes_js_1.ContainerMessageType.GC: {
|
|
1495
1499
|
// GC op is only sent in summarizer which should never have stashed ops.
|
|
1496
|
-
throw new
|
|
1500
|
+
throw new internal_8.LoggingError("GC op not expected to be stashed in summarizer");
|
|
1497
1501
|
}
|
|
1498
1502
|
default: {
|
|
1499
1503
|
const error = getUnknownMessageTypeError(opContents.type, "applyStashedOp" /* codePath */);
|
|
@@ -1502,27 +1506,18 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1502
1506
|
}
|
|
1503
1507
|
}
|
|
1504
1508
|
}
|
|
1505
|
-
|
|
1509
|
+
loadIdCompressor() {
|
|
1506
1510
|
if (this._idCompressor === undefined &&
|
|
1507
|
-
this.sessionSchema.idCompressorMode !== undefined
|
|
1508
|
-
this.
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
this.
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
}
|
|
1517
|
-
(0, internal_2.assert)(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
|
|
1518
|
-
this._idCompressor = compressor;
|
|
1519
|
-
})
|
|
1520
|
-
.catch((error) => {
|
|
1521
|
-
this.mc.logger.sendErrorEvent({ eventName: "IdCompressorDelayedLoad" }, error);
|
|
1522
|
-
throw error;
|
|
1523
|
-
});
|
|
1511
|
+
this.sessionSchema.idCompressorMode !== undefined) {
|
|
1512
|
+
this._idCompressor = this.createIdCompressorFn();
|
|
1513
|
+
// Finalize any ranges we received while the compressor was turned off.
|
|
1514
|
+
const ops = this.pendingIdCompressorOps;
|
|
1515
|
+
this.pendingIdCompressorOps = [];
|
|
1516
|
+
for (const range of ops) {
|
|
1517
|
+
this._idCompressor.finalizeCreationRange(range);
|
|
1518
|
+
}
|
|
1519
|
+
(0, internal_2.assert)(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
|
|
1524
1520
|
}
|
|
1525
|
-
return this._loadIdCompressor;
|
|
1526
1521
|
}
|
|
1527
1522
|
setConnectionState(connected, clientId) {
|
|
1528
1523
|
// Validate we have consistent state
|
|
@@ -1530,7 +1525,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1530
1525
|
(0, internal_2.assert)(clientId === currentClientId, 0x977 /* input clientId does not match Audience */);
|
|
1531
1526
|
(0, internal_2.assert)(this.clientId === currentClientId, 0x978 /* this.clientId does not match Audience */);
|
|
1532
1527
|
if (connected && this.sessionSchema.idCompressorMode === "delayed") {
|
|
1533
|
-
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
1534
1528
|
this.loadIdCompressor();
|
|
1535
1529
|
}
|
|
1536
1530
|
if (connected === false && this.delayConnectClientId !== undefined) {
|
|
@@ -1579,7 +1573,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1579
1573
|
if (reconnection) {
|
|
1580
1574
|
this.consecutiveReconnects++;
|
|
1581
1575
|
if (!this.shouldContinueReconnecting()) {
|
|
1582
|
-
this.closeFn(
|
|
1576
|
+
this.closeFn(internal_8.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
|
|
1583
1577
|
dataLoss: 1,
|
|
1584
1578
|
attempts: this.consecutiveReconnects,
|
|
1585
1579
|
pendingMessages: this.pendingMessagesCount,
|
|
@@ -1592,7 +1586,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1592
1586
|
}
|
|
1593
1587
|
this.channelCollection.setConnectionState(connected, clientId);
|
|
1594
1588
|
this.garbageCollector.setConnectionState(connected, clientId);
|
|
1595
|
-
(0,
|
|
1589
|
+
(0, internal_8.raiseConnectedEvent)(this.mc.logger, this, connected, clientId);
|
|
1596
1590
|
}
|
|
1597
1591
|
async notifyOpReplay(message) {
|
|
1598
1592
|
await this.pendingStateManager.applyStashedOpsAt(message.sequenceNumber);
|
|
@@ -1645,7 +1639,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1645
1639
|
const batchStart = inboundResult.batchStart;
|
|
1646
1640
|
const result = this.duplicateBatchDetector?.processInboundBatch(batchStart);
|
|
1647
1641
|
if (result?.duplicate) {
|
|
1648
|
-
const error = new
|
|
1642
|
+
const error = new internal_8.DataCorruptionError("Duplicate batch - The same batch was sequenced twice", { batchId: batchStart.batchId });
|
|
1649
1643
|
this.mc.logger.sendTelemetryEvent({
|
|
1650
1644
|
eventName: "DuplicateBatch",
|
|
1651
1645
|
details: {
|
|
@@ -1654,7 +1648,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1654
1648
|
batchStartCsn: batchStart.batchStartCsn,
|
|
1655
1649
|
size: inboundResult.length,
|
|
1656
1650
|
duplicateBatchSequenceNumber: result.otherSequenceNumber,
|
|
1657
|
-
...(0,
|
|
1651
|
+
...(0, internal_8.extractSafePropertiesFromMessage)(batchStart.keyMessage),
|
|
1658
1652
|
},
|
|
1659
1653
|
}, error);
|
|
1660
1654
|
throw error;
|
|
@@ -1936,23 +1930,20 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1936
1930
|
this.channelCollection.processSignal(transformed, local);
|
|
1937
1931
|
}
|
|
1938
1932
|
/**
|
|
1939
|
-
* Flush the
|
|
1940
|
-
* This method is expected to be called
|
|
1933
|
+
* Flush the current batch of ops to the ordering service for sequencing
|
|
1934
|
+
* This method is not expected to be called in the middle of a batch.
|
|
1941
1935
|
* @remarks - If it throws (e.g. if the batch is too large to send), the container will be closed.
|
|
1942
1936
|
*
|
|
1943
|
-
* @param
|
|
1944
|
-
* with the given Batch ID, which must be preserved
|
|
1945
|
-
* @param resubmittingStagedBatch - If defined, indicates this is a resubmission of a batch that is staged,
|
|
1946
|
-
* meaning it should not be sent to the ordering service yet.
|
|
1937
|
+
* @param resubmitInfo - If defined, indicates this is a resubmission of a batch with the given Batch info needed for resubmit.
|
|
1947
1938
|
*/
|
|
1948
|
-
flush(
|
|
1939
|
+
flush(resubmitInfo) {
|
|
1949
1940
|
try {
|
|
1950
1941
|
(0, internal_2.assert)(!this.batchRunner.running, 0x24c /* "Cannot call `flush()` while manually accumulating a batch (e.g. under orderSequentially) */);
|
|
1951
|
-
this.outbox.flush(
|
|
1942
|
+
this.outbox.flush(resubmitInfo);
|
|
1952
1943
|
(0, internal_2.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
|
|
1953
1944
|
}
|
|
1954
1945
|
catch (error) {
|
|
1955
|
-
const error2 = (0,
|
|
1946
|
+
const error2 = (0, internal_8.normalizeError)(error, {
|
|
1956
1947
|
props: {
|
|
1957
1948
|
orderSequentiallyCalls: this.batchRunner.runs,
|
|
1958
1949
|
},
|
|
@@ -1995,24 +1986,24 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
1995
1986
|
stageControls = undefined;
|
|
1996
1987
|
}
|
|
1997
1988
|
catch (error_) {
|
|
1998
|
-
const error2 = (0,
|
|
1999
|
-
return
|
|
1989
|
+
const error2 = (0, internal_8.wrapError)(error_, (message) => {
|
|
1990
|
+
return internal_8.DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
|
|
2000
1991
|
});
|
|
2001
1992
|
this.closeFn(error2);
|
|
2002
1993
|
throw error2;
|
|
2003
1994
|
}
|
|
2004
1995
|
}
|
|
2005
1996
|
else {
|
|
2006
|
-
this.closeFn((0,
|
|
1997
|
+
this.closeFn((0, internal_8.wrapError)(error, (errorMessage) => new internal_8.GenericError(`orderSequentially callback exception: ${errorMessage}`, error, {
|
|
2007
1998
|
orderSequentiallyCalls: this.batchRunner.runs,
|
|
2008
1999
|
})));
|
|
2009
2000
|
}
|
|
2010
2001
|
throw error; // throw the original error for the consumer of the runtime
|
|
2011
2002
|
}
|
|
2012
2003
|
});
|
|
2013
|
-
stageControls?.commitChanges();
|
|
2004
|
+
stageControls?.commitChanges({ squash: false });
|
|
2014
2005
|
// We don't flush on TurnBased since we expect all messages in the same JS turn to be part of the same batch
|
|
2015
|
-
if (this.flushMode !==
|
|
2006
|
+
if (this.flushMode !== internal_6.FlushMode.TurnBased && !this.batchRunner.running) {
|
|
2016
2007
|
this.flush();
|
|
2017
2008
|
}
|
|
2018
2009
|
return result;
|
|
@@ -2049,7 +2040,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2049
2040
|
}
|
|
2050
2041
|
const channel = await context.realize();
|
|
2051
2042
|
if (channel.entryPoint === undefined) {
|
|
2052
|
-
throw new
|
|
2043
|
+
throw new internal_8.UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
|
|
2053
2044
|
}
|
|
2054
2045
|
this.garbageCollector.nodeUpdated({
|
|
2055
2046
|
node: { type: "DataStore", path: `/${internalId}` },
|
|
@@ -2075,7 +2066,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2075
2066
|
* Typically ops are batched and later flushed together, but in some cases we want to flush immediately.
|
|
2076
2067
|
*/
|
|
2077
2068
|
currentlyBatching() {
|
|
2078
|
-
return this.flushMode !==
|
|
2069
|
+
return this.flushMode !== internal_6.FlushMode.Immediate || this.batchRunner.running;
|
|
2079
2070
|
}
|
|
2080
2071
|
getQuorum() {
|
|
2081
2072
|
return this._quorum;
|
|
@@ -2186,9 +2177,8 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2186
2177
|
const summarizeResult = await this.channelCollection.summarize(fullTree, trackState, telemetryContext);
|
|
2187
2178
|
// Wrap data store summaries in .channels subtree.
|
|
2188
2179
|
(0, index_js_4.wrapSummaryInChannelsTree)(summarizeResult);
|
|
2189
|
-
const pathPartsForChildren = [
|
|
2190
|
-
|
|
2191
|
-
await this.loadIdCompressor();
|
|
2180
|
+
const pathPartsForChildren = [internal_6.channelsTreeName];
|
|
2181
|
+
this.loadIdCompressor();
|
|
2192
2182
|
this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
|
|
2193
2183
|
return {
|
|
2194
2184
|
...summarizeResult,
|
|
@@ -2202,7 +2192,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2202
2192
|
async summarize(options) {
|
|
2203
2193
|
this.verifyNotClosed();
|
|
2204
2194
|
const { fullTree = false, trackState = true, summaryLogger = this.mc.logger, runGC = this.garbageCollector.shouldRunGC, runSweep, fullGC, } = options;
|
|
2205
|
-
const telemetryContext = new
|
|
2195
|
+
const telemetryContext = new internal_7.TelemetryContext();
|
|
2206
2196
|
// Add the options that are used to generate this summary to the telemetry context.
|
|
2207
2197
|
telemetryContext.setMultiple("fluid_Summarize", "Options", {
|
|
2208
2198
|
fullTree,
|
|
@@ -2235,7 +2225,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2235
2225
|
* @see IGarbageCollectionRuntime.getGCData
|
|
2236
2226
|
*/
|
|
2237
2227
|
async getGCData(fullGC) {
|
|
2238
|
-
const builder = new
|
|
2228
|
+
const builder = new internal_7.GCDataBuilder();
|
|
2239
2229
|
const dsGCData = await this.summarizerNode.getGCData(fullGC);
|
|
2240
2230
|
builder.addNodes(dsGCData.gcNodes);
|
|
2241
2231
|
const blobsGCData = this.blobManager.getGCData(fullGC);
|
|
@@ -2362,7 +2352,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2362
2352
|
if (timestampMs === undefined) {
|
|
2363
2353
|
this.mc.logger.sendTelemetryEvent({
|
|
2364
2354
|
eventName: "NoTimestampInGCOutboundRoute",
|
|
2365
|
-
...(0,
|
|
2355
|
+
...(0, internal_8.tagCodeArtifacts)({
|
|
2366
2356
|
id: toPath,
|
|
2367
2357
|
fromId: fromPath,
|
|
2368
2358
|
}),
|
|
@@ -2385,7 +2375,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2385
2375
|
// use it for all events logged during this summary.
|
|
2386
2376
|
const summaryNumber = this.nextSummaryNumber;
|
|
2387
2377
|
let summaryRefSeqNum;
|
|
2388
|
-
const summaryNumberLogger = (0,
|
|
2378
|
+
const summaryNumberLogger = (0, internal_8.createChildLogger)({
|
|
2389
2379
|
logger: summaryLogger,
|
|
2390
2380
|
properties: {
|
|
2391
2381
|
all: {
|
|
@@ -2396,7 +2386,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2396
2386
|
});
|
|
2397
2387
|
// legacy: assert 0x3d1
|
|
2398
2388
|
if (!this.outbox.isEmpty) {
|
|
2399
|
-
throw
|
|
2389
|
+
throw internal_8.DataProcessingError.create("Can't trigger summary in the middle of a batch", "submitSummary", undefined, {
|
|
2400
2390
|
summaryNumber,
|
|
2401
2391
|
pendingMessages: this.pendingMessagesCount,
|
|
2402
2392
|
outboxLength: this.outbox.messageCount,
|
|
@@ -2533,7 +2523,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2533
2523
|
stage: "base",
|
|
2534
2524
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
2535
2525
|
minimumSequenceNumber,
|
|
2536
|
-
error: (0,
|
|
2526
|
+
error: (0, internal_8.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
|
|
2537
2527
|
};
|
|
2538
2528
|
}
|
|
2539
2529
|
// Validate that the summary generated by summarizer nodes is correct before uploading.
|
|
@@ -2560,11 +2550,11 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2560
2550
|
// Counting dataStores and handles
|
|
2561
2551
|
// Because handles are unchanged dataStores in the current logic,
|
|
2562
2552
|
// summarized dataStore count is total dataStore count minus handle count
|
|
2563
|
-
const dataStoreTree = summaryTree.tree[
|
|
2553
|
+
const dataStoreTree = summaryTree.tree[internal_6.channelsTreeName];
|
|
2564
2554
|
(0, internal_2.assert)(dataStoreTree.type === driver_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
|
|
2565
2555
|
const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === driver_definitions_1.SummaryType.Handle).length;
|
|
2566
|
-
const gcSummaryTreeStats = summaryTree.tree[
|
|
2567
|
-
? (0,
|
|
2556
|
+
const gcSummaryTreeStats = summaryTree.tree[internal_6.gcTreeKey]
|
|
2557
|
+
? (0, internal_7.calculateStats)(summaryTree.tree[internal_6.gcTreeKey])
|
|
2568
2558
|
: undefined;
|
|
2569
2559
|
const summaryStats = {
|
|
2570
2560
|
dataStoreCount: this.channelCollection.size,
|
|
@@ -2603,7 +2593,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2603
2593
|
return {
|
|
2604
2594
|
stage: "generate",
|
|
2605
2595
|
...generateSummaryData,
|
|
2606
|
-
error: (0,
|
|
2596
|
+
error: (0, internal_8.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
|
|
2607
2597
|
};
|
|
2608
2598
|
}
|
|
2609
2599
|
const parent = summaryContext.ackHandle;
|
|
@@ -2635,7 +2625,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2635
2625
|
return {
|
|
2636
2626
|
stage: "upload",
|
|
2637
2627
|
...uploadData,
|
|
2638
|
-
error: (0,
|
|
2628
|
+
error: (0, internal_8.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
|
|
2639
2629
|
};
|
|
2640
2630
|
}
|
|
2641
2631
|
const submitData = {
|
|
@@ -2651,7 +2641,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2651
2641
|
return {
|
|
2652
2642
|
stage: "upload",
|
|
2653
2643
|
...uploadData,
|
|
2654
|
-
error: (0,
|
|
2644
|
+
error: (0, internal_8.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
|
|
2655
2645
|
};
|
|
2656
2646
|
}
|
|
2657
2647
|
return submitData;
|
|
@@ -2687,7 +2677,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2687
2677
|
// the summarizer.
|
|
2688
2678
|
if (finalAttempt &&
|
|
2689
2679
|
this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
|
|
2690
|
-
const error =
|
|
2680
|
+
const error = internal_8.DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
|
|
2691
2681
|
logger.sendErrorEvent({
|
|
2692
2682
|
eventName: "SkipFailingIncorrectSummary",
|
|
2693
2683
|
referenceSequenceNumber,
|
|
@@ -2834,7 +2824,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2834
2824
|
}
|
|
2835
2825
|
}
|
|
2836
2826
|
catch (error) {
|
|
2837
|
-
const dpe =
|
|
2827
|
+
const dpe = internal_8.DataProcessingError.wrapIfUnrecognized(error, "ContainerRuntime.submit", {
|
|
2838
2828
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
2839
2829
|
});
|
|
2840
2830
|
this.closeFn(dpe);
|
|
@@ -2856,7 +2846,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2856
2846
|
this.flush();
|
|
2857
2847
|
};
|
|
2858
2848
|
switch (this.flushMode) {
|
|
2859
|
-
case
|
|
2849
|
+
case internal_6.FlushMode.TurnBased: {
|
|
2860
2850
|
// When in TurnBased flush mode the runtime will buffer operations in the current turn and send them as a single
|
|
2861
2851
|
// batch at the end of the turn
|
|
2862
2852
|
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
@@ -2864,7 +2854,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2864
2854
|
break;
|
|
2865
2855
|
}
|
|
2866
2856
|
// FlushModeExperimental is experimental and not exposed directly in the runtime APIs
|
|
2867
|
-
case
|
|
2857
|
+
case internal_6.FlushModeExperimental.Async: {
|
|
2868
2858
|
// When in Async flush mode, the runtime will accumulate all operations across JS turns and send them as a single
|
|
2869
2859
|
// batch when all micro-tasks are complete.
|
|
2870
2860
|
// Compared to TurnBased, this flush mode will capture more ops into the same batch.
|
|
@@ -2899,18 +2889,18 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2899
2889
|
* @remarks - If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
|
|
2900
2890
|
* for correlation to detect container forking.
|
|
2901
2891
|
*/
|
|
2902
|
-
reSubmitBatch(batch, batchId, staged) {
|
|
2892
|
+
reSubmitBatch(batch, { batchId, staged, squash }) {
|
|
2903
2893
|
this.batchRunner.run(() => {
|
|
2904
2894
|
for (const message of batch) {
|
|
2905
|
-
this.reSubmit(message);
|
|
2895
|
+
this.reSubmit(message, squash);
|
|
2906
2896
|
}
|
|
2907
2897
|
});
|
|
2908
2898
|
// Only include Batch ID if "Offline Load" feature is enabled
|
|
2909
2899
|
// It's only needed to identify batches across container forks arising from misuse of offline load.
|
|
2910
|
-
this.flush(this.offlineEnabled ? batchId : undefined, staged);
|
|
2900
|
+
this.flush({ batchId: this.offlineEnabled ? batchId : undefined, staged });
|
|
2911
2901
|
}
|
|
2912
|
-
reSubmit(message) {
|
|
2913
|
-
this.reSubmitCore(message.runtimeOp, message.localOpMetadata, message.opMetadata);
|
|
2902
|
+
reSubmit(message, squash) {
|
|
2903
|
+
this.reSubmitCore(message.runtimeOp, message.localOpMetadata, message.opMetadata, squash);
|
|
2914
2904
|
}
|
|
2915
2905
|
/**
|
|
2916
2906
|
* Finds the right store and asks it to resubmit the message. This typically happens when we
|
|
@@ -2919,7 +2909,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2919
2909
|
* @param message - The original LocalContainerRuntimeMessage.
|
|
2920
2910
|
* @param localOpMetadata - The local metadata associated with the original message.
|
|
2921
2911
|
*/
|
|
2922
|
-
reSubmitCore(message, localOpMetadata, opMetadata) {
|
|
2912
|
+
reSubmitCore(message, localOpMetadata, opMetadata, squash) {
|
|
2923
2913
|
(0, internal_2.assert)(this._summarizer === undefined, 0x8f2 /* Summarizer never reconnects so should never resubmit */);
|
|
2924
2914
|
switch (message.type) {
|
|
2925
2915
|
case messageTypes_js_1.ContainerMessageType.FluidDataStoreOp:
|
|
@@ -2927,7 +2917,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
2927
2917
|
case messageTypes_js_1.ContainerMessageType.Alias: {
|
|
2928
2918
|
// For Operations, call resubmitDataStoreOp which will find the right store
|
|
2929
2919
|
// and trigger resubmission on it.
|
|
2930
|
-
this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata);
|
|
2920
|
+
this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata, squash);
|
|
2931
2921
|
break;
|
|
2932
2922
|
}
|
|
2933
2923
|
case messageTypes_js_1.ContainerMessageType.IdAllocation: {
|
|
@@ -3029,7 +3019,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
3029
3019
|
* happen in scenarios where the snapshot for the ack was lost in storage in scenarios like DB rollback, etc.
|
|
3030
3020
|
*/
|
|
3031
3021
|
async fetchLatestSnapshotAndMaybeClose(targetRefSeq, targetAckHandle, logger) {
|
|
3032
|
-
const fetchedSnapshotRefSeq = await
|
|
3022
|
+
const fetchedSnapshotRefSeq = await internal_8.PerformanceEvent.timedExecAsync(logger, { eventName: "RefreshLatestSummaryAckFetch" }, async (perfEvent) => {
|
|
3033
3023
|
const props = { targetRefSeq, targetAckHandle };
|
|
3034
3024
|
const trace = client_utils_1.Trace.start();
|
|
3035
3025
|
let snapshotTree;
|
|
@@ -3060,7 +3050,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
3060
3050
|
props.snapshotVersion = versions[0].id;
|
|
3061
3051
|
}
|
|
3062
3052
|
props.getSnapshotDuration = trace.trace().duration;
|
|
3063
|
-
const snapshotRefSeq = await (0,
|
|
3053
|
+
const snapshotRefSeq = await (0, internal_7.seqFromTree)(snapshotTree, this.readAndParseBlob);
|
|
3064
3054
|
props.snapshotRefSeq = snapshotRefSeq;
|
|
3065
3055
|
props.newerSnapshotPresent = snapshotRefSeq >= targetRefSeq;
|
|
3066
3056
|
perfEvent.end({ details: props });
|
|
@@ -3079,7 +3069,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
3079
3069
|
getPendingLocalState(props) {
|
|
3080
3070
|
this.verifyNotClosed();
|
|
3081
3071
|
if (this.batchRunner.running) {
|
|
3082
|
-
throw new
|
|
3072
|
+
throw new internal_8.UsageError("can't get state while manually accumulating a batch");
|
|
3083
3073
|
}
|
|
3084
3074
|
this.imminentClosure ||= props?.notifyImminentClosure ?? false;
|
|
3085
3075
|
const getSyncState = (pendingAttachmentBlobs) => {
|
|
@@ -3109,8 +3099,8 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
3109
3099
|
// to close current batch.
|
|
3110
3100
|
this.flush();
|
|
3111
3101
|
return props?.notifyImminentClosure === true
|
|
3112
|
-
?
|
|
3113
|
-
:
|
|
3102
|
+
? internal_8.PerformanceEvent.timedExecAsync(this.mc.logger, perfEvent, async (event) => logAndReturnPendingState(event, getSyncState(await this.blobManager.attachAndGetPendingBlobs(props?.stopBlobAttachingSignal))))
|
|
3103
|
+
: internal_8.PerformanceEvent.timedExec(this.mc.logger, perfEvent, (event) => logAndReturnPendingState(event, getSyncState()));
|
|
3114
3104
|
}
|
|
3115
3105
|
summarizeOnDemand(options) {
|
|
3116
3106
|
if (this._summarizer !== undefined) {
|
|
@@ -3120,7 +3110,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
3120
3110
|
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
3121
3111
|
// disableSummaries is turned on. We are throwing instead of returning a failure here,
|
|
3122
3112
|
// because it is a misuse of the API rather than an expected failure.
|
|
3123
|
-
throw new
|
|
3113
|
+
throw new internal_8.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
3124
3114
|
}
|
|
3125
3115
|
else {
|
|
3126
3116
|
return this.summaryManager.summarizeOnDemand(options);
|
|
@@ -3134,7 +3124,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
|
3134
3124
|
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
3135
3125
|
// generateSummaries is turned off. We are throwing instead of returning a failure here,
|
|
3136
3126
|
// because it is a misuse of the API rather than an expected failure.
|
|
3137
|
-
throw new
|
|
3127
|
+
throw new internal_8.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
3138
3128
|
}
|
|
3139
3129
|
else {
|
|
3140
3130
|
return this.summaryManager.enqueueSummarize(options);
|