@fluidframework/container-runtime 2.0.0-internal.3.1.0 → 2.0.0-internal.3.2.1

package/dist/containerRuntime.js

@@ -14,27 +14,19 @@ const garbage_collector_1 = require("@fluidframework/garbage-collector");
 const uuid_1 = require("uuid");
 const containerHandleContext_1 = require("./containerHandleContext");
 const dataStoreRegistry_1 = require("./dataStoreRegistry");
-const summarizer_1 = require("./summarizer");
-const summaryManager_1 = require("./summaryManager");
 const connectionTelemetry_1 = require("./connectionTelemetry");
 const pendingStateManager_1 = require("./pendingStateManager");
 const packageVersion_1 = require("./packageVersion");
 const blobManager_1 = require("./blobManager");
 const dataStores_1 = require("./dataStores");
-const summaryFormat_1 = require("./summaryFormat");
-const summaryCollection_1 = require("./summaryCollection");
-const orderedClientElection_1 = require("./orderedClientElection");
-const summarizerClientElection_1 = require("./summarizerClientElection");
+const summary_1 = require("./summary");
 const throttler_1 = require("./throttler");
-const runWhileConnectedCoordinator_1 = require("./runWhileConnectedCoordinator");
-const garbageCollection_1 = require("./garbageCollection");
+const gc_1 = require("./gc");
 const dataStore_1 = require("./dataStore");
 const batchTracker_1 = require("./batchTracker");
 const serializedSnapshotStorage_1 = require("./serializedSnapshotStorage");
 const scheduleManager_1 = require("./scheduleManager");
 const opLifecycle_1 = require("./opLifecycle");
-const garbageCollectionHelpers_1 = require("./garbageCollectionHelpers");
-const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
 var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -71,11 +63,6 @@ var RuntimeHeaders;
 (function (RuntimeHeaders) {
     /** True to wait for a data store to be created and loaded before returning it. */
     RuntimeHeaders["wait"] = "wait";
-    /**
-     * True if the request is from an external app. Used for GC to handle scenarios where a data store
-     * is deleted and requested via an external app.
-     */
-    RuntimeHeaders["externalRequest"] = "externalRequest";
     /** True if the request is coming from an IFluidHandle. */
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders = exports.RuntimeHeaders || (exports.RuntimeHeaders = {}));
@@ -86,7 +73,6 @@ exports.TombstoneResponseHeaderKey = "isTombstoned";
 /** Default values for Runtime Headers */
 exports.defaultRuntimeHeaderData = {
     wait: true,
-    externalRequest: false,
     viaHandle: false,
     allowTombstone: false,
 };
@@ -153,7 +139,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * @internal
      */
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
-        var _a, _b, _c, _d, _e, _f, _g;
+        var _a, _b, _c, _d, _e, _f, _g, _h;
         if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, exports.DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
         super();
         this.context = context;
@@ -166,7 +152,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.summaryConfiguration = summaryConfiguration;
         this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
-        this.flushMicroTaskExists = false;
+        this.flushTaskExists = false;
         this.savedOps = [];
         this.consecutiveReconnects = 0;
         this.ensureNoDataModelChangesCalls = 0;
@@ -187,7 +173,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             trackingSignalSequenceNumber: undefined,
         };
         this.summarizeOnDemand = (...args) => {
-            if (this.clientDetails.type === summarizerClientElection_1.summarizerClientType) {
+            if (this.clientDetails.type === summary_1.summarizerClientType) {
                 return this.summarizer.summarizeOnDemand(...args);
             }
             else if (this.summaryManager !== undefined) {
@@ -201,7 +187,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
         };
         this.enqueueSummarize = (...args) => {
-            if (this.clientDetails.type === summarizerClientElection_1.summarizerClientType) {
+            if (this.clientDetails.type === summary_1.summarizerClientType) {
                 return this.summarizer.enqueueSummarize(...args);
             }
             else if (this.summaryManager !== undefined) {
@@ -236,37 +222,46 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.nextSummaryNumber = loadSummaryNumber + 1;
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         this._connected = this.context.connected;
-        this.gcTombstoneEnforcementAllowed = (0, garbageCollectionHelpers_1.shouldAllowGcTombstoneEnforcement)((_c = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix) === null || _c === void 0 ? void 0 : _c.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[garbageCollectionConstants_1.gcTombstoneGenerationOptionName] /* current */);
+        this.gcTombstoneEnforcementAllowed = (0, gc_1.shouldAllowGcTombstoneEnforcement)((_c = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix) === null || _c === void 0 ? void 0 : _c.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gc_1.gcTombstoneGenerationOptionName] /* current */);
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.logger, "ContainerRuntime"));
         this.mc.logger.sendTelemetryEvent({
             eventName: "GCFeatureMatrix",
             metadataValue: JSON.stringify(metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix),
             inputs: JSON.stringify({
-                gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[garbageCollectionConstants_1.gcTombstoneGenerationOptionName],
+                gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gc_1.gcTombstoneGenerationOptionName],
             }),
         });
-        const opSplitter = new opLifecycle_1.OpSplitter(chunks, this.context.submitBatchFn, this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true
-            ? Number.POSITIVE_INFINITY
-            : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
+        this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
+        const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking");
+        const opSplitter = new opLifecycle_1.OpSplitter(chunks, this.context.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
         this.remoteMessageProcessor = new opLifecycle_1.RemoteMessageProcessor(opSplitter, new opLifecycle_1.OpDecompressor());
         this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
+        const disableOpReentryCheck = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck");
         this.enableOpReentryCheck =
             runtimeOptions.enableOpReentryCheck === true &&
                 // Allow for a break-glass config to override the options
-                this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
+                disableOpReentryCheck !== true;
         this.summariesDisabled = this.isSummariesDisabled();
         this.heuristicsDisabled = this.isHeuristicsDisabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
         this.maxConsecutiveReconnects =
             (_d = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _d !== void 0 ? _d : this.defaultMaxConsecutiveReconnects;
-        this._flushMode = runtimeOptions.flushMode;
+        if (runtimeOptions.flushMode === runtime_definitions_1.FlushModeExperimental.Async &&
+            ((_e = context.supportedFeatures) === null || _e === void 0 ? void 0 : _e.get("referenceSequenceNumbers")) !== true) {
+            // The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
+            this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
+            this._flushMode = runtime_definitions_1.FlushMode.TurnBased;
+        }
+        else {
+            this._flushMode = runtimeOptions.flushMode;
+        }
         const pendingRuntimeState = context.pendingLocalState;
-        const baseSnapshot = (_e = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _e !== void 0 ? _e : context.baseSnapshot;
-        const maxSnapshotCacheDurationMs = (_g = (_f = this._storage) === null || _f === void 0 ? void 0 : _f.policies) === null || _g === void 0 ? void 0 : _g.maximumCacheDurationMs;
+        const baseSnapshot = (_f = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _f !== void 0 ? _f : context.baseSnapshot;
+        const maxSnapshotCacheDurationMs = (_h = (_g = this._storage) === null || _g === void 0 ? void 0 : _g.policies) === null || _h === void 0 ? void 0 : _h.maximumCacheDurationMs;
         if (maxSnapshotCacheDurationMs !== undefined &&
             maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
@@ -274,7 +269,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
             throw new container_utils_1.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
         }
-        this.garbageCollector = garbageCollection_1.GarbageCollector.create({
+        this.garbageCollector = gc_1.GarbageCollector.create({
             runtime: this,
             gcOptions: this.runtimeOptions.gcOptions,
             baseSnapshot,
@@ -282,7 +277,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             existing,
             metadata,
             createContainerMetadata: this.createContainerMetadata,
-            isSummarizerClient: this.context.clientDetails.type === summarizerClientElection_1.summarizerClientType,
+            isSummarizerClient: this.context.clientDetails.type === summary_1.summarizerClientType,
             getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
             getLastSummaryTimestampMs: () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; },
             readAndParseBlob: async (id) => (0, driver_utils_1.readAndParse)(this.storage, id),
@@ -313,7 +308,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         if (baseSnapshot) {
             this.summarizerNode.updateBaseSummaryState(baseSnapshot);
         }
-        this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap));
+        this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap));
         this.blobManager = new blobManager_1.BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
             if (!this.disposed) {
                 this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
@@ -332,12 +327,14 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             rollback: this.rollback.bind(this),
             orderSequentially: this.orderSequentially.bind(this),
         }, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
-        const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true
+        const disableCompression = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression");
+        const compressionOptions = disableCompression === true
             ? {
                 minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
                 compressionAlgorithm: CompressionAlgorithms.lz4,
             }
             : runtimeOptions.compressionOptions;
+        const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
         this.outbox = new opLifecycle_1.Outbox({
             shouldSend: () => this.canSendOps(),
             pendingStateManager: this.pendingStateManager,
@@ -347,14 +344,14 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             config: {
                 compressionOptions,
                 maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
-                enableOpReentryCheck: this.enableOpReentryCheck,
+                disablePartialFlush: disablePartialFlush === true,
             },
             logger: this.mc.logger,
         });
         this.context.quorum.on("removeMember", (clientId) => {
             this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
         });
-        this.summaryCollection = new summaryCollection_1.SummaryCollection(this.deltaManager, this.logger);
+        this.summaryCollection = new summary_1.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
             this.context.attachState !== container_definitions_1.AttachState.Attached ||
                 this.pendingStateManager.hasPendingMessages();
@@ -364,13 +361,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         else {
             const orderedClientLogger = telemetry_utils_1.ChildLogger.create(this.logger, "OrderedClientElection");
-            const orderedClientCollection = new orderedClientElection_1.OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
-            const orderedClientElectionForSummarizer = new orderedClientElection_1.OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, summarizerClientElection_1.SummarizerClientElection.isClientEligible);
-            this.summarizerClientElection = new summarizerClientElection_1.SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
-            if (this.context.clientDetails.type === summarizerClientElection_1.summarizerClientType) {
-                this._summarizer = new summarizer_1.Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => runWhileConnectedCoordinator_1.RunWhileConnectedCoordinator.create(runtime));
+            const orderedClientCollection = new summary_1.OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
+            const orderedClientElectionForSummarizer = new summary_1.OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, summary_1.SummarizerClientElection.isClientEligible);
+            this.summarizerClientElection = new summary_1.SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
+            if (this.context.clientDetails.type === summary_1.summarizerClientType) {
+                this._summarizer = new summary_1.Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => summary_1.RunWhileConnectedCoordinator.create(runtime));
             }
-            else if (summarizerClientElection_1.SummarizerClientElection.clientDetailsPermitElection(this.context.clientDetails)) {
+            else if (summary_1.SummarizerClientElection.clientDetailsPermitElection(this.context.clientDetails)) {
                 // Only create a SummaryManager and SummarizerClientElection
                 // if summaries are enabled and we are not the summarizer client.
                 const defaultAction = () => {
@@ -390,7 +387,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 };
                 this.summaryCollection.on("default", defaultAction);
                 // Create the SummaryManager and mark the initial state
-                this.summaryManager = new summaryManager_1.SummaryManager(this.summarizerClientElection, this, // IConnectedState
+                this.summaryManager = new summary_1.SummaryManager(this.summarizerClientElection, this, // IConnectedState
                 this.summaryCollection, this.logger, this.formRequestSummarizerFn(this.context.loader), new throttler_1.Throttler(60 * 1000, // 60 sec delay window
                 30 * 1000, // 30 sec max delay
                 // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
@@ -421,7 +418,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         });
         // logging hardware telemetry
         logger.sendTelemetryEvent(Object.assign({ eventName: "DeviceSpec" }, getDeviceSpec()));
-        this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryNumber: loadSummaryNumber, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature }));
+        this.logger.sendTelemetryEvent(Object.assign(Object.assign(Object.assign({ eventName: "ContainerLoadStats" }, this.createContainerMetadata), this.dataStores.containerLoadStats), { summaryNumber: loadSummaryNumber, summaryFormatVersion: metadata === null || metadata === void 0 ? void 0 : metadata.summaryFormatVersion, disableIsolatedChannels: metadata === null || metadata === void 0 ? void 0 : metadata.disableIsolatedChannels, gcVersion: metadata === null || metadata === void 0 ? void 0 : metadata.gcFeature, options: JSON.stringify(runtimeOptions), featureGates: JSON.stringify({
+                disableCompression,
+                disableOpReentryCheck,
+                disableChunking,
+                disableAttachReorder: this.disableAttachReorder,
+                disablePartialFlush,
+            }) }));
         (0, connectionTelemetry_1.ReportOpPerfTelemetry)(this.context.clientId, this.deltaManager, this.logger);
         (0, batchTracker_1.BindBatchTracker)(this, this.logger);
     }
@@ -503,14 +506,14 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
         };
         const [chunks, metadata, electedSummarizerData, aliases] = await Promise.all([
-            tryFetchBlob(summaryFormat_1.chunksBlobName),
-            tryFetchBlob(summaryFormat_1.metadataBlobName),
-            tryFetchBlob(summaryFormat_1.electedSummarizerBlobName),
-            tryFetchBlob(summaryFormat_1.aliasBlobName),
+            tryFetchBlob(summary_1.chunksBlobName),
+            tryFetchBlob(summary_1.metadataBlobName),
+            tryFetchBlob(summary_1.electedSummarizerBlobName),
+            tryFetchBlob(summary_1.aliasBlobName),
         ]);
         const loadExisting = existing === true || context.existing === true;
         // read snapshot blobs needed for BlobManager to load
-        const blobManagerSnapshot = await blobManager_1.BlobManager.load(baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[summaryFormat_1.blobsTreeName], async (id) => {
+        const blobManagerSnapshot = await blobManager_1.BlobManager.load(baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[summary_1.blobsTreeName], async (id) => {
             // IContainerContext storage api return type still has undefined in 0.39 package version.
             // So once we release 0.40 container-defn package we can remove this check.
             (0, common_utils_1.assert)(storage !== undefined, 0x256 /* "storage undefined in attached container" */);
@@ -687,6 +690,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.emit("dispose");
         this.removeAllListeners();
     }
+    /**
+     * @deprecated 2.0.0-internal.3.2.0 ContainerRuntime is not an IFluidTokenProvider.  Token providers should be accessed using normal provider patterns.
+     */
     get IFluidTokenProvider() {
         var _a;
         if ((_a = this.options) === null || _a === void 0 ? void 0 : _a.intelligence) {
@@ -764,7 +770,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return (_a = this.dataStores.aliases.get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
     }
     async getDataStoreFromRequest(id, request) {
-        var _a, _b, _c, _d;
+        var _a, _b, _c;
         const headerData = {};
         if (typeof ((_a = request.headers) === null || _a === void 0 ? void 0 : _a[RuntimeHeaders.wait]) === "boolean") {
             headerData.wait = request.headers[RuntimeHeaders.wait];
@@ -778,23 +784,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
         const dataStoreContext = await this.dataStores.getDataStore(internalId, headerData);
-        /**
-         * If GC should run and this an external app request with "externalRequest" header, we need to return
-         * an error if the data store being requested is marked as unreferenced as per the data store's base
-         * GC data.
-         *
-         * This is a workaround to handle scenarios where a data store shared with an external app is deleted
-         * and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
-         */
-        if (((_d = request.headers) === null || _d === void 0 ? void 0 : _d[RuntimeHeaders.externalRequest]) &&
-            this.garbageCollector.shouldRunGC) {
-            // The data store is referenced if used routes in the base summary has a route to self.
-            // Older documents may not have used routes in the summary. They are considered referenced.
-            const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
-            if (!(usedRoutes === undefined || usedRoutes.includes("") || usedRoutes.includes("/"))) {
-                throw (0, runtime_utils_1.responseToException)((0, runtime_utils_1.create404Response)(request), request);
-            }
-        }
         const dataStoreChannel = await dataStoreContext.realize();
         // Remove query params, leading and trailing slashes from the url. This is done to make sure the format is
         // the same as GC nodes id.
@@ -810,29 +799,29 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             summaryNumber: this.nextSummaryNumber++, summaryFormatVersion: 1 }), this.garbageCollector.getMetadata()), { 
             // The last message processed at the time of summary. If there are no new messages, use the message from the
             // last summary.
-            message: (_a = (0, summaryFormat_1.extractSummaryMetadataMessage)(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary });
-        (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.metadataBlobName, JSON.stringify(metadata));
+            message: (_a = (0, summary_1.extractSummaryMetadataMessage)(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary });
+        (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.metadataBlobName, JSON.stringify(metadata));
     }
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         var _a;
         this.addMetadataToSummary(summaryTree);
         if (this.remoteMessageProcessor.partialMessages.size > 0) {
             const content = JSON.stringify([...this.remoteMessageProcessor.partialMessages]);
-            (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.chunksBlobName, content);
+            (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.chunksBlobName, content);
         }
         const dataStoreAliases = this.dataStores.aliases;
         if (dataStoreAliases.size > 0) {
-            (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.aliasBlobName, JSON.stringify([...dataStoreAliases]));
+            (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.aliasBlobName, JSON.stringify([...dataStoreAliases]));
         }
         if (this.summarizerClientElection) {
             const electedSummarizerContent = JSON.stringify((_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.serialize());
-            (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.electedSummarizerBlobName, electedSummarizerContent);
+            (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.electedSummarizerBlobName, electedSummarizerContent);
         }
         const blobManagerSummary = this.blobManager.summarize();
         // Some storage (like git) doesn't allow empty tree, so we can omit it.
         // and the blob manager can handle the tree not existing when loading
         if (Object.keys(blobManagerSummary.summary.tree).length > 0) {
-            (0, runtime_utils_1.addTreeToSummary)(summaryTree, summaryFormat_1.blobsTreeName, blobManagerSummary);
+            (0, runtime_utils_1.addTreeToSummary)(summaryTree, summary_1.blobsTreeName, blobManagerSummary);
         }
         const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
         if (gcSummary !== undefined) {
@@ -1080,8 +1069,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     clientSignalSequenceNumber: envelope.clientSignalSequenceNumber,
                 });
             }
-            else if (envelope.clientSignalSequenceNumber ===
-                this._perfSignalData.trackingSignalSequenceNumber) {
+            else if (this.consecutiveReconnects === 0 &&
+                envelope.clientSignalSequenceNumber ===
+                    this._perfSignalData.trackingSignalSequenceNumber) {
+                // only logging for the first connection and the trackingSignalSequenceNUmber.
                 this.sendSignalTelemetryEvent(envelope.clientSignalSequenceNumber);
                 this._perfSignalData.trackingSignalSequenceNumber = undefined;
             }
@@ -1185,7 +1176,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * Are we in the middle of batching ops together?
      */
     currentlyBatching() {
-        return this.flushMode === runtime_definitions_1.FlushMode.TurnBased || this._orderSequentiallyCalls !== 0;
+        return this.flushMode !== runtime_definitions_1.FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
     }
     getQuorum() {
         return this.context.quorum;
@@ -1274,7 +1265,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         const summarizeResult = this.dataStores.createSummary(telemetryContext);
         // Wrap data store summaries in .channels subtree.
-        (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
+        (0, summary_1.wrapSummaryInChannelsTree)(summarizeResult);
         this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
         return summarizeResult.summary;
     }
@@ -1290,7 +1281,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     async summarizeInternal(fullTree, trackState, telemetryContext) {
         const summarizeResult = await this.dataStores.summarize(fullTree, trackState, telemetryContext);
         // Wrap data store summaries in .channels subtree.
-        (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
+        (0, summary_1.wrapSummaryInChannelsTree)(summarizeResult);
         const pathPartsForChildren = [runtime_definitions_1.channelsTreeName];
         this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
         return Object.assign(Object.assign({}, summarizeResult), { id: "", pathPartsForChildren });
@@ -1301,11 +1292,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     async summarize(options) {
         this.verifyNotClosed();
         const { fullTree = false, trackState = true, summaryLogger = this.mc.logger, runGC = this.garbageCollector.shouldRunGC, runSweep, fullGC, } = options;
+        const telemetryContext = new runtime_utils_1.TelemetryContext();
+        // Add the options that are used to generate this summary to the telemetry context.
+        telemetryContext.setMultiple("fluid_Summarize", "Options", {
+            fullTree,
+            trackState,
+            runGC,
+            fullGC,
+            runSweep,
+        });
         let gcStats;
         if (runGC) {
-            gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
+            gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
         }
-        const telemetryContext = new runtime_utils_1.TelemetryContext();
         const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
         this.logger.sendTelemetryEvent({
             eventName: "SummarizeTelemetry",
@@ -1362,16 +1361,20 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.dataStores.updateUnusedRoutes(dataStoreRoutes);
     }
     /**
-     * This is called to delete objects from the runtime
-     * @param unusedRoutes - object routes and sub routes that can be deleted
-     * @returns - routes of objects deleted from the runtime
+     * @deprecated - Replaced by deleteSweepReadyNodes.
      */
     deleteUnusedNodes(unusedRoutes) {
-        const { dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
-        const deletedRoutes = [];
-        const deletedDataStoreRoutes = this.dataStores.deleteUnusedNodes(dataStoreRoutes);
-        deletedRoutes.push(...deletedDataStoreRoutes);
-        return deletedRoutes;
+        throw new Error("deleteUnusedRoutes should not be called");
+    }
+    /**
+     * After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
+     * @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
+     * @returns - The routes of nodes that were deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyRoutes) {
+        const { dataStoreRoutes, blobManagerRoutes } = this.getDataStoreAndBlobManagerRoutes(sweepReadyRoutes);
+        const deletedRoutes = this.dataStores.deleteSweepReadyNodes(dataStoreRoutes);
+        return deletedRoutes.concat(this.blobManager.deleteSweepReadyNodes(blobManagerRoutes));
     }
     /**
      * This is called to update objects that are tombstones.
@@ -1398,9 +1401,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     getNodeType(nodePath) {
         var _a;
         if (this.isBlobPath(nodePath)) {
-            return garbageCollection_1.GCNodeType.Blob;
+            return gc_1.GCNodeType.Blob;
         }
-        return (_a = this.dataStores.getGCNodeType(nodePath)) !== null && _a !== void 0 ? _a : garbageCollection_1.GCNodeType.Other;
+        return (_a = this.dataStores.getGCNodeType(nodePath)) !== null && _a !== void 0 ? _a : gc_1.GCNodeType.Other;
     }
     /**
      * Called by GC to retrieve the package path of the node with the given path. The node should belong to a
@@ -1408,10 +1411,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      */
     async getGCNodePackagePath(nodePath) {
         switch (this.getNodeType(nodePath)) {
-            case garbageCollection_1.GCNodeType.Blob:
+            case gc_1.GCNodeType.Blob:
                 return [blobManager_1.BlobManager.basePath];
-            case garbageCollection_1.GCNodeType.DataStore:
-            case garbageCollection_1.GCNodeType.SubDataStore:
+            case gc_1.GCNodeType.DataStore:
+            case gc_1.GCNodeType.SubDataStore:
                 return this.dataStores.getDataStorePackagePath(nodePath);
             default:
                 (0, common_utils_1.assert)(false, 0x2de /* "Package path requested for unsupported node type." */);
@@ -1450,8 +1453,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
      * @returns the statistics of the garbage collection run; undefined if GC did not run.
      */
-    async collectGarbage(options) {
-        return this.garbageCollector.collectGarbage(options);
+    async collectGarbage(options, telemetryContext) {
+        return this.garbageCollector.collectGarbage(options, telemetryContext);
     }
     /**
      * Called when a new outbound reference is added to another node. This is used by garbage collection to identify
@@ -1638,7 +1641,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
             let clientSequenceNumber;
             try {
-                clientSequenceNumber = this.submitSummaryMessage(summaryMessage);
+                clientSequenceNumber = this.submitSummaryMessage(summaryMessage, summaryRefSeqNum);
             }
             catch (error) {
                 return Object.assign(Object.assign({ stage: "upload" }, uploadData), { error });
@@ -1700,8 +1703,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
         (0, common_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
-        const deserializedContent = { type, contents };
-        const serializedContent = JSON.stringify(deserializedContent);
+        const serializedContent = JSON.stringify({ type, contents });
         if (this.deltaManager.readOnlyInfo.readonly) {
             this.logger.sendTelemetryEvent({
                 eventName: "SubmitOpInReadonly",
@@ -1710,7 +1712,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         const message = {
             contents: serializedContent,
-            deserializedContent,
+            deserializedContent: JSON.parse(serializedContent),
             metadata,
             localOpMetadata,
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
@@ -1738,7 +1740,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // optimization no longer makes sense (for example, batch compression may make it less appealing).
             if (this.currentlyBatching() &&
                 type === ContainerMessageType.Attach &&
-                this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
+                this.disableAttachReorder !== true) {
                 this.outbox.submitAttach(message);
             }
             else {
@@ -1747,17 +1749,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (!this.currentlyBatching()) {
                 this.flush();
             }
-            else if (!this.flushMicroTaskExists) {
-                this.flushMicroTaskExists = true;
-                // Queue a microtask to detect the end of the turn and force a flush.
-                Promise.resolve()
-                    .then(() => {
-                    this.flushMicroTaskExists = false;
-                    this.flush();
-                })
-                    .catch((error) => {
-                    this.closeFn(error);
-                });
+            else {
+                this.scheduleFlush();
             }
         }
         catch (error) {
@@ -1768,14 +1761,47 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.updateDocumentDirtyState(true);
         }
     }
-    submitSummaryMessage(contents) {
+    scheduleFlush() {
+        if (this.flushTaskExists) {
+            return;
+        }
+        this.flushTaskExists = true;
+        const flush = () => {
+            this.flushTaskExists = false;
+            try {
+                this.flush();
+            }
+            catch (error) {
+                this.closeFn(error);
+            }
+        };
+        switch (this.flushMode) {
+            case runtime_definitions_1.FlushMode.TurnBased:
+                // When in TurnBased flush mode the runtime will buffer operations in the current turn and send them as a single
+                // batch at the end of the turn
+                // eslint-disable-next-line @typescript-eslint/no-floating-promises
+                Promise.resolve().then(flush);
+                break;
+            // FlushModeExperimental is experimental and not exposed directly in the runtime APIs
+            case runtime_definitions_1.FlushModeExperimental.Async:
+                // When in Async flush mode, the runtime will accumulate all operations across JS turns and send them as a single
+                // batch when all micro-tasks are complete.
+                // Compared to TurnBased, this flush mode will capture more ops into the same batch.
+                setTimeout(flush, 0);
+                break;
+            default:
+                (0, common_utils_1.assert)(this._orderSequentiallyCalls > 0, 0x587 /* Unreachable unless running under orderSequentially */);
+                break;
+        }
+    }
+    submitSummaryMessage(contents, referenceSequenceNumber) {
         this.verifyNotClosed();
         (0, common_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
         (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
         // back-compat: ADO #1385: Make this call unconditional in the future
         return this.context.submitSummaryFn !== undefined
-            ? this.context.submitSummaryFn(contents)
+            ? this.context.submitSummaryFn(contents, referenceSequenceNumber)
             : this.context.submitFn(protocol_definitions_1.MessageType.Summarize, contents, false);
     }
     /**
@@ -1872,7 +1898,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
         const fetchLatestSnapshot = async () => {
-            const fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+            let fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
@@ -1888,13 +1914,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
              * state.
              */
             if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                const error = container_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+                /* before failing, let's try to retrieve the latest snapshot for that specific ackHandle */
+                fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
+                    eventName: "RefreshLatestSummaryAckFetch",
                     ackHandle,
-                    summaryRefSeq,
-                    latestSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
-                });
-                this.closeFn(error);
-                throw error;
+                    targetSequenceNumber: summaryRefSeq,
+                }, readAndParseBlob, ackHandle);
+                if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
+                    const error = container_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+                        ackHandle,
+                        summaryRefSeq,
+                        fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
             }
             // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
             // wait for the delta manager to catch up before refreshing the latest Summary.
@@ -1929,10 +1963,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
     async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
+        return this.fetchSnapshotFromStorage(logger, event, readAndParseBlob, null /* latest */);
+    }
+    async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
         return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = common_utils_1.Trace.start();
-            const versions = await this.storage.getVersions(null, 1, "refreshLatestSummaryAckFromServer", driver_definitions_1.FetchSource.noCache);
+            const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? driver_definitions_1.FetchSource.noCache : undefined);
             (0, common_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
@@ -2009,7 +2046,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     [container_definitions_1.LoaderHeader.cache]: false,
                     [container_definitions_1.LoaderHeader.clientDetails]: {
                         capabilities: { interactive: false },
-                        type: summarizerClientElection_1.summarizerClientType,
+                        type: summary_1.summarizerClientType,
                     },
                     [driver_definitions_1.DriverHeader.summarizingClient]: true,
                     [container_definitions_1.LoaderHeader.reconnect]: false,