@fluidframework/container-runtime 2.0.0-internal.3.0.1 → 2.0.0-internal.3.1.0

package/lib/containerRuntime.js

@@ -1,5 +1,5 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
+import { assert, Trace, TypedEventEmitter, unreachableCase } from "@fluidframework/common-utils";
 import { ChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, loggerToMonitoringContext, wrapError, } from "@fluidframework/telemetry-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
@@ -13,23 +13,25 @@ import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
 import { Summarizer } from "./summarizer";
 import { SummaryManager } from "./summaryManager";
-import { ReportOpPerfTelemetry, } from "./connectionTelemetry";
-import { PendingStateManager, } from "./pendingStateManager";
+import { ReportOpPerfTelemetry } from "./connectionTelemetry";
+import { PendingStateManager } from "./pendingStateManager";
 import { pkgVersion } from "./packageVersion";
 import { BlobManager } from "./blobManager";
 import { DataStores, getSummaryForDatastores } from "./dataStores";
 import { aliasBlobName, blobsTreeName, chunksBlobName, electedSummarizerBlobName, extractSummaryMetadataMessage, metadataBlobName, wrapSummaryInChannelsTree, } from "./summaryFormat";
 import { SummaryCollection } from "./summaryCollection";
-import { OrderedClientCollection, OrderedClientElection } from "./orderedClientElection";
+import { OrderedClientCollection, OrderedClientElection, } from "./orderedClientElection";
 import { SummarizerClientElection, summarizerClientType } from "./summarizerClientElection";
 import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
 import { GarbageCollector, GCNodeType, } from "./garbageCollection";
-import { channelToDataStore, isDataStoreAliasMessage, } from "./dataStore";
+import { channelToDataStore, isDataStoreAliasMessage } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
-import { SerializedSnapshotStorage } from "./serializedSnapshotStorage";
+import { SerializedSnapshotStorage, } from "./serializedSnapshotStorage";
 import { ScheduleManager } from "./scheduleManager";
 import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, } from "./opLifecycle";
+import { shouldAllowGcTombstoneEnforcement } from "./garbageCollectionHelpers";
+import { gcTombstoneGenerationOptionName } from "./garbageCollectionConstants";
 export var ContainerMessageType;
 (function (ContainerMessageType) {
     // An op to be delivered to store
@@ -134,8 +136,7 @@ export function getDeviceSpec() {
             };
         }
     }
-    catch (_a) {
-    }
+    catch (_a) { }
     return {};
 }
 /**
@@ -147,7 +148,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @internal
      */
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
-        var _a, _b, _c, _d, _e, _f;
+        var _a, _b, _c, _d, _e, _f, _g;
         if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
         super();
         this.context = context;
@@ -230,28 +231,39 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.nextSummaryNumber = loadSummaryNumber + 1;
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         this._connected = this.context.connected;
+        this.gcTombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement((_c = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix) === null || _c === void 0 ? void 0 : _c.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName] /* current */);
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
-        const opSplitter = new OpSplitter(chunks, this.context.submitBatchFn, this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true ?
-            Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "GCFeatureMatrix",
+            metadataValue: JSON.stringify(metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix),
+            inputs: JSON.stringify({
+                gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName],
+            }),
+        });
+        const opSplitter = new OpSplitter(chunks, this.context.submitBatchFn, this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true
+            ? Number.POSITIVE_INFINITY
+            : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
         this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor());
         this.handleContext = new ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
-        this.enableOpReentryCheck = runtimeOptions.enableOpReentryCheck === true
-            // Allow for a break-glass config to override the options
-            && this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
+        this.enableOpReentryCheck =
+            runtimeOptions.enableOpReentryCheck === true &&
+                // Allow for a break-glass config to override the options
+                this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
         this.summariesDisabled = this.isSummariesDisabled();
         this.heuristicsDisabled = this.isHeuristicsDisabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
         this.maxConsecutiveReconnects =
-            (_c = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _c !== void 0 ? _c : this.defaultMaxConsecutiveReconnects;
+            (_d = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _d !== void 0 ? _d : this.defaultMaxConsecutiveReconnects;
         this._flushMode = runtimeOptions.flushMode;
         const pendingRuntimeState = context.pendingLocalState;
-        const baseSnapshot = (_d = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _d !== void 0 ? _d : context.baseSnapshot;
-        const maxSnapshotCacheDurationMs = (_f = (_e = this._storage) === null || _e === void 0 ? void 0 : _e.policies) === null || _f === void 0 ? void 0 : _f.maximumCacheDurationMs;
-        if (maxSnapshotCacheDurationMs !== undefined && maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
+        const baseSnapshot = (_e = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _e !== void 0 ? _e : context.baseSnapshot;
+        const maxSnapshotCacheDurationMs = (_g = (_f = this._storage) === null || _f === void 0 ? void 0 : _f.policies) === null || _g === void 0 ? void 0 : _g.maximumCacheDurationMs;
+        if (maxSnapshotCacheDurationMs !== undefined &&
+            maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
             // which dictates the value is either undefined or exactly 5 days in ms.
             // As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
@@ -296,12 +308,15 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (baseSnapshot) {
             this.summarizerNode.updateBaseSummaryState(baseSnapshot);
         }
-        this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap));
+        this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap));
         this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
             if (!this.disposed) {
-                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { localId, blobId });
+                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
+                    localId,
+                    blobId,
+                });
             }
-        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (fromPath, toPath) => this.garbageCollector.addedOutboundReference(fromPath, toPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs, () => this.getCurrentReferenceTimestampMs());
+        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (fromPath, toPath) => this.garbageCollector.addedOutboundReference(fromPath, toPath), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs, () => this.getCurrentReferenceTimestampMs());
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, () => this.clientId, ChildLogger.create(this.logger, "ScheduleManager"));
         this.pendingStateManager = new PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
@@ -312,11 +327,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             rollback: this.rollback.bind(this),
             orderSequentially: this.orderSequentially.bind(this),
         }, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
-        const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true ?
-            {
+        const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true
+            ? {
                 minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
-                compressionAlgorithm: CompressionAlgorithms.lz4
-            } : runtimeOptions.compressionOptions;
+                compressionAlgorithm: CompressionAlgorithms.lz4,
+            }
+            : runtimeOptions.compressionOptions;
         this.outbox = new Outbox({
             shouldSend: () => this.canSendOps(),
             pendingStateManager: this.pendingStateManager,
@@ -334,8 +350,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
         });
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
-        this.dirtyContainer = this.context.attachState !== AttachState.Attached
-            || this.pendingStateManager.hasPendingMessages();
+        this.dirtyContainer =
+            this.context.attachState !== AttachState.Attached ||
+                this.pendingStateManager.hasPendingMessages();
         this.context.updateDirtyContainerState(this.dirtyContainer);
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
@@ -403,8 +420,12 @@ export class ContainerRuntime extends TypedEventEmitter {
         ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
         BindBatchTracker(this, this.logger);
     }
-    get IContainerRuntime() { return this; }
-    get IFluidRouter() { return this; }
+    get IContainerRuntime() {
+        return this;
+    }
+    get IFluidRouter() {
+        return this;
+    }
     /**
      * @deprecated - use loadRuntime instead.
      * Load the stores from a snapshot and returns the runtime.
@@ -445,7 +466,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     static async loadRuntime(params) {
         var _a, _b, _c, _d;
-        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime } = params;
+        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
@@ -457,13 +478,15 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = {
             minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
-            compressionAlgorithm: CompressionAlgorithms.lz4
+            compressionAlgorithm: CompressionAlgorithms.lz4,
         }, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, chunkSizeInBytes = Number.POSITIVE_INFINITY, enableOpReentryCheck = false, } = runtimeOptions;
         const pendingRuntimeState = context.pendingLocalState;
         const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
-        const storage = !pendingRuntimeState ?
-            context.storage :
-            new SerializedSnapshotStorage(() => { return context.storage; }, pendingRuntimeState.snapshotBlobs);
+        const storage = !pendingRuntimeState
+            ? context.storage
+            : new SerializedSnapshotStorage(() => {
+                return context.storage;
+            }, pendingRuntimeState.snapshotBlobs);
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.blobs[blobName];
@@ -494,7 +517,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (!pendingRuntimeState && runtimeSequenceNumber !== undefined) {
             const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
-            if (loadSequenceNumberVerification !== "bypass" && runtimeSequenceNumber !== protocolSequenceNumber) {
+            if (loadSequenceNumberVerification !== "bypass" &&
+                runtimeSequenceNumber !== protocolSequenceNumber) {
                 // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
                 const error = new DataCorruptionError(
                 // pre-0.58 error message: SummaryMetadataMismatch
@@ -601,7 +625,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         var _a;
         return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
     }
-    get disposed() { return this._disposed; }
+    get disposed() {
+        return this._disposed;
+    }
     get summarizer() {
         assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
@@ -711,7 +737,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                         status: 200,
                         mimeType: "fluid/object",
                         value: blob,
-                    } : create404Response(request);
+                    }
+                    : create404Response(request);
             }
             else if (requestParser.pathParts.length > 0) {
                 const dataStore = await this.getDataStoreFromRequest(id, request);
@@ -754,7 +781,8 @@ export class ContainerRuntime extends TypedEventEmitter {
          * This is a workaround to handle scenarios where a data store shared with an external app is deleted
          * and marked as unreferenced by GC. Returning an error will fail to load the data store for the app.
          */
-        if (((_d = request.headers) === null || _d === void 0 ? void 0 : _d[RuntimeHeaders.externalRequest]) && this.garbageCollector.shouldRunGC) {
+        if (((_d = request.headers) === null || _d === void 0 ? void 0 : _d[RuntimeHeaders.externalRequest]) &&
+            this.garbageCollector.shouldRunGC) {
             // The data store is referenced if used routes in the base summary has a route to self.
             // Older documents may not have used routes in the summary. They are considered referenced.
             const usedRoutes = (await dataStoreContext.getBaseGCDetails()).usedRoutes;
@@ -928,7 +956,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (reconnection) {
             this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
+                this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
                     pendingMessages: this.pendingStateManager.pendingMessagesCount,
@@ -1035,7 +1063,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (message.clientId === this.clientId && this.connected) {
             // Check to see if the signal was lost.
             if (this._perfSignalData.trackingSignalSequenceNumber !== undefined &&
-                envelope.clientSignalSequenceNumber > this._perfSignalData.trackingSignalSequenceNumber) {
+                envelope.clientSignalSequenceNumber >
+                    this._perfSignalData.trackingSignalSequenceNumber) {
                 this._perfSignalData.signalsLost++;
                 this._perfSignalData.trackingSignalSequenceNumber = undefined;
                 this.logger.sendErrorEvent({
@@ -1046,7 +1075,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                     clientSignalSequenceNumber: envelope.clientSignalSequenceNumber,
                 });
             }
-            else if (envelope.clientSignalSequenceNumber === this._perfSignalData.trackingSignalSequenceNumber) {
+            else if (envelope.clientSignalSequenceNumber ===
+                this._perfSignalData.trackingSignalSequenceNumber) {
                 this.sendSignalTelemetryEvent(envelope.clientSignalSequenceNumber);
                 this._perfSignalData.trackingSignalSequenceNumber = undefined;
             }
@@ -1133,7 +1163,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.dataStores.createDetachedDataStoreCore(pkg, false);
     }
     async _createDataStoreWithProps(pkg, props, id = uuid()) {
-        const fluidDataStore = await this.dataStores._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props).realize();
+        const fluidDataStore = await this.dataStores
+            ._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
+            .realize();
         return channelToDataStore(fluidDataStore, id, this, this.dataStores, this.mc.logger);
     }
     async _createDataStore(pkg, id = uuid(), props) {
@@ -1270,7 +1302,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         const telemetryContext = new TelemetryContext();
         const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
-        this.logger.sendTelemetryEvent({ eventName: "SummarizeTelemetry", details: telemetryContext.serialize() });
+        this.logger.sendTelemetryEvent({
+            eventName: "SummarizeTelemetry",
+            details: telemetryContext.serialize(),
+        });
         assert(summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
         return { stats, summary, gcStats };
     }
@@ -1321,6 +1356,18 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.blobManager.updateUnusedRoutes(blobManagerRoutes);
         this.dataStores.updateUnusedRoutes(dataStoreRoutes);
     }
+    /**
+     * This is called to delete objects from the runtime
+     * @param unusedRoutes - object routes and sub routes that can be deleted
+     * @returns - routes of objects deleted from the runtime
+     */
+    deleteUnusedNodes(unusedRoutes) {
+        const { dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
+        const deletedRoutes = [];
+        const deletedDataStoreRoutes = this.dataStores.deleteUnusedNodes(dataStoreRoutes);
+        deletedRoutes.push(...deletedDataStoreRoutes);
+        return deletedRoutes;
+    }
     /**
      * This is called to update objects that are tombstones.
      * @param tombstonedRoutes - Data store and attachment blob routes that are tombstones in this Container.
@@ -1436,8 +1483,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             // We might need to catch up to the latest summary's reference sequence number before pausing.
             await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
         }
+        const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         try {
             await this.deltaManager.inbound.pause();
+            if (shouldPauseInboundSignal) {
+                await this.deltaManager.inboundSignal.pause();
+            }
             const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
             const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
@@ -1538,8 +1589,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             // latestSnapshotVersionId from storage and it does not match with the lastAck ackHandle, then use
             // the one fetched from storage as parent as that is the latest.
             let summaryContext;
-            if ((lastAck === null || lastAck === void 0 ? void 0 : lastAck.summaryAck.contents.handle) !== latestSnapshotVersionId
-                && latestSnapshotVersionId !== undefined) {
+            if ((lastAck === null || lastAck === void 0 ? void 0 : lastAck.summaryAck.contents.handle) !== latestSnapshotVersionId &&
+                latestSnapshotVersionId !== undefined) {
                 summaryContext = {
                     proposalHandle: undefined,
                     ackHandle: latestSnapshotVersionId,
@@ -1596,6 +1647,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.summarizerNode.clearSummary();
             // Restart the delta manager
             this.deltaManager.inbound.resume();
+            if (shouldPauseInboundSignal) {
+                this.deltaManager.inboundSignal.resume();
+            }
         }
     }
     hasPendingMessages() {
@@ -1644,7 +1698,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         const deserializedContent = { type, contents };
         const serializedContent = JSON.stringify(deserializedContent);
         if (this.deltaManager.readOnlyInfo.readonly) {
-            this.logger.sendTelemetryEvent({ eventName: "SubmitOpInReadonly", connected: this.connected });
+            this.logger.sendTelemetryEvent({
+                eventName: "SubmitOpInReadonly",
+                connected: this.connected,
+            });
         }
         const message = {
             contents: serializedContent,
@@ -1674,7 +1731,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             //    issue than sending.
             // Please note that this does not change file format, so it can be disabled in the future if this
             // optimization no longer makes sense (for example, batch compression may make it less appealing).
-            if (this.currentlyBatching() && type === ContainerMessageType.Attach &&
+            if (this.currentlyBatching() &&
+                type === ContainerMessageType.Attach &&
                 this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder") !== true) {
                 this.outbox.submitAttach(message);
             }
@@ -1687,10 +1745,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             else if (!this.flushMicroTaskExists) {
                 this.flushMicroTaskExists = true;
                 // Queue a microtask to detect the end of the turn and force a flush.
-                Promise.resolve().then(() => {
+                Promise.resolve()
+                    .then(() => {
                     this.flushMicroTaskExists = false;
                     this.flush();
-                }).catch((error) => { this.closeFn(error); });
+                })
+                    .catch((error) => {
+                    this.closeFn(error);
+                });
             }
         }
         catch (error) {
@@ -1804,14 +1866,12 @@ export class ContainerRuntime extends TypedEventEmitter {
         // The call to fetch the snapshot is very expensive and not always needed.
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
-        const snapshotTreeFetcher = async () => {
+        const fetchLatestSnapshot = async () => {
             const fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
-                eventName: "RefreshLatestSummaryGetSnapshot",
+                eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
-                summaryRefSeq,
-                fetchLatest: true,
-            });
-            const latestSnapshotRefSeq = await seqFromTree(fetchResult.snapshotTree, readAndParseBlob);
+                targetSequenceNumber: summaryRefSeq,
+            }, readAndParseBlob);
             /**
              * If the fetched snapshot is older than the one for which the ack was received, close the container.
              * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -1822,29 +1882,26 @@ export class ContainerRuntime extends TypedEventEmitter {
              * such cases, the file will be rolled back along with the ack and we will eventually reach a consistent
              * state.
              */
-            if (latestSnapshotRefSeq < summaryRefSeq) {
+            if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
                 const error = DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
                     ackHandle,
                     summaryRefSeq,
-                    latestSnapshotRefSeq,
+                    latestSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
                 });
                 this.closeFn(error);
                 throw error;
             }
-            summaryLogger.sendTelemetryEvent({
-                eventName: "LatestSummaryRetrieved",
-                ackHandle,
-                lastSequenceNumber: latestSnapshotRefSeq,
-                targetSequenceNumber: summaryRefSeq,
-            });
             // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
             // wait for the delta manager to catch up before refreshing the latest Summary.
-            await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryLogger);
-            return fetchResult.snapshotTree;
+            await this.waitForDeltaManagerToCatchup(fetchResult.latestSnapshotRefSeq, summaryLogger);
+            return {
+                snapshotTree: fetchResult.snapshotTree,
+                snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
+            };
         };
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, snapshotTreeFetcher, readAndParseBlob, summaryLogger);
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(result, proposalHandle, summaryRefSeq, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -1853,18 +1910,20 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @returns downloaded snapshot's reference sequence number
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
-        const { snapshotTree, versionId } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
-            eventName: "RefreshLatestSummaryGetSnapshot",
-            fetchLatest: true,
-        });
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        const latestSnapshotRefSeq = await seqFromTree(snapshotTree, readAndParseBlob);
-        const result = await this.summarizerNode.refreshLatestSummary(undefined, latestSnapshotRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
+        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+            eventName: "RefreshLatestSummaryFromServerFetch",
+        }, readAndParseBlob);
+        const fetchLatestSnapshot = {
+            snapshotTree,
+            snapshotRefSeq: latestSnapshotRefSeq,
+        };
+        const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(result, undefined, latestSnapshotRefSeq, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchLatestSnapshotFromStorage(logger, event) {
+    async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
         return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
@@ -1874,20 +1933,29 @@ export class ContainerRuntime extends TypedEventEmitter {
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
             assert(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
             stats.getSnapshotDuration = trace.trace().duration;
+            const latestSnapshotRefSeq = await seqFromTree(maybeSnapshot, readAndParseBlob);
+            stats.snapshotRefSeq = latestSnapshotRefSeq;
+            stats.snapshotVersion = versions[0].id;
             perfEvent.end(stats);
-            return { snapshotTree: maybeSnapshot, versionId: versions[0].id };
+            return {
+                snapshotTree: maybeSnapshot,
+                versionId: versions[0].id,
+                latestSnapshotRefSeq,
+            };
         });
     }
     notifyAttaching(snapshot) {
         var _a;
         if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
-            this.baseSnapshotBlobs = SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
+            this.baseSnapshotBlobs =
+                SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
         }
     }
     async initializeBaseSnapshotBlobs() {
         var _a;
         if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) ||
-            this.attachState !== AttachState.Attached || this.context.pendingLocalState) {
+            this.attachState !== AttachState.Attached ||
+            this.context.pendingLocalState) {
             return;
         }
         assert(!!this.context.baseSnapshot, 0x2e5 /* "Must have a base snapshot" */);