@fluidframework/container-runtime 2.0.0-dev.4.1.0.148229 → 2.0.0-dev.4.3.0.157531

package/dist/containerRuntime.js

@@ -1,4 +1,23 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.TombstoneResponseHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
 const container_definitions_1 = require("@fluidframework/container-definitions");
@@ -40,6 +59,12 @@ var ContainerMessageType;
     ContainerMessageType["Rejoin"] = "rejoin";
     // Sets the alias of a root data store
     ContainerMessageType["Alias"] = "alias";
+    /**
+     * An op containing an IdRange of Ids allocated using the runtime's IdCompressor since
+     * the last allocation op was sent.
+     * See the [IdCompressor README](./id-compressor/README.md) for more details.
+     */
+    ContainerMessageType["IdAllocation"] = "idAllocation";
 })(ContainerMessageType = exports.ContainerMessageType || (exports.ContainerMessageType = {}));
 exports.DefaultSummaryConfiguration = {
     state: "enabled",
@@ -48,7 +73,7 @@ exports.DefaultSummaryConfiguration = {
     maxTime: 60 * 1000,
     maxOps: 100,
     minOpsForLastSummaryAttempt: 10,
-    maxAckWaitTime: 10 * 60 * 1000,
+    maxAckWaitTime: 3 * 60 * 1000,
     maxOpsSinceLastSummary: 7000,
     initialSummarizerDelayMs: 5 * 1000,
     nonRuntimeOpWeight: 0.1,
@@ -95,6 +120,12 @@ const defaultCompressionConfig = {
     compressionAlgorithm: CompressionAlgorithms.lz4,
 };
 const defaultChunkSizeInBytes = 204800;
+/**
+ * Instead of refreshing from latest because we do not have 100% confidence in the state
+ * of the current system, we should close the summarizer and let it recover.
+ * This delay's goal is to prevent tight restart loops
+ */
+const defaultCloseSummarizerDelayMs = 10000; // 10 seconds
 /**
  * @deprecated - use ContainerRuntimeMessage instead
  */
@@ -143,8 +174,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     /**
      * @internal
      */
-    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration, initializeEntryPoint) {
-        var _a, _b, _c, _d, _e, _f, _g, _h;
+    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, requestHandler, summaryConfiguration, initializeEntryPoint) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
         if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, exports.DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
         super();
         this.context = context;
@@ -206,6 +237,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         };
         this.innerDeltaManager = context.deltaManager;
         this.deltaManager = new deltaManagerSummarizerProxy_1.DeltaManagerSummarizerProxy(context.deltaManager);
+        this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.logger, "ContainerRuntime"));
         let loadSummaryNumber;
         // Get the container creation metadata. For new container, we initialize these. For existing containers,
         // get the values from the metadata blob.
@@ -217,6 +249,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
             // the count is reset to 0.
             loadSummaryNumber = (_b = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _b !== void 0 ? _b : 0;
+            // Enabling the IdCompressor is a one-way operation and we only want to
+            // allow new containers to turn it on
+            this.idCompressorEnabled = (_c = metadata === null || metadata === void 0 ? void 0 : metadata.idCompressorEnabled) !== null && _c !== void 0 ? _c : false;
         }
         else {
             this.createContainerMetadata = {
@@ -224,12 +259,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 createContainerTimestamp: Date.now(),
             };
             loadSummaryNumber = 0;
+            this.idCompressorEnabled =
+                (_d = this.mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled")) !== null && _d !== void 0 ? _d : idCompressor !== undefined;
         }
         this.nextSummaryNumber = loadSummaryNumber + 1;
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         this._connected = this.context.connected;
-        this.gcTombstoneEnforcementAllowed = (0, gc_1.shouldAllowGcTombstoneEnforcement)((_c = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix) === null || _c === void 0 ? void 0 : _c.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gc_1.gcTombstoneGenerationOptionName] /* current */);
-        this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.logger, "ContainerRuntime"));
+        this.gcTombstoneEnforcementAllowed = (0, gc_1.shouldAllowGcTombstoneEnforcement)((_e = metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix) === null || _e === void 0 ? void 0 : _e.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gc_1.gcTombstoneGenerationOptionName] /* current */);
         this.mc.logger.sendTelemetryEvent({
             eventName: "GCFeatureMatrix",
             metadataValue: JSON.stringify(metadata === null || metadata === void 0 ? void 0 : metadata.gcFeatureMatrix),
@@ -237,11 +273,12 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gc_1.gcTombstoneGenerationOptionName],
             }),
         });
-        this.telemetryDocumentId = (_d = metadata === null || metadata === void 0 ? void 0 : metadata.telemetryDocumentId) !== null && _d !== void 0 ? _d : (0, uuid_1.v4)();
+        this.telemetryDocumentId = (_f = metadata === null || metadata === void 0 ? void 0 : metadata.telemetryDocumentId) !== null && _f !== void 0 ? _f : (0, uuid_1.v4)();
         this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
         const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionChunkingDisabled");
+        const opGroupingManager = new opLifecycle_1.OpGroupingManager(this.groupedBatchingEnabled);
         const opSplitter = new opLifecycle_1.OpSplitter(chunks, this.context.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
-        this.remoteMessageProcessor = new opLifecycle_1.RemoteMessageProcessor(opSplitter, new opLifecycle_1.OpDecompressor(this.mc.logger));
+        this.remoteMessageProcessor = new opLifecycle_1.RemoteMessageProcessor(opSplitter, new opLifecycle_1.OpDecompressor(this.mc.logger), opGroupingManager);
         this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
@@ -255,10 +292,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.heuristicsDisabled = this.isHeuristicsDisabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
+        if (this.idCompressorEnabled) {
+            this.idCompressor = idCompressor;
+        }
         this.maxConsecutiveReconnects =
-            (_e = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _e !== void 0 ? _e : this.defaultMaxConsecutiveReconnects;
+            (_g = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _g !== void 0 ? _g : this.defaultMaxConsecutiveReconnects;
         if (runtimeOptions.flushMode === runtime_definitions_1.FlushModeExperimental.Async &&
-            ((_f = context.supportedFeatures) === null || _f === void 0 ? void 0 : _f.get("referenceSequenceNumbers")) !== true) {
+            ((_h = context.supportedFeatures) === null || _h === void 0 ? void 0 : _h.get("referenceSequenceNumbers")) !== true) {
             // The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
             this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
             this._flushMode = runtime_definitions_1.FlushMode.TurnBased;
@@ -267,7 +307,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this._flushMode = runtimeOptions.flushMode;
         }
         const pendingRuntimeState = context.pendingLocalState;
-        const maxSnapshotCacheDurationMs = (_h = (_g = this._storage) === null || _g === void 0 ? void 0 : _g.policies) === null || _h === void 0 ? void 0 : _h.maximumCacheDurationMs;
+        const maxSnapshotCacheDurationMs = (_k = (_j = this._storage) === null || _j === void 0 ? void 0 : _j.policies) === null || _k === void 0 ? void 0 : _k.maximumCacheDurationMs;
         if (maxSnapshotCacheDurationMs !== undefined &&
             maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
@@ -324,7 +364,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     blobId,
                 });
             }
-        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs, () => this.getCurrentReferenceTimestampMs());
+        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs, (error) => this.closeFn(error));
         this.scheduleManager = new scheduleManager_1.ScheduleManager(context.deltaManager, this, () => this.clientId, telemetry_utils_1.ChildLogger.create(this.logger, "ScheduleManager"));
         this.pendingStateManager = new pendingStateManager_1.PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
@@ -355,10 +395,18 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 disablePartialFlush: disablePartialFlush === true,
             },
             logger: this.mc.logger,
+            groupingManager: opGroupingManager,
+            getCurrentSequenceNumbers: () => ({
+                referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                clientSequenceNumber: this._processedClientSequenceNumber,
+            }),
         });
         this.context.quorum.on("removeMember", (clientId) => {
             this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
         });
+        this.summaryStateUpdateMethod = this.mc.config.getString("Fluid.ContainerRuntime.Test.SummaryStateUpdateMethod");
+        const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
+        this.closeSummarizerDelayMs = closeSummarizerDelayOverride !== null && closeSummarizerDelayOverride !== void 0 ? closeSummarizerDelayOverride : defaultCloseSummarizerDelayMs;
         this.summaryCollection = new summary_1.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
             this.context.attachState !== container_definitions_1.AttachState.Attached ||
@@ -436,7 +484,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 disableChunking,
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
-            }), telemetryDocumentId: this.telemetryDocumentId }));
+                idCompressorEnabled: this.idCompressorEnabled,
+                summaryStateUpdateMethod: this.summaryStateUpdateMethod,
+                closeSummarizerDelayOverride,
+            }), telemetryDocumentId: this.telemetryDocumentId, groupedBatchingEnabled: this.groupedBatchingEnabled }));
         (0, connectionTelemetry_1.ReportOpPerfTelemetry)(this.context.clientId, this.deltaManager, this.logger);
         (0, batchTracker_1.BindBatchTracker)(this, this.logger);
         this.entryPoint = new common_utils_1.LazyPromise(async () => {
@@ -495,7 +546,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * This object should provide all the functionality that the Container is expected to provide to the loader layer.
      */
     static async loadRuntime(params) {
-        var _a, _b, _c, _d;
+        var _a, _b, _c, _d, _e, _f;
         const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, initializeEntryPoint, } = params;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
@@ -506,7 +557,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 runtimeVersion: packageVersion_1.pkgVersion,
             },
         });
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor = false, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, } = runtimeOptions;
         const registry = new dataStoreRegistry_1.FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             var _a;
@@ -518,11 +569,12 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 return (0, driver_utils_1.readAndParse)(context.storage, blobId);
             }
         };
-        const [chunks, metadata, electedSummarizerData, aliases] = await Promise.all([
+        const [chunks, metadata, electedSummarizerData, aliases, serializedIdCompressor] = await Promise.all([
             tryFetchBlob(summary_1.chunksBlobName),
             tryFetchBlob(summary_1.metadataBlobName),
             tryFetchBlob(summary_1.electedSummarizerBlobName),
             tryFetchBlob(summary_1.aliasBlobName),
+            tryFetchBlob(summary_1.idCompressorBlobName),
         ]);
         const loadExisting = existing === true || context.existing === true;
         // read snapshot blobs needed for BlobManager to load
@@ -554,6 +606,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 }
             }
         }
+        const idCompressorEnabled = (_f = (_e = metadata === null || metadata === void 0 ? void 0 : metadata.idCompressorEnabled) !== null && _e !== void 0 ? _e : runtimeOptions.enableRuntimeIdCompressor) !== null && _f !== void 0 ? _f : false;
+        let idCompressor;
+        if (idCompressorEnabled) {
+            const { IdCompressor, createSessionId } = await Promise.resolve().then(() => __importStar(require("./id-compressor")));
+            idCompressor =
+                serializedIdCompressor !== undefined
+                    ? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
+                    : new IdCompressor(createSessionId(), logger);
+        }
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks !== null && chunks !== void 0 ? chunks : [], aliases !== null && aliases !== void 0 ? aliases : [], {
             summaryOptions,
             gcOptions,
@@ -562,8 +623,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             compressionOptions,
             maxBatchSizeInBytes,
             chunkSizeInBytes,
+            enableRuntimeIdCompressor,
             enableOpReentryCheck,
-        }, containerScope, logger, loadExisting, blobManagerSnapshot, context.storage, requestHandler, undefined, // summaryConfiguration
+            enableGroupedBatching,
+        }, containerScope, logger, loadExisting, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
         initializeEntryPoint);
         // It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
         // at 1, so we won't see a replayed saved op with a sequence number of 0.
@@ -799,12 +862,17 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             summaryNumber: this.nextSummaryNumber++, summaryFormatVersion: 1 }), this.garbageCollector.getMetadata()), { 
             // The last message processed at the time of summary. If there are no new messages, use the message from the
             // last summary.
-            message: (_a = (0, summary_1.extractSummaryMetadataMessage)(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary, telemetryDocumentId: this.telemetryDocumentId });
+            message: (_a = (0, summary_1.extractSummaryMetadataMessage)(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary, telemetryDocumentId: this.telemetryDocumentId, idCompressorEnabled: this.idCompressorEnabled ? true : undefined });
         (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.metadataBlobName, JSON.stringify(metadata));
     }
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         var _a;
         this.addMetadataToSummary(summaryTree);
+        if (this.idCompressorEnabled) {
+            (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
+            const idCompressorState = JSON.stringify(this.idCompressor.serialize(false));
+            (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.idCompressorBlobName, idCompressorState);
+        }
         if (this.remoteMessageProcessor.partialMessages.size > 0) {
             const content = JSON.stringify([...this.remoteMessageProcessor.partialMessages]);
             (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.chunksBlobName, content);
@@ -886,12 +954,28 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // Officially transition from the old state to the new state.
         this.updateDocumentDirtyState(newState);
     }
-    async applyStashedOp(type, op) {
+    /**
+     * Updates the runtime's IdCompressor with the stashed state present in the given op. This is a bit of a
+     * hack and is unnecessarily expensive. As it stands, every locally stashed op (all ops that get stored in
+     * the PendingStateManager) will store their serialized representation locally until ack'd. Upon receiving
+     * this stashed state, the IdCompressor blindly deserializes to the stashed state and assumes the session.
+     * Technically only the last stashed state is needed to do this correctly, but we would have to write some
+     * more hacky code to modify the batch before it gets sent out.
+     * @param content - An IdAllocationOp with "stashedState", which is a representation of un-ack'd local state.
+     */
+    async applyStashedIdAllocationOp(op) {
+        const { IdCompressor } = await Promise.resolve().then(() => __importStar(require("./id-compressor")));
+        this.idCompressor = IdCompressor.deserialize(op.stashedState);
+    }
+    async applyStashedOp(type, contents) {
         switch (type) {
             case ContainerMessageType.FluidDataStoreOp:
-                return this.dataStores.applyStashedOp(op);
+                return this.dataStores.applyStashedOp(contents);
             case ContainerMessageType.Attach:
-                return this.dataStores.applyStashedAttachOp(op);
+                return this.dataStores.applyStashedAttachOp(contents);
+            case ContainerMessageType.IdAllocation:
+                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67b /* IdCompressor should be defined if enabled */);
+                return this.applyStashedIdAllocationOp(contents);
             case ContainerMessageType.Alias:
             case ContainerMessageType.BlobAttach:
                 return;
@@ -970,7 +1054,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         await this.pendingStateManager.applyStashedOpsAt(message.sequenceNumber);
     }
     process(messageArg, local) {
-        var _a;
         this.verifyNotClosed();
         // Whether or not the message is actually a runtime message.
         // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
@@ -978,11 +1061,17 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         const runtimeMessage = messageArg.type === protocol_definitions_1.MessageType.Operation;
         // Do shallow copy of message, as the processing flow will modify it.
         const messageCopy = Object.assign({}, messageArg);
-        const message = this.remoteMessageProcessor.process(messageCopy);
+        for (const message of this.remoteMessageProcessor.process(messageCopy)) {
+            this.processCore(message, local, runtimeMessage);
+        }
+    }
+    processCore(message, local, runtimeMessage) {
+        var _a;
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
         this.scheduleManager.beforeOpProcessing(message);
+        this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
             let localOpMetadata;
             if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
@@ -1007,6 +1096,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 case ContainerMessageType.BlobAttach:
                     this.blobManager.processBlobAttachOp(message, local);
                     break;
+                case ContainerMessageType.IdAllocation:
+                    (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
+                    this.idCompressor.finalizeCreationRange(message.contents);
+                    break;
                 case ContainerMessageType.ChunkedOp:
                 case ContainerMessageType.Rejoin:
                     break;
@@ -1025,8 +1118,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                         throw error;
                     }
             }
-            // For back-compat, notify only about runtime messages for now.
-            if (runtimeMessage) {
+            if (runtimeMessage || this.groupedBatchingEnabled) {
                 this.emit("op", message, runtimeMessage);
             }
             this.scheduleManager.afterOpProcessing(undefined, message);
@@ -1317,17 +1409,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             fullGC,
             runSweep,
         });
-        let gcStats;
-        if (runGC) {
-            gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
+        try {
+            let gcStats;
+            if (runGC) {
+                gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
+            }
+            const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
+            (0, common_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
+            return { stats, summary, gcStats };
+        }
+        finally {
+            this.logger.sendTelemetryEvent({
+                eventName: "SummarizeTelemetry",
+                details: telemetryContext.serialize(),
+            });
         }
-        const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
-        this.logger.sendTelemetryEvent({
-            eventName: "SummarizeTelemetry",
-            details: telemetryContext.serialize(),
-        });
-        (0, common_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
-        return { stats, summary, gcStats };
     }
     /**
      * Before GC runs, called by the garbage collector to update any pending GC state. This is mainly used to notify
@@ -1714,6 +1810,35 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyNotClosed();
         return this.blobManager.createBlob(blob);
     }
+    maybeSubmitIdAllocationOp(type) {
+        var _a, _b;
+        if (type !== ContainerMessageType.IdAllocation) {
+            let idAllocationBatchMessage;
+            let idRange;
+            if (this.idCompressorEnabled) {
+                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
+                idRange = this.idCompressor.takeNextCreationRange();
+                // Don't include the idRange if there weren't any Ids allocated
+                idRange = ((_a = idRange === null || idRange === void 0 ? void 0 : idRange.ids) === null || _a === void 0 ? void 0 : _a.first) !== undefined ? idRange : undefined;
+            }
+            if (idRange !== undefined) {
+                const idAllocationMessage = {
+                    type: ContainerMessageType.IdAllocation,
+                    contents: idRange,
+                };
+                idAllocationBatchMessage = {
+                    contents: JSON.stringify(idAllocationMessage),
+                    deserializedContent: idAllocationMessage,
+                    referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                    metadata: undefined,
+                    localOpMetadata: (_b = this.idCompressor) === null || _b === void 0 ? void 0 : _b.serialize(true),
+                };
+            }
+            if (idAllocationBatchMessage !== undefined) {
+                this.outbox.submit(idAllocationBatchMessage);
+            }
+        }
+    }
     submit(type, contents, localOpMetadata = undefined, metadata = undefined) {
         this.verifyNotClosed();
         this.verifyCanSubmitOps();
@@ -1736,6 +1861,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
         };
         try {
+            // Submit an IdAllocation op if any Ids have been generated since
+            // the last op was submitted. Don't submit another if it's an IdAllocation
+            // op as that means we're in resubmission flow and we don't want to send
+            // IdRanges out of order.
+            this.maybeSubmitIdAllocationOp(type);
             // If this is attach message for new data store, and we are in a batch, send this op out of order
             // Is it safe:
             //    Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
@@ -1872,6 +2002,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 break;
             case ContainerMessageType.Attach:
             case ContainerMessageType.Alias:
+            case ContainerMessageType.IdAllocation:
                 this.submit(type, content, localOpMetadata);
                 break;
             case ContainerMessageType.ChunkedOp:
@@ -1921,6 +2052,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
             }, readAndParseBlob);
+            /**
+             * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
+             * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
+             * these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
+             * change that started fetching latest snapshot always.
+             */
+            if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
+                fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
+                    eventName: "RefreshLatestSummaryAckFetchBackCompat",
+                    ackHandle,
+                    targetSequenceNumber: summaryRefSeq,
+                }, readAndParseBlob, ackHandle);
+            }
             /**
              * If the fetched snapshot is older than the one for which the ack was received, close the container.
              * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -1932,21 +2076,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
              * state.
              */
             if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                /* before failing, let's try to retrieve the latest snapshot for that specific ackHandle */
-                fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
-                    eventName: "RefreshLatestSummaryAckFetch",
+                const error = container_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
                     ackHandle,
-                    targetSequenceNumber: summaryRefSeq,
-                }, readAndParseBlob, ackHandle);
-                if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                    const error = container_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
-                        ackHandle,
-                        summaryRefSeq,
-                        fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
-                    });
-                    this.closeFn(error);
-                    throw error;
-                }
+                    summaryRefSeq,
+                    fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
+                });
+                this.closeFn(error);
+                throw error;
             }
             // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
             // wait for the delta manager to catch up before refreshing the latest Summary.
@@ -1984,7 +2120,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return this.fetchSnapshotFromStorage(logger, event, readAndParseBlob, null /* latest */);
     }
     async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
-        return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
+        var _a;
+        const snapshotResults = await telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = common_utils_1.Trace.start();
             const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? driver_definitions_1.FetchSource.noCache : undefined);
@@ -2003,6 +2140,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 latestSnapshotRefSeq,
             };
         });
+        // We choose to close the summarizer after the snapshot cache is updated to avoid
+        // situations which the main client (which is likely to be re-elected as the leader again)
+        // loads the summarizer from cache.
+        if (this.summaryStateUpdateMethod === "restart") {
+            const error = new container_utils_1.GenericError("Restarting summarizer instead of refreshing");
+            this.mc.logger.sendTelemetryEvent(Object.assign(Object.assign({}, event), { eventName: "ClosingSummarizerOnSummaryStale", codePath: event.eventName, message: "Stopping fetch from storage", versionId: versionId != null ? versionId : undefined, closeSummarizerDelayMs: this.closeSummarizerDelayMs }), error);
+            // Delay 10 seconds before restarting summarizer to prevent the summarizer from restarting too frequently.
+            await (0, common_utils_1.delay)(this.closeSummarizerDelayMs);
+            (_a = this._summarizer) === null || _a === void 0 ? void 0 : _a.stop("latestSummaryStateStale");
+            this.closeFn();
+            throw error;
+        }
+        return snapshotResults;
     }
     notifyAttaching() { } // do nothing (deprecated method)
     getPendingLocalState() {
@@ -2055,6 +2205,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             throw new container_utils_1.UsageError(`"minIdleTime" [${configuration.minIdleTime}] cannot be greater than "maxIdleTime" [${configuration.maxIdleTime}]`);
         }
     }
+    get groupedBatchingEnabled() {
+        const killSwitch = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
+        return killSwitch !== true && this.runtimeOptions.enableGroupedBatching;
+    }
 }
 exports.ContainerRuntime = ContainerRuntime;
 /**