@fluidframework/container-runtime 2.0.0-dev-rc.3.0.0.254866 → 2.0.0-dev-rc.4.0.0.261659

package/dist/containerRuntime.js

@@ -4,7 +4,7 @@
  * Licensed under the MIT License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.defaultPendingOpsRetryDelayMs = exports.defaultPendingOpsWaitTimeoutMs = exports.disabledCompressionConfig = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.InactiveResponseHeaderKey = exports.TombstoneResponseHeaderKey = exports.DefaultSummaryConfiguration = void 0;
+exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.defaultPendingOpsRetryDelayMs = exports.defaultPendingOpsWaitTimeoutMs = exports.disabledCompressionConfig = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.InactiveResponseHeaderKey = exports.TombstoneResponseHeaderKey = exports.DeletedResponseHeaderKey = exports.DefaultSummaryConfiguration = void 0;
 const client_utils_1 = require("@fluid-internal/client-utils");
 const container_definitions_1 = require("@fluidframework/container-definitions");
 const internal_1 = require("@fluidframework/container-definitions/internal");
@@ -61,6 +61,11 @@ exports.DefaultSummaryConfiguration = {
     runtimeOpWeight: 1.0,
     nonRuntimeHeuristicThreshold: 20,
 };
+/**
+ * Error responses when requesting a deleted object will have this header set to true
+ * @alpha
+ */
+exports.DeletedResponseHeaderKey = "wasDeleted";
 /**
  * Tombstone error responses will have this header set to true
  * @alpha
@@ -245,7 +250,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             },
         });
         const mc = (0, internal_7.loggerToMonitoringContext)(logger);
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, explicitSchemaControl = false, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableGroupedBatching = false, explicitSchemaControl = false, } = runtimeOptions;
         const registry = new dataStoreRegistry_js_1.FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = context.baseSnapshot?.blobs[blobName];
@@ -273,9 +278,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         const messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Verify summary runtime sequence number matches protocol sequence number.
         const runtimeSequenceNumber = messageAtLastSummary?.sequenceNumber;
+        const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
         // When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
         if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
-            const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" &&
                 runtimeSequenceNumber !== protocolSequenceNumber) {
@@ -327,7 +332,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 ?.idCompressorMode;
             // This is the only exception to the rule above - we have proper plumbing to load ID compressor on schema change
             // event. It is loaded async (relative to op processing), so this conversion is only safe for off -> delayed conversion!
-            if (idCompressorMode === undefined && desiredIdCompressorMode === "delayed") {
+            // Clients do not expect ID compressor ops unless ID compressor is On for them, and that could be achieved only through
+            // explicit schema change, i.e. only if explicitSchemaControl is on.
+            // Note: it would be better if we throw on combination of options (explicitSchemaControl = off, desiredIdCompressorMode === "delayed")
+            // that is not supported. But our service tests are oblivious to these problems and throwing here will cause a ton of failures
+            // We ignored incompatible ID compressor changes from the start (they were sticky), so that's not a new problem being introduced...
+            if (idCompressorMode === undefined &&
+                desiredIdCompressorMode === "delayed" &&
+                explicitSchemaControl) {
                 idCompressorMode = desiredIdCompressorMode;
             }
         }
@@ -366,7 +378,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             compressionOptions.minimumBatchSizeInBytes !== Infinity &&
             compressionOptions.compressionAlgorithm === "lz4";
         const opGroupingEnabled = disableGroupedBatching !== true && enableGroupedBatching;
-        const documentSchemaController = new index_js_3.DocumentsSchemaController(existing, metadata?.documentSchema, {
+        const documentSchemaController = new index_js_3.DocumentsSchemaController(existing, protocolSequenceNumber, metadata?.documentSchema, {
             explicitSchemaControl,
             compressionLz4,
             idCompressorMode,
@@ -389,7 +401,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             chunkSizeInBytes,
             // Requires<> drops undefined from IdCompressorType
             enableRuntimeIdCompressor: enableRuntimeIdCompressor,
-            enableOpReentryCheck,
             enableGroupedBatching,
             explicitSchemaControl,
         }, containerScope, logger, existing, blobManagerSnapshot, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, undefined);
@@ -428,11 +439,21 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     get attachState() {
         return this._getAttachState();
     }
-    get documentSchema() {
+    /**
+     * Current session schema - defines what options are on & off.
+     * It's overlap of document schema (controlled by summary & ops) and options controlling this session.
+     * For example, document schema might have compression ON, but feature gates / runtime options turn it Off.
+     * In such case it will be off in session schema (i.e. this session should not use compression), but this client
+     * has to deal with compressed ops as other clients might send them.
+     * And in reverse, session schema can have compression Off, but feature gates / runtime options want it On.
+     * In such case it will be off in session schema, however this client will propose change to schema, and once / if
+     * this op rountrips, compression will be On. Client can't send compressed ops until it's change in schema.
+     */
+    get sessionSchema() {
         return this.documentsSchemaController.sessionSchema.runtime;
     }
     get idCompressorMode() {
-        return this.documentSchema.idCompressorMode;
+        return this.sessionSchema.idCompressorMode;
     }
     /**
      * See IContainerRuntimeBase.idCompressor() for details.
@@ -540,13 +561,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         this.flushTaskExists = false;
         this.consecutiveReconnects = 0;
         this.ensureNoDataModelChangesCalls = 0;
-        /**
-         * Tracks the number of detected reentrant ops to report,
-         * in order to self-throttle the telemetry events.
-         *
-         * This should be removed as part of ADO:2322
-         */
-        this.opReentryCallsToReport = 5;
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
         this.defaultTelemetrySignalSampleCount = 100;
@@ -573,7 +587,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // If it's not in the list, then we will need to either use no compression, or fallback to some other (supported by format)
         // compression.
         const compressionOptions = {
-            minimumBatchSizeInBytes: this.documentSchema.compressionLz4
+            minimumBatchSizeInBytes: this.sessionSchema.compressionLz4
                 ? runtimeOptions.compressionOptions.minimumBatchSizeInBytes
                 : Number.POSITIVE_INFINITY,
             compressionAlgorithm: CompressionAlgorithms.lz4,
@@ -657,11 +671,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
-        const disableOpReentryCheck = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck");
-        this.enableOpReentryCheck =
-            runtimeOptions.enableOpReentryCheck === true &&
-                // Allow for a break-glass config to override the options
-                disableOpReentryCheck !== true;
         this.summariesDisabled = this.isSummariesDisabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
@@ -733,7 +742,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             const envelope2 = this.createNewSignalEnvelope(envelope1.address, type, envelope1.contents);
             return this.submitSignalFn(envelope2, targetClientId);
         };
-        this.channelCollection = new channelCollection_js_1.ChannelCollection((0, channelCollection_js_1.getSummaryForDatastores)(baseSnapshot, metadata), parentContext, this.mc.logger, (path, reason, timestampMs, packagePath, request, headerData) => this.garbageCollector.nodeUpdated(path, reason, timestampMs, packagePath, request, headerData), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
+        this.channelCollection = new channelCollection_js_1.ChannelCollection((0, channelCollection_js_1.getSummaryForDatastores)(baseSnapshot, metadata), parentContext, this.mc.logger, (props) => this.garbageCollector.nodeUpdated(props), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
         this.blobManager = new blobManager_js_1.BlobManager({
             routeContext: this.handleContext,
             snapshot: blobManagerSnapshot,
@@ -746,7 +755,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                     });
                 }
             },
-            blobRequested: (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
+            blobRequested: (blobPath) => this.garbageCollector.nodeUpdated({
+                node: { type: "Blob", path: blobPath },
+                reason: "Loaded",
+            }),
             isBlobDeleted: (blobPath) => this.garbageCollector.isNodeDeleted(blobPath),
             runtime: this,
             stashedBlobs: pendingRuntimeState?.pendingAttachmentBlobs,
@@ -794,12 +806,29 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         this._quorum.on("removeMember", (clientId) => {
             this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
         });
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         this._audience = audience;
+        if (audience.getSelf === undefined) {
+            // back-compat, added in 2.0 RC3.
+            // Purpose: deal with cases when we run against old loader that does not have newly added capabilities
+            audience.getSelf = () => {
+                const clientId = this._getClientId();
+                return clientId === undefined
+                    ? undefined
+                    : ({
+                        clientId,
+                        client: audience.getMember(clientId),
+                    });
+            };
+            let oldClientId = this.clientId;
+            this.on("connected", () => {
+                const clientId = this.clientId;
+                (0, internal_2.assert)(clientId !== undefined, "can't be undefined");
+                audience.emit("selfChanged", { clientId: oldClientId }, { clientId, client: audience.getMember(clientId) });
+                oldClientId = clientId;
+            });
+        }
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
-        this.validateSummaryBeforeUpload =
-            this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
         this.summaryCollection = new index_js_3.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
             this.attachState !== container_definitions_1.AttachState.Attached || this.hasPendingMessages();
@@ -872,9 +901,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             options: JSON.stringify(runtimeOptions),
             idCompressorModeMetadata: metadata?.documentSchema?.runtime?.idCompressorMode,
             idCompressorMode: this.idCompressorMode,
+            sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
             featureGates: JSON.stringify({
                 ...featureGatesForTelemetry,
-                disableOpReentryCheck,
                 disableChunking,
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
@@ -882,6 +911,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             }),
             telemetryDocumentId: this.telemetryDocumentId,
             groupedBatchingEnabled: this.groupedBatchingEnabled,
+            initialSequenceNumber: this.deltaManager.initialSequenceNumber,
         });
         (0, connectionTelemetry_js_1.ReportOpPerfTelemetry)(this.clientId, this.deltaManager, this, this.logger);
         (0, batchTracker_js_1.BindBatchTracker)(this, this.logger);
@@ -897,6 +927,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         this.skipSavedCompressorOps = pendingRuntimeState?.pendingIdCompressorState !== undefined;
     }
     onSchemaChange(schema) {
+        this.logger.sendTelemetryEvent({
+            eventName: "SchemaChangeAccept",
+            sessionRuntimeSchema: JSON.stringify(schema),
+        });
         // Most of the settings will be picked up only by new sessions (i.e. after reload).
         // We can make it better in the future (i.e. start to use op compression right away), but for simplicity
         // this is not done.
@@ -929,9 +963,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     async initializeBaseState() {
         if (this.idCompressorMode === "on" ||
             (this.idCompressorMode === "delayed" && this.connected)) {
+            this._idCompressor = await this.createIdCompressor();
             // This is called from loadRuntime(), long before we process any ops, so there should be no ops accumulated yet.
             (0, internal_2.assert)(this.pendingIdCompressorOps.length === 0, 0x8ec /* no pending ops */);
-            this._idCompressor = await this.createIdCompressor();
         }
         await this.garbageCollector.initializeBaseState();
     }
@@ -1309,12 +1343,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             this._loadIdCompressor === undefined) {
             this._loadIdCompressor = this.createIdCompressor()
                 .then((compressor) => {
-                this._idCompressor = compressor;
                 // Finalize any ranges we received while the compressor was turned off.
-                for (const range of this.pendingIdCompressorOps) {
-                    this._idCompressor.finalizeCreationRange(range);
-                }
+                const ops = this.pendingIdCompressorOps;
                 this.pendingIdCompressorOps = [];
+                for (const range of ops) {
+                    compressor.finalizeCreationRange(range);
+                }
+                (0, internal_2.assert)(this.pendingIdCompressorOps.length === 0, "No new ops added");
+                this._idCompressor = compressor;
             })
                 .catch((error) => {
                 this.logger.sendErrorEvent({ eventName: "IdCompressorDelayedLoad" }, error);
@@ -1324,6 +1360,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         return this._loadIdCompressor;
     }
     setConnectionState(connected, clientId) {
+        // Validate we have consistent state
+        const currentClientId = this._audience.getSelf()?.clientId;
+        (0, internal_2.assert)(clientId === currentClientId, "input clientId does not match Audience");
+        (0, internal_2.assert)(this.clientId === currentClientId, "this.clientId does not match Audience");
         if (connected && this.idCompressorMode === "delayed") {
             // eslint-disable-next-line @typescript-eslint/no-floating-promises
             this.loadIdCompressor();
@@ -1405,9 +1445,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // We do not need to make a deep copy. Each layer will just replace message.contents itself,
         // but will not modify the contents object (likely it will replace it on the message).
         const messageCopy = { ...messageArg };
+        const savedOp = messageCopy.metadata?.savedOp;
         for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-            if (modernRuntimeMessage) {
-                this.processCore({
+            const msg = modernRuntimeMessage
+                ? {
                     // Cast it since we expect it to be this based on modernRuntimeMessage computation above.
                     // There is nothing really ensuring that anytime original message.type is Operation that
                     // the result messages will be so. In the end modern bool being true only directs to
@@ -1415,12 +1456,16 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                     message: message,
                     local,
                     modernRuntimeMessage,
-                });
-            }
-            else {
-                // Unrecognized message will be ignored.
-                this.processCore({ message, local, modernRuntimeMessage });
-            }
+                }
+                : // Unrecognized message will be ignored.
+                    {
+                        message,
+                        local,
+                        modernRuntimeMessage,
+                    };
+            msg.savedOp = savedOp;
+            // ensure that we observe any re-entrancy, and if needed, rebase ops
+            this.ensureNoDataModelChanges(() => this.processCore(msg));
         }
     }
     /**
@@ -1438,7 +1483,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             // These calls should be made for all but chunked ops:
             // 1) this.pendingStateManager.processPendingLocalMessage() below
             // 2) this.resetReconnectCount() below
-            (0, internal_2.assert)(message.type !== messageTypes_js_1.ContainerMessageType.ChunkedOp, "we should never get here with chunked ops");
+            (0, internal_2.assert)(message.type !== messageTypes_js_1.ContainerMessageType.ChunkedOp, 0x93b /* we should never get here with chunked ops */);
             let localOpMetadata;
             if (local && messageWithContext.modernRuntimeMessage) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(messageWithContext.message);
@@ -1485,17 +1530,16 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 // stashed ops flow. The compressor is stashed with these ops already processed.
                 // That said, in idCompressorMode === "delayed", we might not serialize ID compressor, and
                 // thus we need to process all the ops.
-                if (!(this.skipSavedCompressorOps &&
-                    messageWithContext.message.metadata?.savedOp ===
-                        true)) {
+                if (!(this.skipSavedCompressorOps && messageWithContext.savedOp === true)) {
                     const range = messageWithContext.message.contents;
                     // Some other client turned on the id compressor. If we have not turned it on,
                     // put it in a pending queue and delay finalization.
                     if (this._idCompressor === undefined) {
-                        (0, internal_2.assert)(this.idCompressorMode !== undefined, "id compressor should be enabled");
+                        (0, internal_2.assert)(this.idCompressorMode !== undefined, 0x93c /* id compressor should be enabled */);
                         this.pendingIdCompressorOps.push(range);
                     }
                     else {
+                        (0, internal_2.assert)(this.pendingIdCompressorOps.length === 0, "there should be no pending ops!");
                         this._idCompressor.finalizeCreationRange(range);
                     }
                 }
@@ -1506,7 +1550,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             case messageTypes_js_1.ContainerMessageType.ChunkedOp:
                 // From observability POV, we should not exppse the rest of the system (including "op" events on object) to these messages.
                 // Also resetReconnectCount() would be wrong - see comment that was there before this change was made.
-                (0, internal_2.assert)(false, "should not even get here");
+                (0, internal_2.assert)(false, 0x93d /* should not even get here */);
             case messageTypes_js_1.ContainerMessageType.Rejoin:
                 break;
             case messageTypes_js_1.ContainerMessageType.DocumentSchemaChange:
@@ -1616,9 +1660,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         let checkpoint;
         let result;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
-            // Note: we are not touching this.pendingAttachBatch here, for two reasons:
-            // 1. It would not help, as we flush attach ops as they become available.
-            // 2. There is no way to undo process of data store creation.
+            // Note: we are not touching any batches other than mainBatch here, for two reasons:
+            // 1. It would not help, as other batches are flushed independently from main batch.
+            // 2. There is no way to undo process of data store creation, blob creation, ID compressor ops, or other things tracked by other batches.
             checkpoint = this.outbox.checkpoint().mainBatch;
         }
         try {
@@ -1681,7 +1725,11 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         if (channel.entryPoint === undefined) {
             throw new internal_7.UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
         }
-        this.garbageCollector.nodeUpdated(`/${internalId}`, "Loaded", undefined /* timestampMs */, context.packagePath);
+        this.garbageCollector.nodeUpdated({
+            node: { type: "DataStore", path: `/${internalId}` },
+            reason: "Loaded",
+            packagePath: context.packagePath,
+        });
         return channel.entryPoint;
     }
     createDetachedDataStore(pkg, loadingGroupId) {
@@ -1805,7 +1853,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // We can finalize any allocated IDs since we're the only client
         const idRange = this._idCompressor?.takeNextCreationRange();
         if (idRange !== undefined) {
-            (0, internal_2.assert)(idRange.ids === undefined || idRange.ids.firstGenCount === 1, "No other ranges should be taken while container is detached.");
+            (0, internal_2.assert)(idRange.ids === undefined || idRange.ids.firstGenCount === 1, 0x93e /* No other ranges should be taken while container is detached. */);
             this._idCompressor?.finalizeCreationRange(idRange);
         }
         const summarizeResult = this.channelCollection.getAttachSummary(telemetryContext);
@@ -2015,10 +2063,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
+        let summaryRefSeqNum;
         const summaryNumberLogger = (0, internal_7.createChildLogger)({
             logger: summaryLogger,
             properties: {
-                all: { summaryNumber },
+                all: {
+                    summaryNumber,
+                    referenceSequenceNumber: () => summaryRefSeqNum,
+                },
             },
         });
         (0, internal_2.assert)(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
@@ -2032,7 +2084,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // If the container is dirty, i.e., there are pending unacked ops, the summary will not be eventual consistent
         // and it may even be incorrect. So, wait for the container to be saved with a timeout. If the container is not
         // saved within the timeout, check if it should be failed or can continue.
-        if (this.validateSummaryBeforeUpload && this.isDirty) {
+        if (this.isDirty) {
             const countBefore = this.pendingMessagesCount;
             // The timeout for waiting for pending ops can be overridden via configurations.
             const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
@@ -2065,7 +2117,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         }
         const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         const shouldValidatePreSummaryState = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.shouldValidatePreSummaryState") === true;
-        let summaryRefSeqNum;
         try {
             await this.deltaManager.inbound.pause();
             if (shouldPauseInboundSignal) {
@@ -2076,9 +2127,17 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
             const lastAck = this.summaryCollection.latestAck;
             const startSummaryResult = this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger, latestSummaryRefSeqNum);
+            /**
+             * This was added to validate that the summarizer node tree has the same reference sequence number from the
+             * top running summarizer down to the lowest summarizer node.
+             *
+             * The order of mismatch numbers goes (validate sequence number)-(node sequence number).
+             * Generally the validate sequence number comes from the running summarizer and the node sequence number comes from the
+             * summarizer nodes.
+             */
             if (startSummaryResult.invalidNodes > 0 ||
                 startSummaryResult.mismatchNumbers.size > 0) {
-                summaryLogger.sendErrorEvent({
+                summaryLogger.sendTelemetryEvent({
                     eventName: "LatestSummaryRefSeqNumMismatch",
                     details: {
                         ...startSummaryResult,
@@ -2090,7 +2149,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                         stage: "base",
                         referenceSequenceNumber: summaryRefSeqNum,
                         minimumSequenceNumber,
-                        error: `Summarizer node state inconsistent with summarizer state.`,
+                        error: new internal_7.LoggingError(`Summarizer node state inconsistent with summarizer state.`),
                     };
                 }
             }
@@ -2133,7 +2192,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
                     minimumSequenceNumber,
-                    error: continueResult.error,
+                    error: new internal_7.LoggingError(continueResult.error),
                 };
             }
             const trace = client_utils_1.Trace.start();
@@ -2150,6 +2209,18 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 });
             }
             catch (error) {
+                return {
+                    stage: "base",
+                    referenceSequenceNumber: summaryRefSeqNum,
+                    minimumSequenceNumber,
+                    error: (0, internal_7.wrapError)(error, (msg) => new internal_7.LoggingError(msg)),
+                };
+            }
+            // Validate that the summary generated by summarizer nodes is correct before uploading.
+            const validateResult = this.summarizerNode.validateSummary();
+            if (!validateResult.success) {
+                const { success, ...loggingProps } = validateResult;
+                const error = new index_js_3.RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
                 return {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
@@ -2157,24 +2228,11 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                     error,
                 };
             }
-            // If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
-            if (this.validateSummaryBeforeUpload) {
-                // Validate that the summaries generated by summarize nodes is correct.
-                const validateResult = this.summarizerNode.validateSummary();
-                if (!validateResult.success) {
-                    const { success, ...loggingProps } = validateResult;
-                    const error = new index_js_3.RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
-                    return {
-                        stage: "base",
-                        referenceSequenceNumber: summaryRefSeqNum,
-                        minimumSequenceNumber,
-                        error,
-                    };
-                }
-                const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
-                if (pendingMessagesFailResult !== undefined) {
-                    return pendingMessagesFailResult;
-                }
+            // If there are pending unacked ops, this summary attempt may fail as the uploaded
+            // summary would be eventually inconsistent.
+            const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
+            if (pendingMessagesFailResult !== undefined) {
+                return pendingMessagesFailResult;
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
             // Now that we have generated the summary, update the message at last summary to the last message processed.
@@ -2207,7 +2265,11 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             };
             continueResult = checkContinue();
             if (!continueResult.continue) {
-                return { stage: "generate", ...generateSummaryData, error: continueResult.error };
+                return {
+                    stage: "generate",
+                    ...generateSummaryData,
+                    error: new internal_7.LoggingError(continueResult.error),
+                };
             }
             const summaryContext = lastAck === undefined
                 ? {
@@ -2225,7 +2287,11 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 handle = await this.storage.uploadSummaryWithContext(summarizeResult.summary, summaryContext);
             }
             catch (error) {
-                return { stage: "generate", ...generateSummaryData, error };
+                return {
+                    stage: "generate",
+                    ...generateSummaryData,
+                    error: (0, internal_7.wrapError)(error, (msg) => new internal_7.LoggingError(msg)),
+                };
             }
             const parent = summaryContext.ackHandle;
             const summaryMessage = {
@@ -2242,14 +2308,22 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             };
             continueResult = checkContinue();
             if (!continueResult.continue) {
-                return { stage: "upload", ...uploadData, error: continueResult.error };
+                return {
+                    stage: "upload",
+                    ...uploadData,
+                    error: new internal_7.LoggingError(continueResult.error),
+                };
             }
             let clientSequenceNumber;
             try {
                 clientSequenceNumber = this.submitSummaryMessage(summaryMessage, summaryRefSeqNum);
             }
             catch (error) {
-                return { stage: "upload", ...uploadData, error };
+                return {
+                    stage: "upload",
+                    ...uploadData,
+                    error: (0, internal_7.wrapError)(error, (msg) => new internal_7.LoggingError(msg)),
+                };
             }
             const submitData = {
                 stage: "submit",
@@ -2258,11 +2332,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 submitOpDuration: trace.trace().duration,
             };
             try {
-                // If validateSummaryBeforeUpload is false, the summary should be validated in this step.
-                this.summarizerNode.completeSummary(handle, !this.validateSummaryBeforeUpload /* validate */);
+                this.summarizerNode.completeSummary(handle);
             }
             catch (error) {
-                return { stage: "upload", ...uploadData, error };
+                return {
+                    stage: "upload",
+                    ...uploadData,
+                    error: (0, internal_7.wrapError)(error, (msg) => new internal_7.LoggingError(msg)),
+                };
             }
             return submitData;
         }
@@ -2370,11 +2447,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     }
     submit(containerRuntimeMessage, localOpMetadata = undefined, metadata) {
         this.verifyNotClosed();
-        this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
         (0, internal_2.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         (0, internal_2.assert)(metadata === undefined ||
-            containerRuntimeMessage.type === messageTypes_js_1.ContainerMessageType.BlobAttach, "metadata");
+            containerRuntimeMessage.type === messageTypes_js_1.ContainerMessageType.BlobAttach, 0x93f /* metadata */);
         const serializedContent = JSON.stringify(containerRuntimeMessage);
         // Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
         // container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
@@ -2406,6 +2482,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 // on this callback to do actual sending.
                 const contents = this.documentsSchemaController.maybeSendSchemaMessage();
                 if (contents) {
+                    this.logger.sendTelemetryEvent({
+                        eventName: "SchemaChangeProposal",
+                        refSeq: contents.refSeq,
+                        version: contents.version,
+                        newRuntimeSchema: JSON.stringify(contents.runtime),
+                        sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
+                        oldRuntimeSchema: JSON.stringify(this.metadata?.documentSchema?.runtime),
+                    });
                     const msg = {
                         type: messageTypes_js_1.ContainerMessageType.DocumentSchemaChange,
                         contents,
@@ -2415,32 +2499,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                         referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
                     });
                 }
-                // If this is attach message for new data store, and we are in a batch, send this op out of order
-                // Is it safe:
-                //    Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
-                //    They become visible only when aliased, or handle to some sub-element of newly created datastore
-                //    is stored in some DDS, i.e. only after some other op.
-                // Why:
-                //    Attach ops are large, and expensive to process. Plus there are scenarios where a lot of new data
-                //    stores are created, causing issues like relay service throttling (too many ops) and catastrophic
-                //    failure (batch is too large). Pushing them earlier and outside of main batch should alleviate
-                //    these issues.
-                // Cons:
-                //    1. With large batches, relay service may throttle clients. Clients may disconnect while throttled.
-                //    This change creates new possibility of a lot of newly created data stores never being referenced
-                //    because client died before it had a change to submit the rest of the ops. This will create more
-                //    garbage that needs to be collected leveraging GC (Garbage Collection) feature.
-                //    2. Sending ops out of order means they are excluded from rollback functionality. This is not an issue
-                //    today as rollback can't undo creation of data store. To some extent not sending them is a bigger
-                //    issue than sending.
-                // Please note that this does not change file format, so it can be disabled in the future if this
-                // optimization no longer makes sense (for example, batch compression may make it less appealing).
-                if (this.currentlyBatching() &&
-                    type === messageTypes_js_1.ContainerMessageType.Attach &&
-                    this.disableAttachReorder !== true) {
-                    this.outbox.submitAttach(message);
-                }
-                else if (type === messageTypes_js_1.ContainerMessageType.BlobAttach) {
+                if (type === messageTypes_js_1.ContainerMessageType.BlobAttach) {
                     // BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
                     this.outbox.submitBlobAttach(message);
                 }
@@ -2515,32 +2574,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             throw new Error("Runtime is closed");
         }
     }
-    verifyCanSubmitOps() {
-        if (this.ensureNoDataModelChangesCalls > 0) {
-            const errorMessage = "Op was submitted from within a `ensureNoDataModelChanges` callback";
-            if (this.opReentryCallsToReport > 0) {
-                this.mc.logger.sendTelemetryEvent({ eventName: "OpReentry" }, 
-                // We need to capture the call stack in order to inspect the source of this usage pattern
-                (0, index_js_2.getLongStack)(() => new internal_7.UsageError(errorMessage)));
-                this.opReentryCallsToReport--;
-            }
-            // Creating ops while processing ops can lead
-            // to undefined behavior and events observed in the wrong order.
-            // For example, we have two callbacks registered for a DDS, A and B.
-            // Then if on change #1 callback A creates change #2, the invocation flow will be:
-            //
-            // A because of #1
-            // A because of #2
-            // B because of #2
-            // B because of #1
-            //
-            // The runtime must enforce op coherence by not allowing ops to be submitted
-            // while ops are being processed.
-            if (this.enableOpReentryCheck) {
-                throw new internal_7.UsageError(errorMessage);
-            }
-        }
-    }
     reSubmitBatch(batch) {
         this.orderSequentially(() => {
             for (const message of batch) {
@@ -2670,7 +2703,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         await this.closeStaleSummarizer();
         return {
             stage: "base",
-            error: "summary state stale - Unsupported option 'refreshLatestAck'",
+            error: new internal_7.LoggingError("summary state stale - Unsupported option 'refreshLatestAck'"),
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
             minimumSequenceNumber: this.deltaManager.minimumSequenceNumber,
         };
@@ -2714,16 +2747,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         }
         this.imminentClosure || (this.imminentClosure = props?.notifyImminentClosure ?? false);
         const getSyncState = (pendingAttachmentBlobs) => {
-            const pending = this.pendingStateManager.getLocalState();
-            if (pendingAttachmentBlobs === undefined && !this.hasPendingMessages()) {
-                return; // no pending state to save
-            }
+            const pending = this.pendingStateManager.getLocalState(props?.snapshotSequenceNumber);
+            const sessionExpiryTimerStarted = props?.sessionExpiryTimerStarted ?? this.garbageCollector.sessionExpiryTimerStarted;
             const pendingIdCompressorState = this._idCompressor?.serialize(true);
             return {
                 pending,
                 pendingIdCompressorState,
                 pendingAttachmentBlobs,
-                sessionExpiryTimerStarted: this.garbageCollector.sessionExpiryTimerStarted,
+                sessionExpiryTimerStarted,
             };
         };
         const perfEvent = {
@@ -2793,7 +2824,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         }
     }
     get groupedBatchingEnabled() {
-        return this.documentSchema.opGroupingEnabled === true;
+        return this.sessionSchema.opGroupingEnabled === true;
     }
 }
 exports.ContainerRuntime = ContainerRuntime;