@fluidframework/container-runtime 2.0.0-dev-rc.3.0.0.254866 → 2.0.0-dev-rc.5.0.0.263932

package/lib/containerRuntime.js

@@ -22,10 +22,10 @@ import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
 import { ContainerFluidHandleContext } from "./containerHandleContext.js";
 import { channelToDataStore } from "./dataStore.js";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
-import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy.js";
+import { DeltaManagerPendingOpsProxy, DeltaManagerSummarizerProxy } from "./deltaManagerProxies.js";
 import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
 import { ContainerMessageType, } from "./messageTypes.js";
-import { OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, getLongStack, } from "./opLifecycle/index.js";
+import { OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
 import { pkgVersion } from "./packageVersion.js";
 import { PendingStateManager, } from "./pendingStateManager.js";
 import { ScheduleManager } from "./scheduleManager.js";
@@ -60,6 +60,11 @@ export const DefaultSummaryConfiguration = {
     runtimeOpWeight: 1.0,
     nonRuntimeHeuristicThreshold: 20,
 };
+/**
+ * Error responses when requesting a deleted object will have this header set to true
+ * @alpha
+ */
+export const DeletedResponseHeaderKey = "wasDeleted";
 /**
  * Tombstone error responses will have this header set to true
  * @alpha
@@ -241,7 +246,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             },
         });
         const mc = loggerToMonitoringContext(logger);
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, explicitSchemaControl = false, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableGroupedBatching = true, explicitSchemaControl = false, } = runtimeOptions;
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = context.baseSnapshot?.blobs[blobName];
@@ -269,9 +274,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         const messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Verify summary runtime sequence number matches protocol sequence number.
         const runtimeSequenceNumber = messageAtLastSummary?.sequenceNumber;
+        const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
         // When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
         if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
-            const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" &&
                 runtimeSequenceNumber !== protocolSequenceNumber) {
@@ -323,7 +328,14 @@ export class ContainerRuntime extends TypedEventEmitter {
                 ?.idCompressorMode;
             // This is the only exception to the rule above - we have proper plumbing to load ID compressor on schema change
             // event. It is loaded async (relative to op processing), so this conversion is only safe for off -> delayed conversion!
-            if (idCompressorMode === undefined && desiredIdCompressorMode === "delayed") {
+            // Clients do not expect ID compressor ops unless ID compressor is On for them, and that could be achieved only through
+            // explicit schema change, i.e. only if explicitSchemaControl is on.
+            // Note: it would be better if we throw on combination of options (explicitSchemaControl = off, desiredIdCompressorMode === "delayed")
+            // that is not supported. But our service tests are oblivious to these problems and throwing here will cause a ton of failures
+            // We ignored incompatible ID compressor changes from the start (they were sticky), so that's not a new problem being introduced...
+            if (idCompressorMode === undefined &&
+                desiredIdCompressorMode === "delayed" &&
+                explicitSchemaControl) {
                 idCompressorMode = desiredIdCompressorMode;
             }
         }
@@ -356,23 +368,20 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return createIdCompressor(compressorLogger);
             }
         };
-        const disableGroupedBatching = mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
         const disableCompression = mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
         const compressionLz4 = disableCompression !== true &&
             compressionOptions.minimumBatchSizeInBytes !== Infinity &&
             compressionOptions.compressionAlgorithm === "lz4";
-        const opGroupingEnabled = disableGroupedBatching !== true && enableGroupedBatching;
-        const documentSchemaController = new DocumentsSchemaController(existing, metadata?.documentSchema, {
+        const documentSchemaController = new DocumentsSchemaController(existing, protocolSequenceNumber, metadata?.documentSchema, {
             explicitSchemaControl,
             compressionLz4,
             idCompressorMode,
-            opGroupingEnabled,
+            opGroupingEnabled: enableGroupedBatching,
             disallowedVersions: [],
         }, (schema) => {
             runtime.onSchemaChange(schema);
         });
         const featureGatesForTelemetry = {
-            disableGroupedBatching,
             disableCompression,
         };
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
@@ -385,7 +394,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             chunkSizeInBytes,
             // Requires<> drops undefined from IdCompressorType
             enableRuntimeIdCompressor: enableRuntimeIdCompressor,
-            enableOpReentryCheck,
             enableGroupedBatching,
             explicitSchemaControl,
         }, containerScope, logger, existing, blobManagerSnapshot, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, undefined);
@@ -424,11 +432,21 @@ export class ContainerRuntime extends TypedEventEmitter {
     get attachState() {
         return this._getAttachState();
     }
-    get documentSchema() {
+    /**
+     * Current session schema - defines what options are on & off.
+     * It's overlap of document schema (controlled by summary & ops) and options controlling this session.
+     * For example, document schema might have compression ON, but feature gates / runtime options turn it Off.
+     * In such case it will be off in session schema (i.e. this session should not use compression), but this client
+     * has to deal with compressed ops as other clients might send them.
+     * And in reverse, session schema can have compression Off, but feature gates / runtime options want it On.
+     * In such case it will be off in session schema, however this client will propose change to schema, and once / if
+     * this op rountrips, compression will be On. Client can't send compressed ops until it's change in schema.
+     */
+    get sessionSchema() {
         return this.documentsSchemaController.sessionSchema.runtime;
     }
     get idCompressorMode() {
-        return this.documentSchema.idCompressorMode;
+        return this.sessionSchema.idCompressorMode;
     }
     /**
      * See IContainerRuntimeBase.idCompressor() for details.
@@ -536,13 +554,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.flushTaskExists = false;
         this.consecutiveReconnects = 0;
         this.ensureNoDataModelChangesCalls = 0;
-        /**
-         * Tracks the number of detected reentrant ops to report,
-         * in order to self-throttle the telemetry events.
-         *
-         * This should be removed as part of ADO:2322
-         */
-        this.opReentryCallsToReport = 5;
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
         this.defaultTelemetrySignalSampleCount = 100;
@@ -559,7 +570,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.snapshotCacheForLoadingGroupIds = new PromiseCache({
             expiry: { policy: "absolute", durationMs: 60000 },
         });
-        const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
+        const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, snapshotWithContents, } = context;
         this.mc = createChildMonitoringContext({
             logger: this.logger,
             namespace: "ContainerRuntime",
@@ -569,13 +580,12 @@ export class ContainerRuntime extends TypedEventEmitter {
         // If it's not in the list, then we will need to either use no compression, or fallback to some other (supported by format)
         // compression.
         const compressionOptions = {
-            minimumBatchSizeInBytes: this.documentSchema.compressionLz4
+            minimumBatchSizeInBytes: this.sessionSchema.compressionLz4
                 ? runtimeOptions.compressionOptions.minimumBatchSizeInBytes
                 : Number.POSITIVE_INFINITY,
             compressionAlgorithm: CompressionAlgorithms.lz4,
         };
         this.innerDeltaManager = deltaManager;
-        this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
         // Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
         // This makes ContainerRuntime the final gatekeeper for outgoing messages.
         this.submitFn = submitFn;
@@ -649,15 +659,36 @@ export class ContainerRuntime extends TypedEventEmitter {
         }, this.mc.logger);
         const opSplitter = new OpSplitter(chunks, this.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
         this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor(this.mc.logger), opGroupingManager);
+        const pendingRuntimeState = pendingLocalState;
+        this.pendingStateManager = new PendingStateManager({
+            applyStashedOp: this.applyStashedOp.bind(this),
+            clientId: () => this.clientId,
+            close: this.closeFn,
+            connected: () => this.connected,
+            reSubmit: (message) => {
+                this.reSubmit(message);
+                this.flush();
+            },
+            reSubmitBatch: this.reSubmitBatch.bind(this),
+            isActiveConnection: () => this.innerDeltaManager.active,
+            isAttached: () => this.attachState !== AttachState.Detached,
+        }, pendingRuntimeState?.pending, this.logger);
+        let outerDeltaManager;
+        const useDeltaManagerOpsProxy = this.mc.config.getBoolean("Fluid.ContainerRuntime.DeltaManagerOpsProxy") !== false;
+        // The summarizerDeltaManager Proxy is used to lie to the summarizer to convince it is in the right state as a summarizer client.
+        const summarizerDeltaManagerProxy = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
+        outerDeltaManager = summarizerDeltaManagerProxy;
+        // The DeltaManagerPendingOpsProxy is used to control the minimum sequence number
+        // It allows us to lie to the layers below so that they can maintain enough local state for rebasing ops.
+        if (useDeltaManagerOpsProxy) {
+            const pendingOpsDeltaManagerProxy = new DeltaManagerPendingOpsProxy(summarizerDeltaManagerProxy, this.pendingStateManager);
+            outerDeltaManager = pendingOpsDeltaManagerProxy;
+        }
+        this.deltaManager = outerDeltaManager;
         this.handleContext = new ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
-        const disableOpReentryCheck = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck");
-        this.enableOpReentryCheck =
-            runtimeOptions.enableOpReentryCheck === true &&
-                // Allow for a break-glass config to override the options
-                disableOpReentryCheck !== true;
         this.summariesDisabled = this.isSummariesDisabled();
         this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
         this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
@@ -673,7 +704,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         else {
             this._flushMode = runtimeOptions.flushMode;
         }
-        const pendingRuntimeState = pendingLocalState;
         if (context.attachState === AttachState.Attached) {
             const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
             if (maxSnapshotCacheDurationMs !== undefined &&
@@ -729,7 +759,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             const envelope2 = this.createNewSignalEnvelope(envelope1.address, type, envelope1.contents);
             return this.submitSignalFn(envelope2, targetClientId);
         };
-        this.channelCollection = new ChannelCollection(getSummaryForDatastores(baseSnapshot, metadata), parentContext, this.mc.logger, (path, reason, timestampMs, packagePath, request, headerData) => this.garbageCollector.nodeUpdated(path, reason, timestampMs, packagePath, request, headerData), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
+        let snapshot = getSummaryForDatastores(baseSnapshot, metadata);
+        if (snapshot !== undefined && snapshotWithContents !== undefined) {
+            snapshot = {
+                ...snapshotWithContents,
+                snapshotTree: snapshot,
+            };
+        }
+        this.channelCollection = new ChannelCollection(snapshot, parentContext, this.mc.logger, (props) => this.garbageCollector.nodeUpdated(props), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
         this.blobManager = new BlobManager({
             routeContext: this.handleContext,
             snapshot: blobManagerSnapshot,
@@ -742,26 +779,16 @@ export class ContainerRuntime extends TypedEventEmitter {
                     });
                 }
             },
-            blobRequested: (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
+            blobRequested: (blobPath) => this.garbageCollector.nodeUpdated({
+                node: { type: "Blob", path: blobPath },
+                reason: "Loaded",
+            }),
             isBlobDeleted: (blobPath) => this.garbageCollector.isNodeDeleted(blobPath),
             runtime: this,
             stashedBlobs: pendingRuntimeState?.pendingAttachmentBlobs,
             closeContainer: (error) => this.closeFn(error),
         });
         this.scheduleManager = new ScheduleManager(this.innerDeltaManager, this, () => this.clientId, createChildLogger({ logger: this.logger, namespace: "ScheduleManager" }));
-        this.pendingStateManager = new PendingStateManager({
-            applyStashedOp: this.applyStashedOp.bind(this),
-            clientId: () => this.clientId,
-            close: this.closeFn,
-            connected: () => this.connected,
-            reSubmit: (message) => {
-                this.reSubmit(message);
-                this.flush();
-            },
-            reSubmitBatch: this.reSubmitBatch.bind(this),
-            isActiveConnection: () => this.innerDeltaManager.active,
-            isAttached: () => this.attachState !== AttachState.Detached,
-        }, pendingRuntimeState?.pending, this.logger);
         const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
         const legacySendBatchFn = makeLegacySendBatchFn(this.submitFn, this.innerDeltaManager);
         this.outbox = new Outbox({
@@ -790,12 +817,29 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._quorum.on("removeMember", (clientId) => {
             this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
         });
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         this._audience = audience;
+        if (audience.getSelf === undefined) {
+            // back-compat, added in 2.0 RC3.
+            // Purpose: deal with cases when we run against old loader that does not have newly added capabilities
+            audience.getSelf = () => {
+                const clientId = this._getClientId();
+                return clientId === undefined
+                    ? undefined
+                    : ({
+                        clientId,
+                        client: audience.getMember(clientId),
+                    });
+            };
+            let oldClientId = this.clientId;
+            this.on("connected", () => {
+                const clientId = this.clientId;
+                assert(clientId !== undefined, 0x975 /* can't be undefined */);
+                audience.emit("selfChanged", { clientId: oldClientId }, { clientId, client: audience.getMember(clientId) });
+                oldClientId = clientId;
+            });
+        }
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
-        this.validateSummaryBeforeUpload =
-            this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
             this.attachState !== AttachState.Attached || this.hasPendingMessages();
@@ -868,9 +912,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             options: JSON.stringify(runtimeOptions),
             idCompressorModeMetadata: metadata?.documentSchema?.runtime?.idCompressorMode,
             idCompressorMode: this.idCompressorMode,
+            sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
             featureGates: JSON.stringify({
                 ...featureGatesForTelemetry,
-                disableOpReentryCheck,
                 disableChunking,
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
@@ -878,6 +922,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             }),
             telemetryDocumentId: this.telemetryDocumentId,
             groupedBatchingEnabled: this.groupedBatchingEnabled,
+            initialSequenceNumber: this.deltaManager.initialSequenceNumber,
         });
         ReportOpPerfTelemetry(this.clientId, this.deltaManager, this, this.logger);
         BindBatchTracker(this, this.logger);
@@ -893,6 +938,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.skipSavedCompressorOps = pendingRuntimeState?.pendingIdCompressorState !== undefined;
     }
     onSchemaChange(schema) {
+        this.logger.sendTelemetryEvent({
+            eventName: "SchemaChangeAccept",
+            sessionRuntimeSchema: JSON.stringify(schema),
+        });
         // Most of the settings will be picked up only by new sessions (i.e. after reload).
         // We can make it better in the future (i.e. start to use op compression right away), but for simplicity
         // this is not done.
@@ -925,9 +974,9 @@ export class ContainerRuntime extends TypedEventEmitter {
     async initializeBaseState() {
         if (this.idCompressorMode === "on" ||
             (this.idCompressorMode === "delayed" && this.connected)) {
+            this._idCompressor = await this.createIdCompressor();
             // This is called from loadRuntime(), long before we process any ops, so there should be no ops accumulated yet.
             assert(this.pendingIdCompressorOps.length === 0, 0x8ec /* no pending ops */);
-            this._idCompressor = await this.createIdCompressor();
         }
         await this.garbageCollector.initializeBaseState();
     }
@@ -1225,6 +1274,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.emitDirtyDocumentEvent = false;
         let newState;
         try {
+            this.submitIdAllocationOpIfNeeded(true);
             // replay the ops
             this.pendingStateManager.replayPendingStates();
         }
@@ -1272,8 +1322,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return;
             case ContainerMessageType.BlobAttach:
                 return;
-            case ContainerMessageType.ChunkedOp:
-                throw new Error("chunkedOp not expected here");
             case ContainerMessageType.Rejoin:
                 throw new Error("rejoin not expected here");
             case ContainerMessageType.GC:
@@ -1285,7 +1333,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // e.g. if an app rolled back its container version
                 const compatBehavior = opContents.compatDetails?.behavior;
                 if (!compatBehaviorAllowsMessageType(opContents.type, compatBehavior)) {
-                    const error = DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
+                    const error = DataProcessingError.create("Stashed runtime message of unexpected type", "applyStashedOp", undefined /* sequencedMessage */, {
                         messageDetails: JSON.stringify({
                             type: opContents.type,
                             compatBehavior,
@@ -1305,12 +1353,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             this._loadIdCompressor === undefined) {
             this._loadIdCompressor = this.createIdCompressor()
                 .then((compressor) => {
-                this._idCompressor = compressor;
                 // Finalize any ranges we received while the compressor was turned off.
-                for (const range of this.pendingIdCompressorOps) {
-                    this._idCompressor.finalizeCreationRange(range);
-                }
+                const ops = this.pendingIdCompressorOps;
                 this.pendingIdCompressorOps = [];
+                for (const range of ops) {
+                    compressor.finalizeCreationRange(range);
+                }
+                assert(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
+                this._idCompressor = compressor;
             })
                 .catch((error) => {
                 this.logger.sendErrorEvent({ eventName: "IdCompressorDelayedLoad" }, error);
@@ -1320,6 +1370,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this._loadIdCompressor;
     }
     setConnectionState(connected, clientId) {
+        // Validate we have consistent state
+        const currentClientId = this._audience.getSelf()?.clientId;
+        assert(clientId === currentClientId, 0x977 /* input clientId does not match Audience */);
+        assert(this.clientId === currentClientId, 0x978 /* this.clientId does not match Audience */);
         if (connected && this.idCompressorMode === "delayed") {
             // eslint-disable-next-line @typescript-eslint/no-floating-promises
             this.loadIdCompressor();
@@ -1401,9 +1455,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         // We do not need to make a deep copy. Each layer will just replace message.contents itself,
         // but will not modify the contents object (likely it will replace it on the message).
         const messageCopy = { ...messageArg };
+        const savedOp = messageCopy.metadata?.savedOp;
         for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-            if (modernRuntimeMessage) {
-                this.processCore({
+            const msg = modernRuntimeMessage
+                ? {
                     // Cast it since we expect it to be this based on modernRuntimeMessage computation above.
                     // There is nothing really ensuring that anytime original message.type is Operation that
                     // the result messages will be so. In the end modern bool being true only directs to
@@ -1411,12 +1466,16 @@ export class ContainerRuntime extends TypedEventEmitter {
                     message: message,
                     local,
                     modernRuntimeMessage,
-                });
-            }
-            else {
-                // Unrecognized message will be ignored.
-                this.processCore({ message, local, modernRuntimeMessage });
-            }
+                }
+                : // Unrecognized message will be ignored.
+                    {
+                        message,
+                        local,
+                        modernRuntimeMessage,
+                    };
+            msg.savedOp = savedOp;
+            // ensure that we observe any re-entrancy, and if needed, rebase ops
+            this.ensureNoDataModelChanges(() => this.processCore(msg));
         }
     }
     /**
@@ -1424,6 +1483,13 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     processCore(messageWithContext) {
         const { message, local } = messageWithContext;
+        // Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
+        // Sequence numbers are not guaranteed to follow any sort of order. Re-entrancy is one of those situations
+        if (this.deltaManager.minimumSequenceNumber <
+            messageWithContext.message.minimumSequenceNumber) {
+            messageWithContext.message.minimumSequenceNumber =
+                this.deltaManager.minimumSequenceNumber;
+        }
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
@@ -1434,7 +1500,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // These calls should be made for all but chunked ops:
             // 1) this.pendingStateManager.processPendingLocalMessage() below
             // 2) this.resetReconnectCount() below
-            assert(message.type !== ContainerMessageType.ChunkedOp, "we should never get here with chunked ops");
+            assert(message.type !== ContainerMessageType.ChunkedOp, 0x93b /* we should never get here with chunked ops */);
             let localOpMetadata;
             if (local && messageWithContext.modernRuntimeMessage) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(messageWithContext.message);
@@ -1481,17 +1547,16 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // stashed ops flow. The compressor is stashed with these ops already processed.
                 // That said, in idCompressorMode === "delayed", we might not serialize ID compressor, and
                 // thus we need to process all the ops.
-                if (!(this.skipSavedCompressorOps &&
-                    messageWithContext.message.metadata?.savedOp ===
-                        true)) {
+                if (!(this.skipSavedCompressorOps && messageWithContext.savedOp === true)) {
                     const range = messageWithContext.message.contents;
                     // Some other client turned on the id compressor. If we have not turned it on,
                     // put it in a pending queue and delay finalization.
                     if (this._idCompressor === undefined) {
-                        assert(this.idCompressorMode !== undefined, "id compressor should be enabled");
+                        assert(this.idCompressorMode !== undefined, 0x93c /* id compressor should be enabled */);
                         this.pendingIdCompressorOps.push(range);
                     }
                     else {
+                        assert(this.pendingIdCompressorOps.length === 0, 0x979 /* there should be no pending ops! */);
                         this._idCompressor.finalizeCreationRange(range);
                     }
                 }
@@ -1502,7 +1567,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.ChunkedOp:
                 // From observability POV, we should not exppse the rest of the system (including "op" events on object) to these messages.
                 // Also resetReconnectCount() would be wrong - see comment that was there before this change was made.
-                assert(false, "should not even get here");
+                assert(false, 0x93d /* should not even get here */);
             case ContainerMessageType.Rejoin:
                 break;
             case ContainerMessageType.DocumentSchemaChange:
@@ -1612,9 +1677,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         let checkpoint;
         let result;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
-            // Note: we are not touching this.pendingAttachBatch here, for two reasons:
-            // 1. It would not help, as we flush attach ops as they become available.
-            // 2. There is no way to undo process of data store creation.
+            // Note: we are not touching any batches other than mainBatch here, for two reasons:
+            // 1. It would not help, as other batches are flushed independently from main batch.
+            // 2. There is no way to undo process of data store creation, blob creation, ID compressor ops, or other things tracked by other batches.
             checkpoint = this.outbox.checkpoint().mainBatch;
         }
         try {
@@ -1677,7 +1742,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (channel.entryPoint === undefined) {
             throw new UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
         }
-        this.garbageCollector.nodeUpdated(`/${internalId}`, "Loaded", undefined /* timestampMs */, context.packagePath);
+        this.garbageCollector.nodeUpdated({
+            node: { type: "DataStore", path: `/${internalId}` },
+            reason: "Loaded",
+            packagePath: context.packagePath,
+        });
         return channel.entryPoint;
     }
     createDetachedDataStore(pkg, loadingGroupId) {
@@ -1801,7 +1870,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // We can finalize any allocated IDs since we're the only client
         const idRange = this._idCompressor?.takeNextCreationRange();
         if (idRange !== undefined) {
-            assert(idRange.ids === undefined || idRange.ids.firstGenCount === 1, "No other ranges should be taken while container is detached.");
+            assert(idRange.ids === undefined || idRange.ids.firstGenCount === 1, 0x93e /* No other ranges should be taken while container is detached. */);
             this._idCompressor?.finalizeCreationRange(idRange);
         }
         const summarizeResult = this.channelCollection.getAttachSummary(telemetryContext);
@@ -1848,7 +1917,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             return { stats, summary };
         }
         finally {
-            this.mc.logger.sendTelemetryEvent({
+            summaryLogger.sendTelemetryEvent({
                 eventName: "SummarizeTelemetry",
                 details: telemetryContext.serialize(),
             });
@@ -2011,10 +2080,14 @@ export class ContainerRuntime extends TypedEventEmitter {
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
+        let summaryRefSeqNum;
         const summaryNumberLogger = createChildLogger({
             logger: summaryLogger,
             properties: {
-                all: { summaryNumber },
+                all: {
+                    summaryNumber,
+                    referenceSequenceNumber: () => summaryRefSeqNum,
+                },
             },
         });
         assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
@@ -2028,7 +2101,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // If the container is dirty, i.e., there are pending unacked ops, the summary will not be eventual consistent
         // and it may even be incorrect. So, wait for the container to be saved with a timeout. If the container is not
         // saved within the timeout, check if it should be failed or can continue.
-        if (this.validateSummaryBeforeUpload && this.isDirty) {
+        if (this.isDirty) {
             const countBefore = this.pendingMessagesCount;
             // The timeout for waiting for pending ops can be overridden via configurations.
             const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
@@ -2061,7 +2134,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         const shouldValidatePreSummaryState = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.shouldValidatePreSummaryState") === true;
-        let summaryRefSeqNum;
         try {
             await this.deltaManager.inbound.pause();
             if (shouldPauseInboundSignal) {
@@ -2072,9 +2144,17 @@ export class ContainerRuntime extends TypedEventEmitter {
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
             const lastAck = this.summaryCollection.latestAck;
             const startSummaryResult = this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger, latestSummaryRefSeqNum);
+            /**
+             * This was added to validate that the summarizer node tree has the same reference sequence number from the
+             * top running summarizer down to the lowest summarizer node.
+             *
+             * The order of mismatch numbers goes (validate sequence number)-(node sequence number).
+             * Generally the validate sequence number comes from the running summarizer and the node sequence number comes from the
+             * summarizer nodes.
+             */
             if (startSummaryResult.invalidNodes > 0 ||
                 startSummaryResult.mismatchNumbers.size > 0) {
-                summaryLogger.sendErrorEvent({
+                summaryLogger.sendTelemetryEvent({
                     eventName: "LatestSummaryRefSeqNumMismatch",
                     details: {
                         ...startSummaryResult,
@@ -2086,7 +2166,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                         stage: "base",
                         referenceSequenceNumber: summaryRefSeqNum,
                         minimumSequenceNumber,
-                        error: `Summarizer node state inconsistent with summarizer state.`,
+                        error: new LoggingError(`Summarizer node state inconsistent with summarizer state.`),
                     };
                 }
             }
@@ -2129,7 +2209,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
                     minimumSequenceNumber,
-                    error: continueResult.error,
+                    error: new LoggingError(continueResult.error),
                 };
             }
             const trace = Trace.start();
@@ -2146,6 +2226,18 @@ export class ContainerRuntime extends TypedEventEmitter {
                 });
             }
             catch (error) {
+                return {
+                    stage: "base",
+                    referenceSequenceNumber: summaryRefSeqNum,
+                    minimumSequenceNumber,
+                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                };
+            }
+            // Validate that the summary generated by summarizer nodes is correct before uploading.
+            const validateResult = this.summarizerNode.validateSummary();
+            if (!validateResult.success) {
+                const { success, ...loggingProps } = validateResult;
+                const error = new RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
                 return {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
@@ -2153,24 +2245,11 @@ export class ContainerRuntime extends TypedEventEmitter {
                     error,
                 };
             }
-            // If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
-            if (this.validateSummaryBeforeUpload) {
-                // Validate that the summaries generated by summarize nodes is correct.
-                const validateResult = this.summarizerNode.validateSummary();
-                if (!validateResult.success) {
-                    const { success, ...loggingProps } = validateResult;
-                    const error = new RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
-                    return {
-                        stage: "base",
-                        referenceSequenceNumber: summaryRefSeqNum,
-                        minimumSequenceNumber,
-                        error,
-                    };
-                }
-                const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
-                if (pendingMessagesFailResult !== undefined) {
-                    return pendingMessagesFailResult;
-                }
+            // If there are pending unacked ops, this summary attempt may fail as the uploaded
+            // summary would be eventually inconsistent.
+            const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
+            if (pendingMessagesFailResult !== undefined) {
+                return pendingMessagesFailResult;
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
             // Now that we have generated the summary, update the message at last summary to the last message processed.
@@ -2203,7 +2282,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             };
             continueResult = checkContinue();
             if (!continueResult.continue) {
-                return { stage: "generate", ...generateSummaryData, error: continueResult.error };
+                return {
+                    stage: "generate",
+                    ...generateSummaryData,
+                    error: new LoggingError(continueResult.error),
+                };
             }
             const summaryContext = lastAck === undefined
                 ? {
@@ -2221,7 +2304,11 @@ export class ContainerRuntime extends TypedEventEmitter {
                 handle = await this.storage.uploadSummaryWithContext(summarizeResult.summary, summaryContext);
             }
             catch (error) {
-                return { stage: "generate", ...generateSummaryData, error };
+                return {
+                    stage: "generate",
+                    ...generateSummaryData,
+                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                };
             }
             const parent = summaryContext.ackHandle;
             const summaryMessage = {
@@ -2238,14 +2325,22 @@ export class ContainerRuntime extends TypedEventEmitter {
             };
             continueResult = checkContinue();
             if (!continueResult.continue) {
-                return { stage: "upload", ...uploadData, error: continueResult.error };
+                return {
+                    stage: "upload",
+                    ...uploadData,
+                    error: new LoggingError(continueResult.error),
+                };
             }
             let clientSequenceNumber;
             try {
                 clientSequenceNumber = this.submitSummaryMessage(summaryMessage, summaryRefSeqNum);
             }
             catch (error) {
-                return { stage: "upload", ...uploadData, error };
+                return {
+                    stage: "upload",
+                    ...uploadData,
+                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                };
             }
             const submitData = {
                 stage: "submit",
@@ -2254,11 +2349,14 @@ export class ContainerRuntime extends TypedEventEmitter {
                 submitOpDuration: trace.trace().duration,
             };
             try {
-                // If validateSummaryBeforeUpload is false, the summary should be validated in this step.
-                this.summarizerNode.completeSummary(handle, !this.validateSummaryBeforeUpload /* validate */);
+                this.summarizerNode.completeSummary(handle);
             }
             catch (error) {
-                return { stage: "upload", ...uploadData, error };
+                return {
+                    stage: "upload",
+                    ...uploadData,
+                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                };
             }
             return submitData;
         }
@@ -2347,9 +2445,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.verifyNotClosed();
         return this.blobManager.createBlob(blob, signal);
     }
-    submitIdAllocationOpIfNeeded() {
+    submitIdAllocationOpIfNeeded(resubmitOutstandingRanges = false) {
         if (this._idCompressor) {
-            const idRange = this._idCompressor.takeNextCreationRange();
+            const idRange = resubmitOutstandingRanges
+                ? this.idCompressor?.takeUnfinalizedCreationRange()
+                : this._idCompressor.takeNextCreationRange();
             // Don't include the idRange if there weren't any Ids allocated
             if (idRange?.ids !== undefined) {
                 const idAllocationMessage = {
@@ -2366,11 +2466,10 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     submit(containerRuntimeMessage, localOpMetadata = undefined, metadata) {
         this.verifyNotClosed();
-        this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
         assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         assert(metadata === undefined ||
-            containerRuntimeMessage.type === ContainerMessageType.BlobAttach, "metadata");
+            containerRuntimeMessage.type === ContainerMessageType.BlobAttach, 0x93f /* metadata */);
         const serializedContent = JSON.stringify(containerRuntimeMessage);
         // Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
         // container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
@@ -2402,6 +2501,14 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // on this callback to do actual sending.
                 const contents = this.documentsSchemaController.maybeSendSchemaMessage();
                 if (contents) {
+                    this.logger.sendTelemetryEvent({
+                        eventName: "SchemaChangeProposal",
+                        refSeq: contents.refSeq,
+                        version: contents.version,
+                        newRuntimeSchema: JSON.stringify(contents.runtime),
+                        sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
+                        oldRuntimeSchema: JSON.stringify(this.metadata?.documentSchema?.runtime),
+                    });
                     const msg = {
                         type: ContainerMessageType.DocumentSchemaChange,
                         contents,
@@ -2411,32 +2518,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                         referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
                     });
                 }
-                // If this is attach message for new data store, and we are in a batch, send this op out of order
-                // Is it safe:
-                //    Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
-                //    They become visible only when aliased, or handle to some sub-element of newly created datastore
-                //    is stored in some DDS, i.e. only after some other op.
-                // Why:
-                //    Attach ops are large, and expensive to process. Plus there are scenarios where a lot of new data
-                //    stores are created, causing issues like relay service throttling (too many ops) and catastrophic
-                //    failure (batch is too large). Pushing them earlier and outside of main batch should alleviate
-                //    these issues.
-                // Cons:
-                //    1. With large batches, relay service may throttle clients. Clients may disconnect while throttled.
-                //    This change creates new possibility of a lot of newly created data stores never being referenced
-                //    because client died before it had a change to submit the rest of the ops. This will create more
-                //    garbage that needs to be collected leveraging GC (Garbage Collection) feature.
-                //    2. Sending ops out of order means they are excluded from rollback functionality. This is not an issue
-                //    today as rollback can't undo creation of data store. To some extent not sending them is a bigger
-                //    issue than sending.
-                // Please note that this does not change file format, so it can be disabled in the future if this
-                // optimization no longer makes sense (for example, batch compression may make it less appealing).
-                if (this.currentlyBatching() &&
-                    type === ContainerMessageType.Attach &&
-                    this.disableAttachReorder !== true) {
-                    this.outbox.submitAttach(message);
-                }
-                else if (type === ContainerMessageType.BlobAttach) {
+                if (type === ContainerMessageType.BlobAttach) {
                     // BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
                     this.outbox.submitBlobAttach(message);
                 }
@@ -2511,32 +2593,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw new Error("Runtime is closed");
         }
     }
-    verifyCanSubmitOps() {
-        if (this.ensureNoDataModelChangesCalls > 0) {
-            const errorMessage = "Op was submitted from within a `ensureNoDataModelChanges` callback";
-            if (this.opReentryCallsToReport > 0) {
-                this.mc.logger.sendTelemetryEvent({ eventName: "OpReentry" }, 
-                // We need to capture the call stack in order to inspect the source of this usage pattern
-                getLongStack(() => new UsageError(errorMessage)));
-                this.opReentryCallsToReport--;
-            }
-            // Creating ops while processing ops can lead
-            // to undefined behavior and events observed in the wrong order.
-            // For example, we have two callbacks registered for a DDS, A and B.
-            // Then if on change #1 callback A creates change #2, the invocation flow will be:
-            //
-            // A because of #1
-            // A because of #2
-            // B because of #2
-            // B because of #1
-            //
-            // The runtime must enforce op coherence by not allowing ops to be submitted
-            // while ops are being processed.
-            if (this.enableOpReentryCheck) {
-                throw new UsageError(errorMessage);
-            }
-        }
-    }
     reSubmitBatch(batch) {
         this.orderSequentially(() => {
             for (const message of batch) {
@@ -2568,11 +2624,15 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata);
                 break;
             case ContainerMessageType.IdAllocation: {
-                this.submit(message, localOpMetadata);
+                // Allocation ops are never resubmitted/rebased. This is because they require special handling to
+                // avoid being submitted out of order. For example, if the pending state manager contained
+                // [idOp1, dataOp1, idOp2, dataOp2] and the resubmission of dataOp1 generated idOp3, that would be
+                // placed into the outbox in the same batch as idOp1, but before idOp2 is resubmitted.
+                // To avoid this, allocation ops are simply never resubmitted. Prior to invoking the pending state
+                // manager to replay pending ops, the runtime will always submit a new allocation range that includes
+                // all pending IDs. The resubmitted allocation ops are then ignored here.
                 break;
             }
-            case ContainerMessageType.ChunkedOp:
-                throw new Error(`chunkedOp not expected here`);
             case ContainerMessageType.BlobAttach:
                 this.blobManager.reSubmit(opMetadata);
                 break;
@@ -2599,7 +2659,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                     });
                 }
                 else {
-                    const error = DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
+                    const error = DataProcessingError.create("Resubmitting runtime message of unexpected type", "reSubmitCore", undefined /* sequencedMessage */, {
                         messageDetails: JSON.stringify({
                             type: message.type,
                             compatBehavior,
@@ -2666,7 +2726,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         await this.closeStaleSummarizer();
         return {
             stage: "base",
-            error: "summary state stale - Unsupported option 'refreshLatestAck'",
+            error: new LoggingError("summary state stale - Unsupported option 'refreshLatestAck'"),
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
             minimumSequenceNumber: this.deltaManager.minimumSequenceNumber,
         };
@@ -2710,16 +2770,14 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         this.imminentClosure || (this.imminentClosure = props?.notifyImminentClosure ?? false);
         const getSyncState = (pendingAttachmentBlobs) => {
-            const pending = this.pendingStateManager.getLocalState();
-            if (pendingAttachmentBlobs === undefined && !this.hasPendingMessages()) {
-                return; // no pending state to save
-            }
+            const pending = this.pendingStateManager.getLocalState(props?.snapshotSequenceNumber);
+            const sessionExpiryTimerStarted = props?.sessionExpiryTimerStarted ?? this.garbageCollector.sessionExpiryTimerStarted;
             const pendingIdCompressorState = this._idCompressor?.serialize(true);
             return {
                 pending,
                 pendingIdCompressorState,
                 pendingAttachmentBlobs,
-                sessionExpiryTimerStarted: this.garbageCollector.sessionExpiryTimerStarted,
+                sessionExpiryTimerStarted,
             };
         };
         const perfEvent = {
@@ -2789,7 +2847,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
     }
     get groupedBatchingEnabled() {
-        return this.documentSchema.opGroupingEnabled === true;
+        return this.sessionSchema.opGroupingEnabled === true;
     }
 }
 //# sourceMappingURL=containerRuntime.js.map