@fluidframework/container-runtime 2.0.0-dev-rc.2.0.0.245554 → 2.0.0-dev-rc.3.0.0.250606

package/lib/containerRuntime.js

@@ -1,29 +1,33 @@
-import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, Deferred, delay, LazyPromise, PromiseCache } from "@fluidframework/core-utils";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
 import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
-import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, wrapError, UsageError, LoggingError, createSampledLogger, loggerToMonitoringContext, } from "@fluidframework/telemetry-utils";
+import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
+import { assert, Deferred, LazyPromise, PromiseCache, delay } from "@fluidframework/core-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
-import { FlushMode, FlushModeExperimental, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, addSummarizeResultToSummary, RequestParser, create404Response, exceptionToResponse, GCDataBuilder, seqFromTree, calculateStats, TelemetryContext, responseToException, } from "@fluidframework/runtime-utils";
+import { FlushMode, FlushModeExperimental, channelsTreeName, gcTreeKey, } from "@fluidframework/runtime-definitions";
+import { GCDataBuilder, RequestParser, TelemetryContext, addBlobToSummary, addSummarizeResultToSummary, calculateStats, create404Response, exceptionToResponse, responseToException, seqFromTree, } from "@fluidframework/runtime-utils";
+import { DataCorruptionError, DataProcessingError, GenericError, LoggingError, PerformanceEvent, TaggedLoggerAdapter, UsageError, createChildLogger, createChildMonitoringContext, createSampledLogger, loggerToMonitoringContext, raiseConnectedEvent, wrapError, } from "@fluidframework/telemetry-utils";
 import { v4 as uuid } from "uuid";
-import { ContainerFluidHandleContext } from "./containerHandleContext.js";
-import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
-import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
-import { PendingStateManager, } from "./pendingStateManager.js";
-import { pkgVersion } from "./packageVersion.js";
+import { BindBatchTracker } from "./batchTracker.js";
 import { BlobManager } from "./blobManager.js";
 import { ChannelCollection, getSummaryForDatastores, wrapContext } from "./channelCollection.js";
-import { aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, Summarizer, SummaryManager, wrapSummaryInChannelsTree, SummaryCollection, OrderedClientCollection, OrderedClientElection, SummarizerClientElection, summarizerClientType, RunWhileConnectedCoordinator, RetriableSummaryError, rootHasIsolatedChannels, } from "./summary/index.js";
-import { formExponentialFn, Throttler } from "./throttler.js";
-import { GarbageCollector, GCNodeType, gcGenerationOptionName, } from "./gc/index.js";
+import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
+import { ContainerFluidHandleContext } from "./containerHandleContext.js";
 import { channelToDataStore } from "./dataStore.js";
-import { BindBatchTracker } from "./batchTracker.js";
-import { ScheduleManager } from "./scheduleManager.js";
-import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, OpGroupingManager, getLongStack, } from "./opLifecycle/index.js";
+import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
 import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy.js";
+import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
 import { ContainerMessageType, } from "./messageTypes.js";
+import { OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, getLongStack, } from "./opLifecycle/index.js";
+import { pkgVersion } from "./packageVersion.js";
+import { PendingStateManager, } from "./pendingStateManager.js";
+import { ScheduleManager } from "./scheduleManager.js";
+import { DocumentsSchemaController, OrderedClientCollection, OrderedClientElection, RetriableSummaryError, RunWhileConnectedCoordinator, Summarizer, SummarizerClientElection, SummaryCollection, SummaryManager, aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, rootHasIsolatedChannels, summarizerClientType, wrapSummaryInChannelsTree, } from "./summary/index.js";
+import { Throttler, formExponentialFn } from "./throttler.js";
 /**
  * Utility to implement compat behaviors given an unknown message type
  * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
@@ -78,6 +82,11 @@ export var CompressionAlgorithms;
 (function (CompressionAlgorithms) {
     CompressionAlgorithms["lz4"] = "lz4";
 })(CompressionAlgorithms || (CompressionAlgorithms = {}));
+/** @alpha */
+export const disabledCompressionConfig = {
+    minimumBatchSizeInBytes: Infinity,
+    compressionAlgorithm: CompressionAlgorithms.lz4,
+};
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
 const defaultFlushMode = FlushMode.TurnBased;
 // The actual limit is 1Mb (socket.io and Kafka limits)
@@ -101,26 +110,12 @@ export const defaultPendingOpsRetryDelayMs = 1000;
  * This delay's goal is to prevent tight restart loops
  */
 const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
-/**
- * @deprecated use ContainerRuntimeMessageType instead
- * @internal
- */
-export var RuntimeMessage;
-(function (RuntimeMessage) {
-    RuntimeMessage["FluidDataStoreOp"] = "component";
-    RuntimeMessage["Attach"] = "attach";
-    RuntimeMessage["ChunkedOp"] = "chunkedOp";
-    RuntimeMessage["BlobAttach"] = "blobAttach";
-    RuntimeMessage["Rejoin"] = "rejoin";
-    RuntimeMessage["Alias"] = "alias";
-    RuntimeMessage["Operation"] = "op";
-})(RuntimeMessage || (RuntimeMessage = {}));
 /**
  * @deprecated please use version in driver-utils
  * @internal
  */
 export function isRuntimeMessage(message) {
-    return Object.values(RuntimeMessage).includes(message.type);
+    return Object.values(ContainerMessageType).includes(message.type);
 }
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
@@ -194,6 +189,17 @@ async function createSummarizer(loader, url) {
     }
     return fluidObject.ISummarizer;
 }
+/**
+ * Extract last message from the snapshot metadata.
+ * Uses legacy property if not using explicit schema control, otherwise uses the new property.
+ * This allows new runtime to make documents not openable for old runtimes, one explicit document schema control is enabled.
+ * Please see addMetadataToSummary() as well
+ */
+function lastMessageFromMetadata(metadata) {
+    return metadata?.documentSchema?.runtime?.explicitSchemaControl
+        ? metadata?.lastMessage
+        : metadata?.message;
+}
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
@@ -231,7 +237,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             },
         });
         const mc = loggerToMonitoringContext(logger);
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor = "off", chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, explicitSchemaControl = false, } = runtimeOptions;
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = context.baseSnapshot?.blobs[blobName];
@@ -256,14 +262,21 @@ export class ContainerRuntime extends TypedEventEmitter {
             assert(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
             return readAndParse(context.storage, id);
         });
+        const messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Verify summary runtime sequence number matches protocol sequence number.
-        const runtimeSequenceNumber = metadata?.message?.sequenceNumber;
+        const runtimeSequenceNumber = messageAtLastSummary?.sequenceNumber;
         // When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
         if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
             const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" &&
                 runtimeSequenceNumber !== protocolSequenceNumber) {
+                // Message to OCEs:
+                // You can hit this error with runtimeSequenceNumber === -1 in < 2.0 RC3 builds.
+                // This would indicate that explicit schema control is enabled in current (2.0 RC3+) builds and it
+                // results in addMetadataToSummary() creating a poison pill for older runtimes in the form of a -1 sequence number.
+                // Older runtimes do not understand new schema, and thus could corrupt document if they proceed, thus we are using
+                // this poison pill to prevent them from proceeding.
                 // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
                 const error = new DataCorruptionError(
                 // pre-0.58 error message: SummaryMetadataMismatch
@@ -277,7 +290,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
         // Enabling the IdCompressor is a one-way operation and we only want to
-        // allow new containers to turn it on
+        // allow new containers to turn it on.
         let idCompressorMode;
         if (existing) {
             // This setting has to be sticky for correctness:
@@ -286,20 +299,20 @@ export class ContainerRuntime extends TypedEventEmitter {
             // 2) if it's ON, then all sessions should load compressor right away
             // 3) Same logic applies for "delayed" mode
             // Maybe in the future we will need to enabled (and figure how to do it safely) "delayed" -> "on" change.
-            // We could do "off" -> "on" transtition too, if all clients start loading compressor (but not using it initially) and do so for a while -
-            // this will allow clients to eventually to disregard "off" setting (when it's safe so) and start using compressor in future sessions.
+            // We could do "off" -> "on" transition too, if all clients start loading compressor (but not using it initially) and
+            // do so for a while - this will allow clients to eventually to disregard "off" setting (when it's safe so) and start
+            // using compressor in future sessions.
             // Everyting is possible, but it needs to be designed and executed carefully, when such need arises.
-            idCompressorMode = metadata?.idCompressorMode ?? "off";
+            idCompressorMode = metadata?.documentSchema?.runtime
+                ?.idCompressorMode;
         }
         else {
-            // FG overwrite
-            const enabled = mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled");
-            switch (enabled) {
+            switch (mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled")) {
                 case true:
                     idCompressorMode = "on";
                     break;
                 case false:
-                    idCompressorMode = "off";
+                    idCompressorMode = undefined;
                     break;
                 default:
                     idCompressorMode = enableRuntimeIdCompressor;
@@ -332,6 +345,24 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return createIdCompressor(compressorLogger);
             }
         };
+        const disableGroupedBatching = mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
+        const disableCompression = mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
+        const compressionLz4 = disableCompression !== true &&
+            compressionOptions.minimumBatchSizeInBytes !== Infinity &&
+            compressionOptions.compressionAlgorithm === "lz4";
+        const opGroupingEnabled = disableGroupedBatching !== true && enableGroupedBatching;
+        const documentSchemaController = new DocumentsSchemaController(existing, metadata?.documentSchema, {
+            explicitSchemaControl,
+            compressionLz4,
+            idCompressorMode,
+            opGroupingEnabled,
+        }, (schema) => {
+            runtime.onSchemaChange(schema);
+        });
+        const featureGatesForTelemetry = {
+            disableGroupedBatching,
+            disableCompression,
+        };
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
             gcOptions,
@@ -340,10 +371,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             compressionOptions,
             maxBatchSizeInBytes,
             chunkSizeInBytes,
-            enableRuntimeIdCompressor,
+            // Requires<> drops undefined from IdCompressorType
+            enableRuntimeIdCompressor: enableRuntimeIdCompressor,
             enableOpReentryCheck,
             enableGroupedBatching,
-        }, containerScope, logger, existing, blobManagerSnapshot, context.storage, createIdCompressorFn, idCompressorMode, provideEntryPoint, requestHandler, undefined);
+            explicitSchemaControl,
+        }, containerScope, logger, existing, blobManagerSnapshot, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, undefined);
         // Apply stashed ops with a reference sequence number equal to the sequence number of the snapshot,
         // or zero. This must be done before Container replays saved ops.
         await runtime.pendingStateManager.applyStashedOpsAt(runtimeSequenceNumber ?? 0);
@@ -372,6 +405,12 @@ export class ContainerRuntime extends TypedEventEmitter {
     get attachState() {
         return this._getAttachState();
     }
+    get documentSchema() {
+        return this.documentsSchemaController.sessionSchema.runtime;
+    }
+    get idCompressorMode() {
+        return this.documentSchema.idCompressorMode;
+    }
     /**
      * See IContainerRuntimeBase.idCompressor() for details.
      */
@@ -452,7 +491,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.garbageCollector.throwOnTombstoneUsage;
     }
     /***/
-    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, createIdCompressor, idCompressorMode, provideEntryPoint, requestHandler, summaryConfiguration = {
+    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, createIdCompressor, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, summaryConfiguration = {
         // the defaults
         ...DefaultSummaryConfiguration,
         // the runtime configuration overrides
@@ -466,18 +505,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.logger = logger;
         this._storage = _storage;
         this.createIdCompressor = createIdCompressor;
-        this.idCompressorMode = idCompressorMode;
+        this.documentsSchemaController = documentsSchemaController;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
         this.imminentClosure = false;
         // We accumulate Id compressor Ops while Id compressor is not loaded yet (only for "delayed" mode)
         // Once it loads, it will process all such ops and we will stop accumulating further ops - ops will be processes as they come in.
         this.pendingIdCompressorOps = [];
-        /**
-         * True if we have ID compressor loading in-flight (async operation). Useful only for
-         * this.idCompressorMode === "delayed" mode
-         */
-        this.compressorLoadInitiated = false;
         this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
         this.flushTaskExists = false;
@@ -507,6 +541,20 @@ export class ContainerRuntime extends TypedEventEmitter {
             expiry: { policy: "absolute", durationMs: 60000 },
         });
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
+        this.mc = createChildMonitoringContext({
+            logger: this.logger,
+            namespace: "ContainerRuntime",
+        });
+        // If we support multiple algorithms in the future, then we would need to manage it here carefully.
+        // We can use runtimeOptions.compressionOptions.compressionAlgorithm, but only if it's in the schema list!
+        // If it's not in the list, then we will need to either use no compression, or fallback to some other (supported by format)
+        // compression.
+        const compressionOptions = {
+            minimumBatchSizeInBytes: this.documentSchema.compressionLz4
+                ? runtimeOptions.compressionOptions.minimumBatchSizeInBytes
+                : Number.POSITIVE_INFINITY,
+            compressionAlgorithm: CompressionAlgorithms.lz4,
+        };
         this.innerDeltaManager = deltaManager;
         this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
         // Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
@@ -540,10 +588,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.disposeFn = disposeFn ?? closeFn;
         // In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
         this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
-        this.mc = createChildMonitoringContext({
-            logger: this.logger,
-            namespace: "ContainerRuntime",
-        });
         let loadSummaryNumber;
         // Get the container creation metadata. For new container, we initialize these. For existing containers,
         // get the values from the metadata blob.
@@ -564,7 +608,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             loadSummaryNumber = 0;
         }
         this.nextSummaryNumber = loadSummaryNumber + 1;
-        this.messageAtLastSummary = metadata?.message;
+        this.messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Note that we only need to pull the *initial* connected state from the context.
         // Later updates come through calls to setConnectionState.
         this._connected = connected;
@@ -681,18 +725,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             clientId: () => this.clientId,
             close: this.closeFn,
             connected: () => this.connected,
-            reSubmit: this.reSubmit.bind(this),
+            reSubmit: (message) => {
+                this.reSubmit(message);
+                this.flush();
+            },
             reSubmitBatch: this.reSubmitBatch.bind(this),
             isActiveConnection: () => this.innerDeltaManager.active,
             isAttached: () => this.attachState !== AttachState.Detached,
         }, pendingRuntimeState?.pending, this.logger);
-        const disableCompression = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
-        const compressionOptions = disableCompression === true
-            ? {
-                minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
-                compressionAlgorithm: CompressionAlgorithms.lz4,
-            }
-            : runtimeOptions.compressionOptions;
         const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
         const legacySendBatchFn = makeLegacySendBatchFn(this.submitFn, this.innerDeltaManager);
         this.outbox = new Outbox({
@@ -797,10 +837,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             disableIsolatedChannels: metadata?.disableIsolatedChannels,
             gcVersion: metadata?.gcFeature,
             options: JSON.stringify(runtimeOptions),
-            idCompressorModeMetadata: metadata?.idCompressorMode,
+            idCompressorModeMetadata: metadata?.documentSchema?.runtime?.idCompressorMode,
             idCompressorMode: this.idCompressorMode,
             featureGates: JSON.stringify({
-                disableCompression,
+                ...featureGatesForTelemetry,
                 disableOpReentryCheck,
                 disableChunking,
                 disableAttachReorder: this.disableAttachReorder,
@@ -823,6 +863,20 @@ export class ContainerRuntime extends TypedEventEmitter {
         // saved state, i.e. all the ops marked by Loader layer sa savedOp === true.
         this.skipSavedCompressorOps = pendingRuntimeState?.pendingIdCompressorState !== undefined;
     }
+    onSchemaChange(schema) {
+        // Most of the settings will be picked up only by new sessions (i.e. after reload).
+        // We can make it better in the future (i.e. start to use op compression right away), but for simplicity
+        // this is not done.
+        // But ID compressor is special. It's possible, that in future, we will remove "stickiness" of ID compressor setting
+        // and will allow to start using it. If that were to happen, we want to ensure that we do not break eventual consistency
+        // promises. To do so, we need to initialize id compressor right away.
+        // As it's implemented right now (with async initialization), this will only work for "off" -> "delayed" transitions.
+        // Anything else is too risky, and requires ability to initialize ID compressor synchronously!
+        if (schema.runtime.idCompressorMode !== undefined) {
+            // eslint-disable-next-line @typescript-eslint/no-floating-promises
+            this.loadIdCompressor();
+        }
+    }
     getCreateChildSummarizerNodeFn(id, createParam) {
         return (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn);
     }
@@ -834,7 +888,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         assert(false, 0x8eb /* should not be called */);
     }
     setChannelDirty(address) {
-        assert(false, "should not be called");
+        assert(false, 0x909 /* should not be called */);
     }
     /**
      * Initializes the state from the base snapshot this container runtime loaded from.
@@ -1040,18 +1094,30 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     /** Adds the container's metadata to the given summary tree. */
     addMetadataToSummary(summaryTree) {
+        // The last message processed at the time of summary. If there are no new messages, use the message from the
+        // last summary.
+        const message = extractSummaryMetadataMessage(this.deltaManager.lastMessage) ??
+            this.messageAtLastSummary;
+        const documentSchema = this.documentsSchemaController.summarizeDocumentSchema(this.deltaManager.lastSequenceNumber);
+        // Is document schema explicit control on?
+        const explitiSchemaControl = documentSchema?.runtime.explicitSchemaControl;
         const metadata = {
             ...this.createContainerMetadata,
             // Increment the summary number for the next summary that will be generated.
             summaryNumber: this.nextSummaryNumber++,
             summaryFormatVersion: 1,
             ...this.garbageCollector.getMetadata(),
-            // The last message processed at the time of summary. If there are no new messages, use the message from the
-            // last summary.
-            message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ??
-                this.messageAtLastSummary,
             telemetryDocumentId: this.telemetryDocumentId,
-            idCompressorMode: this.idCompressorMode,
+            // If explicit document schema control is not on, use legacy way to supply last message (using 'message' property).
+            // Otherwise use new 'lastMessage' property, but also put content into the 'message' property that cases old
+            // runtimes (that preceed document schema control capabilities) to close container on load due to mismatch in
+            // last message's sequence number.
+            // See also lastMessageFromMetadata()
+            message: explitiSchemaControl
+                ? { sequenceNumber: -1 }
+                : message,
+            lastMessage: explitiSchemaControl ? message : undefined,
+            documentSchema,
         };
         addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
     }
@@ -1112,14 +1178,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
     }
-    resetReconnectCount(message) {
-        // Chunked ops don't count towards making progress as they are sent
-        // in their own batches before the originating batch is sent.
-        // Therefore, receiving them while attempting to send the originating batch
-        // does not mean that the container is making any progress.
-        if (message?.type !== ContainerMessageType.ChunkedOp) {
-            this.consecutiveReconnects = 0;
-        }
+    resetReconnectCount() {
+        this.consecutiveReconnects = 0;
     }
     replayPendingStates() {
         // We need to be able to send ops to replay states
@@ -1168,7 +1228,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.Alias:
                 return this.channelCollection.applyStashedOp(opContents);
             case ContainerMessageType.IdAllocation:
-                assert(this.idCompressorMode !== "off", 0x8f1 /* ID compressor should be in use */);
+                assert(this.idCompressorMode !== undefined, 0x8f1 /* ID compressor should be in use */);
+                return;
+            case ContainerMessageType.DocumentSchemaChange:
                 return;
             case ContainerMessageType.BlobAttach:
                 return;
@@ -1199,10 +1261,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
-    setConnectionState(connected, clientId) {
-        if (connected && this.idCompressorMode === "delayed" && !this.compressorLoadInitiated) {
-            this.compressorLoadInitiated = true;
-            this.createIdCompressor()
+    async loadIdCompressor() {
+        if (this._idCompressor === undefined &&
+            this.idCompressorMode !== undefined &&
+            this._loadIdCompressor === undefined) {
+            this._loadIdCompressor = this.createIdCompressor()
                 .then((compressor) => {
                 this._idCompressor = compressor;
                 for (const range of this.pendingIdCompressorOps) {
@@ -1212,8 +1275,16 @@ export class ContainerRuntime extends TypedEventEmitter {
             })
                 .catch((error) => {
                 this.logger.sendErrorEvent({ eventName: "IdCompressorDelayedLoad" }, error);
+                throw error;
             });
         }
+        return this._loadIdCompressor;
+    }
+    setConnectionState(connected, clientId) {
+        if (connected && this.idCompressorMode === "delayed") {
+            // eslint-disable-next-line @typescript-eslint/no-floating-promises
+            this.loadIdCompressor();
+        }
         if (connected === false && this.delayConnectClientId !== undefined) {
             this.delayConnectClientId = undefined;
             this.mc.logger.sendTelemetryEvent({
@@ -1222,6 +1293,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
             return;
         }
+        if (!connected) {
+            this.documentsSchemaController.onDisconnect();
+        }
         // If there are stashed blobs in the pending state, we need to delay
         // propagation of the "connected" event until we have uploaded them to
         // ensure we don't submit ops referencing a blob that has not been uploaded
@@ -1324,10 +1398,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.scheduleManager.beforeOpProcessing(message);
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
+            // See commit that added this assert for more details.
+            // These calls should be made for all but chunked ops:
+            // 1) this.pendingStateManager.processPendingLocalMessage() below
+            // 2) this.resetReconnectCount() below
+            assert(message.type !== ContainerMessageType.ChunkedOp, "we should never get here with chunked ops");
             let localOpMetadata;
-            if (local &&
-                messageWithContext.modernRuntimeMessage &&
-                message.type !== ContainerMessageType.ChunkedOp) {
+            if (local && messageWithContext.modernRuntimeMessage) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(messageWithContext.message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1342,7 +1419,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // If we have processed a local op, this means that the container is
                 // making progress and we can reset the counter for how many times
                 // we have consecutively replayed the pending states
-                this.resetReconnectCount(message);
+                this.resetReconnectCount();
             }
         }
         catch (e) {
@@ -1388,8 +1465,14 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.garbageCollector.processMessage(messageWithContext.message, local);
                 break;
             case ContainerMessageType.ChunkedOp:
+                // From observability POV, we should not exppse the rest of the system (including "op" events on object) to these messages.
+                // Also resetReconnectCount() would be wrong - see comment that was there before this change was made.
+                assert(false, "should not even get here");
             case ContainerMessageType.Rejoin:
                 break;
+            case ContainerMessageType.DocumentSchemaChange:
+                this.documentsSchemaController.processDocumentSchemaOp(messageWithContext.message.contents, messageWithContext.local, messageWithContext.message.sequenceNumber);
+                break;
             default: {
                 // If we didn't necessarily expect a runtime message type, then no worries - just return
                 // e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
@@ -1619,6 +1702,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                 }
                 break;
             }
+            case ContainerMessageType.IdAllocation:
+            case ContainerMessageType.DocumentSchemaChange:
             case ContainerMessageType.GC: {
                 return false;
             }
@@ -1694,6 +1779,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         // Wrap data store summaries in .channels subtree.
         wrapSummaryInChannelsTree(summarizeResult);
         const pathPartsForChildren = [channelsTreeName];
+        // Ensure that ID compressor had a chance to load, if we are using delayed mode.
+        await this.loadIdCompressor();
         this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
         return {
             ...summarizeResult,
@@ -2236,19 +2323,18 @@ export class ContainerRuntime extends TypedEventEmitter {
                 const idAllocationBatchMessage = {
                     contents: JSON.stringify(idAllocationMessage),
                     referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
-                    metadata: undefined,
-                    localOpMetadata: undefined,
-                    type: ContainerMessageType.IdAllocation,
                 };
                 this.outbox.submitIdAllocation(idAllocationBatchMessage);
             }
         }
     }
-    submit(containerRuntimeMessage, localOpMetadata = undefined, metadata = undefined) {
+    submit(containerRuntimeMessage, localOpMetadata = undefined, metadata) {
         this.verifyNotClosed();
         this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
         assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
+        assert(metadata === undefined ||
+            containerRuntimeMessage.type === ContainerMessageType.BlobAttach, "metadata");
         const serializedContent = JSON.stringify(containerRuntimeMessage);
         // Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
         // container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
@@ -2261,7 +2347,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         const type = containerRuntimeMessage.type;
         const message = {
             contents: serializedContent,
-            type,
             metadata,
             localOpMetadata,
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
@@ -2276,6 +2361,19 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             else {
                 this.submitIdAllocationOpIfNeeded();
+                // Allow document schema controller to send a message if it needs to propose change in document schema.
+                // If it needs to send a message, it will call provided callback with payload of such message and rely
+                // on this callback to do actual sending.
+                this.documentsSchemaController.onMessageSent((contents) => {
+                    const msg = {
+                        type: ContainerMessageType.DocumentSchemaChange,
+                        contents,
+                    };
+                    this.outbox.submit({
+                        contents: JSON.stringify(msg),
+                        referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                    });
+                });
                 // If this is attach message for new data store, and we are in a batch, send this op out of order
                 // Is it safe:
                 //    Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
@@ -2447,6 +2545,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.GC:
                 this.submit(message);
                 break;
+            case ContainerMessageType.DocumentSchemaChange:
+                // There is no need to resend this message. Document schema controller will properly resend it again (if needed)
+                // on a first occasion (any ops sent after reconnect). There is a good chance, though, that it will not want to
+                // send any ops, as some other client already changed schema.
+                break;
             default: {
                 // This case should be very rare - it would imply an op was stashed from a
                 // future version of runtime code and now is being applied on an older version.
@@ -2649,8 +2752,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
     }
     get groupedBatchingEnabled() {
-        const killSwitch = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
-        return killSwitch !== true && this.runtimeOptions.enableGroupedBatching;
+        return this.documentSchema.opGroupingEnabled === true;
     }
 }
 //# sourceMappingURL=containerRuntime.js.map