@fluidframework/container-runtime 2.0.0-rc.2.0.2 → 2.0.0-rc.3.0.1

package/lib/containerRuntime.js

@@ -1,29 +1,36 @@
-import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, Deferred, delay, LazyPromise, PromiseCache } from "@fluidframework/core-utils";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
 import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
-import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, wrapError, UsageError, LoggingError, createSampledLogger, loggerToMonitoringContext, } from "@fluidframework/telemetry-utils";
-import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
-import { readAndParse } from "@fluidframework/driver-utils";
+import { AttachState, } from "@fluidframework/container-definitions";
+import { LoaderHeader, } from "@fluidframework/container-definitions/internal";
+import { assert, Deferred, LazyPromise, PromiseCache, delay, } from "@fluidframework/core-utils/internal";
+import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions/internal";
+import { readAndParse } from "@fluidframework/driver-utils/internal";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
-import { FlushMode, FlushModeExperimental, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, addSummarizeResultToSummary, RequestParser, create404Response, exceptionToResponse, GCDataBuilder, seqFromTree, calculateStats, TelemetryContext, responseToException, } from "@fluidframework/runtime-utils";
+import { FlushMode, FlushModeExperimental, channelsTreeName, gcTreeKey, } from "@fluidframework/runtime-definitions/internal";
+import { GCDataBuilder, RequestParser, TelemetryContext, addBlobToSummary, addSummarizeResultToSummary, calculateStats, create404Response, exceptionToResponse, responseToException, seqFromTree, } from "@fluidframework/runtime-utils/internal";
+import { DataCorruptionError, DataProcessingError, GenericError, LoggingError, PerformanceEvent, 
+// eslint-disable-next-line import/no-deprecated
+TaggedLoggerAdapter, UsageError, createChildLogger, createChildMonitoringContext, createSampledLogger, loggerToMonitoringContext, raiseConnectedEvent, wrapError, } from "@fluidframework/telemetry-utils/internal";
 import { v4 as uuid } from "uuid";
-import { ContainerFluidHandleContext } from "./containerHandleContext.js";
-import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
-import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
-import { PendingStateManager, } from "./pendingStateManager.js";
-import { pkgVersion } from "./packageVersion.js";
+import { BindBatchTracker } from "./batchTracker.js";
 import { BlobManager } from "./blobManager.js";
 import { ChannelCollection, getSummaryForDatastores, wrapContext } from "./channelCollection.js";
-import { aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, Summarizer, SummaryManager, wrapSummaryInChannelsTree, SummaryCollection, OrderedClientCollection, OrderedClientElection, SummarizerClientElection, summarizerClientType, RunWhileConnectedCoordinator, RetriableSummaryError, rootHasIsolatedChannels, } from "./summary/index.js";
-import { formExponentialFn, Throttler } from "./throttler.js";
-import { GarbageCollector, GCNodeType, gcGenerationOptionName, } from "./gc/index.js";
+import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
+import { ContainerFluidHandleContext } from "./containerHandleContext.js";
 import { channelToDataStore } from "./dataStore.js";
-import { BindBatchTracker } from "./batchTracker.js";
-import { ScheduleManager } from "./scheduleManager.js";
-import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, OpGroupingManager, getLongStack, } from "./opLifecycle/index.js";
+import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
 import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy.js";
+import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
 import { ContainerMessageType, } from "./messageTypes.js";
+import { OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, getLongStack, } from "./opLifecycle/index.js";
+import { pkgVersion } from "./packageVersion.js";
+import { PendingStateManager, } from "./pendingStateManager.js";
+import { ScheduleManager } from "./scheduleManager.js";
+import { DocumentsSchemaController, OrderedClientCollection, OrderedClientElection, RetriableSummaryError, RunWhileConnectedCoordinator, Summarizer, SummarizerClientElection, SummaryCollection, SummaryManager, aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, rootHasIsolatedChannels, summarizerClientType, wrapSummaryInChannelsTree, } from "./summary/index.js";
+import { Throttler, formExponentialFn } from "./throttler.js";
 /**
  * Utility to implement compat behaviors given an unknown message type
  * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
@@ -78,6 +85,11 @@ export var CompressionAlgorithms;
 (function (CompressionAlgorithms) {
     CompressionAlgorithms["lz4"] = "lz4";
 })(CompressionAlgorithms || (CompressionAlgorithms = {}));
+/** @alpha */
+export const disabledCompressionConfig = {
+    minimumBatchSizeInBytes: Infinity,
+    compressionAlgorithm: CompressionAlgorithms.lz4,
+};
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
 const defaultFlushMode = FlushMode.TurnBased;
 // The actual limit is 1Mb (socket.io and Kafka limits)
@@ -101,26 +113,12 @@ export const defaultPendingOpsRetryDelayMs = 1000;
  * This delay's goal is to prevent tight restart loops
  */
 const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
-/**
- * @deprecated use ContainerRuntimeMessageType instead
- * @internal
- */
-export var RuntimeMessage;
-(function (RuntimeMessage) {
-    RuntimeMessage["FluidDataStoreOp"] = "component";
-    RuntimeMessage["Attach"] = "attach";
-    RuntimeMessage["ChunkedOp"] = "chunkedOp";
-    RuntimeMessage["BlobAttach"] = "blobAttach";
-    RuntimeMessage["Rejoin"] = "rejoin";
-    RuntimeMessage["Alias"] = "alias";
-    RuntimeMessage["Operation"] = "op";
-})(RuntimeMessage || (RuntimeMessage = {}));
 /**
  * @deprecated please use version in driver-utils
  * @internal
  */
 export function isRuntimeMessage(message) {
-    return Object.values(RuntimeMessage).includes(message.type);
+    return Object.values(ContainerMessageType).includes(message.type);
 }
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
@@ -194,6 +192,17 @@ async function createSummarizer(loader, url) {
     }
     return fluidObject.ISummarizer;
 }
+/**
+ * Extract last message from the snapshot metadata.
+ * Uses legacy property if not using explicit schema control, otherwise uses the new property.
+ * This allows new runtime to make documents not openable for old runtimes, one explicit document schema control is enabled.
+ * Please see addMetadataToSummary() as well
+ */
+function lastMessageFromMetadata(metadata) {
+    return metadata?.documentSchema?.runtime?.explicitSchemaControl
+        ? metadata?.lastMessage
+        : metadata?.message;
+}
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
@@ -221,6 +230,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
         const passLogger = backCompatContext.taggedLogger ??
+            // eslint-disable-next-line import/no-deprecated
             new TaggedLoggerAdapter(backCompatContext.logger);
         const logger = createChildLogger({
             logger: passLogger,
@@ -231,7 +241,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             },
         });
         const mc = loggerToMonitoringContext(logger);
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor = "off", chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, explicitSchemaControl = false, } = runtimeOptions;
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = context.baseSnapshot?.blobs[blobName];
@@ -256,14 +266,21 @@ export class ContainerRuntime extends TypedEventEmitter {
             assert(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
             return readAndParse(context.storage, id);
         });
+        const messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Verify summary runtime sequence number matches protocol sequence number.
-        const runtimeSequenceNumber = metadata?.message?.sequenceNumber;
+        const runtimeSequenceNumber = messageAtLastSummary?.sequenceNumber;
         // When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
         if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
             const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
             // Unless bypass is explicitly set, then take action when sequence numbers mismatch.
             if (loadSequenceNumberVerification !== "bypass" &&
                 runtimeSequenceNumber !== protocolSequenceNumber) {
+                // Message to OCEs:
+                // You can hit this error with runtimeSequenceNumber === -1 in < 2.0 RC3 builds.
+                // This would indicate that explicit schema control is enabled in current (2.0 RC3+) builds and it
+                // results in addMetadataToSummary() creating a poison pill for older runtimes in the form of a -1 sequence number.
+                // Older runtimes do not understand new schema, and thus could corrupt document if they proceed, thus we are using
+                // this poison pill to prevent them from proceeding.
                 // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
                 const error = new DataCorruptionError(
                 // pre-0.58 error message: SummaryMetadataMismatch
@@ -276,8 +293,20 @@ export class ContainerRuntime extends TypedEventEmitter {
                 }
             }
         }
+        let desiredIdCompressorMode;
+        switch (mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled")) {
+            case true:
+                desiredIdCompressorMode = "on";
+                break;
+            case false:
+                desiredIdCompressorMode = undefined;
+                break;
+            default:
+                desiredIdCompressorMode = enableRuntimeIdCompressor;
+                break;
+        }
         // Enabling the IdCompressor is a one-way operation and we only want to
-        // allow new containers to turn it on
+        // allow new containers to turn it on.
         let idCompressorMode;
         if (existing) {
             // This setting has to be sticky for correctness:
@@ -286,28 +315,30 @@ export class ContainerRuntime extends TypedEventEmitter {
             // 2) if it's ON, then all sessions should load compressor right away
             // 3) Same logic applies for "delayed" mode
             // Maybe in the future we will need to enabled (and figure how to do it safely) "delayed" -> "on" change.
-            // We could do "off" -> "on" transtition too, if all clients start loading compressor (but not using it initially) and do so for a while -
-            // this will allow clients to eventually to disregard "off" setting (when it's safe so) and start using compressor in future sessions.
+            // We could do "off" -> "on" transition too, if all clients start loading compressor (but not using it initially) and
+            // do so for a while - this will allow clients to eventually disregard "off" setting (when it's safe so) and start
+            // using compressor in future sessions.
             // Everyting is possible, but it needs to be designed and executed carefully, when such need arises.
-            idCompressorMode = metadata?.idCompressorMode ?? "off";
+            idCompressorMode = metadata?.documentSchema?.runtime
+                ?.idCompressorMode;
+            // This is the only exception to the rule above - we have proper plumbing to load ID compressor on schema change
+            // event. It is loaded async (relative to op processing), so this conversion is only safe for off -> delayed conversion!
+            // Clients do not expect ID compressor ops unless ID compressor is On for them, and that could be achieved only through
+            // explicit schema change, i.e. only if explicitSchemaControl is on.
+            // Note: it would be better if we throw on combination of options (explicitSchemaControl = off, desiredIdCompressorMode === "delayed")
+            // that is not supported. But our service tests are oblivious to these problems and throwing here will cause a ton of failures
+            // We ignored incompatible ID compressor changes from the start (they were sticky), so that's not a new problem being introduced...
+            if (idCompressorMode === undefined &&
+                desiredIdCompressorMode === "delayed" &&
+                explicitSchemaControl) {
+                idCompressorMode = desiredIdCompressorMode;
+            }
         }
         else {
-            // FG overwrite
-            const enabled = mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled");
-            switch (enabled) {
-                case true:
-                    idCompressorMode = "on";
-                    break;
-                case false:
-                    idCompressorMode = "off";
-                    break;
-                default:
-                    idCompressorMode = enableRuntimeIdCompressor;
-                    break;
-            }
+            idCompressorMode = desiredIdCompressorMode;
         }
         const createIdCompressorFn = async () => {
-            const { createIdCompressor, deserializeIdCompressor, createSessionId } = await import("@fluidframework/id-compressor");
+            const { createIdCompressor, deserializeIdCompressor, createSessionId } = await import("@fluidframework/id-compressor/internal");
             /**
              * Because the IdCompressor emits so much telemetry, this function is used to sample
              * approximately 5% of all clients. Only the given percentage of sessions will emit telemetry.
@@ -332,6 +363,25 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return createIdCompressor(compressorLogger);
             }
         };
+        const disableGroupedBatching = mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
+        const disableCompression = mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
+        const compressionLz4 = disableCompression !== true &&
+            compressionOptions.minimumBatchSizeInBytes !== Infinity &&
+            compressionOptions.compressionAlgorithm === "lz4";
+        const opGroupingEnabled = disableGroupedBatching !== true && enableGroupedBatching;
+        const documentSchemaController = new DocumentsSchemaController(existing, metadata?.documentSchema, {
+            explicitSchemaControl,
+            compressionLz4,
+            idCompressorMode,
+            opGroupingEnabled,
+            disallowedVersions: [],
+        }, (schema) => {
+            runtime.onSchemaChange(schema);
+        });
+        const featureGatesForTelemetry = {
+            disableGroupedBatching,
+            disableCompression,
+        };
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
             gcOptions,
@@ -340,10 +390,19 @@ export class ContainerRuntime extends TypedEventEmitter {
             compressionOptions,
             maxBatchSizeInBytes,
             chunkSizeInBytes,
-            enableRuntimeIdCompressor,
+            // Requires<> drops undefined from IdCompressorType
+            enableRuntimeIdCompressor: enableRuntimeIdCompressor,
             enableOpReentryCheck,
             enableGroupedBatching,
-        }, containerScope, logger, existing, blobManagerSnapshot, context.storage, createIdCompressorFn, idCompressorMode, provideEntryPoint, requestHandler, undefined);
+            explicitSchemaControl,
+        }, containerScope, logger, existing, blobManagerSnapshot, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, undefined);
+        runtime.blobManager.trackPendingStashedUploads().then(() => {
+            // make sure we didn't reconnect before the promise resolved
+            if (runtime.delayConnectClientId !== undefined && !runtime.disposed) {
+                runtime.delayConnectClientId = undefined;
+                runtime.setConnectionStateCore(true, runtime.delayConnectClientId);
+            }
+        }, (error) => runtime.closeFn(error));
         // Apply stashed ops with a reference sequence number equal to the sequence number of the snapshot,
         // or zero. This must be done before Container replays saved ops.
         await runtime.pendingStateManager.applyStashedOpsAt(runtimeSequenceNumber ?? 0);
@@ -372,6 +431,12 @@ export class ContainerRuntime extends TypedEventEmitter {
     get attachState() {
         return this._getAttachState();
     }
+    get documentSchema() {
+        return this.documentsSchemaController.sessionSchema.runtime;
+    }
+    get idCompressorMode() {
+        return this.documentSchema.idCompressorMode;
+    }
     /**
      * See IContainerRuntimeBase.idCompressor() for details.
      */
@@ -452,7 +517,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.garbageCollector.throwOnTombstoneUsage;
     }
     /***/
-    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, createIdCompressor, idCompressorMode, provideEntryPoint, requestHandler, summaryConfiguration = {
+    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, createIdCompressor, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, summaryConfiguration = {
         // the defaults
         ...DefaultSummaryConfiguration,
         // the runtime configuration overrides
@@ -466,18 +531,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.logger = logger;
         this._storage = _storage;
         this.createIdCompressor = createIdCompressor;
-        this.idCompressorMode = idCompressorMode;
+        this.documentsSchemaController = documentsSchemaController;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
         this.imminentClosure = false;
         // We accumulate Id compressor Ops while Id compressor is not loaded yet (only for "delayed" mode)
         // Once it loads, it will process all such ops and we will stop accumulating further ops - ops will be processes as they come in.
         this.pendingIdCompressorOps = [];
-        /**
-         * True if we have ID compressor loading in-flight (async operation). Useful only for
-         * this.idCompressorMode === "delayed" mode
-         */
-        this.compressorLoadInitiated = false;
         this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
         this.flushTaskExists = false;
@@ -507,6 +567,20 @@ export class ContainerRuntime extends TypedEventEmitter {
             expiry: { policy: "absolute", durationMs: 60000 },
         });
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
+        this.mc = createChildMonitoringContext({
+            logger: this.logger,
+            namespace: "ContainerRuntime",
+        });
+        // If we support multiple algorithms in the future, then we would need to manage it here carefully.
+        // We can use runtimeOptions.compressionOptions.compressionAlgorithm, but only if it's in the schema list!
+        // If it's not in the list, then we will need to either use no compression, or fallback to some other (supported by format)
+        // compression.
+        const compressionOptions = {
+            minimumBatchSizeInBytes: this.documentSchema.compressionLz4
+                ? runtimeOptions.compressionOptions.minimumBatchSizeInBytes
+                : Number.POSITIVE_INFINITY,
+            compressionAlgorithm: CompressionAlgorithms.lz4,
+        };
         this.innerDeltaManager = deltaManager;
         this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
         // Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
@@ -540,10 +614,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.disposeFn = disposeFn ?? closeFn;
         // In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
         this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
-        this.mc = createChildMonitoringContext({
-            logger: this.logger,
-            namespace: "ContainerRuntime",
-        });
         let loadSummaryNumber;
         // Get the container creation metadata. For new container, we initialize these. For existing containers,
         // get the values from the metadata blob.
@@ -564,7 +634,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             loadSummaryNumber = 0;
         }
         this.nextSummaryNumber = loadSummaryNumber + 1;
-        this.messageAtLastSummary = metadata?.message;
+        this.messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Note that we only need to pull the *initial* connected state from the context.
         // Later updates come through calls to setConnectionState.
         this._connected = connected;
@@ -667,32 +737,38 @@ export class ContainerRuntime extends TypedEventEmitter {
             return this.submitSignalFn(envelope2, targetClientId);
         };
         this.channelCollection = new ChannelCollection(getSummaryForDatastores(baseSnapshot, metadata), parentContext, this.mc.logger, (path, reason, timestampMs, packagePath, request, headerData) => this.garbageCollector.nodeUpdated(path, reason, timestampMs, packagePath, request, headerData), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
-        this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
-            if (!this.disposed) {
-                this.submit({ type: ContainerMessageType.BlobAttach, contents: undefined }, undefined, {
-                    localId,
-                    blobId,
-                });
-            }
-        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState?.pendingAttachmentBlobs, (error) => this.closeFn(error));
+        this.blobManager = new BlobManager({
+            routeContext: this.handleContext,
+            snapshot: blobManagerSnapshot,
+            getStorage: () => this.storage,
+            sendBlobAttachOp: (localId, blobId) => {
+                if (!this.disposed) {
+                    this.submit({ type: ContainerMessageType.BlobAttach, contents: undefined }, undefined, {
+                        localId,
+                        blobId,
+                    });
+                }
+            },
+            blobRequested: (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
+            isBlobDeleted: (blobPath) => this.garbageCollector.isNodeDeleted(blobPath),
+            runtime: this,
+            stashedBlobs: pendingRuntimeState?.pendingAttachmentBlobs,
+            closeContainer: (error) => this.closeFn(error),
+        });
         this.scheduleManager = new ScheduleManager(this.innerDeltaManager, this, () => this.clientId, createChildLogger({ logger: this.logger, namespace: "ScheduleManager" }));
         this.pendingStateManager = new PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
             clientId: () => this.clientId,
             close: this.closeFn,
             connected: () => this.connected,
-            reSubmit: this.reSubmit.bind(this),
+            reSubmit: (message) => {
+                this.reSubmit(message);
+                this.flush();
+            },
             reSubmitBatch: this.reSubmitBatch.bind(this),
             isActiveConnection: () => this.innerDeltaManager.active,
             isAttached: () => this.attachState !== AttachState.Detached,
         }, pendingRuntimeState?.pending, this.logger);
-        const disableCompression = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
-        const compressionOptions = disableCompression === true
-            ? {
-                minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
-                compressionAlgorithm: CompressionAlgorithms.lz4,
-            }
-            : runtimeOptions.compressionOptions;
         const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
         const legacySendBatchFn = makeLegacySendBatchFn(this.submitFn, this.innerDeltaManager);
         this.outbox = new Outbox({
@@ -797,10 +873,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             disableIsolatedChannels: metadata?.disableIsolatedChannels,
             gcVersion: metadata?.gcFeature,
             options: JSON.stringify(runtimeOptions),
-            idCompressorModeMetadata: metadata?.idCompressorMode,
+            idCompressorModeMetadata: metadata?.documentSchema?.runtime?.idCompressorMode,
             idCompressorMode: this.idCompressorMode,
             featureGates: JSON.stringify({
-                disableCompression,
+                ...featureGatesForTelemetry,
                 disableOpReentryCheck,
                 disableChunking,
                 disableAttachReorder: this.disableAttachReorder,
@@ -823,15 +899,33 @@ export class ContainerRuntime extends TypedEventEmitter {
         // saved state, i.e. all the ops marked by Loader layer sa savedOp === true.
         this.skipSavedCompressorOps = pendingRuntimeState?.pendingIdCompressorState !== undefined;
     }
+    onSchemaChange(schema) {
+        // Most of the settings will be picked up only by new sessions (i.e. after reload).
+        // We can make it better in the future (i.e. start to use op compression right away), but for simplicity
+        // this is not done.
+        // But ID compressor is special. It's possible, that in future, we will remove "stickiness" of ID compressor setting
+        // and will allow to start using it. If that were to happen, we want to ensure that we do not break eventual consistency
+        // promises. To do so, we need to initialize id compressor right away.
+        // As it's implemented right now (with async initialization), this will only work for "off" -> "delayed" transitions.
+        // Anything else is too risky, and requires ability to initialize ID compressor synchronously!
+        if (schema.runtime.idCompressorMode !== undefined) {
+            // eslint-disable-next-line @typescript-eslint/no-floating-promises
+            this.loadIdCompressor();
+        }
+    }
     getCreateChildSummarizerNodeFn(id, createParam) {
         return (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn);
     }
     deleteChildSummarizerNode(id) {
         return this.summarizerNode.deleteChild(id);
     }
+    /* IFluidParentContext APIs that should not be called on Root */
     makeLocallyVisible() {
         assert(false, 0x8eb /* should not be called */);
     }
+    setChannelDirty(address) {
+        assert(false, 0x909 /* should not be called */);
+    }
     /**
      * Initializes the state from the base snapshot this container runtime loaded from.
      */
@@ -1036,18 +1130,30 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     /** Adds the container's metadata to the given summary tree. */
     addMetadataToSummary(summaryTree) {
+        // The last message processed at the time of summary. If there are no new messages, use the message from the
+        // last summary.
+        const message = extractSummaryMetadataMessage(this.deltaManager.lastMessage) ??
+            this.messageAtLastSummary;
+        const documentSchema = this.documentsSchemaController.summarizeDocumentSchema(this.deltaManager.lastSequenceNumber);
+        // Is document schema explicit control on?
+        const explitiSchemaControl = documentSchema?.runtime.explicitSchemaControl;
         const metadata = {
             ...this.createContainerMetadata,
             // Increment the summary number for the next summary that will be generated.
             summaryNumber: this.nextSummaryNumber++,
             summaryFormatVersion: 1,
             ...this.garbageCollector.getMetadata(),
-            // The last message processed at the time of summary. If there are no new messages, use the message from the
-            // last summary.
-            message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ??
-                this.messageAtLastSummary,
             telemetryDocumentId: this.telemetryDocumentId,
-            idCompressorMode: this.idCompressorMode,
+            // If explicit document schema control is not on, use legacy way to supply last message (using 'message' property).
+            // Otherwise use new 'lastMessage' property, but also put content into the 'message' property that cases old
+            // runtimes (that preceed document schema control capabilities) to close container on load due to mismatch in
+            // last message's sequence number.
+            // See also lastMessageFromMetadata()
+            message: explitiSchemaControl
+                ? { sequenceNumber: -1 }
+                : message,
+            lastMessage: explitiSchemaControl ? message : undefined,
+            documentSchema,
         };
         addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
     }
@@ -1108,14 +1214,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
     }
-    resetReconnectCount(message) {
-        // Chunked ops don't count towards making progress as they are sent
-        // in their own batches before the originating batch is sent.
-        // Therefore, receiving them while attempting to send the originating batch
-        // does not mean that the container is making any progress.
-        if (message?.type !== ContainerMessageType.ChunkedOp) {
-            this.consecutiveReconnects = 0;
-        }
+    resetReconnectCount() {
+        this.consecutiveReconnects = 0;
     }
     replayPendingStates() {
         // We need to be able to send ops to replay states
@@ -1164,7 +1264,18 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.Alias:
                 return this.channelCollection.applyStashedOp(opContents);
             case ContainerMessageType.IdAllocation:
-                assert(this.idCompressorMode !== "off", 0x8f1 /* ID compressor should be in use */);
+                // IDs allocation ops in stashed state are ignored because the tip state of the compressor
+                // is serialized into the pending state. This is done because generation of new IDs during
+                // stashed op application (or, later, resubmit) must generate new IDs and if the compressor
+                // was loaded from a state serialized at the same time as the summary tree in the stashed state
+                // then it would generate IDs that collide with any in later stashed ops.
+                // In the future, IdCompressor could be extended to have an "applyStashedOp" or similar method
+                // and the runtime could filter out all ID allocation ops from the stashed state and apply them
+                // before applying the rest of the stashed ops. This would accomplish the same thing but with
+                // better performance in future incremental stashed state creation.
+                assert(this.idCompressorMode !== undefined, 0x8f1 /* ID compressor should be in use */);
+                return;
+            case ContainerMessageType.DocumentSchemaChange:
                 return;
             case ContainerMessageType.BlobAttach:
                 return;
@@ -1195,12 +1306,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
-    setConnectionState(connected, clientId) {
-        if (connected && this.idCompressorMode === "delayed" && !this.compressorLoadInitiated) {
-            this.compressorLoadInitiated = true;
-            this.createIdCompressor()
+    async loadIdCompressor() {
+        if (this._idCompressor === undefined &&
+            this.idCompressorMode !== undefined &&
+            this._loadIdCompressor === undefined) {
+            this._loadIdCompressor = this.createIdCompressor()
                 .then((compressor) => {
                 this._idCompressor = compressor;
+                // Finalize any ranges we received while the compressor was turned off.
                 for (const range of this.pendingIdCompressorOps) {
                     this._idCompressor.finalizeCreationRange(range);
                 }
@@ -1208,8 +1321,16 @@ export class ContainerRuntime extends TypedEventEmitter {
             })
                 .catch((error) => {
                 this.logger.sendErrorEvent({ eventName: "IdCompressorDelayedLoad" }, error);
+                throw error;
             });
         }
+        return this._loadIdCompressor;
+    }
+    setConnectionState(connected, clientId) {
+        if (connected && this.idCompressorMode === "delayed") {
+            // eslint-disable-next-line @typescript-eslint/no-floating-promises
+            this.loadIdCompressor();
+        }
         if (connected === false && this.delayConnectClientId !== undefined) {
             this.delayConnectClientId = undefined;
             this.mc.logger.sendTelemetryEvent({
@@ -1218,21 +1339,17 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
             return;
         }
+        if (!connected) {
+            this.documentsSchemaController.onDisconnect();
+        }
         // If there are stashed blobs in the pending state, we need to delay
         // propagation of the "connected" event until we have uploaded them to
         // ensure we don't submit ops referencing a blob that has not been uploaded
         const connecting = connected && !this._connected;
-        if (connecting && this.blobManager.hasPendingStashedBlobs()) {
+        if (connecting && this.blobManager.hasPendingStashedUploads()) {
             assert(!this.delayConnectClientId, 0x791 /* Connect event delay must be canceled before subsequent connect event */);
             assert(!!clientId, 0x792 /* Must have clientId when connecting */);
             this.delayConnectClientId = clientId;
-            this.blobManager.processStashedChanges().then(() => {
-                // make sure we didn't reconnect before the promise resolved
-                if (this.delayConnectClientId === clientId && !this.disposed) {
-                    this.delayConnectClientId = undefined;
-                    this.setConnectionStateCore(connected, clientId);
-                }
-            }, (error) => this.closeFn(error));
             return;
         }
         this.setConnectionStateCore(connected, clientId);
@@ -1320,10 +1437,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.scheduleManager.beforeOpProcessing(message);
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
+            // See commit that added this assert for more details.
+            // These calls should be made for all but chunked ops:
+            // 1) this.pendingStateManager.processPendingLocalMessage() below
+            // 2) this.resetReconnectCount() below
+            assert(message.type !== ContainerMessageType.ChunkedOp, 0x93b /* we should never get here with chunked ops */);
             let localOpMetadata;
-            if (local &&
-                messageWithContext.modernRuntimeMessage &&
-                message.type !== ContainerMessageType.ChunkedOp) {
+            if (local && messageWithContext.modernRuntimeMessage) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(messageWithContext.message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1338,7 +1458,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // If we have processed a local op, this means that the container is
                 // making progress and we can reset the counter for how many times
                 // we have consecutively replayed the pending states
-                this.resetReconnectCount(message);
+                this.resetReconnectCount();
             }
         }
         catch (e) {
@@ -1372,7 +1492,10 @@ export class ContainerRuntime extends TypedEventEmitter {
                     messageWithContext.message.metadata?.savedOp ===
                         true)) {
                     const range = messageWithContext.message.contents;
+                    // Some other client turned on the id compressor. If we have not turned it on,
+                    // put it in a pending queue and delay finalization.
                     if (this._idCompressor === undefined) {
+                        assert(this.idCompressorMode !== undefined, 0x93c /* id compressor should be enabled */);
                         this.pendingIdCompressorOps.push(range);
                     }
                     else {
@@ -1384,8 +1507,14 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.garbageCollector.processMessage(messageWithContext.message, local);
                 break;
             case ContainerMessageType.ChunkedOp:
+                // From observability POV, we should not exppse the rest of the system (including "op" events on object) to these messages.
+                // Also resetReconnectCount() would be wrong - see comment that was there before this change was made.
+                assert(false, 0x93d /* should not even get here */);
             case ContainerMessageType.Rejoin:
                 break;
+            case ContainerMessageType.DocumentSchemaChange:
+                this.documentsSchemaController.processDocumentSchemaOp(messageWithContext.message.contents, messageWithContext.local, messageWithContext.message.sequenceNumber);
+                break;
             default: {
                 // If we didn't necessarily expect a runtime message type, then no worries - just return
                 // e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
@@ -1559,10 +1688,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         return channel.entryPoint;
     }
     createDetachedDataStore(pkg, loadingGroupId) {
-        return this.channelCollection.createDetachedDataStoreCore(pkg, loadingGroupId);
+        return this.channelCollection.createDetachedDataStore(pkg, loadingGroupId);
     }
     async createDataStore(pkg, loadingGroupId) {
-        const context = this.channelCollection._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], undefined, // props
+        const context = this.channelCollection.createDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], undefined, // props
         loadingGroupId);
         return channelToDataStore(await context.realize(), context.id, this.channelCollection, this.mc.logger);
     }
@@ -1570,7 +1699,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @deprecated 0.16 Issue #1537, #3631
      */
     async _createDataStoreWithProps(pkg, props) {
-        const context = this.channelCollection._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], props);
+        const context = this.channelCollection.createDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], props);
         return channelToDataStore(await context.realize(), context.id, this.channelCollection, this.mc.logger);
     }
     canSendOps() {
@@ -1615,6 +1744,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                 }
                 break;
             }
+            case ContainerMessageType.IdAllocation:
+            case ContainerMessageType.DocumentSchemaChange:
             case ContainerMessageType.GC: {
                 return false;
             }
@@ -1641,7 +1772,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     /**
      * Submits the signal to be sent to other clients.
      * @param type - Type of the signal.
-     * @param content - Content of the signal.
+     * @param content - Content of the signal. Should be a JSON serializable object or primitive.
      * @param targetClientId - When specified, the signal is only sent to the provided client id.
      */
     submitSignal(type, content, targetClientId) {
@@ -1677,6 +1808,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // We can finalize any allocated IDs since we're the only client
         const idRange = this._idCompressor?.takeNextCreationRange();
         if (idRange !== undefined) {
+            assert(idRange.ids === undefined || idRange.ids.firstGenCount === 1, 0x93e /* No other ranges should be taken while container is detached. */);
             this._idCompressor?.finalizeCreationRange(idRange);
         }
         const summarizeResult = this.channelCollection.getAttachSummary(telemetryContext);
@@ -1690,6 +1822,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         // Wrap data store summaries in .channels subtree.
         wrapSummaryInChannelsTree(summarizeResult);
         const pathPartsForChildren = [channelsTreeName];
+        // Ensure that ID compressor had a chance to load, if we are using delayed mode.
+        await this.loadIdCompressor();
         this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
         return {
             ...summarizeResult,
@@ -1765,21 +1899,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         const { dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(usedRoutes);
         this.channelCollection.updateUsedRoutes(dataStoreRoutes);
     }
-    /**
-     * This is called to update objects whose routes are unused.
-     * @param unusedRoutes - Data store and attachment blob routes that are unused in this Container.
-     */
-    updateUnusedRoutes(unusedRoutes) {
-        const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
-        this.blobManager.updateUnusedRoutes(blobManagerRoutes);
-        this.channelCollection.updateUnusedRoutes(dataStoreRoutes);
-    }
-    /**
-     * @deprecated Replaced by deleteSweepReadyNodes.
-     */
-    deleteUnusedNodes(unusedRoutes) {
-        throw new Error("deleteUnusedRoutes should not be called");
-    }
     /**
      * After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
      * @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
@@ -2247,19 +2366,18 @@ export class ContainerRuntime extends TypedEventEmitter {
                 const idAllocationBatchMessage = {
                     contents: JSON.stringify(idAllocationMessage),
                     referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
-                    metadata: undefined,
-                    localOpMetadata: undefined,
-                    type: ContainerMessageType.IdAllocation,
                 };
                 this.outbox.submitIdAllocation(idAllocationBatchMessage);
             }
         }
     }
-    submit(containerRuntimeMessage, localOpMetadata = undefined, metadata = undefined) {
+    submit(containerRuntimeMessage, localOpMetadata = undefined, metadata) {
         this.verifyNotClosed();
         this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
         assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
+        assert(metadata === undefined ||
+            containerRuntimeMessage.type === ContainerMessageType.BlobAttach, 0x93f /* metadata */);
         const serializedContent = JSON.stringify(containerRuntimeMessage);
         // Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
         // container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
@@ -2272,7 +2390,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         const type = containerRuntimeMessage.type;
         const message = {
             contents: serializedContent,
-            type,
             metadata,
             localOpMetadata,
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
@@ -2287,6 +2404,20 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             else {
                 this.submitIdAllocationOpIfNeeded();
+                // Allow document schema controller to send a message if it needs to propose change in document schema.
+                // If it needs to send a message, it will call provided callback with payload of such message and rely
+                // on this callback to do actual sending.
+                const contents = this.documentsSchemaController.maybeSendSchemaMessage();
+                if (contents) {
+                    const msg = {
+                        type: ContainerMessageType.DocumentSchemaChange,
+                        contents,
+                    };
+                    this.outbox.submit({
+                        contents: JSON.stringify(msg),
+                        referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                    });
+                }
                 // If this is attach message for new data store, and we are in a batch, send this op out of order
                 // Is it safe:
                 //    Yes, this should be safe reordering. Newly created data stores are not visible through API surface.
@@ -2458,6 +2589,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.GC:
                 this.submit(message);
                 break;
+            case ContainerMessageType.DocumentSchemaChange:
+                // There is no need to resend this message. Document schema controller will properly resend it again (if needed)
+                // on a first occasion (any ops sent after reconnect). There is a good chance, though, that it will not want to
+                // send any ops, as some other client already changed schema.
+                break;
             default: {
                 // This case should be very rare - it would imply an op was stashed from a
                 // future version of runtime code and now is being applied on an older version.
@@ -2660,8 +2796,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
     }
     get groupedBatchingEnabled() {
-        const killSwitch = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
-        return killSwitch !== true && this.runtimeOptions.enableGroupedBatching;
+        return this.documentSchema.opGroupingEnabled === true;
     }
 }
 //# sourceMappingURL=containerRuntime.js.map