@fluidframework/container-runtime 2.41.0-338401 → 2.42.0

package/lib/containerRuntime.js

@@ -2,10 +2,10 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
+import { createEmitter, Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
 import { AttachState } from "@fluidframework/container-definitions";
 import { isIDeltaManagerFull } from "@fluidframework/container-definitions/internal";
-import { assert, Deferred, LazyPromise, PromiseCache, delay, } from "@fluidframework/core-utils/internal";
+import { assert, Deferred, Lazy, LazyPromise, PromiseCache, delay, fail, unreachableCase, } from "@fluidframework/core-utils/internal";
 import { SummaryType } from "@fluidframework/driver-definitions";
 import { FetchSource, MessageType } from "@fluidframework/driver-definitions/internal";
 import { readAndParse } from "@fluidframework/driver-utils/internal";
@@ -19,7 +19,7 @@ import { v4 as uuid } from "uuid";
 import { BindBatchTracker } from "./batchTracker.js";
 import { BlobManager, blobManagerBasePath, blobsTreeName, isBlobPath, loadBlobManagerLoadInfo, } from "./blobManager/index.js";
 import { ChannelCollection, getSummaryForDatastores, wrapContext, } from "./channelCollection.js";
-import { defaultMinVersionForCollab, getMinVersionForCollabDefaults, isValidMinVersionForCollab, } from "./compatUtils.js";
+import { defaultMinVersionForCollab, getMinVersionForCollabDefaults, isValidMinVersionForCollab, validateRuntimeOptions, } from "./compatUtils.js";
 import { CompressionAlgorithms, disabledCompressionConfig } from "./compressionDefinitions.js";
 import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
 import { ContainerFluidHandleContext } from "./containerHandleContext.js";
@@ -191,6 +191,20 @@ export async function loadContainerRuntime(params) {
     return ContainerRuntime.loadRuntime(params);
 }
 const defaultMaxConsecutiveReconnects = 7;
+/**
+ * These are the ONLY message types that are allowed to be submitted while in staging mode
+ * (Does not apply to pre-StagingMode batches that are resubmitted, those are not considered to be staged)
+ */
+function canStageMessageOfType(type) {
+    return (
+    // These are user changes coming up from the runtime's DataStores
+    type === ContainerMessageType.FluidDataStoreOp ||
+        // GC ops are used to detect issues in the reference graph so all clients can repair their GC state.
+        // These can be submitted at any time, including while in Staging Mode.
+        type === ContainerMessageType.GC ||
+        // These are typically sent shortly after boot and will not be common in Staging Mode, but it's possible.
+        type === ContainerMessageType.DocumentSchemaChange);
+}
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
@@ -239,6 +253,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (!isValidMinVersionForCollab(minVersionForCollab)) {
             throw new UsageError(`Invalid minVersionForCollab: ${minVersionForCollab}. It must be an existing FF version (i.e. 2.22.1).`);
         }
+        // We also validate that there is not a mismatch between `minVersionForCollab` and runtime options that
+        // were manually set.
+        validateRuntimeOptions(minVersionForCollab, runtimeOptions);
         const defaultsAffectingDocSchema = getMinVersionForCollabDefaults(minVersionForCollab);
         // The following are the default values for the options that do not affect the DocumentSchema.
         const defaultsNotAffectingDocSchema = {
@@ -497,15 +514,19 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     /**
      * Invokes the given callback and expects that no ops are submitted
-     * until execution finishes. If an op is submitted, an error will be raised.
+     * until execution finishes. If an op is submitted, it will be marked as reentrant.
      *
      * @param callback - the callback to be invoked
      */
     ensureNoDataModelChanges(callback) {
         return this.dataModelChangeRunner.run(callback);
     }
+    /**
+     * Indicates whether the container is in a state where it is able to send
+     * ops (connected to op stream and not in readonly mode).
+     */
     get connected() {
-        return this._connected;
+        return this.canSendOps;
     }
     /**
      * clientId of parent (non-summarizing) container that owns summarizer container
@@ -569,7 +590,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // Once it loads, it will process all such ops and we will stop accumulating further ops - ops will be processes as they come in.
         this.pendingIdCompressorOps = [];
         this.batchRunner = new BatchRunCounter();
-        this.flushTaskExists = false;
+        this.flushScheduled = false;
         this.consecutiveReconnects = 0;
         this.dataModelChangeRunner = new RunCounter();
         this._disposed = false;
@@ -582,6 +603,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.snapshotCacheForLoadingGroupIds = new PromiseCache({
             expiry: { policy: "absolute", durationMs: 60000 },
         });
+        this.extensions = new Map();
         this.notifyReadOnlyState = (readonly) => this.channelCollection.notifyReadOnlyState(readonly);
         /**
          * Enter Staging Mode, such that ops submitted to the ContainerRuntime will not be sent to the ordering service.
@@ -592,7 +614,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         // eslint-disable-next-line import/no-deprecated
         this.enterStagingMode = () => {
             if (this.stageControls !== undefined) {
-                throw new Error("already in staging mode");
+                throw new UsageError("already in staging mode");
+            }
+            if (this.attachState === AttachState.Detached) {
+                throw new UsageError("cannot enter staging mode while detached");
             }
             // Make sure all BatchManagers are empty before entering staging mode,
             // since we mark whole batches as "staged" or not to indicate whether to submit them.
@@ -613,11 +638,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                     // Pop all staged batches from the PSM and roll them back in LIFO order
                     this.pendingStateManager.popStagedBatches(({ runtimeOp, localOpMetadata }) => {
                         assert(runtimeOp !== undefined, 0xb82 /* Staged batches expected to have runtimeOp defined */);
-                        this.rollback(runtimeOp, localOpMetadata);
+                        this.rollbackStagedChanges(runtimeOp, localOpMetadata);
                     });
-                    if (this.attachState === AttachState.Attached) {
-                        this.updateDocumentDirtyState(this.pendingMessagesCount !== 0);
-                    }
+                    this.updateDocumentDirtyState();
                 }),
                 commitChanges: (optionsParam) => {
                     const options = { ...defaultStagingCommitOptions, ...optionsParam };
@@ -634,6 +657,15 @@ export class ContainerRuntime extends TypedEventEmitter {
             return this.stageControls;
         };
         this.readAndParseBlob = async (id) => readAndParse(this.storage, id);
+        // While internal, ContainerRuntime has not been converted to use the new events support.
+        // Recreate the required events (new pattern) with injected, wrapper new emitter.
+        // It is lazily create to avoid listeners (old events) that ultimately go nowhere.
+        this.lazyEventsForExtensions = new Lazy(() => {
+            const eventEmitter = createEmitter();
+            this.on("connected", (clientId) => eventEmitter.emit("connected", clientId));
+            this.on("disconnected", () => eventEmitter.emit("disconnected"));
+            return eventEmitter;
+        });
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, pendingLocalState, supportedFeatures, snapshotWithContents, } = context;
         // In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
         this.disposeFn = disposeFn ?? closeFn;
@@ -643,6 +675,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.mc = createChildMonitoringContext({
             logger: this.baseLogger,
             namespace: "ContainerRuntime",
+            properties: {
+                all: {
+                    inStagingMode: this.inStagingMode,
+                },
+            },
         });
         // If we support multiple algorithms in the future, then we would need to manage it here carefully.
         // We can use runtimeOptions.compressionOptions.compressionAlgorithm, but only if it's in the schema list!
@@ -662,12 +699,28 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.submitSummaryFn =
             submitSummaryFn ??
                 ((summaryOp, refseq) => submitFn(MessageType.Summarize, summaryOp, false));
-        this.submitSignalFn = submitSignalFn;
+        const sequenceAndSubmitSignal = (envelope, targetClientId) => {
+            if (targetClientId === undefined) {
+                this.signalTelemetryManager.applyTrackingToBroadcastSignalEnvelope(envelope);
+            }
+            submitSignalFn(envelope, targetClientId);
+        };
+        this.submitSignalFn = (envelope, targetClientId) => {
+            if (envelope.address?.startsWith("/")) {
+                throw new Error("General path based addressing is not implemented");
+            }
+            sequenceAndSubmitSignal(envelope, targetClientId);
+        };
+        this.submitExtensionSignal = (id, addressChain, message) => {
+            this.verifyNotClosed();
+            const envelope = createNewSignalEnvelope(`/ext/${id}/${addressChain.join("/")}`, message.type, message.content);
+            sequenceAndSubmitSignal(envelope, message.targetClientId);
+        };
         // TODO: After IContainerContext.options is removed, we'll just create a new blank object {} here.
         // Values are generally expected to be set from the runtime side.
         this.options = options ?? {};
         this.clientDetails = clientDetails;
-        const isSummarizerClient = this.clientDetails.type === summarizerClientType;
+        this.isSummarizerClient = this.clientDetails.type === summarizerClientType;
         this.loadedFromVersionId = context.getLoadedFromVersion()?.id;
         // eslint-disable-next-line unicorn/consistent-destructuring
         this._getClientId = () => context.clientId;
@@ -694,8 +747,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.mc.logger.sendTelemetryEvent({
                 eventName: "Attached",
                 details: {
-                    dirtyContainer: this.dirtyContainer,
-                    hasPendingMessages: this.hasPendingMessages(),
+                    lastEmittedDirty: this.lastEmittedDirty,
+                    currentDirtyState: this.computeCurrentDirtyState(),
                 },
             });
         });
@@ -704,7 +757,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             details: { attachState: this.attachState },
         }));
         // In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
-        this.closeFn = isSummarizerClient ? this.disposeFn : closeFn;
+        this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
         let loadSummaryNumber;
         // Get the container creation metadata. For new container, we initialize these. For existing containers,
         // get the values from the metadata blob.
@@ -728,7 +781,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Note that we only need to pull the *initial* connected state from the context.
         // Later updates come through calls to setConnectionState.
-        this._connected = connected;
+        this.canSendOps = connected;
         this.mc.logger.sendTelemetryEvent({
             eventName: "GCFeatureMatrix",
             metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
@@ -821,7 +874,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             existing,
             metadata,
             createContainerMetadata: this.createContainerMetadata,
-            isSummarizerClient,
+            isSummarizerClient: this.isSummarizerClient,
             getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
             getLastSummaryTimestampMs: () => this.messageAtLastSummary?.timestamp,
             readAndParseBlob: async (id) => readAndParse(this.storage, id),
@@ -862,9 +915,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             // verifyNotClosed is called in FluidDataStoreContext, which is *the* expected caller.
             const envelope1 = content;
             const envelope2 = createNewSignalEnvelope(envelope1.address, type, envelope1.contents);
-            if (targetClientId === undefined) {
-                this.signalTelemetryManager.applyTrackingToBroadcastSignalEnvelope(envelope2);
-            }
             this.submitSignalFn(envelope2, targetClientId);
         };
         let snapshot = getSummaryForDatastores(baseSnapshot, metadata);
@@ -908,7 +958,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Keep the old flag name even though we renamed the class member (it shipped in 2.31.0)
             this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableFlushBeforeProcess") === true;
         this.outbox = new Outbox({
-            shouldSend: () => this.canSendOps(),
+            shouldSend: () => this.shouldSendOps(),
             pendingStateManager: this.pendingStateManager,
             submitBatchFn,
             legacySendBatchFn,
@@ -958,9 +1008,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs =
             closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
-        this.dirtyContainer =
-            this.attachState !== AttachState.Attached || this.hasPendingMessages();
-        context.updateDirtyContainerState(this.dirtyContainer);
+        // We haven't emitted dirty/saved yet, but this is the baseline so we know to emit when it changes
+        this.lastEmittedDirty = this.computeCurrentDirtyState();
+        context.updateDirtyContainerState(this.lastEmittedDirty);
         if (!this.skipSafetyFlushDuringProcessStack) {
             // Reference Sequence Number may have just changed, and it must be consistent across a batch,
             // so we should flush now to clear the way for the next ops.
@@ -1047,7 +1097,14 @@ export class ContainerRuntime extends TypedEventEmitter {
         await this.initializeSummarizer(loader);
         if (this.sessionSchema.idCompressorMode === "on" ||
             (this.sessionSchema.idCompressorMode === "delayed" && this.connected)) {
-            this._idCompressor = this.createIdCompressorFn();
+            PerformanceEvent.timedExec(this.mc.logger, { eventName: "CreateIdCompressorOnBoot" }, (event) => {
+                this._idCompressor = this.createIdCompressorFn();
+                event.end({
+                    details: {
+                        idCompressorMode: this.sessionSchema.idCompressorMode,
+                    },
+                });
+            });
             // This is called from loadRuntime(), long before we process any ops, so there should be no ops accumulated yet.
             assert(this.pendingIdCompressorOps.length === 0, 0x8ec /* no pending ops */);
         }
@@ -1076,8 +1133,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.innerDeltaManager, this._quorum);
         const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, this.electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible, this.mc.config.getBoolean("Fluid.ContainerRuntime.OrderedClientElection.EnablePerformanceEvents"));
         this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, summaryCollection, orderedClientElectionForSummarizer, maxOpsSinceLastSummary);
-        const isSummarizerClient = this.clientDetails.type === summarizerClientType;
-        if (isSummarizerClient) {
+        if (this.isSummarizerClient) {
             // We want to dynamically import any thing inside summaryDelayLoadedModule module only when we are the summarizer client,
             // so that all non summarizer clients don't have to load the code inside this module.
             const module = await import(
@@ -1425,35 +1481,32 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     replayPendingStates() {
         // We need to be able to send ops to replay states
-        if (!this.canSendOps()) {
+        if (!this.shouldSendOps()) {
             return;
         }
-        // We need to temporary clear the dirty flags and disable
-        // dirty state change events to detect whether replaying ops
-        // has any effect.
-        // Save the old state, reset to false, disable event emit
-        const oldState = this.dirtyContainer;
-        this.dirtyContainer = false;
+        // Replaying is an internal operation and we don't want to generate noise while doing it.
+        // So temporarily disable dirty state change events, and save the old state.
+        // When we're done, we'll emit the event if the state changed.
+        const oldState = this.lastEmittedDirty;
         assert(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
         this.emitDirtyDocumentEvent = false;
-        let newState;
         try {
             // Any ID Allocation ops that failed to submit after the pending state was queued need to have
             // the corresponding ranges resubmitted (note this call replaces the typical resubmit flow).
             // Since we don't submit ID Allocation ops when staged, any outstanding ranges would be from
             // before staging mode so we can simply say staged: false.
             this.submitIdAllocationOpIfNeeded({ resubmitOutstandingRanges: true, staged: false });
+            this.scheduleFlush();
             // replay the ops
             this.pendingStateManager.replayPendingStates();
         }
         finally {
-            // Save the new start and restore the old state, re-enable event emit
-            newState = this.dirtyContainer;
-            this.dirtyContainer = oldState;
+            // Restore the old state, re-enable event emit
+            this.lastEmittedDirty = oldState;
             this.emitDirtyDocumentEvent = true;
         }
-        // Officially transition from the old state to the new state.
-        this.updateDocumentDirtyState(newState);
+        // This will emit an event if the state changed relative to before replay
+        this.updateDocumentDirtyState();
     }
     /**
      * Parse an op's type and actual content from given serialized content
@@ -1510,25 +1563,35 @@ export class ContainerRuntime extends TypedEventEmitter {
     loadIdCompressor() {
         if (this._idCompressor === undefined &&
             this.sessionSchema.idCompressorMode !== undefined) {
-            this._idCompressor = this.createIdCompressorFn();
-            // Finalize any ranges we received while the compressor was turned off.
-            const ops = this.pendingIdCompressorOps;
-            this.pendingIdCompressorOps = [];
-            for (const range of ops) {
-                this._idCompressor.finalizeCreationRange(range);
-            }
+            PerformanceEvent.timedExec(this.mc.logger, { eventName: "CreateIdCompressorOnDelayedLoad" }, (event) => {
+                this._idCompressor = this.createIdCompressorFn();
+                // Finalize any ranges we received while the compressor was turned off.
+                const ops = this.pendingIdCompressorOps;
+                this.pendingIdCompressorOps = [];
+                const trace = Trace.start();
+                for (const range of ops) {
+                    this._idCompressor.finalizeCreationRange(range);
+                }
+                event.end({
+                    details: {
+                        finalizeCreationRangeDuration: trace.trace().duration,
+                        idCompressorMode: this.sessionSchema.idCompressorMode,
+                        pendingIdCompressorOps: ops.length,
+                    },
+                });
+            });
             assert(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
         }
     }
-    setConnectionState(connected, clientId) {
+    setConnectionState(canSendOps, clientId) {
         // Validate we have consistent state
         const currentClientId = this._audience.getSelf()?.clientId;
         assert(clientId === currentClientId, 0x977 /* input clientId does not match Audience */);
         assert(this.clientId === currentClientId, 0x978 /* this.clientId does not match Audience */);
-        if (connected && this.sessionSchema.idCompressorMode === "delayed") {
+        if (canSendOps && this.sessionSchema.idCompressorMode === "delayed") {
             this.loadIdCompressor();
         }
-        if (connected === false && this.delayConnectClientId !== undefined) {
+        if (canSendOps === false && this.delayConnectClientId !== undefined) {
             this.delayConnectClientId = undefined;
             this.mc.logger.sendTelemetryEvent({
                 eventName: "UnsuccessfulConnectedTransition",
@@ -1536,37 +1599,39 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
             return;
         }
-        if (!connected) {
-            this.documentsSchemaController.onDisconnect();
-        }
         // If there are stashed blobs in the pending state, we need to delay
         // propagation of the "connected" event until we have uploaded them to
         // ensure we don't submit ops referencing a blob that has not been uploaded
-        const connecting = connected && !this._connected;
+        const connecting = canSendOps && !this.canSendOps;
         if (connecting && this.blobManager.hasPendingStashedUploads()) {
             assert(!this.delayConnectClientId, 0x791 /* Connect event delay must be canceled before subsequent connect event */);
             assert(!!clientId, 0x792 /* Must have clientId when connecting */);
             this.delayConnectClientId = clientId;
             return;
         }
-        this.setConnectionStateCore(connected, clientId);
+        this.setConnectionStateCore(canSendOps, clientId);
     }
-    setConnectionStateCore(connected, clientId) {
+    /**
+     * Raises and propagates connected events.
+     * @param canSendOps - Indicates whether the container can send ops or not (connected and not readonly).
+     * @remarks The connection state from container context used here when raising connected events.
+     */
+    setConnectionStateCore(canSendOps, clientId) {
         assert(!this.delayConnectClientId, 0x394 /* connect event delay must be cleared before propagating connect event */);
         this.verifyNotClosed();
         // There might be no change of state due to Container calling this API after loading runtime.
-        const changeOfState = this._connected !== connected;
-        const reconnection = changeOfState && !connected;
+        const canSendOpsChanged = this.canSendOps !== canSendOps;
+        const reconnection = canSendOpsChanged && !canSendOps;
         // We need to flush the ops currently collected by Outbox to preserve original order.
         // This flush NEEDS to happen before we set the ContainerRuntime to "connected".
         // We want these ops to get to the PendingStateManager without sending to service and have them return to the Outbox upon calling "replayPendingStates".
-        if (changeOfState && connected) {
+        if (canSendOpsChanged && canSendOps) {
             this.flush();
         }
-        this._connected = connected;
-        if (connected) {
+        this.canSendOps = canSendOps;
+        if (canSendOps) {
             assert(this.attachState === AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
-            if (changeOfState) {
+            if (canSendOpsChanged) {
                 this.signalTelemetryManager.resetTracking();
             }
         }
@@ -1582,12 +1647,12 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return;
             }
         }
-        if (changeOfState) {
+        if (canSendOpsChanged) {
             this.replayPendingStates();
         }
-        this.channelCollection.setConnectionState(connected, clientId);
-        this.garbageCollector.setConnectionState(connected, clientId);
-        raiseConnectedEvent(this.mc.logger, this, connected, clientId);
+        this.channelCollection.setConnectionState(canSendOps, clientId);
+        this.garbageCollector.setConnectionState(canSendOps, clientId);
+        raiseConnectedEvent(this.mc.logger, this, this.connected /* canSendOps */, clientId);
     }
     async notifyOpReplay(message) {
         await this.pendingStateManager.applyStashedOpsAt(message.sequenceNumber);
@@ -1715,6 +1780,8 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param groupedBatch - true if these messages are part of a grouped op batch.
      */
     processInboundMessages(messagesWithMetadata, locationInBatch, local, savedOp, runtimeBatch, groupedBatch) {
+        // This message could have been the last pending local (dirtyable) message, in which case we need to update dirty state to "saved"
+        this.updateDocumentDirtyState();
         if (locationInBatch.batchStart) {
             const firstMessage = messagesWithMetadata[0]?.message;
             assert(firstMessage !== undefined, 0xa31 /* Batch must have at least one message */);
@@ -1809,11 +1876,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             message.minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
         }
         this._processedClientSequenceNumber = message.clientSequenceNumber;
-        // If there are no more pending messages after processing a local message,
-        // the document is no longer dirty.
-        if (!this.hasPendingMessages()) {
-            this.updateDocumentDirtyState(false);
-        }
         // The DeltaManager used to do this, but doesn't anymore as of Loader v2.4
         // Anyone listening to our "op" event would expect the contents to be parsed per this same logic
         if (typeof message.contents === "string" &&
@@ -1835,11 +1897,6 @@ export class ContainerRuntime extends TypedEventEmitter {
      *
      */
     validateAndProcessRuntimeMessages(message, messagesContent, local, savedOp) {
-        // If there are no more pending messages after processing a local message,
-        // the document is no longer dirty.
-        if (!this.hasPendingMessages()) {
-            this.updateDocumentDirtyState(false);
-        }
         // Get the contents without the localOpMetadata because not all message types know about localOpMetadata.
         const contents = messagesContent.map((c) => c.contents);
         switch (message.type) {
@@ -1915,20 +1972,44 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (message.clientId === this.clientId) {
             this.signalTelemetryManager.trackReceivedSignal(envelope, this.mc.logger, this.consecutiveReconnects);
         }
-        if (envelope.address === undefined) {
+        const fullAddress = envelope.address;
+        if (fullAddress === undefined) {
             // No address indicates a container signal message.
             this.emit("signal", transformed, local);
             return;
         }
-        // Due to a mismatch between different layers in terms of
-        // what is the interface of passing signals, we need to adjust
-        // the signal envelope before sending it to the datastores to be processed
-        const envelope2 = {
-            address: envelope.address,
-            contents: transformed.content,
-        };
-        transformed.content = envelope2;
-        this.channelCollection.processSignal(transformed, local);
+        this.routeNonContainerSignal(fullAddress, transformed, local);
+    }
+    routeNonContainerSignal(address, signalMessage, local) {
+        // channelCollection signals are identified by no starting `/` in address.
+        if (!address.startsWith("/")) {
+            // Due to a mismatch between different layers in terms of
+            // what is the interface of passing signals, we need to adjust
+            // the signal envelope before sending it to the datastores to be processed
+            const envelope = {
+                address,
+                contents: signalMessage.content,
+            };
+            signalMessage.content = envelope;
+            this.channelCollection.processSignal(signalMessage, local);
+            return;
+        }
+        const addresses = address.split("/");
+        if (addresses.length > 2 && addresses[1] === "ext") {
+            const id = addresses[2];
+            const entry = this.extensions.get(id);
+            if (entry !== undefined) {
+                entry.extension.processSignal?.(addresses.slice(3), signalMessage, local);
+                return;
+            }
+        }
+        assert(!local, 0xba0 /* No recipient found for local signal */);
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "SignalAddressNotFound",
+            ...tagCodeArtifacts({
+                address,
+            }),
+        });
     }
     /**
      * Flush the current batch of ops to the ordering service for sequencing
@@ -1938,6 +2019,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param resubmitInfo - If defined, indicates this is a resubmission of a batch with the given Batch info needed for resubmit.
      */
     flush(resubmitInfo) {
+        this.flushScheduled = false;
         try {
             assert(!this.batchRunner.running, 0x24c /* "Cannot call `flush()` while manually accumulating a batch (e.g. under orderSequentially) */);
             this.outbox.flush(resubmitInfo);
@@ -1958,7 +2040,6 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     orderSequentially(callback) {
         let checkpoint;
-        const checkpointDirtyState = this.dirtyContainer;
         // eslint-disable-next-line import/no-deprecated
         let stageControls;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
@@ -1978,11 +2059,10 @@ export class ContainerRuntime extends TypedEventEmitter {
                 if (checkpoint) {
                     // This will throw and close the container if rollback fails
                     try {
-                        checkpoint.rollback((message) => this.rollback(message.runtimeOp, message.localOpMetadata));
-                        // reset the dirty state after rollback to what it was before to keep it consistent
-                        if (this.dirtyContainer !== checkpointDirtyState) {
-                            this.updateDocumentDirtyState(checkpointDirtyState);
-                        }
+                        checkpoint.rollback((message) => 
+                        // These changes are staged since we entered staging mode above
+                        this.rollbackStagedChanges(message.runtimeOp, message.localOpMetadata));
+                        this.updateDocumentDirtyState();
                         stageControls?.discardChanges();
                         stageControls = undefined;
                     }
@@ -2058,17 +2138,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         const context = this.channelCollection.createDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], loadingGroupId);
         return channelToDataStore(await context.realize(), context.id, this.channelCollection, this.mc.logger);
     }
-    canSendOps() {
+    shouldSendOps() {
         // Note that the real (non-proxy) delta manager is needed here to get the readonly info. This is because
         // container runtime's ability to send ops depend on the actual readonly state of the delta manager.
         return (this.connected && !this.innerDeltaManager.readOnlyInfo.readonly && !this.imminentClosure);
     }
-    /**
-     * Typically ops are batched and later flushed together, but in some cases we want to flush immediately.
-     */
-    currentlyBatching() {
-        return this.flushMode !== FlushMode.Immediate || this.batchRunner.running;
-    }
     getQuorum() {
         return this._quorum;
     }
@@ -2080,36 +2154,17 @@ export class ContainerRuntime extends TypedEventEmitter {
      * either were not sent out to delta stream or were not yet acknowledged.
      */
     get isDirty() {
-        return this.dirtyContainer;
+        // Rather than recomputing the dirty state in this moment,
+        // just regurgitate the last emitted dirty state.
+        return this.lastEmittedDirty;
     }
-    isContainerMessageDirtyable({ type, contents, }) {
-        // Certain container runtime messages should not mark the container dirty such as the old built-in
-        // AgentScheduler and Garbage collector messages.
-        switch (type) {
-            case ContainerMessageType.Attach: {
-                const attachMessage = contents;
-                if (attachMessage.id === agentSchedulerId) {
-                    return false;
-                }
-                break;
-            }
-            case ContainerMessageType.FluidDataStoreOp: {
-                const envelope = contents;
-                if (envelope.address === agentSchedulerId) {
-                    return false;
-                }
-                break;
-            }
-            case ContainerMessageType.IdAllocation:
-            case ContainerMessageType.DocumentSchemaChange:
-            case ContainerMessageType.GC: {
-                return false;
-            }
-            default: {
-                break;
-            }
-        }
-        return true;
+    /**
+     * Returns true if the container is dirty: not attached, or no pending user messages (could be some "non-dirtyable" ones though)
+     */
+    computeCurrentDirtyState() {
+        return (this.attachState !== AttachState.Attached ||
+            this.pendingStateManager.hasPendingUserChanges() ||
+            this.outbox.containsUserChanges());
     }
     /**
      * Submits the signal to be sent to other clients.
@@ -2126,9 +2181,6 @@ export class ContainerRuntime extends TypedEventEmitter {
     submitSignal(type, content, targetClientId) {
         this.verifyNotClosed();
         const envelope = createNewSignalEnvelope(undefined /* address */, type, content);
-        if (targetClientId === undefined) {
-            this.signalTelemetryManager.applyTrackingToBroadcastSignalEnvelope(envelope);
-        }
         this.submitSignalFn(envelope, targetClientId);
     }
     setAttachState(attachState) {
@@ -2139,9 +2191,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             assert(this.attachState === AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
             this.emit("attached");
         }
-        if (attachState === AttachState.Attached && !this.hasPendingMessages()) {
-            this.updateDocumentDirtyState(false);
-        }
+        this.updateDocumentDirtyState();
         this.channelCollection.setAttachState(attachState);
     }
     /**
@@ -2709,18 +2759,20 @@ export class ContainerRuntime extends TypedEventEmitter {
     hasPendingMessages() {
         return this.pendingMessagesCount !== 0;
     }
-    updateDocumentDirtyState(dirty) {
-        if (this.attachState === AttachState.Attached) {
-            // Other way is not true = see this.isContainerMessageDirtyable()
-            assert(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
-        }
-        else {
-            assert(dirty, 0x3d2 /* Non-attached container is dirty */);
-        }
-        if (this.dirtyContainer === dirty) {
+    /**
+     * Emit "dirty" or "saved" event based on the current dirty state of the document.
+     * This must be called every time the states underlying the dirty state change.
+     *
+     * @privateRemarks - It's helpful to think of this as an event handler registered
+     * for hypothetical "changed" events for PendingStateManager, Outbox, and Container Attach machinery.
+     * But those events don't exist so we manually call this wherever we know those changes happen.
+     */
+    updateDocumentDirtyState() {
+        const dirty = this.computeCurrentDirtyState();
+        if (this.lastEmittedDirty === dirty) {
             return;
         }
-        this.dirtyContainer = dirty;
+        this.lastEmittedDirty = dirty;
         if (this.emitDirtyDocumentEvent) {
             this.emit(dirty ? "dirty" : "saved");
         }
@@ -2775,6 +2827,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         try {
             // If we're resubmitting a batch, keep the same "staged" value as before.  Otherwise, use the current "global" state.
             const staged = this.batchRunner.resubmitInfo?.staged ?? this.inStagingMode;
+            assert(!staged || canStageMessageOfType(type), 0xbba /* Unexpected message type submitted in Staging Mode */);
             // Before submitting any non-staged change, submit the ID Allocation op to cover any compressed IDs included in the op.
             if (!staged) {
                 this.submitIdAllocationOpIfNeeded({ staged: false });
@@ -2782,7 +2835,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Allow document schema controller to send a message if it needs to propose change in document schema.
             // If it needs to send a message, it will call provided callback with payload of such message and rely
             // on this callback to do actual sending.
-            const schemaChangeMessage = this.documentsSchemaController.maybeSendSchemaMessage();
+            const schemaChangeMessage = this.documentsSchemaController.maybeGenerateSchemaMessage();
             if (schemaChangeMessage) {
                 this.mc.logger.sendTelemetryEvent({
                     eventName: "SchemaChangeProposal",
@@ -2818,14 +2871,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             else {
                 this.outbox.submit(message);
             }
-            // Note: Technically, the system "always" batches - if this case is true we'll just have a single-message batch.
-            const flushImmediatelyOnSubmit = !this.currentlyBatching();
-            if (flushImmediatelyOnSubmit) {
-                this.flush();
-            }
-            else {
-                this.scheduleFlush();
-            }
+            this.scheduleFlush();
         }
         catch (error) {
             const dpe = DataProcessingError.wrapIfUnrecognized(error, "ContainerRuntime.submit", {
@@ -2834,27 +2880,26 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.closeFn(dpe);
             throw dpe;
         }
-        if (this.isContainerMessageDirtyable(containerRuntimeMessage)) {
-            this.updateDocumentDirtyState(true);
-        }
+        this.updateDocumentDirtyState();
     }
     scheduleFlush() {
-        if (this.flushTaskExists) {
+        if (this.flushScheduled) {
             return;
         }
-        this.flushTaskExists = true;
-        // TODO: hoist this out of the function scope to save unnecessary allocations
-        // eslint-disable-next-line unicorn/consistent-function-scoping -- Separate `flush` method already exists in outer scope
-        const flush = () => {
-            this.flushTaskExists = false;
-            this.flush();
-        };
+        this.flushScheduled = true;
         switch (this.flushMode) {
+            case FlushMode.Immediate: {
+                // When in Immediate flush mode, flush immediately unless we are intentionally batching multiple ops (e.g. via orderSequentially)
+                if (!this.batchRunner.running) {
+                    this.flush();
+                }
+                break;
+            }
             case FlushMode.TurnBased: {
                 // When in TurnBased flush mode the runtime will buffer operations in the current turn and send them as a single
                 // batch at the end of the turn
-                // eslint-disable-next-line @typescript-eslint/no-floating-promises
-                Promise.resolve().then(flush);
+                // eslint-disable-next-line @typescript-eslint/no-floating-promises -- Container will close if flush throws
+                Promise.resolve().then(() => this.flush());
                 break;
             }
             // FlushModeExperimental is experimental and not exposed directly in the runtime APIs
@@ -2862,12 +2907,11 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // When in Async flush mode, the runtime will accumulate all operations across JS turns and send them as a single
                 // batch when all micro-tasks are complete.
                 // Compared to TurnBased, this flush mode will capture more ops into the same batch.
-                setTimeout(flush, 0);
+                setTimeout(() => this.flush(), 0);
                 break;
             }
             default: {
-                assert(this.batchRunner.running, 0x587 /* Unreachable unless manually accumulating a batch */);
-                break;
+                fail(0x587 /* Unreachable unless manually accumulating a batch */);
             }
         }
     }
@@ -2889,43 +2933,72 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     /**
      * Resubmits each message in the batch, and then flushes the outbox.
+     * This typically happens when we reconnect and there are pending messages.
+     *
+     * @remarks
+     * Attempting to resubmit a batch that has been successfully sequenced will not happen due to
+     * checks in the ConnectionStateHandler (Loader layer)
      *
-     * @remarks - If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
+     * The only exception to this would be if the Container "forks" due to misuse of the "Offline Load" feature.
+     * If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
      * for correlation to detect container forking.
      */
     reSubmitBatch(batch, { batchId, staged, squash }) {
+        assert(this._summarizer === undefined, 0x8f2 /* Summarizer never reconnects so should never resubmit */);
         const resubmitInfo = {
             // Only include Batch ID if "Offline Load" feature is enabled
             // It's only needed to identify batches across container forks arising from misuse of offline load.
             batchId: this.offlineEnabled ? batchId : undefined,
             staged,
         };
+        const resubmitFn = squash
+            ? this.reSubmitWithSquashing.bind(this)
+            : this.reSubmit.bind(this);
         this.batchRunner.run(() => {
             for (const message of batch) {
-                this.reSubmit(message, squash);
+                resubmitFn(message);
             }
         }, resubmitInfo);
         this.flush(resubmitInfo);
     }
-    reSubmit(message, squash) {
-        this.reSubmitCore(message.runtimeOp, message.localOpMetadata, message.opMetadata, squash);
+    /**
+     * Resubmit the given message as part of a squash rebase upon exiting Staging Mode.
+     * How exactly to resubmit the message is up to the subsystem that submitted the op to begin with.
+     */
+    reSubmitWithSquashing(resubmitData) {
+        const message = resubmitData.runtimeOp;
+        assert(canStageMessageOfType(message.type), 0xbbb /* Expected message type to be compatible with staging */);
+        switch (message.type) {
+            case ContainerMessageType.FluidDataStoreOp: {
+                this.channelCollection.reSubmit(message.type, message.contents, resubmitData.localOpMetadata, 
+                /* squash: */ true);
+                break;
+            }
+            // NOTE: Squash doesn't apply to GC or DocumentSchemaChange ops, fallback to typical resubmit logic.
+            case ContainerMessageType.GC:
+            case ContainerMessageType.DocumentSchemaChange: {
+                this.reSubmit(resubmitData);
+                break;
+            }
+            default: {
+                unreachableCase(message.type);
+            }
+        }
     }
     /**
-     * Finds the right store and asks it to resubmit the message. This typically happens when we
-     * reconnect and there are pending messages.
-     * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
-     * @param message - The original LocalContainerRuntimeMessage.
-     * @param localOpMetadata - The local metadata associated with the original message.
+     * Resubmit the given message which was previously submitted to the ContainerRuntime but not successfully
+     * transmitted to the ordering service (e.g. due to a disconnect, or being in Staging Mode)
+     * How to resubmit is up to the subsystem that submitted the op to begin with
      */
-    reSubmitCore(message, localOpMetadata, opMetadata, squash) {
-        assert(this._summarizer === undefined, 0x8f2 /* Summarizer never reconnects so should never resubmit */);
+    reSubmit({ runtimeOp: message, localOpMetadata, opMetadata, }) {
         switch (message.type) {
             case ContainerMessageType.FluidDataStoreOp:
             case ContainerMessageType.Attach:
             case ContainerMessageType.Alias: {
                 // For Operations, call resubmitDataStoreOp which will find the right store
                 // and trigger resubmission on it.
-                this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata, squash);
+                this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata, 
+                /* squash: */ false);
                 break;
             }
             case ContainerMessageType.IdAllocation: {
@@ -2951,9 +3024,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                 break;
             }
             case ContainerMessageType.DocumentSchemaChange: {
-                // There is no need to resend this message. Document schema controller will properly resend it again (if needed)
-                // on a first occasion (any ops sent after reconnect). There is a good chance, though, that it will not want to
-                // send any ops, as some other client already changed schema.
+                // We shouldn't directly resubmit due to Compare-And-Swap semantics.
+                // If needed it will be generated from scratch before other ops are submitted.
+                this.documentsSchemaController.pendingOpNotAcked();
                 break;
             }
             default: {
@@ -2963,8 +3036,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
-    rollback(runtimeOp, localOpMetadata) {
-        const { type, contents } = runtimeOp;
+    /**
+     * Rollback the given op which was only staged but not yet submitted.
+     */
+    rollbackStagedChanges({ type, contents }, localOpMetadata) {
+        assert(canStageMessageOfType(type), 0xbbc /* Unexpected message type to be rolled back */);
         switch (type) {
             case ContainerMessageType.FluidDataStoreOp: {
                 // For operations, call rollbackDataStoreOp which will find the right store
@@ -2972,8 +3048,24 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.channelCollection.rollback(type, contents, localOpMetadata);
                 break;
             }
+            case ContainerMessageType.GC: {
+                // Just drop it, but log an error, this is not expected and not ideal, but not critical failure either.
+                // Currently the only expected type here is TombstoneLoaded, which will have been preceded by one of these events as well:
+                // GC_Tombstone_DataStore_Requested, GC_Tombstone_SubDataStore_Requested, GC_Tombstone_Blob_Requested
+                this.mc.logger.sendErrorEvent({
+                    eventName: "GC_OpDiscarded",
+                    details: { subType: contents.type },
+                });
+                break;
+            }
+            case ContainerMessageType.DocumentSchemaChange: {
+                // Notify the document schema controller that the pending op was not acked.
+                // This will allow it to propose the schema change again if needed.
+                this.documentsSchemaController.pendingOpNotAcked();
+                break;
+            }
             default: {
-                throw new Error(`Can't rollback ${type}`);
+                unreachableCase(type);
             }
         }
     }
@@ -3138,6 +3230,30 @@ export class ContainerRuntime extends TypedEventEmitter {
             return this.summaryManager.enqueueSummarize(options);
         }
     }
+    acquireExtension(id, factory, ...useContext) {
+        let entry = this.extensions.get(id);
+        if (entry === undefined) {
+            const runtime = {
+                isConnected: () => this.connected,
+                getClientId: () => this.clientId,
+                events: this.lazyEventsForExtensions.value,
+                logger: this.baseLogger,
+                submitAddressedSignal: (addressChain, message) => {
+                    this.submitExtensionSignal(id, addressChain, message);
+                },
+                getQuorum: this.getQuorum.bind(this),
+                getAudience: this.getAudience.bind(this),
+                supportedFeatures: this.ILayerCompatDetails.supportedFeatures,
+            };
+            entry = new factory(runtime, ...useContext);
+            this.extensions.set(id, entry);
+        }
+        else {
+            assert(entry instanceof factory, 0xba1 /* Extension entry is not of the expected type */);
+            entry.extension.onNewUse(...useContext);
+        }
+        return entry.interface;
+    }
     get groupedBatchingEnabled() {
         return this.sessionSchema.opGroupingEnabled === true;
     }
@@ -3149,4 +3265,33 @@ export function createNewSignalEnvelope(address, type, content) {
     };
     return newEnvelope;
 }
+export function isContainerMessageDirtyable({ type, contents, }) {
+    // Certain container runtime messages should not mark the container dirty such as the old built-in
+    // AgentScheduler and Garbage collector messages.
+    switch (type) {
+        case ContainerMessageType.Attach: {
+            const attachMessage = contents;
+            if (attachMessage.id === agentSchedulerId) {
+                return false;
+            }
+            break;
+        }
+        case ContainerMessageType.FluidDataStoreOp: {
+            const envelope = contents;
+            if (envelope.address === agentSchedulerId) {
+                return false;
+            }
+            break;
+        }
+        case ContainerMessageType.IdAllocation:
+        case ContainerMessageType.DocumentSchemaChange:
+        case ContainerMessageType.GC: {
+            return false;
+        }
+        default: {
+            break;
+        }
+    }
+    return true;
+}
 //# sourceMappingURL=containerRuntime.js.map