@fluidframework/container-runtime 2.91.0 → 2.92.0

package/lib/containerRuntime.js

@@ -28,10 +28,10 @@ import { channelToDataStore } from "./dataStore.js";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
 import { BaseDeltaManagerProxy, DeltaManagerPendingOpsProxy, DeltaManagerSummarizerProxy, } from "./deltaManagerProxies.js";
 import { DeltaScheduler } from "./deltaScheduler.js";
-import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
+import { GCNodeType, GarbageCollector, } from "./gc/index.js";
 import { InboundBatchAggregator } from "./inboundBatchAggregator.js";
 import { ContainerMessageType, } from "./messageTypes.js";
-import { DuplicateBatchDetector, ensureContentsDeserialized, OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
+import { DuplicateBatchDetector, ensureContentsDeserialized, largeBatchThreshold, OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
 import { pkgVersion } from "./packageVersion.js";
 import { PendingStateManager, } from "./pendingStateManager.js";
 import { BatchRunCounter, RunCounter } from "./runCounter.js";
@@ -104,6 +104,15 @@ const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconn
 // - we do not stringify final op, thus we do not know how much escaping will be added.
 const defaultMaxBatchSizeInBytes = 700 * 1024;
 const defaultChunkSizeInBytes = 204800;
+/**
+ * Default maximum ops per staging-mode batch before automatic flush scheduling resumes.
+ *
+ * Chosen based on production telemetry: copy-paste operations routinely produce batches
+ * of 1000+ ops (435K instances over 30 days), and receivers on modern Fluid versions
+ * handle them without issues. Uses {@link largeBatchThreshold} to stay aligned with
+ * the existing "large batch" telemetry threshold ({@link OpGroupingManager}).
+ */
+const defaultStagingModeAutoFlushThreshold = largeBatchThreshold;
 /**
  * The default time to wait for pending ops to be processed during summarization
  */
@@ -200,6 +209,21 @@ export let getSingleUseLegacyLogCallback = (logger, type) => {
 export async function loadContainerRuntime(params) {
     return ContainerRuntime.loadRuntime(params);
 }
+/**
+ * Alpha variant of {@link loadContainerRuntime} that returns the runtime in an
+ * extendable object, allowing additional properties to be added in the future.
+ *
+ * @param params - An object which specifies all required and optional params necessary to instantiate a runtime.
+ * @returns An object containing the runtime.
+ *
+ * @legacy @alpha
+ */
+export async function loadContainerRuntimeAlpha(params) {
+    return ContainerRuntime.loadRuntime2({
+        ...params,
+        registry: new FluidDataStoreRegistry(params.registryEntries),
+    });
+}
 const defaultMaxConsecutiveReconnects = 7;
 /**
  * These are the ONLY message types that are allowed to be submitted while in staging mode
@@ -239,13 +263,14 @@ export class ContainerRuntime extends TypedEventEmitter {
         return ContainerRuntime.loadRuntime2({
             ...params,
             registry: new FluidDataStoreRegistry(params.registryEntries),
-        });
+        }).then((r) => r.runtime);
     }
     /**
-     * Load the stores from a snapshot and returns the runtime.
+     * Load the stores from a snapshot and returns an object containing the runtime.
      * @remarks
      * Same as {@link ContainerRuntime.loadRuntime},
      * but with `registry` instead of `registryEntries` and more `runtimeOptions`.
+     * Returns `{ runtime }` to allow future extensions (e.g. staging mode controls).
      */
     static async loadRuntime2(params) {
         const { context, registry, existing, requestHandler, provideEntryPoint, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, minVersionForCollab = defaultMinVersionForCollab, } = params;
@@ -282,6 +307,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             loadSequenceNumberVerification: "close",
             maxBatchSizeInBytes: defaultMaxBatchSizeInBytes,
             chunkSizeInBytes: defaultChunkSizeInBytes,
+            stagingModeAutoFlushThreshold: defaultStagingModeAutoFlushThreshold,
+            disableSchemaUpgrade: false,
         };
         const defaultConfigs = {
             ...defaultsAffectingDocSchema,
@@ -295,7 +322,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // is enabled via runtimeOptions, we will throw an error later.
         compressionOptions = enableGroupedBatching === false
             ? disabledCompressionConfig
-            : defaultConfigs.compressionOptions, createBlobPayloadPending = defaultConfigs.createBlobPayloadPending, } = runtimeOptions;
+            : defaultConfigs.compressionOptions, createBlobPayloadPending = defaultConfigs.createBlobPayloadPending, stagingModeAutoFlushThreshold = defaultConfigs.stagingModeAutoFlushThreshold, disableSchemaUpgrade = defaultConfigs.disableSchemaUpgrade, } = runtimeOptions;
         // If explicitSchemaControl is off, ensure that options which require explicitSchemaControl are not enabled.
         if (!explicitSchemaControl) {
             const disallowedKeys = Object.keys(runtimeOptions).filter((key) => runtimeOptionKeysThatRequireExplicitSchemaControl.includes(key) && runtimeOptions[key] !== undefined);
@@ -402,6 +429,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         else {
             idCompressorMode = desiredIdCompressorMode;
         }
+        // eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
         const createIdCompressorFn = () => {
             /**
              * Because the IdCompressor emits so much telemetry, this function is used to sample
@@ -438,7 +466,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             disallowedVersions: [],
         }, (schema) => {
             runtime.onSchemaChange(schema);
-        }, { minVersionForCollab }, logger);
+        }, { minVersionForCollab }, logger, disableSchemaUpgrade);
         // If the minVersionForCollab for this client is greater than the existing one, we should use that one going forward.
         const existingMinVersionForCollab = documentSchemaController.sessionSchema.info.minVersionForCollab;
         const updatedMinVersionForCollab = existingMinVersionForCollab === undefined ||
@@ -462,6 +490,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             enableGroupedBatching,
             explicitSchemaControl,
             createBlobPayloadPending,
+            stagingModeAutoFlushThreshold,
+            disableSchemaUpgrade,
         };
         validateMinimumVersionForCollab(updatedMinVersionForCollab);
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], internalRuntimeOptions, containerScope, logger, existing, blobManagerLoadInfo, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, updatedMinVersionForCollab, requestHandler, undefined, // summaryConfiguration
@@ -472,7 +502,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // Apply stashed ops with a reference sequence number equal to the sequence number of the snapshot,
         // or zero. This must be done before Container replays saved ops.
         await runtime.pendingStateManager.applyStashedOpsAt(runtimeSequenceNumber ?? 0);
-        return runtime;
+        return { runtime };
     }
     get clientId() {
         return this._getClientId();
@@ -511,6 +541,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     /**
      * {@inheritDoc @fluidframework/runtime-definitions#IContainerRuntimeBase.idCompressor}
      */
+    // eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
     get idCompressor() {
         // Expose ID Compressor only if it's On from the start.
         // If container uses delayed mode, then we can only expose generateDocumentUniqueId() and nothing else.
@@ -589,7 +620,9 @@ export class ContainerRuntime extends TypedEventEmitter {
     /***/
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, 
     // Create a custom ITelemetryBaseLogger to output telemetry events.
-    baseLogger, existing, blobManagerLoadInfo, _storage, createIdCompressorFn, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, minVersionForCollab, requestHandler, 
+    baseLogger, existing, blobManagerLoadInfo, _storage, 
+    // eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
+    createIdCompressorFn, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, minVersionForCollab, requestHandler, 
     // // eslint-disable-next-line unicorn/no-object-as-default-parameter
     summaryConfiguration = {
         // the defaults
@@ -646,17 +679,28 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Make sure Outbox is empty before entering staging mode,
             // since we mark whole batches as "staged" or not to indicate whether to submit them.
             this.flush();
-            const exitStagingMode = (discardOrCommit) => {
+            const exitStagingMode = (discardOrCommit, exitMethod) => {
                 try {
-                    // Final flush of any last staged changes
-                    // NOTE: We can't use this.flush() here, because orderSequentially uses StagingMode and in the rollback case we'll hit assert 0x24c
-                    this.outbox.flush();
-                    this.stageControls = undefined;
-                    // During Staging Mode, we avoid submitting any ID Allocation ops (apart from resubmitting pre-staging ops).
-                    // Now that we've exited, we need to submit an ID Allocation op for any IDs that were generated while in Staging Mode.
-                    this.submitIdAllocationOpIfNeeded({ staged: false });
-                    discardOrCommit();
-                    this.channelCollection.notifyStagingMode(false);
+                    PerformanceEvent.timedExec(this.mc.logger, {
+                        eventName: `ExitStagingMode_${exitMethod}`,
+                    }, (event) => {
+                        // Final flush of any last staged changes
+                        // NOTE: We can't use this.flush() here, because orderSequentially uses StagingMode and in the rollback case we'll hit assert 0x24c
+                        this.outbox.flush();
+                        this.stageControls = undefined;
+                        // During Staging Mode, we avoid submitting any ID Allocation ops (apart from resubmitting pre-staging ops).
+                        // Now that we've exited, we need to submit an ID Allocation op for any IDs that were generated while in Staging Mode.
+                        this.submitIdAllocationOpIfNeeded({ staged: false });
+                        const batchInfos = discardOrCommit();
+                        event.reportProgress({
+                            details: {
+                                autoFlushThreshold: this.stagingModeAutoFlushThreshold,
+                                batches: batchInfos.length,
+                                batchesAtOrOverThreshold: batchInfos.filter((b) => b.length >= this.stagingModeAutoFlushThreshold).length,
+                            },
+                        });
+                        this.channelCollection.notifyStagingMode(false);
+                    });
                 }
                 catch (error) {
                     const normalizedError = normalizeError(error);
@@ -667,21 +711,22 @@ export class ContainerRuntime extends TypedEventEmitter {
             const stageControls = {
                 discardChanges: () => exitStagingMode(() => {
                     // Pop all staged batches from the PSM and roll them back in LIFO order
-                    this.pendingStateManager.popStagedBatches(({ runtimeOp, localOpMetadata }) => {
+                    const batchInfos = this.pendingStateManager.popStagedBatches(({ runtimeOp, localOpMetadata }) => {
                         this.rollbackStagedChange(runtimeOp, localOpMetadata);
                     });
                     this.updateDocumentDirtyState();
-                }),
+                    return batchInfos;
+                }, "discard"),
                 commitChanges: (options) => {
                     const { squash } = { ...defaultStagingCommitOptions, ...options };
                     exitStagingMode(() => {
                         // Replay all staged batches in typical FIFO order.
                         // We'll be out of staging mode so they'll be sent to the service finally.
-                        this.pendingStateManager.replayPendingStates({
+                        return this.pendingStateManager.replayPendingStates({
                             committingStagedBatches: true,
                             squash,
                         });
-                    });
+                    }, "commit");
                 },
             };
             this.stageControls = stageControls;
@@ -864,14 +909,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             ? this.getConnectionState() === ConnectionState.Connected ||
                 this.getConnectionState() === ConnectionState.CatchingUp
             : undefined;
-        this.mc.logger.sendTelemetryEvent({
-            eventName: "GCFeatureMatrix",
-            metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
-            inputs: JSON.stringify({
-                // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-                gcOptions_gcGeneration: runtimeOptions.gcOptions[gcGenerationOptionName],
-            }),
-        });
         this.telemetryDocumentId = metadata?.telemetryDocumentId ?? uuid();
         const opGroupingManager = new OpGroupingManager({
             groupedBatchingEnabled: this.groupedBatchingEnabled,
@@ -919,6 +956,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.closeFn(error);
             throw error;
         }
+        this.stagingModeAutoFlushThreshold =
+            this.mc.config.getNumber("Fluid.ContainerRuntime.StagingModeAutoFlushThreshold") ??
+                runtimeOptions.stagingModeAutoFlushThreshold ??
+                defaultStagingModeAutoFlushThreshold;
         this.batchIdTrackingEnabled =
             this.mc.config.getBoolean("Fluid.Container.enableOfflineFull") ??
                 this.mc.config.getBoolean("Fluid.ContainerRuntime.enableBatchIdTracking") ??
@@ -1028,9 +1069,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.deltaScheduler = new DeltaScheduler(this.innerDeltaManager, this, createChildLogger({ logger: this.baseLogger, namespace: "DeltaScheduler" }));
         this.inboundBatchAggregator = new InboundBatchAggregator(this.innerDeltaManager, () => this.clientId, createChildLogger({ logger: this.baseLogger, namespace: "InboundBatchAggregator" }));
         const legacySendBatchFn = makeLegacySendBatchFn(submitFn, this.innerDeltaManager);
-        this.skipSafetyFlushDuringProcessStack =
-            // Keep the old flag name even though we renamed the class member (it shipped in 2.31.0)
-            this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableFlushBeforeProcess") === true;
         this.outbox = new Outbox({
             shouldSend: () => this.shouldSendOps(),
             pendingStateManager: this.pendingStateManager,
@@ -1041,8 +1079,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             config: {
                 compressionOptions,
                 maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
-                // If we disable flush before process, we must be ready to flush partial batches
-                flushPartialBatches: this.skipSafetyFlushDuringProcessStack,
             },
             logger: this.mc.logger,
             groupingManager: opGroupingManager,
@@ -1086,14 +1122,12 @@ export class ContainerRuntime extends TypedEventEmitter {
         // We haven't emitted dirty/saved yet, but this is the baseline so we know to emit when it changes
         this.lastEmittedDirty = this.computeCurrentDirtyState();
         context.updateDirtyContainerState(this.lastEmittedDirty);
-        if (!this.skipSafetyFlushDuringProcessStack) {
-            // Reference Sequence Number may have just changed, and it must be consistent across a batch,
-            // so we should flush now to clear the way for the next ops.
-            // NOTE: This will be redundant whenever CR.process was called for the op (since we flush there too) -
-            // But we need this coverage for old loaders that don't call ContainerRuntime.process for non-runtime messages.
-            // (We have to call flush _before_ processing a runtime op, but after is ok for non-runtime op)
-            this.deltaManager.on("op", () => this.flush());
-        }
+        // Reference Sequence Number may have just changed, and it must be consistent across a batch,
+        // so we should flush now to clear the way for the next ops.
+        // NOTE: This will be redundant whenever CR.process was called for the op (since we flush there too) -
+        // But we need this coverage for old loaders that don't call ContainerRuntime.process for non-runtime messages.
+        // (We have to call flush _before_ processing a runtime op, but after is ok for non-runtime op)
+        this.deltaManager.on("op", () => this.flush());
         // logging hardware telemetry
         this.baseLogger.send({
             category: "generic",
@@ -1107,7 +1141,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             summaryNumber: loadSummaryNumber,
             summaryFormatVersion: metadata?.summaryFormatVersion,
             disableIsolatedChannels: metadata?.disableIsolatedChannels,
+            // This is useful even for interactive clients since they track unreferenced nodes and log errors.
             gcVersion: metadata?.gcFeature,
+            gcConfigs: this.garbageCollector.serializedConfigs,
             options: JSON.stringify(runtimeOptions),
             idCompressorModeMetadata: metadata?.documentSchema?.runtime?.idCompressorMode,
             idCompressorMode: this.sessionSchema.idCompressorMode,
@@ -1115,7 +1151,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             featureGates: JSON.stringify({
                 ...featureGatesForTelemetry,
                 closeSummarizerDelayOverride,
-                disableFlushBeforeProcess: this.skipSafetyFlushDuringProcessStack,
             }),
             telemetryDocumentId: this.telemetryDocumentId,
             groupedBatchingEnabled: this.groupedBatchingEnabled,
@@ -1572,12 +1607,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         assert(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
         this.emitDirtyDocumentEvent = false;
         try {
-            // Any ID Allocation ops that failed to submit after the pending state was queued need to have
-            // the corresponding ranges resubmitted (note this call replaces the typical resubmit flow).
-            // Since we don't submit ID Allocation ops when staged, any outstanding ranges would be from
-            // before staging mode so we can simply say staged: false.
-            this.submitIdAllocationOpIfNeeded({ resubmitOutstandingRanges: true, staged: false });
-            this.scheduleFlush();
+            // Any ID Allocation ops that failed to submit need to have their ranges included
+            // in the next allocation op. Reset the compressor's unfinalized range cursor so that the next
+            // call to takeNextCreationRange (during replay) will include those unfinalized ranges.
+            this._idCompressor?.resetUnfinalizedCreationRange();
             // replay the ops
             this.pendingStateManager.replayPendingStates();
         }
@@ -1798,10 +1831,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         // spread operator above ensure we make a shallow copy of message, as the processing flow will modify it.
         // There might be multiple container instances receiving the same message.
         this.verifyNotClosed();
-        if (!this.skipSafetyFlushDuringProcessStack) {
-            // Reference Sequence Number may be about to change, and it must be consistent across a batch, so flush now
-            this.flush();
-        }
+        // Reference Sequence Number may be about to change, and it must be consistent across a batch, so flush now
+        this.flush();
         this.ensureNoDataModelChanges(() => {
             this.processInboundMessageOrBatch(messageCopy, local);
         });
@@ -3032,6 +3063,17 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.updateDocumentDirtyState();
     }
     scheduleFlush() {
+        // During staging mode, suppress automatic flush scheduling until the main batch
+        // reaches or exceeds the threshold.
+        // Incoming ops still break the batch via direct this.flush() calls elsewhere
+        // (deltaManager "op" handler, process(), connection changes, getPendingLocalState,
+        // exitStagingMode). Those all bypass scheduleFlush(), so they're unaffected by this check.
+        // Additionally, outbox.maybeFlushPartialBatch() (called on every submit) detects
+        // sequence number changes and throws if unexpected changes are detected.
+        if (this.inStagingMode &&
+            this.outbox.mainBatchMessageCount < this.stagingModeAutoFlushThreshold) {
+            return;
+        }
         if (this.flushScheduled) {
             return;
         }
@@ -3326,7 +3368,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         return PerformanceEvent.timedExec(this.mc.logger, {
             eventName: "getPendingLocalState",
         }, (event) => {
-            const pending = this.pendingStateManager.getLocalState(props?.snapshotSequenceNumber);
+            const { pending } = this.pendingStateManager.getLocalState(props?.snapshotSequenceNumber);
             const sessionExpiryTimerStarted = props?.sessionExpiryTimerStarted ?? this.garbageCollector.sessionExpiryTimerStarted;
             const pendingIdCompressorState = this._idCompressor?.serialize(true);
             const pendingAttachmentBlobs = this.blobManager.getPendingBlobs();