@fluidframework/container-runtime 2.40.0-336023 → 2.41.0-337492

package/lib/containerRuntime.js

@@ -9,6 +9,7 @@ import { assert, Deferred, LazyPromise, PromiseCache, delay, } from "@fluidframe
 import { SummaryType } from "@fluidframework/driver-definitions";
 import { FetchSource, MessageType } from "@fluidframework/driver-definitions/internal";
 import { readAndParse } from "@fluidframework/driver-utils/internal";
+import { createIdCompressor, createSessionId, deserializeIdCompressor, } from "@fluidframework/id-compressor/internal";
 import { FlushMode, FlushModeExperimental, channelsTreeName, gcTreeKey, } from "@fluidframework/runtime-definitions/internal";
 import { GCDataBuilder, RequestParser, RuntimeHeaders, TelemetryContext, addBlobToSummary, addSummarizeResultToSummary, calculateStats, create404Response, exceptionToResponse, seqFromTree, } from "@fluidframework/runtime-utils/internal";
 import { DataCorruptionError, DataProcessingError, extractSafePropertiesFromMessage, GenericError, LoggingError, PerformanceEvent, 
@@ -18,7 +19,7 @@ import { v4 as uuid } from "uuid";
 import { BindBatchTracker } from "./batchTracker.js";
 import { BlobManager, blobManagerBasePath, blobsTreeName, isBlobPath, loadBlobManagerLoadInfo, } from "./blobManager/index.js";
 import { ChannelCollection, getSummaryForDatastores, wrapContext, } from "./channelCollection.js";
-import { defaultCompatibilityVersion, getCompatibilityVersionDefaults, isValidCompatVersion, } from "./compatUtils.js";
+import { defaultMinVersionForCollab, getMinVersionForCollabDefaults, isValidMinVersionForCollab, } from "./compatUtils.js";
 import { CompressionAlgorithms, disabledCompressionConfig } from "./compressionDefinitions.js";
 import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
 import { ContainerFluidHandleContext } from "./containerHandleContext.js";
@@ -84,6 +85,7 @@ export const defaultRuntimeHeaderData = {
     viaHandle: false,
     allowTombstone: false,
 };
+const defaultStagingCommitOptions = { squash: false };
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
 // The actual limit is 1Mb (socket.io and Kafka limits)
 // We can't estimate it fully, as we
@@ -209,10 +211,11 @@ export class ContainerRuntime extends TypedEventEmitter {
      * - containerRuntimeCtor - Constructor to use to create the ContainerRuntime instance.
      * This allows mixin classes to leverage this method to define their own async initializer.
      * - provideEntryPoint - Promise that resolves to an object which will act as entryPoint for the Container.
+     * - minVersionForCollab - Minimum version of the FF runtime that this runtime supports collaboration with.
      * This object should provide all the functionality that the Container is expected to provide to the loader layer.
      */
     static async loadRuntime(params) {
-        const { context, registryEntries, existing, requestHandler, provideEntryPoint, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
+        const { context, registryEntries, existing, requestHandler, provideEntryPoint, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, minVersionForCollab = defaultMinVersionForCollab, } = params;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
@@ -229,26 +232,24 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         const mc = loggerToMonitoringContext(logger);
         // Some options require a minimum version of the FF runtime to operate, so the default configs will be generated
-        // based on the compatibility mode.
-        // For example, if compatibility mode is set to "1.0.0", the default configs will ensure compatibility with FF runtime
-        // 1.0.0 or later. If the compatibility mode is set to "2.10.0", the default values will be generated to ensure compatibility
+        // based on the minVersionForCollab.
+        // For example, if minVersionForCollab is set to "1.0.0", the default configs will ensure compatibility with FF runtime
+        // 1.0.0 or later. If the minVersionForCollab is set to "2.10.0", the default values will be generated to ensure compatibility
         // with FF runtime 2.10.0 or later.
-        // TODO: We will add in a way for users to pass in compatibilityVersion in a follow up PR.
-        const compatibilityVersion = defaultCompatibilityVersion;
-        if (!isValidCompatVersion(compatibilityVersion)) {
-            throw new UsageError(`Invalid compatibility version: ${compatibilityVersion}. It must be an existing FF version (i.e. 2.22.1).`);
+        if (!isValidMinVersionForCollab(minVersionForCollab)) {
+            throw new UsageError(`Invalid minVersionForCollab: ${minVersionForCollab}. It must be an existing FF version (i.e. 2.22.1).`);
         }
-        const defaultVersionDependentConfigs = getCompatibilityVersionDefaults(compatibilityVersion);
+        const defaultsAffectingDocSchema = getMinVersionForCollabDefaults(minVersionForCollab);
         // The following are the default values for the options that do not affect the DocumentSchema.
-        const defaultConfigsNonVersionDependent = {
+        const defaultsNotAffectingDocSchema = {
             summaryOptions: {},
             loadSequenceNumberVerification: "close",
             maxBatchSizeInBytes: defaultMaxBatchSizeInBytes,
             chunkSizeInBytes: defaultChunkSizeInBytes,
         };
         const defaultConfigs = {
-            ...defaultVersionDependentConfigs,
-            ...defaultConfigsNonVersionDependent,
+            ...defaultsAffectingDocSchema,
+            ...defaultsNotAffectingDocSchema,
         };
         // Here we set each option to its corresponding default config value if it's not provided in runtimeOptions.
         // Note: We cannot do a simple object merge of defaultConfigs/runtimeOptions because in most cases we don't want
@@ -359,8 +360,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         else {
             idCompressorMode = desiredIdCompressorMode;
         }
-        const createIdCompressorFn = async () => {
-            const { createIdCompressor, deserializeIdCompressor, createSessionId } = await import("@fluidframework/id-compressor/internal");
+        const createIdCompressorFn = () => {
             /**
              * Because the IdCompressor emits so much telemetry, this function is used to sample
              * approximately 5% of all clients. Only the given percentage of sessions will emit telemetry.
@@ -415,7 +415,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             explicitSchemaControl,
             createBlobPayloadPending,
         };
-        const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], internalRuntimeOptions, containerScope, logger, existing, blobManagerLoadInfo, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, undefined, // summaryConfiguration
+        const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], internalRuntimeOptions, containerScope, logger, existing, blobManagerLoadInfo, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, minVersionForCollab, requestHandler, undefined, // summaryConfiguration
         recentBatchInfo);
         runtime.blobManager.stashedBlobsUploadP.then(() => {
             // make sure we didn't reconnect before the promise resolved
@@ -542,7 +542,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     /***/
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, 
     // Create a custom ITelemetryBaseLogger to output telemetry events.
-    baseLogger, existing, blobManagerLoadInfo, _storage, createIdCompressor, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, 
+    baseLogger, existing, blobManagerLoadInfo, _storage, createIdCompressorFn, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, minVersionForCollab, requestHandler, 
     // // eslint-disable-next-line unicorn/no-object-as-default-parameter
     summaryConfiguration = {
         // the defaults
@@ -558,8 +558,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.containerScope = containerScope;
         this.baseLogger = baseLogger;
         this._storage = _storage;
-        this.createIdCompressor = createIdCompressor;
+        this.createIdCompressorFn = createIdCompressorFn;
         this.documentsSchemaController = documentsSchemaController;
+        this.minVersionForCollab = minVersionForCollab;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
         this.imminentClosure = false;
@@ -598,10 +599,12 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.outbox.flush();
             const exitStagingMode = (discardOrCommit) => () => {
                 // Final flush of any last staged changes
-                this.outbox.flush(undefined, true /* staged */);
+                this.outbox.flush();
                 this.stageControls = undefined;
                 discardOrCommit();
+                this.channelCollection.notifyStagingMode(false);
             };
+            // eslint-disable-next-line import/no-deprecated
             const stageControls = {
                 discardChanges: exitStagingMode(() => {
                     // Pop all staged batches from the PSM and roll them back in LIFO order
@@ -613,18 +616,19 @@ export class ContainerRuntime extends TypedEventEmitter {
                         this.updateDocumentDirtyState(this.pendingMessagesCount !== 0);
                     }
                 }),
-                commitChanges: exitStagingMode(() => {
-                    // All staged changes are in the PSM, so just replay them (ignore pre-staging batches)
-                    // FUTURE: Have this do squash-rebase instead of resubmitting all intermediate changes
-                    if (this.connected) {
-                        this.pendingStateManager.replayPendingStates(true /* onlyStagedBatched */);
-                    }
-                    else {
-                        this.pendingStateManager.clearStagingFlags();
-                    }
-                }),
+                commitChanges: (optionsParam) => {
+                    const options = { ...defaultStagingCommitOptions, ...optionsParam };
+                    return exitStagingMode(() => {
+                        this.pendingStateManager.replayPendingStates({
+                            onlyStagedBatches: true,
+                            squash: options.squash ?? false,
+                        });
+                    })();
+                },
             };
-            return (this.stageControls = stageControls);
+            this.stageControls = stageControls;
+            this.channelCollection.notifyStagingMode(true);
+            return this.stageControls;
         };
         this.readAndParseBlob = async (id) => readAndParse(this.storage, id);
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, pendingLocalState, supportedFeatures, snapshotWithContents, } = context;
@@ -988,6 +992,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             telemetryDocumentId: this.telemetryDocumentId,
             groupedBatchingEnabled: this.groupedBatchingEnabled,
             initialSequenceNumber: this.deltaManager.initialSequenceNumber,
+            minVersionForCollab: this.minVersionForCollab,
         });
         ReportOpPerfTelemetry(this.clientId, this._deltaManager, this, this.baseLogger);
         BindBatchTracker(this, this.baseLogger);
@@ -1015,7 +1020,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         // As it's implemented right now (with async initialization), this will only work for "off" -> "delayed" transitions.
         // Anything else is too risky, and requires ability to initialize ID compressor synchronously!
         if (schema.runtime.idCompressorMode !== undefined) {
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
             this.loadIdCompressor();
         }
     }
@@ -1040,7 +1044,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         await this.initializeSummarizer(loader);
         if (this.sessionSchema.idCompressorMode === "on" ||
             (this.sessionSchema.idCompressorMode === "delayed" && this.connected)) {
-            this._idCompressor = await this.createIdCompressor();
+            this._idCompressor = this.createIdCompressorFn();
             // This is called from loadRuntime(), long before we process any ops, so there should be no ops accumulated yet.
             assert(this.pendingIdCompressorOps.length === 0, 0x8ec /* no pending ops */);
         }
@@ -1496,27 +1500,18 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
-    async loadIdCompressor() {
+    loadIdCompressor() {
         if (this._idCompressor === undefined &&
-            this.sessionSchema.idCompressorMode !== undefined &&
-            this._loadIdCompressor === undefined) {
-            this._loadIdCompressor = this.createIdCompressor()
-                .then((compressor) => {
-                // Finalize any ranges we received while the compressor was turned off.
-                const ops = this.pendingIdCompressorOps;
-                this.pendingIdCompressorOps = [];
-                for (const range of ops) {
-                    compressor.finalizeCreationRange(range);
-                }
-                assert(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
-                this._idCompressor = compressor;
-            })
-                .catch((error) => {
-                this.mc.logger.sendErrorEvent({ eventName: "IdCompressorDelayedLoad" }, error);
-                throw error;
-            });
+            this.sessionSchema.idCompressorMode !== undefined) {
+            this._idCompressor = this.createIdCompressorFn();
+            // Finalize any ranges we received while the compressor was turned off.
+            const ops = this.pendingIdCompressorOps;
+            this.pendingIdCompressorOps = [];
+            for (const range of ops) {
+                this._idCompressor.finalizeCreationRange(range);
+            }
+            assert(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
         }
-        return this._loadIdCompressor;
     }
     setConnectionState(connected, clientId) {
         // Validate we have consistent state
@@ -1524,7 +1519,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         assert(clientId === currentClientId, 0x977 /* input clientId does not match Audience */);
         assert(this.clientId === currentClientId, 0x978 /* this.clientId does not match Audience */);
         if (connected && this.sessionSchema.idCompressorMode === "delayed") {
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
             this.loadIdCompressor();
         }
         if (connected === false && this.delayConnectClientId !== undefined) {
@@ -1930,19 +1924,16 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.channelCollection.processSignal(transformed, local);
     }
     /**
-     * Flush the pending ops manually.
-     * This method is expected to be called at the end of a batch.
+     * Flush the current batch of ops to the ordering service for sequencing
+     * This method is not expected to be called in the middle of a batch.
      * @remarks - If it throws (e.g. if the batch is too large to send), the container will be closed.
      *
-     * @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
-     * with the given Batch ID, which must be preserved
-     * @param resubmittingStagedBatch - If defined, indicates this is a resubmission of a batch that is staged,
-     * meaning it should not be sent to the ordering service yet.
+     * @param resubmitInfo - If defined, indicates this is a resubmission of a batch with the given Batch info needed for resubmit.
      */
-    flush(resubmittingBatchId, resubmittingStagedBatch) {
+    flush(resubmitInfo) {
         try {
             assert(!this.batchRunner.running, 0x24c /* "Cannot call `flush()` while manually accumulating a batch (e.g. under orderSequentially) */);
-            this.outbox.flush(resubmittingBatchId, resubmittingStagedBatch);
+            this.outbox.flush(resubmitInfo);
             assert(this.outbox.isEmpty, 0x3cf /* reentrancy */);
         }
         catch (error) {
@@ -2004,7 +1995,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 throw error; // throw the original error for the consumer of the runtime
             }
         });
-        stageControls?.commitChanges();
+        stageControls?.commitChanges({ squash: false });
         // We don't flush on TurnBased since we expect all messages in the same JS turn to be part of the same batch
         if (this.flushMode !== FlushMode.TurnBased && !this.batchRunner.running) {
             this.flush();
@@ -2181,8 +2172,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         // Wrap data store summaries in .channels subtree.
         wrapSummaryInChannelsTree(summarizeResult);
         const pathPartsForChildren = [channelsTreeName];
-        // Ensure that ID compressor had a chance to load, if we are using delayed mode.
-        await this.loadIdCompressor();
+        this.loadIdCompressor();
         this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
         return {
             ...summarizeResult,
@@ -2893,18 +2883,18 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @remarks - If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
      * for correlation to detect container forking.
      */
-    reSubmitBatch(batch, batchId, staged) {
+    reSubmitBatch(batch, { batchId, staged, squash }) {
         this.batchRunner.run(() => {
             for (const message of batch) {
-                this.reSubmit(message);
+                this.reSubmit(message, squash);
             }
         });
         // Only include Batch ID if "Offline Load" feature is enabled
         // It's only needed to identify batches across container forks arising from misuse of offline load.
-        this.flush(this.offlineEnabled ? batchId : undefined, staged);
+        this.flush({ batchId: this.offlineEnabled ? batchId : undefined, staged });
     }
-    reSubmit(message) {
-        this.reSubmitCore(message.runtimeOp, message.localOpMetadata, message.opMetadata);
+    reSubmit(message, squash) {
+        this.reSubmitCore(message.runtimeOp, message.localOpMetadata, message.opMetadata, squash);
     }
     /**
      * Finds the right store and asks it to resubmit the message. This typically happens when we
@@ -2913,7 +2903,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param message - The original LocalContainerRuntimeMessage.
      * @param localOpMetadata - The local metadata associated with the original message.
      */
-    reSubmitCore(message, localOpMetadata, opMetadata) {
+    reSubmitCore(message, localOpMetadata, opMetadata, squash) {
         assert(this._summarizer === undefined, 0x8f2 /* Summarizer never reconnects so should never resubmit */);
         switch (message.type) {
             case ContainerMessageType.FluidDataStoreOp:
@@ -2921,7 +2911,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.Alias: {
                 // For Operations, call resubmitDataStoreOp which will find the right store
                 // and trigger resubmission on it.
-                this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata);
+                this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata, squash);
                 break;
             }
             case ContainerMessageType.IdAllocation: {