@fluidframework/container-runtime 2.33.0-333010 → 2.33.0

package/lib/containerRuntime.js

@@ -10,7 +10,7 @@ import { SummaryType } from "@fluidframework/driver-definitions";
 import { FetchSource, MessageType } from "@fluidframework/driver-definitions/internal";
 import { readAndParse } from "@fluidframework/driver-utils/internal";
 import { FlushMode, FlushModeExperimental, channelsTreeName, gcTreeKey, } from "@fluidframework/runtime-definitions/internal";
-import { GCDataBuilder, RequestParser, TelemetryContext, addBlobToSummary, addSummarizeResultToSummary, calculateStats, create404Response, exceptionToResponse, seqFromTree, } from "@fluidframework/runtime-utils/internal";
+import { GCDataBuilder, RequestParser, RuntimeHeaders, TelemetryContext, addBlobToSummary, addSummarizeResultToSummary, calculateStats, create404Response, exceptionToResponse, seqFromTree, } from "@fluidframework/runtime-utils/internal";
 import { DataCorruptionError, DataProcessingError, extractSafePropertiesFromMessage, GenericError, LoggingError, PerformanceEvent, 
 // eslint-disable-next-line import/no-deprecated
 TaggedLoggerAdapter, UsageError, createChildLogger, createChildMonitoringContext, createSampledLogger, loggerToMonitoringContext, raiseConnectedEvent, wrapError, tagCodeArtifacts, normalizeError, } from "@fluidframework/telemetry-utils/internal";
@@ -18,22 +18,24 @@ import { v4 as uuid } from "uuid";
 import { BindBatchTracker } from "./batchTracker.js";
 import { BlobManager, blobManagerBasePath, blobsTreeName, isBlobPath, loadBlobManagerLoadInfo, } from "./blobManager/index.js";
 import { ChannelCollection, getSummaryForDatastores, wrapContext, } from "./channelCollection.js";
+import { defaultCompatibilityVersion, getCompatibilityVersionDefaults, isValidCompatVersion, } from "./compatUtils.js";
+import { CompressionAlgorithms, disabledCompressionConfig } from "./compressionDefinitions.js";
 import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
 import { ContainerFluidHandleContext } from "./containerHandleContext.js";
 import { channelToDataStore } from "./dataStore.js";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
-import { DeltaManagerPendingOpsProxy, DeltaManagerSummarizerProxy, } from "./deltaManagerProxies.js";
+import { BaseDeltaManagerProxy, DeltaManagerPendingOpsProxy, DeltaManagerSummarizerProxy, } from "./deltaManagerProxies.js";
 import { DeltaScheduler } from "./deltaScheduler.js";
 import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
 import { InboundBatchAggregator } from "./inboundBatchAggregator.js";
 import { ContainerMessageType, } from "./messageTypes.js";
-import { DuplicateBatchDetector, ensureContentsDeserialized, OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, serializeOp, } from "./opLifecycle/index.js";
+import { DuplicateBatchDetector, ensureContentsDeserialized, OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
 import { pkgVersion } from "./packageVersion.js";
 import { PendingStateManager, } from "./pendingStateManager.js";
 import { RunCounter } from "./runCounter.js";
 import { runtimeCompatDetailsForLoader, validateLoaderCompatibility, } from "./runtimeLayerCompatState.js";
 import { SignalTelemetryManager } from "./signalTelemetryProcessing.js";
-import { DocumentsSchemaController, OrderedClientElection, RetriableSummaryError, aliasBlobName, chunksBlobName, recentBatchInfoBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, rootHasIsolatedChannels, summarizerClientType, wrapSummaryInChannelsTree, formCreateSummarizerFn, summarizerRequestUrl, SummaryManager, SummarizerClientElection, SummaryCollection, OrderedClientCollection, validateSummaryHeuristicConfiguration, DefaultSummaryConfiguration, isSummariesDisabled, } from "./summary/index.js";
+import { DocumentsSchemaController, OrderedClientElection, RetriableSummaryError, aliasBlobName, chunksBlobName, recentBatchInfoBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, rootHasIsolatedChannels, wrapSummaryInChannelsTree, formCreateSummarizerFn, summarizerRequestUrl, SummaryManager, SummarizerClientElection, SummaryCollection, OrderedClientCollection, validateSummaryHeuristicConfiguration, DefaultSummaryConfiguration, isSummariesDisabled, summarizerClientType, } from "./summary/index.js";
 import { Throttler, formExponentialFn } from "./throttler.js";
 /**
  * Creates an error object to be thrown / passed to Container's close fn in case of an unknown message type.
@@ -82,35 +84,12 @@ export const defaultRuntimeHeaderData = {
     viaHandle: false,
     allowTombstone: false,
 };
-/**
- * Available compression algorithms for op compression.
- * @legacy
- * @alpha
- */
-export var CompressionAlgorithms;
-(function (CompressionAlgorithms) {
-    CompressionAlgorithms["lz4"] = "lz4";
-})(CompressionAlgorithms || (CompressionAlgorithms = {}));
-/**
- * @legacy
- * @alpha
- */
-export const disabledCompressionConfig = {
-    minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
-    compressionAlgorithm: CompressionAlgorithms.lz4,
-};
 const maxConsecutiveReconnectsKey = "Fluid.ContainerRuntime.MaxConsecutiveReconnects";
-const defaultFlushMode = FlushMode.TurnBased;
 // The actual limit is 1Mb (socket.io and Kafka limits)
 // We can't estimate it fully, as we
 // - do not know what properties relay service will add
 // - we do not stringify final op, thus we do not know how much escaping will be added.
 const defaultMaxBatchSizeInBytes = 700 * 1024;
-const defaultCompressionConfig = {
-    // Batches with content size exceeding this value will be compressed
-    minimumBatchSizeInBytes: 614400,
-    compressionAlgorithm: CompressionAlgorithms.lz4,
-};
 const defaultChunkSizeInBytes = 204800;
 /**
  * The default time to wait for pending ops to be processed during summarization
@@ -249,9 +228,42 @@ export class ContainerRuntime extends TypedEventEmitter {
             },
         });
         const mc = loggerToMonitoringContext(logger);
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = runtimeOptions.enableGroupedBatching === false
-            ? disabledCompressionConfig // Compression must be disabled if Grouping is disabled
-            : defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableGroupedBatching = true, explicitSchemaControl = false, } = runtimeOptions;
+        // Some options require a minimum version of the FF runtime to operate, so the default configs will be generated
+        // based on the compatibility mode.
+        // For example, if compatibility mode is set to "1.0.0", the default configs will ensure compatibility with FF runtime
+        // 1.0.0 or later. If the compatibility mode is set to "2.10.0", the default values will be generated to ensure compatibility
+        // with FF runtime 2.10.0 or later.
+        // TODO: We will add in a way for users to pass in compatibilityVersion in a follow up PR.
+        const compatibilityVersion = defaultCompatibilityVersion;
+        if (!isValidCompatVersion(compatibilityVersion)) {
+            throw new UsageError(`Invalid compatibility version: ${compatibilityVersion}. It must be an existing FF version (i.e. 2.22.1).`);
+        }
+        const defaultVersionDependentConfigs = getCompatibilityVersionDefaults(compatibilityVersion);
+        // The following are the default values for the options that do not affect the DocumentSchema.
+        const defaultConfigsNonVersionDependent = {
+            summaryOptions: {},
+            loadSequenceNumberVerification: "close",
+            maxBatchSizeInBytes: defaultMaxBatchSizeInBytes,
+            chunkSizeInBytes: defaultChunkSizeInBytes,
+        };
+        const defaultConfigs = {
+            ...defaultVersionDependentConfigs,
+            ...defaultConfigsNonVersionDependent,
+        };
+        // Here we set each option to its corresponding default config value if it's not provided in runtimeOptions.
+        // Note: We cannot do a simple object merge of defaultConfigs/runtimeOptions because in most cases we don't want
+        // a option that is undefined in runtimeOptions to override the default value (except for idCompressor, see below).
+        const { summaryOptions = defaultConfigs.summaryOptions, gcOptions = defaultConfigs.gcOptions, loadSequenceNumberVerification = defaultConfigs.loadSequenceNumberVerification, maxBatchSizeInBytes = defaultConfigs.maxBatchSizeInBytes, chunkSizeInBytes = defaultConfigs.chunkSizeInBytes, explicitSchemaControl = defaultConfigs.explicitSchemaControl, enableGroupedBatching = defaultConfigs.enableGroupedBatching, flushMode = defaultConfigs.flushMode, 
+        // If batching is disabled then we should disable compression as well. If batching is disabled and compression
+        // is enabled via runtimeOptions, we will throw an error later.
+        compressionOptions = enableGroupedBatching === false
+            ? disabledCompressionConfig
+            : defaultConfigs.compressionOptions, createBlobPayloadPending = defaultConfigs.createBlobPayloadPending, } = runtimeOptions;
+        // The logic for enableRuntimeIdCompressor is a bit different. Since `undefined` represents a logical state (off)
+        // we need to check it's explicitly set in runtimeOptions. If so, we should use that value even if it's undefined.
+        const enableRuntimeIdCompressor = "enableRuntimeIdCompressor" in runtimeOptions
+            ? runtimeOptions.enableRuntimeIdCompressor
+            : defaultConfigs.enableRuntimeIdCompressor;
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = context.baseSnapshot?.blobs[blobName];
@@ -380,6 +392,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             compressionLz4,
             idCompressorMode,
             opGroupingEnabled: enableGroupedBatching,
+            createBlobPayloadPending,
             disallowedVersions: [],
         }, (schema) => {
             runtime.onSchemaChange(schema);
@@ -397,10 +410,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             compressionOptions,
             maxBatchSizeInBytes,
             chunkSizeInBytes,
-            // Requires<> drops undefined from IdCompressorType
-            enableRuntimeIdCompressor: enableRuntimeIdCompressor,
+            enableRuntimeIdCompressor,
             enableGroupedBatching,
             explicitSchemaControl,
+            createBlobPayloadPending,
         };
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], internalRuntimeOptions, containerScope, logger, existing, blobManagerLoadInfo, context.storage, createIdCompressorFn, documentSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, undefined, // summaryConfiguration
         recentBatchInfo);
@@ -550,6 +563,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
         this.imminentClosure = false;
+        this.isReadOnly = () => this.deltaManager.readOnlyInfo.readonly === true;
         // We accumulate Id compressor Ops while Id compressor is not loaded yet (only for "delayed" mode)
         // Once it loads, it will process all such ops and we will stop accumulating further ops - ops will be processes as they come in.
         this.pendingIdCompressorOps = [];
@@ -567,6 +581,51 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.snapshotCacheForLoadingGroupIds = new PromiseCache({
             expiry: { policy: "absolute", durationMs: 60000 },
         });
+        this.notifyReadOnlyState = (readonly) => this.channelCollection.notifyReadOnlyState(readonly);
+        /**
+         * Enter Staging Mode, such that ops submitted to the ContainerRuntime will not be sent to the ordering service.
+         * To exit Staging Mode, call either discardChanges or commitChanges on the Stage Controls returned from this method.
+         *
+         * @returns StageControlsExperimental - Controls for exiting Staging Mode.
+         */
+        // eslint-disable-next-line import/no-deprecated
+        this.enterStagingMode = () => {
+            if (this.stageControls !== undefined) {
+                throw new Error("already in staging mode");
+            }
+            // Make sure all BatchManagers are empty before entering staging mode,
+            // since we mark whole batches as "staged" or not to indicate whether to submit them.
+            this.outbox.flush();
+            const exitStagingMode = (discardOrCommit) => () => {
+                // Final flush of any last staged changes
+                this.outbox.flush(undefined, true /* staged */);
+                this.stageControls = undefined;
+                discardOrCommit();
+            };
+            const stageControls = {
+                discardChanges: exitStagingMode(() => {
+                    // Pop all staged batches from the PSM and roll them back in LIFO order
+                    this.pendingStateManager.popStagedBatches(({ runtimeOp, localOpMetadata }) => {
+                        assert(runtimeOp !== undefined, 0xb82 /* Staged batches expected to have runtimeOp defined */);
+                        this.rollback(runtimeOp, localOpMetadata);
+                    });
+                    if (this.attachState === AttachState.Attached) {
+                        this.updateDocumentDirtyState(this.pendingMessagesCount !== 0);
+                    }
+                }),
+                commitChanges: exitStagingMode(() => {
+                    // All staged changes are in the PSM, so just replay them (ignore pre-staging batches)
+                    // FUTURE: Have this do squash-rebase instead of resubmitting all intermediate changes
+                    if (this.connected) {
+                        this.pendingStateManager.replayPendingStates(true /* onlyStagedBatched */);
+                    }
+                    else {
+                        this.pendingStateManager.clearStagingFlags();
+                    }
+                }),
+            };
+            return (this.stageControls = stageControls);
+        };
         this.readAndParseBlob = async (id) => readAndParse(this.storage, id);
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, pendingLocalState, supportedFeatures, snapshotWithContents, } = context;
         // In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
@@ -686,19 +745,22 @@ export class ContainerRuntime extends TypedEventEmitter {
             isActiveConnection: () => this.innerDeltaManager.active,
             isAttached: () => this.attachState !== AttachState.Detached,
         }, pendingRuntimeState?.pending, this.baseLogger);
-        let outerDeltaManager;
+        let outerDeltaManager = this.innerDeltaManager;
         this.useDeltaManagerOpsProxy =
             this.mc.config.getBoolean("Fluid.ContainerRuntime.DeltaManagerOpsProxy") === true;
         // The summarizerDeltaManager Proxy is used to lie to the summarizer to convince it is in the right state as a summarizer client.
-        const summarizerDeltaManagerProxy = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
-        outerDeltaManager = summarizerDeltaManagerProxy;
+        outerDeltaManager = DeltaManagerSummarizerProxy.wrapIfSummarizer(outerDeltaManager);
         // The DeltaManagerPendingOpsProxy is used to control the minimum sequence number
         // It allows us to lie to the layers below so that they can maintain enough local state for rebasing ops.
         if (this.useDeltaManagerOpsProxy) {
-            const pendingOpsDeltaManagerProxy = new DeltaManagerPendingOpsProxy(summarizerDeltaManagerProxy, this.pendingStateManager);
+            const pendingOpsDeltaManagerProxy = new DeltaManagerPendingOpsProxy(outerDeltaManager, this.pendingStateManager);
             outerDeltaManager = pendingOpsDeltaManagerProxy;
         }
-        this._deltaManager = outerDeltaManager;
+        // always wrap the exposed delta manager in at least on layer of proxying
+        this._deltaManager =
+            outerDeltaManager instanceof BaseDeltaManagerProxy
+                ? outerDeltaManager
+                : new BaseDeltaManagerProxy(outerDeltaManager);
         this.handleContext = new ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             validateSummaryHeuristicConfiguration(this.summaryConfiguration);
@@ -809,6 +871,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             ...props,
             timestampMs: props.timestampMs ?? this.getCurrentReferenceTimestampMs(),
         }), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
+        this._deltaManager.on("readonly", this.notifyReadOnlyState);
         this.blobManager = new BlobManager({
             routeContext: this.handleContext,
             blobManagerLoadInfo,
@@ -829,6 +892,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             isBlobDeleted: (blobPath) => this.garbageCollector.isNodeDeleted(blobPath),
             runtime: this,
             stashedBlobs: pendingRuntimeState?.pendingAttachmentBlobs,
+            createBlobPayloadPending: this.sessionSchema.createBlobPayloadPending === true,
         });
         this.deltaScheduler = new DeltaScheduler(this.innerDeltaManager, this, createChildLogger({ logger: this.baseLogger, namespace: "DeltaScheduler" }));
         this.inboundBatchAggregator = new InboundBatchAggregator(this.innerDeltaManager, () => this.clientId, createChildLogger({ logger: this.baseLogger, namespace: "InboundBatchAggregator" }));
@@ -1080,6 +1144,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.pendingStateManager.dispose();
         this.inboundBatchAggregator.dispose();
         this.deltaScheduler.dispose();
+        this._deltaManager.dispose();
         this.emit("dispose");
         this.removeAllListeners();
     }
@@ -1225,7 +1290,13 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return this.resolveHandle(requestParser.createSubRequest(1));
             }
             if (id === blobManagerBasePath && requestParser.isLeaf(2)) {
-                const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
+                const localId = requestParser.pathParts[1];
+                const payloadPending = requestParser.headers?.[RuntimeHeaders.payloadPending] === true;
+                if (!this.blobManager.hasBlob(localId) &&
+                    requestParser.headers?.[RuntimeHeaders.wait] === false) {
+                    return create404Response(request);
+                }
+                const blob = await this.blobManager.getBlob(localId, payloadPending);
                 return {
                     status: 200,
                     mimeType: "fluid/object",
@@ -1865,11 +1936,13 @@ export class ContainerRuntime extends TypedEventEmitter {
      *
      * @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
      * with the given Batch ID, which must be preserved
+     * @param resubmittingStagedBatch - If defined, indicates this is a resubmission of a batch that is staged,
+     * meaning it should not be sent to the ordering service yet.
      */
-    flush(resubmittingBatchId) {
+    flush(resubmittingBatchId, resubmittingStagedBatch) {
         try {
             assert(!this.batchRunner.running, 0x24c /* "Cannot call `flush()` while manually accumulating a batch (e.g. under orderSequentially) */);
-            this.outbox.flush(resubmittingBatchId);
+            this.outbox.flush(resubmittingBatchId, resubmittingStagedBatch);
             assert(this.outbox.isEmpty, 0x3cf /* reentrancy */);
         }
         catch (error) {
@@ -1888,7 +1961,12 @@ export class ContainerRuntime extends TypedEventEmitter {
     orderSequentially(callback) {
         let checkpoint;
         const checkpointDirtyState = this.dirtyContainer;
+        // eslint-disable-next-line import/no-deprecated
+        let stageControls;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
+            if (!this.batchRunner.running && !this.inStagingMode) {
+                stageControls = this.enterStagingMode();
+            }
             // Note: we are not touching any batches other than mainBatch here, for two reasons:
             // 1. It would not help, as other batches are flushed independently from main batch.
             // 2. There is no way to undo process of data store creation, blob creation, ID compressor ops, or other things tracked by other batches.
@@ -1902,11 +1980,13 @@ export class ContainerRuntime extends TypedEventEmitter {
                 if (checkpoint) {
                     // This will throw and close the container if rollback fails
                     try {
-                        checkpoint.rollback((message) => this.rollback(message.serializedOp, message.localOpMetadata));
+                        checkpoint.rollback((message) => this.rollback(message.runtimeOp, message.localOpMetadata));
                         // reset the dirty state after rollback to what it was before to keep it consistent
                         if (this.dirtyContainer !== checkpointDirtyState) {
                             this.updateDocumentDirtyState(checkpointDirtyState);
                         }
+                        stageControls?.discardChanges();
+                        stageControls = undefined;
                     }
                     catch (error_) {
                         const error2 = wrapError(error_, (message) => {
@@ -1924,12 +2004,21 @@ export class ContainerRuntime extends TypedEventEmitter {
                 throw error; // throw the original error for the consumer of the runtime
             }
         });
+        stageControls?.commitChanges();
         // We don't flush on TurnBased since we expect all messages in the same JS turn to be part of the same batch
         if (this.flushMode !== FlushMode.TurnBased && !this.batchRunner.running) {
             this.flush();
         }
         return result;
     }
+    /**
+     * If true, the ContainerRuntime is not submitting any new ops to the ordering service.
+     * Ops submitted to the ContainerRuntime while in Staging Mode will be queued in the PendingStateManager,
+     * either to be discarded or committed later (via the Stage Controls returned from enterStagingMode).
+     */
+    get inStagingMode() {
+        return this.stageControls !== undefined;
+    }
     /**
      * Returns the aliased data store's entryPoint, given the alias.
      * @param alias - The alias for the data store.
@@ -2662,8 +2751,11 @@ export class ContainerRuntime extends TypedEventEmitter {
                     contents: idRange,
                 };
                 const idAllocationBatchMessage = {
-                    serializedOp: serializeOp(idAllocationMessage),
+                    runtimeOp: idAllocationMessage,
                     referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                    // Note: For now, we will never stage ID Allocation messages.
+                    // They won't contain personal info and no harm in extra allocations in case of discarding the staged changes
+                    staged: false,
                 };
                 this.outbox.submitIdAllocation(idAllocationBatchMessage);
             }
@@ -2705,17 +2797,19 @@ export class ContainerRuntime extends TypedEventEmitter {
                     contents: schemaChangeMessage,
                 };
                 this.outbox.submit({
-                    serializedOp: serializeOp(msg),
+                    runtimeOp: msg,
                     referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                    staged: this.inStagingMode,
                 });
             }
             const message = {
                 // This will encode any handles present in this op before serializing to string
                 // Note: handles may already have been encoded by the DDS layer, but encoding handles is idempotent so there's no problem.
-                serializedOp: serializeOp(containerRuntimeMessage),
+                runtimeOp: containerRuntimeMessage,
                 metadata,
                 localOpMetadata,
                 referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                staged: this.inStagingMode,
             };
             if (type === ContainerMessageType.BlobAttach) {
                 // BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
@@ -2799,7 +2893,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @remarks - If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
      * for correlation to detect container forking.
      */
-    reSubmitBatch(batch, batchId) {
+    reSubmitBatch(batch, batchId, staged) {
         this.batchRunner.run(() => {
             for (const message of batch) {
                 this.reSubmit(message);
@@ -2807,14 +2901,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         // Only include Batch ID if "Offline Load" feature is enabled
         // It's only needed to identify batches across container forks arising from misuse of offline load.
-        this.flush(this.offlineEnabled ? batchId : undefined);
+        this.flush(this.offlineEnabled ? batchId : undefined, staged);
     }
     reSubmit(message) {
-        // Messages in the PendingStateManager and Outbox (both call resubmit) have serialized contents, so parse it here.
-        // Note that roundtripping handles through string like this means this parsed contents
-        // contains RemoteFluidObjectHandles, not the original handle.
-        const containerRuntimeMessage = this.parseLocalOpContent(message.content);
-        this.reSubmitCore(containerRuntimeMessage, message.localOpMetadata, message.opMetadata);
+        this.reSubmitCore(message.runtimeOp, message.localOpMetadata, message.opMetadata);
     }
     /**
      * Finds the right store and asks it to resubmit the message. This typically happens when we
@@ -2869,11 +2959,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
-    rollback(content, localOpMetadata) {
-        // Messages in the Outbox (which calls rollback) have serialized contents, so parse it here.
-        // Note that roundtripping handles through string like this means this parsed contents
-        // contains RemoteFluidObjectHandles, not the original handle.
-        const { type, contents } = this.parseLocalOpContent(content);
+    rollback(runtimeOp, localOpMetadata) {
+        const { type, contents } = runtimeOp;
         switch (type) {
             case ContainerMessageType.FluidDataStoreOp: {
                 // For operations, call rollbackDataStoreOp which will find the right store