@fluidframework/container-runtime 2.101.1 → 2.103.0

package/dist/containerRuntime.js

@@ -632,14 +632,31 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         this.electedSummarizerData = electedSummarizerData;
         this.runtimeOptions = runtimeOptions;
         this.containerScope = containerScope;
-        this.baseLogger = baseLogger;
         this._storage = _storage;
         this.createIdCompressorFn = createIdCompressorFn;
         this.documentsSchemaController = documentsSchemaController;
         this.minVersionForCollab = minVersionForCollab;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
-        this.isReadOnly = () => this.deltaManager.readOnlyInfo.readonly === true;
+        /**
+         * Whether local op submission is currently disallowed.
+         *
+         * This is `true` in two distinct situations.
+         *
+         * First: the delta manager reports a read-only connection (host/service-imposed permission or connection state — the historical meaning of `readOnly`).
+         *
+         * Second: the `PendingStateManager` is replaying stashed ops (`isApplyingStashedOps`). During this window DDSes must not submit new local ops, as doing so would interleave fresh content ahead of the stashed pending stream and corrupt pending local state. Surfacing it through `isReadOnly()` lets DDSes that consult `readOnly` at realize time self-suppress; see the apply-lifecycle docs on `PendingStateManager` for the full rationale.
+         *
+         * Note this layers a third meaning ("transiently quiescing for stashed-op replay") onto the `readOnly` predicate, which is broader than its host/connection-permission origin.
+         */
+        this.isReadOnly = () => 
+        // `_deltaManager` and `pendingStateManager` are both assigned partway
+        // through the constructor; `baseLogger` is built earlier and stamps
+        // `isReadOnly` on every error event (e.g. layer-compat failures
+        // during construction), so this can be called before either is
+        // assigned. Optional chains keep that window safe.
+        this._deltaManager?.readOnlyInfo.readonly === true ||
+            this.pendingStateManager?.isApplyingStashedOps === true;
         // We accumulate Id compressor Ops while Id compressor is not loaded yet (only for "delayed" mode)
         // Once it loads, it will process all such ops and we will stop accumulating further ops - ops will be processes as they come in.
         this.pendingIdCompressorOps = [];
@@ -658,7 +675,13 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             expiry: { policy: "absolute", durationMs: 60000 },
         });
         this.extensions = new Map();
-        this.notifyReadOnlyState = (readonly) => this.channelCollection.notifyReadOnlyState(readonly);
+        // Boolean payload from the `"readonly"` delta-manager event is intentionally
+        // ignored — `isReadOnly()` aggregates delta-manager readonly with the PSM
+        // apply window, and that aggregation is the source of truth for fanout.
+        // `channelCollection?.` guards against future wiring changes; both callers
+        // today (the `"readonly"` listener and `onAfterStashedOpsApplied`) fire
+        // after `channelCollection` is assigned.
+        this.notifyReadOnlyState = (_readonly) => this.channelCollection?.notifyReadOnlyState(this.isReadOnly());
         /**
          * Enter Staging Mode, such that ops submitted to the ContainerRuntime will not be sent to the ordering service.
          * To exit Staging Mode, call either discardChanges or commitChanges on the Stage Controls returned from this method.
@@ -817,15 +840,20 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
         this.disposeFn = disposeFn ?? closeFn;
         this.isSnapshotInstanceOfISnapshot = snapshotWithContents !== undefined;
-        this.mc = (0, internal_8.createChildMonitoringContext)({
-            logger: this.baseLogger,
-            namespace: "ContainerRuntime",
+        this.baseLogger = (0, internal_8.createChildLogger)({
+            logger: baseLogger,
             properties: {
-                all: {
-                    inStagingMode: this.inStagingMode,
+                error: {
+                    inStagingMode: () => this.inStagingMode,
+                    isApplyingStashedOps: () => this.pendingStateManager?.isApplyingStashedOps,
+                    isReadOnly: () => this.isReadOnly(),
                 },
             },
         });
+        this.mc = (0, internal_8.createChildMonitoringContext)({
+            logger: this.baseLogger,
+            namespace: "ContainerRuntime",
+        });
         // Validate that the Loader is compatible with this Runtime.
         const maybeLoaderCompatDetailsForRuntime = context;
         (0, runtimeLayerCompatState_js_1.validateLoaderCompatibility)(maybeLoaderCompatDetailsForRuntime.ILayerCompatDetails, this.disposeFn, this.mc);
@@ -948,7 +976,18 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             reSubmitBatch: this.reSubmitBatch.bind(this),
             isActiveConnection: () => this.innerDeltaManager.active,
             isAttached: () => this.attachState !== container_definitions_1.AttachState.Detached,
-        }, pendingRuntimeState?.pending, this.baseLogger);
+        }, pendingRuntimeState?.pending, this.baseLogger, {
+            // PSM has cleared `isApplyingStashedOps`; `isReadOnly()` now
+            // reflects the network-readonly state again. Fan out so DDSes
+            // know they can submit once more. No open hook is needed —
+            // the apply window opens before `channelCollection` exists,
+            // so a fanout there would be a no-op; data stores instead
+            // pick up the initial readonly state from `isReadOnly()`
+            // when they're first asked.
+            onAfterStashedOpsApplied: () => {
+                this.notifyReadOnlyState();
+            },
+        });
         let outerDeltaManager = this.innerDeltaManager;
         this.useDeltaManagerOpsProxy =
             this.mc.config.getBoolean("Fluid.ContainerRuntime.DeltaManagerOpsProxy") === true;
@@ -985,18 +1024,39 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             this.mc.config.getNumber("Fluid.ContainerRuntime.StagingModeAutoFlushThreshold") ??
                 runtimeOptions.stagingModeAutoFlushThreshold ??
                 defaultStagingModeAutoFlushThreshold;
-        this.batchIdTrackingEnabled =
-            this.mc.config.getBoolean("Fluid.Container.enableOfflineFull") ??
-                this.mc.config.getBoolean("Fluid.ContainerRuntime.enableBatchIdTracking") ??
-                false;
-        if (this.batchIdTrackingEnabled && this._flushMode !== internal_6.FlushMode.TurnBased) {
+        // BatchId tracking powers DuplicateBatchDetector (catching forked-container duplicates)
+        // and is also a prerequisite for the Offline Load feature. It is enabled by default
+        // when both TurnBased flush mode and grouped batching are active; the kill-switch
+        // below allows disabling it without a code change if a regression is observed.
+        // Grouped batching is required because resubmits can produce empty batches that must
+        // still be sent on the wire as a placeholder grouped batch to preserve their batchId
+        // (see OpGroupingManager.createEmptyGroupedBatch / outbox.flushEmptyBatch).
+        // Offline Load requires all three prerequisites (TurnBased, grouped batching, and
+        // batchId tracking not killed by config), so a consumer that opts into it without
+        // them gets an explicit UsageError rather than silent degradation.
+        const offlineLoadRequested = this.mc.config.getBoolean("Fluid.Container.enableOfflineFull") === true;
+        const disableBatchIdTracking = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableBatchIdTracking") === true;
+        if (offlineLoadRequested && this._flushMode !== internal_6.FlushMode.TurnBased) {
             const error = new internal_8.UsageError("Offline mode is only supported in turn-based mode");
             this.closeFn(error);
             throw error;
         }
-        // DuplicateBatchDetection is only enabled if Offline Load is enabled
-        // It maintains a cache of all batchIds/sequenceNumbers within the collab window.
-        // Don't waste resources doing so if not needed.
+        if (offlineLoadRequested && !this.groupedBatchingEnabled) {
+            const error = new internal_8.UsageError("Offline mode requires grouped batching to be enabled");
+            this.closeFn(error);
+            throw error;
+        }
+        if (offlineLoadRequested && disableBatchIdTracking) {
+            const error = new internal_8.UsageError("Offline mode requires batchId tracking; remove Fluid.ContainerRuntime.DisableBatchIdTracking");
+            this.closeFn(error);
+            throw error;
+        }
+        this.batchIdTrackingEnabled =
+            !disableBatchIdTracking &&
+                this._flushMode === internal_6.FlushMode.TurnBased &&
+                this.groupedBatchingEnabled;
+        // DuplicateBatchDetector maintains a cache of all batchIds/sequenceNumbers within the
+        // collab window. Skip allocating it when batchId tracking is off.
         if (this.batchIdTrackingEnabled) {
             this.duplicateBatchDetector = new index_js_3.DuplicateBatchDetector(recentBatchInfo);
         }
@@ -1193,6 +1253,13 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             telemetryDocumentId: this.telemetryDocumentId,
             groupedBatchingEnabled: this.groupedBatchingEnabled,
             initialSequenceNumber: this.deltaManager.initialSequenceNumber,
+            // Number of ops since the last summary that this client is aware of (including ops still
+            // queued for processing). Computed as the gap between the latest known op sequence number
+            // and the sequence number of the message at which the last summary was taken (per snapshot
+            // metadata). Falls back to lastKnownSeqNumber when no prior summary message is recorded
+            // (e.g. new container or older snapshot without metadata).
+            numUnsummarizedOps: this.deltaManager.lastKnownSeqNumber -
+                (this.messageAtLastSummary?.sequenceNumber ?? 0),
             minVersionForCollab: this.minVersionForCollab,
             // logging hardware telemetry
             deviceSpec: { ...getDeviceSpec() },
@@ -1645,6 +1712,24 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         if (!this.shouldSendOps()) {
             return;
         }
+        // Invariant: the canSendOps edge in `setConnectionStateCore` — the
+        // only caller of this method — cannot fire while
+        // `applyStashedOpsAt` is in flight, because the loader awaits the
+        // apply before transitioning the runtime to a write-capable
+        // connection. If this assert ever fires, that contract has changed
+        // and the submit guard at `submit()` would catch a runtime-internal
+        // resubmit (`Rejoin`, `GC`, `FluidDataStoreOp`) for an op type
+        // outside the apply-window allowlist.
+        //
+        // The precondition is held by the load sequence: `loadRuntime2`
+        // awaits `pendingStateManager.applyStashedOpsAt(...)` before
+        // returning the runtime, and the loader gates `setLoaded` on that
+        // completion before any write-capable connection edge fires. A
+        // maintainer reordering either sequence (or adding a new
+        // `canSendOps` edge that fires before the apply resolves) is what
+        // would trip this assert.
+        // @see {@link ContainerRuntime.loadRuntime2} (awaits `applyStashedOpsAt`)
+        (0, internal_2.assert)(!this.pendingStateManager.isApplyingStashedOps, 0xd01 /* replayPendingStates must not be called during stashed-op apply window */);
         // Replaying is an internal operation and we don't want to generate noise while doing it.
         // So temporarily disable dirty state change events, and save the old state.
         // When we're done, we'll emit the event if the state changed.
@@ -1918,14 +2003,32 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                         eventName: "DuplicateBatch",
                         details: {
                             batchId: batchStart.batchId,
+                            batchIdExplicit: batchStart.batchId !== undefined,
                             clientId: batchStart.clientId,
                             batchStartCsn: batchStart.batchStartCsn,
                             size: inboundResult.length,
                             duplicateBatchSequenceNumber: result.otherSequenceNumber,
+                            // Identifying info for the ORIGINAL occurrence of this batch, so we can
+                            // disambiguate the duplicate's source (e.g. resubmit vs fresh submit, same
+                            // vs different wire clientId). Undefined fields indicate the original was
+                            // loaded from a summary snapshot rather than seen at runtime.
+                            otherClientId: result.otherBatchInfo?.clientId,
+                            otherBatchStartCsn: result.otherBatchInfo?.batchStartCsn,
+                            otherBatchIdExplicit: result.otherBatchInfo?.batchIdExplicit,
+                            otherFromSnapshot: result.otherBatchInfo === undefined,
                             ...(0, internal_8.extractSafePropertiesFromMessage)(batchStart.keyMessage),
+                            // For grouped batches, `keyMessage` is one of the sub-messages produced by
+                            // `OpGroupingManager.ungroupOp`, which overwrites `clientSequenceNumber`
+                            // with a synthetic counter (1, 2, 3, ...). Override with the real outer
+                            // envelope's clientSequenceNumber so downstream telemetry doesn't get a
+                            // misleading "fake csn" value.
+                            messageClientSequenceNumber: batchStart.batchStartCsn,
                         },
                     }, error);
-                    throw error;
+                    // Due to a live incident where we had a bug in the service that caused duplicate batches to be sent to clients, we want to log when we detect a duplicate batch, but we don't want to throw an error
+                    // as it could hit the same service bug. We need to monitor below event to catch legitimate container forking scenarios and reenable throwing the data corruption error once the service bug is fixed and we stop seeing duplicate batches in the wild
+                    // or once we are able to identify batch duplication reason (forking vs service bug).
+                    // throw error;
                 }
             }
             // Reach out to PendingStateManager, either to zip localOpMetadata into the *local* message list,
@@ -3023,6 +3126,45 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     }
     submit(containerRuntimeMessage, localOpMetadata = undefined, metadata) {
         this.verifyNotClosed();
+        // Nothing should be submitting while we're replaying stashed ops.
+        // The runtime is readonly during the apply window (see
+        // `PendingStateManager._applyLifecycle`), so a compliant DDS skips
+        // submits. If we land here anyway, a DDS bypassed the readonly gate
+        // (e.g. a realize-time write that doesn't consult `readOnly`) and
+        // produced a local op that has no counterpart in the saved-op
+        // replay — we cannot reconcile the mismatch, so fail fatally. We
+        // check here (rather than at flush) because outbox flushes are
+        // deferred and the apply window could close before the offending op
+        // reaches the pending queue.
+        //
+        // Allowlist: `BlobAttach` is a runtime-internal op type that may
+        // legitimately fire during apply — produced by `sharePendingBlobs`,
+        // which is invoked from `loadRuntime2` before `applyStashedOpsAt`
+        // resolves. `IdAllocation` is not in this allowlist because the
+        // assert at 0x9a5 below enforces that it never reaches `submit()`
+        // at all; treating that assert as the single source of truth.
+        //
+        // Always surface the error event to telemetry on a bypass so we can
+        // attribute incidents regardless of the on-switch state. The
+        // `EnableSubmitDuringStashedApplyThrow` config opts in to the
+        // throw + container close; by default we log only, so a first- or
+        // third-party DDS that quietly bypasses the readonly gate in
+        // production is observable without escalating to a fatal close.
+        if (this.pendingStateManager.isApplyingStashedOps &&
+            containerRuntimeMessage.type !== messageTypes_js_1.ContainerMessageType.BlobAttach) {
+            const error = new internal_8.UsageError("Local op submitted during stashed-op apply window", {
+                messageType: containerRuntimeMessage.type,
+            });
+            this.mc.logger.sendErrorEvent({ eventName: "SubmitDuringStashedOpApply" }, error);
+            if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableSubmitDuringStashedApplyThrow") === true) {
+                // Close the container before throwing so the "throw + close"
+                // contract is enforced by this code path rather than by
+                // whichever caller happens to wrap the throw in `.catch(closeFn)`.
+                // `closeFn` is idempotent; a caller that also closes won't double-close.
+                this.closeFn(error);
+                throw error;
+            }
+        }
         // There should be no ops in detached container state!
         (0, internal_2.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         (0, internal_2.assert)(metadata === undefined ||