@fluidframework/container-runtime 2.102.0 → 2.110.0

package/dist/containerRuntime.js

@@ -632,14 +632,31 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         this.electedSummarizerData = electedSummarizerData;
         this.runtimeOptions = runtimeOptions;
         this.containerScope = containerScope;
-        this.baseLogger = baseLogger;
         this._storage = _storage;
         this.createIdCompressorFn = createIdCompressorFn;
         this.documentsSchemaController = documentsSchemaController;
         this.minVersionForCollab = minVersionForCollab;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
-        this.isReadOnly = () => this.deltaManager.readOnlyInfo.readonly === true;
+        /**
+         * Whether local op submission is currently disallowed.
+         *
+         * This is `true` in two distinct situations.
+         *
+         * First: the delta manager reports a read-only connection (host/service-imposed permission or connection state — the historical meaning of `readOnly`).
+         *
+         * Second: the `PendingStateManager` is replaying stashed ops (`isApplyingStashedOps`). During this window DDSes must not submit new local ops, as doing so would interleave fresh content ahead of the stashed pending stream and corrupt pending local state. Surfacing it through `isReadOnly()` lets DDSes that consult `readOnly` at realize time self-suppress; see the apply-lifecycle docs on `PendingStateManager` for the full rationale.
+         *
+         * Note this layers a third meaning ("transiently quiescing for stashed-op replay") onto the `readOnly` predicate, which is broader than its host/connection-permission origin.
+         */
+        this.isReadOnly = () => 
+        // `_deltaManager` and `pendingStateManager` are both assigned partway
+        // through the constructor; `baseLogger` is built earlier and stamps
+        // `isReadOnly` on every error event (e.g. layer-compat failures
+        // during construction), so this can be called before either is
+        // assigned. Optional chains keep that window safe.
+        this._deltaManager?.readOnlyInfo.readonly === true ||
+            this.pendingStateManager?.isApplyingStashedOps === true;
         // We accumulate Id compressor Ops while Id compressor is not loaded yet (only for "delayed" mode)
         // Once it loads, it will process all such ops and we will stop accumulating further ops - ops will be processes as they come in.
         this.pendingIdCompressorOps = [];
@@ -658,7 +675,13 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             expiry: { policy: "absolute", durationMs: 60000 },
         });
         this.extensions = new Map();
-        this.notifyReadOnlyState = (readonly) => this.channelCollection.notifyReadOnlyState(readonly);
+        // Boolean payload from the `"readonly"` delta-manager event is intentionally
+        // ignored — `isReadOnly()` aggregates delta-manager readonly with the PSM
+        // apply window, and that aggregation is the source of truth for fanout.
+        // `channelCollection?.` guards against future wiring changes; both callers
+        // today (the `"readonly"` listener and `onAfterStashedOpsApplied`) fire
+        // after `channelCollection` is assigned.
+        this.notifyReadOnlyState = (_readonly) => this.channelCollection?.notifyReadOnlyState(this.isReadOnly());
         /**
          * Enter Staging Mode, such that ops submitted to the ContainerRuntime will not be sent to the ordering service.
          * To exit Staging Mode, call either discardChanges or commitChanges on the Stage Controls returned from this method.
@@ -817,15 +840,20 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
         this.disposeFn = disposeFn ?? closeFn;
         this.isSnapshotInstanceOfISnapshot = snapshotWithContents !== undefined;
-        this.mc = (0, internal_8.createChildMonitoringContext)({
-            logger: this.baseLogger,
-            namespace: "ContainerRuntime",
+        this.baseLogger = (0, internal_8.createChildLogger)({
+            logger: baseLogger,
             properties: {
-                all: {
-                    inStagingMode: this.inStagingMode,
+                error: {
+                    inStagingMode: () => this.inStagingMode,
+                    isApplyingStashedOps: () => this.pendingStateManager?.isApplyingStashedOps,
+                    isReadOnly: () => this.isReadOnly(),
                 },
             },
         });
+        this.mc = (0, internal_8.createChildMonitoringContext)({
+            logger: this.baseLogger,
+            namespace: "ContainerRuntime",
+        });
         // Validate that the Loader is compatible with this Runtime.
         const maybeLoaderCompatDetailsForRuntime = context;
         (0, runtimeLayerCompatState_js_1.validateLoaderCompatibility)(maybeLoaderCompatDetailsForRuntime.ILayerCompatDetails, this.disposeFn, this.mc);
@@ -948,7 +976,18 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             reSubmitBatch: this.reSubmitBatch.bind(this),
             isActiveConnection: () => this.innerDeltaManager.active,
             isAttached: () => this.attachState !== container_definitions_1.AttachState.Detached,
-        }, pendingRuntimeState?.pending, this.baseLogger);
+        }, pendingRuntimeState?.pending, this.baseLogger, {
+            // PSM has cleared `isApplyingStashedOps`; `isReadOnly()` now
+            // reflects the network-readonly state again. Fan out so DDSes
+            // know they can submit once more. No open hook is needed —
+            // the apply window opens before `channelCollection` exists,
+            // so a fanout there would be a no-op; data stores instead
+            // pick up the initial readonly state from `isReadOnly()`
+            // when they're first asked.
+            onAfterStashedOpsApplied: () => {
+                this.notifyReadOnlyState();
+            },
+        });
         let outerDeltaManager = this.innerDeltaManager;
         this.useDeltaManagerOpsProxy =
             this.mc.config.getBoolean("Fluid.ContainerRuntime.DeltaManagerOpsProxy") === true;
@@ -1214,6 +1253,13 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             telemetryDocumentId: this.telemetryDocumentId,
             groupedBatchingEnabled: this.groupedBatchingEnabled,
             initialSequenceNumber: this.deltaManager.initialSequenceNumber,
+            // Number of ops since the last summary that this client is aware of (including ops still
+            // queued for processing). Computed as the gap between the latest known op sequence number
+            // and the sequence number of the message at which the last summary was taken (per snapshot
+            // metadata). Falls back to lastKnownSeqNumber when no prior summary message is recorded
+            // (e.g. new container or older snapshot without metadata).
+            numUnsummarizedOps: this.deltaManager.lastKnownSeqNumber -
+                (this.messageAtLastSummary?.sequenceNumber ?? 0),
             minVersionForCollab: this.minVersionForCollab,
             // logging hardware telemetry
             deviceSpec: { ...getDeviceSpec() },
@@ -1666,6 +1712,24 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         if (!this.shouldSendOps()) {
             return;
         }
+        // Invariant: the canSendOps edge in `setConnectionStateCore` — the
+        // only caller of this method — cannot fire while
+        // `applyStashedOpsAt` is in flight, because the loader awaits the
+        // apply before transitioning the runtime to a write-capable
+        // connection. If this assert ever fires, that contract has changed
+        // and the submit guard at `submit()` would catch a runtime-internal
+        // resubmit (`Rejoin`, `GC`, `FluidDataStoreOp`) for an op type
+        // outside the apply-window allowlist.
+        //
+        // The precondition is held by the load sequence: `loadRuntime2`
+        // awaits `pendingStateManager.applyStashedOpsAt(...)` before
+        // returning the runtime, and the loader gates `setLoaded` on that
+        // completion before any write-capable connection edge fires. A
+        // maintainer reordering either sequence (or adding a new
+        // `canSendOps` edge that fires before the apply resolves) is what
+        // would trip this assert.
+        // @see {@link ContainerRuntime.loadRuntime2} (awaits `applyStashedOpsAt`)
+        (0, internal_2.assert)(!this.pendingStateManager.isApplyingStashedOps, 0xd01 /* replayPendingStates must not be called during stashed-op apply window */);
         // Replaying is an internal operation and we don't want to generate noise while doing it.
         // So temporarily disable dirty state change events, and save the old state.
         // When we're done, we'll emit the event if the state changed.
@@ -3062,6 +3126,45 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     }
     submit(containerRuntimeMessage, localOpMetadata = undefined, metadata) {
         this.verifyNotClosed();
+        // Nothing should be submitting while we're replaying stashed ops.
+        // The runtime is readonly during the apply window (see
+        // `PendingStateManager._applyLifecycle`), so a compliant DDS skips
+        // submits. If we land here anyway, a DDS bypassed the readonly gate
+        // (e.g. a realize-time write that doesn't consult `readOnly`) and
+        // produced a local op that has no counterpart in the saved-op
+        // replay — we cannot reconcile the mismatch, so fail fatally. We
+        // check here (rather than at flush) because outbox flushes are
+        // deferred and the apply window could close before the offending op
+        // reaches the pending queue.
+        //
+        // Allowlist: `BlobAttach` is a runtime-internal op type that may
+        // legitimately fire during apply — produced by `sharePendingBlobs`,
+        // which is invoked from `loadRuntime2` before `applyStashedOpsAt`
+        // resolves. `IdAllocation` is not in this allowlist because the
+        // assert at 0x9a5 below enforces that it never reaches `submit()`
+        // at all; treating that assert as the single source of truth.
+        //
+        // Always surface the error event to telemetry on a bypass so we can
+        // attribute incidents regardless of the on-switch state. The
+        // `EnableSubmitDuringStashedApplyThrow` config opts in to the
+        // throw + container close; by default we log only, so a first- or
+        // third-party DDS that quietly bypasses the readonly gate in
+        // production is observable without escalating to a fatal close.
+        if (this.pendingStateManager.isApplyingStashedOps &&
+            containerRuntimeMessage.type !== messageTypes_js_1.ContainerMessageType.BlobAttach) {
+            const error = new internal_8.UsageError("Local op submitted during stashed-op apply window", {
+                messageType: containerRuntimeMessage.type,
+            });
+            this.mc.logger.sendErrorEvent({ eventName: "SubmitDuringStashedOpApply" }, error);
+            if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableSubmitDuringStashedApplyThrow") === true) {
+                // Close the container before throwing so the "throw + close"
+                // contract is enforced by this code path rather than by
+                // whichever caller happens to wrap the throw in `.catch(closeFn)`.
+                // `closeFn` is idempotent; a caller that also closes won't double-close.
+                this.closeFn(error);
+                throw error;
+            }
+        }
         // There should be no ops in detached container state!
         (0, internal_2.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         (0, internal_2.assert)(metadata === undefined ||