@fluidframework/container-runtime 2.0.0-internal.7.2.2 → 2.0.0-internal.7.4.0

package/lib/containerRuntime.js

@@ -3,7 +3,7 @@ import { assert, delay, LazyPromise } from "@fluidframework/core-utils";
 import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
 import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent, 
 // eslint-disable-next-line import/no-deprecated
-TaggedLoggerAdapter, wrapError, UsageError, } from "@fluidframework/telemetry-utils";
+TaggedLoggerAdapter, wrapError, UsageError, LoggingError, } from "@fluidframework/telemetry-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
@@ -19,7 +19,7 @@ import { BlobManager } from "./blobManager";
 import { DataStores, getSummaryForDatastores } from "./dataStores";
 import { aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, Summarizer, SummaryManager, wrapSummaryInChannelsTree, SummaryCollection, OrderedClientCollection, OrderedClientElection, SummarizerClientElection, summarizerClientType, RunWhileConnectedCoordinator, RetriableSummaryError, } from "./summary";
 import { formExponentialFn, Throttler } from "./throttler";
-import { GarbageCollector, GCNodeType, gcTombstoneGenerationOptionName, trimLeadingAndTrailingSlashes, } from "./gc";
+import { GarbageCollector, GCNodeType, gcGenerationOptionName, trimLeadingAndTrailingSlashes, } from "./gc";
 import { channelToDataStore, isDataStoreAliasMessage } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
 import { ScheduleManager } from "./scheduleManager";
@@ -38,14 +38,8 @@ function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, co
     // undefined defaults to same behavior as "FailToProcess"
     return compatBehavior === "Ignore";
 }
-function prepareLocalContainerRuntimeIdAllocationMessageForTransit(message) {
-    // Remove the stashedState from the op if it's a stashed op
-    if ("stashedState" in message.contents) {
-        delete message.contents.stashedState;
-    }
-}
 /**
- * @public
+ * @internal
  */
 export const DefaultSummaryConfiguration = {
     state: "enabled",
@@ -63,7 +57,7 @@ export const DefaultSummaryConfiguration = {
 };
 /**
  * Accepted header keys for requests coming to the runtime.
- * @public
+ * @internal
  */
 export var RuntimeHeaders;
 (function (RuntimeHeaders) {
@@ -73,22 +67,22 @@ export var RuntimeHeaders;
     RuntimeHeaders["viaHandle"] = "viaHandle";
 })(RuntimeHeaders || (RuntimeHeaders = {}));
 /** True if a tombstoned object should be returned without erroring
- * @public
+ * @internal
  */
 export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
 /**
  * [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring
- * @public
+ * @internal
  */
 export const AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
 /**
  * Tombstone error responses will have this header set to true
- * @public
+ * @internal
  */
 export const TombstoneResponseHeaderKey = "isTombstoned";
 /**
  * Inactive error responses will have this header set to true
- * @public
+ * @internal
  */
 export const InactiveResponseHeaderKey = "isInactive";
 /** Default values for Runtime Headers */
@@ -100,7 +94,7 @@ export const defaultRuntimeHeaderData = {
 };
 /**
  * Available compression algorithms for op compression.
- * @public
+ * @alpha
  */
 export var CompressionAlgorithms;
 (function (CompressionAlgorithms) {
@@ -131,7 +125,7 @@ export const defaultPendingOpsRetryDelayMs = 1000;
 const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
 /**
  * @deprecated use ContainerRuntimeMessageType instead
- * @public
+ * @internal
  */
 export var RuntimeMessage;
 (function (RuntimeMessage) {
@@ -145,7 +139,7 @@ export var RuntimeMessage;
 })(RuntimeMessage || (RuntimeMessage = {}));
 /**
  * @deprecated please use version in driver-utils
- * @public
+ * @internal
  */
 export function isRuntimeMessage(message) {
     return Object.values(RuntimeMessage).includes(message.type);
@@ -154,7 +148,7 @@ export function isRuntimeMessage(message) {
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
  * special-case for document dirty state.  Ultimately we should have no special-cases from the
  * ContainerRuntime's perspective.
- * @public
+ * @internal
  */
 export const agentSchedulerId = "_scheduler";
 // safely check navigator and get the hardware spec value
@@ -230,7 +224,7 @@ export async function TEST_requestSummarizer(loader, url) {
 /**
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
- * @public
+ * @alpha
  */
 export class ContainerRuntime extends TypedEventEmitter {
     /**
@@ -347,11 +341,17 @@ export class ContainerRuntime extends TypedEventEmitter {
         const idCompressorEnabled = metadata?.idCompressorEnabled ?? runtimeOptions.enableRuntimeIdCompressor ?? false;
         let idCompressor;
         if (idCompressorEnabled) {
-            const { IdCompressor, createSessionId } = await import("./id-compressor");
-            idCompressor =
-                serializedIdCompressor !== undefined
-                    ? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
-                    : IdCompressor.create(logger);
+            const { createIdCompressor, deserializeIdCompressor, createSessionId } = await import("@fluidframework/id-compressor");
+            const pendingLocalState = context.pendingLocalState;
+            if (pendingLocalState?.pendingIdCompressorState !== undefined) {
+                idCompressor = deserializeIdCompressor(pendingLocalState.pendingIdCompressorState);
+            }
+            else if (serializedIdCompressor !== undefined) {
+                idCompressor = deserializeIdCompressor(serializedIdCompressor, createSessionId());
+            }
+            else {
+                idCompressor = createIdCompressor(logger);
+            }
         }
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
@@ -453,9 +453,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     get gcThrowOnTombstoneUsage() {
         return this.garbageCollector.throwOnTombstoneUsage;
     }
-    /**
-     * @internal
-     */
+    /***/
     constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, provideEntryPoint, requestHandler, summaryConfiguration = {
         // the defaults
         ...DefaultSummaryConfiguration,
@@ -470,6 +468,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._storage = _storage;
         this.requestHandler = requestHandler;
         this.summaryConfiguration = summaryConfiguration;
+        this.imminentClosure = false;
         this.defaultMaxConsecutiveReconnects = 7;
         this._orderSequentiallyCalls = 0;
         this.flushTaskExists = false;
@@ -561,7 +560,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             eventName: "GCFeatureMatrix",
             metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
             inputs: JSON.stringify({
-                gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName],
+                gcOptions_gcGeneration: this.runtimeOptions.gcOptions[gcGenerationOptionName],
             }),
         });
         this.telemetryDocumentId = metadata?.telemetryDocumentId ?? uuid();
@@ -627,6 +626,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // GC runs in summarizer client and needs access to the real (non-proxy) active information. The proxy
             // delta manager would always return false for summarizer client.
             activeConnection: () => this.innerDeltaManager.active,
+            submitMessage: (message) => this.submit(message),
         });
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = createRootSummarizerNodeWithGC(createChildLogger({ logger: this.logger, namespace: "SummarizerNode" }), 
@@ -1050,19 +1050,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         // Officially transition from the old state to the new state.
         this.updateDocumentDirtyState(newState);
     }
-    /**
-     * Updates the runtime's IdCompressor with the stashed state present in the given op. This is a bit of a
-     * hack and is unnecessarily expensive. As it stands, every locally stashed op (all ops that get stored in
-     * the PendingStateManager) will store their serialized representation locally until ack'd. Upon receiving
-     * this stashed state, the IdCompressor blindly deserializes to the stashed state and assumes the session.
-     * Technically only the last stashed state is needed to do this correctly, but we would have to write some
-     * more hacky code to modify the batch before it gets sent out.
-     * @param content - An IdAllocationOp with "stashedState", which is a representation of un-ack'd local state.
-     */
-    async applyStashedIdAllocationOp(op) {
-        const { IdCompressor } = await import("./id-compressor");
-        this.idCompressor = IdCompressor.deserialize(op.stashedState);
-    }
     /**
      * Parse an op's type and actual content from given serialized content
      * ! Note: this format needs to be in-line with what is set in the "ContainerRuntime.submit(...)" method
@@ -1084,7 +1071,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return this.dataStores.applyStashedAttachOp(opContents.contents);
             case ContainerMessageType.IdAllocation:
                 assert(this.idCompressor !== undefined, 0x67b /* IdCompressor should be defined if enabled */);
-                return this.applyStashedIdAllocationOp(opContents.contents);
+                return;
             case ContainerMessageType.Alias:
             case ContainerMessageType.BlobAttach:
                 return;
@@ -1092,6 +1079,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                 throw new Error("chunkedOp not expected here");
             case ContainerMessageType.Rejoin:
                 throw new Error("rejoin not expected here");
+            case ContainerMessageType.GC:
+                // GC op is only sent in summarizer which should never have stashed ops.
+                throw new LoggingError("GC op not expected to be stashed in summarizer");
             default: {
                 // This should be extremely rare for stashed ops.
                 // It would require a newer runtime stashing ops and then an older one applying them,
@@ -1270,7 +1260,14 @@ export class ContainerRuntime extends TypedEventEmitter {
                 break;
             case ContainerMessageType.IdAllocation:
                 assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
-                this.idCompressor.finalizeCreationRange(messageWithContext.message.contents);
+                // Don't re-finalize the range if we're processing a "savedOp" in
+                // stashed ops flow. The compressor is stashed with these ops already processed.
+                if (messageWithContext.message.metadata?.savedOp !== true) {
+                    this.idCompressor.finalizeCreationRange(messageWithContext.message.contents);
+                }
+                break;
+            case ContainerMessageType.GC:
+                this.garbageCollector.processMessage(messageWithContext.message, local);
                 break;
             case ContainerMessageType.ChunkedOp:
             case ContainerMessageType.Rejoin:
@@ -1461,7 +1458,6 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     /**
      * @deprecated 0.16 Issue #1537, #3631
-     * @internal
      */
     async _createDataStoreWithProps(pkg, props, id = uuid()) {
         return channelToDataStore(await this.dataStores
@@ -1471,7 +1467,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     canSendOps() {
         // Note that the real (non-proxy) delta manager is needed here to get the readonly info. This is because
         // container runtime's ability to send ops depend on the actual readonly state of the delta manager.
-        return this.connected && !this.innerDeltaManager.readOnlyInfo.readonly;
+        return (this.connected && !this.innerDeltaManager.readOnlyInfo.readonly && !this.imminentClosure);
     }
     /**
      * Are we in the middle of batching ops together?
@@ -1493,19 +1489,28 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.dirtyContainer;
     }
     isContainerMessageDirtyable({ type, contents }) {
-        // For legacy purposes, exclude the old built-in AgentScheduler from dirty consideration as a special-case.
-        // Ultimately we should have no special-cases from the ContainerRuntime's perspective.
-        if (type === ContainerMessageType.Attach) {
-            const attachMessage = contents;
-            if (attachMessage.id === agentSchedulerId) {
-                return false;
+        // Certain container runtime messages should not mark the container dirty such as the old built-in
+        // AgentScheduler and Garbage collector messages.
+        switch (type) {
+            case ContainerMessageType.Attach: {
+                const attachMessage = contents;
+                if (attachMessage.id === agentSchedulerId) {
+                    return false;
+                }
+                break;
             }
-        }
-        else if (type === ContainerMessageType.FluidDataStoreOp) {
-            const envelope = contents;
-            if (envelope.address === agentSchedulerId) {
+            case ContainerMessageType.FluidDataStoreOp: {
+                const envelope = contents;
+                if (envelope.address === agentSchedulerId) {
+                    return false;
+                }
+                break;
+            }
+            case ContainerMessageType.GC: {
                 return false;
             }
+            default:
+                break;
         }
         return true;
     }
@@ -1796,10 +1801,10 @@ export class ContainerRuntime extends TypedEventEmitter {
                 properties: { all: { safeSummary: true } },
             }));
         }
-        // If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
-        // incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
-        // within the timeout, check if it should be failed or can continue.
-        if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
+        // If the container is dirty, i.e., there are pending unacked ops, the summary will not be eventual consistent
+        // and it may even be incorrect. So, wait for the container to be saved with a timeout. If the container is not
+        // saved within the timeout, check if it should be failed or can continue.
+        if (this.validateSummaryBeforeUpload && this.isDirty) {
             const countBefore = this.pendingMessagesCount;
             // The timeout for waiting for pending ops can be overridden via configurations.
             const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
@@ -1819,7 +1824,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // happens, whether we attempted to wait for these ops to be acked and what was the result.
             summaryNumberLogger.sendTelemetryEvent({
                 eventName: "PendingOpsWhileSummarizing",
-                saved: this.hasPendingMessages() ? false : true,
+                saved: !this.isDirty,
                 timeout: pendingOpsTimeout,
                 countBefore,
                 countAfter: this.pendingMessagesCount,
@@ -2044,7 +2049,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
     }
     /**
-     * This helper is called during summarization. If there are pending ops, it will return a failed summarize result
+     * This helper is called during summarization. If the container is dirty, it will return a failed summarize result
      * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
      * @param logger - The logger to be used for sending telemetry.
      * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
@@ -2054,7 +2059,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
      */
     async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
-        if (!this.hasPendingMessages()) {
+        if (!this.isDirty) {
             return;
         }
         // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
@@ -2146,12 +2151,12 @@ export class ContainerRuntime extends TypedEventEmitter {
                     contents: JSON.stringify(idAllocationMessage),
                     referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
                     metadata: undefined,
-                    localOpMetadata: this.idCompressor?.serialize(true),
+                    localOpMetadata: undefined,
                     type: ContainerMessageType.IdAllocation,
                 };
             }
             if (idAllocationBatchMessage !== undefined) {
-                this.outbox.submit(idAllocationBatchMessage);
+                this.outbox.submitIdAllocation(idAllocationBatchMessage);
             }
         }
     }
@@ -2324,6 +2329,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     /**
      * Finds the right store and asks it to resubmit the message. This typically happens when we
      * reconnect and there are pending messages.
+     * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
      * @param message - The original LocalContainerRuntimeMessage.
      * @param localOpMetadata - The local metadata associated with the original message.
      */
@@ -2336,10 +2342,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 break;
             case ContainerMessageType.Attach:
             case ContainerMessageType.Alias:
-                this.submit(message, localOpMetadata);
-                break;
             case ContainerMessageType.IdAllocation: {
-                prepareLocalContainerRuntimeIdAllocationMessageForTransit(message);
                 this.submit(message, localOpMetadata);
                 break;
             }
@@ -2351,6 +2354,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.Rejoin:
                 this.submit(message);
                 break;
+            case ContainerMessageType.GC:
+                // GC op is only sent in summarizer which should never reconnect.
+                throw new LoggingError("GC op not expected to be resubmitted in summarizer");
             default: {
                 // This case should be very rare - it would imply an op was stashed from a
                 // future version of runtime code and now is being applied on an older version
@@ -2492,7 +2498,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             notifyImminentClosure: props?.notifyImminentClosure,
         }, async (event) => {
             this.verifyNotClosed();
-            const waitBlobsToAttach = props?.notifyImminentClosure;
+            // in case imminentClosure is set to true by future code, we don't
+            // try to change its value
+            if (!this.imminentClosure) {
+                this.imminentClosure = props?.notifyImminentClosure ?? this.imminentClosure;
+            }
             const stopBlobAttachingSignal = props?.stopBlobAttachingSignal;
             if (this._orderSequentiallyCalls !== 0) {
                 throw new UsageError("can't get state during orderSequentially");
@@ -2501,16 +2511,18 @@ export class ContainerRuntime extends TypedEventEmitter {
             // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
             // to close current batch.
             this.flush();
-            const pendingAttachmentBlobs = waitBlobsToAttach
+            const pendingAttachmentBlobs = this.imminentClosure
                 ? await this.blobManager.attachAndGetPendingBlobs(stopBlobAttachingSignal)
                 : undefined;
             const pending = this.pendingStateManager.getLocalState();
             if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
                 return; // no pending state to save
             }
+            const pendingIdCompressorState = this.idCompressor?.serialize(true);
             const pendingState = {
                 pending,
                 pendingAttachmentBlobs,
+                pendingIdCompressorState,
             };
             event.end({
                 attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,