@fluidframework/container-runtime 2.0.0-internal.2.2.0 → 2.0.0-internal.2.3.0

package/lib/containerRuntime.js

@@ -5,7 +5,7 @@ import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
-import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
+import { FlushMode, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
 import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
 import { GCDataBuilder, trimLeadingAndTrailingSlashes } from "@fluidframework/garbage-collector";
 import { v4 as uuid } from "uuid";
@@ -25,7 +25,6 @@ import { SummarizerClientElection, summarizerClientType } from "./summarizerClie
 import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
 import { GarbageCollector, GCNodeType, } from "./garbageCollection";
-import { gcTreeKey, } from "./garbageCollectionConstants";
 import { channelToDataStore, isDataStoreAliasMessage, } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
 import { SerializedSnapshotStorage } from "./serializedSnapshotStorage";
@@ -213,12 +212,17 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.nextSummaryNumber = loadSummaryNumber + 1;
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         this._connected = this.context.connected;
-        this.remoteMessageProcessor = new RemoteMessageProcessor(new OpSplitter(chunks), new OpDecompressor());
-        this.handleContext = new ContainerFluidHandleContext("", this);
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
+        const opSplitter = new OpSplitter(chunks, this.context.submitBatchFn, this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompressionChunking") === true ?
+            Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
+        this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor());
+        this.handleContext = new ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
         }
+        this.enableOpReentryCheck = runtimeOptions.enableOpReentryCheck === true
+            // Allow for a break-glass config to override the options
+            && this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableOpReentryCheck") !== true;
         this.summariesDisabled = this.isSummariesDisabled();
         this.heuristicsDisabled = this.isHeuristicsDisabled();
         this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
@@ -267,16 +271,20 @@ export class ContainerRuntime extends TypedEventEmitter {
             throwOnFailure: true,
             // If GC should not run, let the summarizer node know so that it does not track GC state.
             gcDisabled: !this.garbageCollector.shouldRunGC,
-        });
+        }, 
+        // Function to get GC data if needed. This will always be called by the root summarizer node to get GC data.
+        async (fullGC) => this.getGCDataInternal(fullGC), 
+        // Function to get the GC details from the base snapshot we loaded from.
+        async () => this.garbageCollector.getBaseGCDetails());
         if (baseSnapshot) {
             this.summarizerNode.updateBaseSummaryState(baseSnapshot);
         }
         this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap));
-        this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId, localId) => {
+        this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
             if (!this.disposed) {
-                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId, localId });
+                this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { localId, blobId });
             }
-        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
+        }, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (fromPath, toPath) => this.garbageCollector.addedOutboundReference(fromPath, toPath), this, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pendingAttachmentBlobs);
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, () => this.clientId, ChildLogger.create(this.logger, "ScheduleManager"));
         this.pendingStateManager = new PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
@@ -288,15 +296,23 @@ export class ContainerRuntime extends TypedEventEmitter {
             rollback: this.rollback.bind(this),
             orderSequentially: this.orderSequentially.bind(this),
         }, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
+        const compressionOptions = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableCompression") === true ?
+            {
+                minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
+                compressionAlgorithm: CompressionAlgorithms.lz4
+            } : runtimeOptions.compressionOptions;
         this.outbox = new Outbox({
             shouldSend: () => this.canSendOps(),
             pendingStateManager: this.pendingStateManager,
             containerContext: this.context,
             compressor: new OpCompressor(this.mc.logger),
+            splitter: opSplitter,
             config: {
-                compressionOptions: runtimeOptions.compressionOptions,
+                compressionOptions,
                 maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
+                enableOpReentryCheck: this.enableOpReentryCheck,
             },
+            logger: this.mc.logger,
         });
         this.context.quorum.on("removeMember", (clientId) => {
             this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
@@ -384,7 +400,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * allows mixin classes to leverage this method to define their own async initializer.
      */
     static async load(context, registryEntries, requestHandler, runtimeOptions = {}, containerScope = context.scope, existing, containerRuntimeCtor = ContainerRuntime) {
-        var _a, _b, _c;
+        var _a, _b, _c, _d;
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
@@ -397,7 +413,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, enableOfflineLoad = false, compressionOptions = {
             minimumBatchSizeInBytes: Number.POSITIVE_INFINITY,
             compressionAlgorithm: CompressionAlgorithms.lz4
-        }, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, } = runtimeOptions;
+        }, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, chunkSizeInBytes = Number.POSITIVE_INFINITY, enableOpReentryCheck = false, } = runtimeOptions;
         const pendingRuntimeState = context.pendingLocalState;
         const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
         const storage = !pendingRuntimeState ?
@@ -442,7 +458,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                     logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
                 }
                 else {
+                    // Call both close and dispose as close implementation will no longer dispose runtime in future (2.0.0-internal.3.0.0)
                     context.closeFn(error);
+                    (_d = context.disposeFn) === null || _d === void 0 ? void 0 : _d.call(context, error);
                 }
             }
         }
@@ -454,6 +472,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             enableOfflineLoad,
             compressionOptions,
             maxBatchSizeInBytes,
+            chunkSizeInBytes,
+            enableOpReentryCheck,
         }, containerScope, logger, loadExisting, blobManagerSnapshot, storage, requestHandler);
         if (pendingRuntimeState) {
             await runtime.processSavedOps(pendingRuntimeState);
@@ -483,8 +503,18 @@ export class ContainerRuntime extends TypedEventEmitter {
         // eslint-disable-next-line @typescript-eslint/unbound-method
         return this.reSubmit;
     }
+    get disposeFn() {
+        var _a;
+        // In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
+        return (_a = this.context.disposeFn) !== null && _a !== void 0 ? _a : this.context.closeFn;
+    }
     get closeFn() {
-        return this.context.closeFn;
+        // Also call disposeFn to retain functionality of runtime being disposed on close
+        return (error) => {
+            var _a, _b;
+            this.context.closeFn(error);
+            (_b = (_a = this.context).disposeFn) === null || _b === void 0 ? void 0 : _b.call(_a, error);
+        };
     }
     get flushMode() {
         return this._flushMode;
@@ -862,9 +892,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (reconnection) {
             this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(DataProcessingError.create(
-                // eslint-disable-next-line max-len
-                "Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
+                this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops. Batch of ops is likely too large (over 1Mb)", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
                     pendingMessages: this.pendingStateManager.pendingMessagesCount,
@@ -898,7 +926,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.scheduleManager.beforeOpProcessing(message);
         try {
             let localOpMetadata;
-            if (local && runtimeMessage) {
+            if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1015,6 +1043,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     orderSequentially(callback) {
         let checkpoint;
+        let result;
         if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
             // Note: we are not touching this.pendingAttachBatch here, for two reasons:
             // 1. It would not help, as we flush attach ops as they become available.
@@ -1023,7 +1052,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         try {
             this._orderSequentiallyCalls++;
-            callback();
+            result = callback();
         }
         catch (error) {
             if (checkpoint) {
@@ -1051,6 +1080,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (this.flushMode === FlushMode.Immediate && this._orderSequentiallyCalls === 0) {
             this.flush();
         }
+        return result;
     }
     async createDataStore(pkg) {
         const internalId = uuid();
@@ -1216,6 +1246,9 @@ export class ContainerRuntime extends TypedEventEmitter {
     async updateStateBeforeGC() {
         return this.dataStores.updateStateBeforeGC();
     }
+    async getGCDataInternal(fullGC) {
+        return this.dataStores.getGCData(fullGC);
+    }
     /**
      * Implementation of IGarbageCollectionRuntime::getGCData.
      * Generates and returns the GC data for this container.
@@ -1223,7 +1256,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     async getGCData(fullGC) {
         const builder = new GCDataBuilder();
-        const dsGCData = await this.dataStores.getGCData(fullGC);
+        const dsGCData = await this.summarizerNode.getGCData(fullGC);
         builder.addNodes(dsGCData.gcNodes);
         const blobsGCData = this.blobManager.getGCData(fullGC);
         builder.addNodes(blobsGCData.gcNodes);
@@ -1239,39 +1272,26 @@ export class ContainerRuntime extends TypedEventEmitter {
         // summarizing is required and asserted by the the summarizer node. We are the root and are
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
-        const blobManagerUsedRoutes = [];
-        const dataStoreUsedRoutes = [];
-        for (const route of usedRoutes) {
-            if (this.isBlobPath(route)) {
-                blobManagerUsedRoutes.push(route);
-            }
-            else {
-                dataStoreUsedRoutes.push(route);
-            }
-        }
-        this.blobManager.updateUsedRoutes(blobManagerUsedRoutes);
-        this.dataStores.updateUsedRoutes(dataStoreUsedRoutes);
+        const { dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(usedRoutes);
+        this.dataStores.updateUsedRoutes(dataStoreRoutes);
     }
     /**
-     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
-     * tombstones.
-     * @param unusedRoutes - The routes that are unused in all data stores and attachment blobs in this Container.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
+     * This is called to update objects whose routes are unused.
+     * @param unusedRoutes - Data store and attachment blob routes that are unused in this Container.
      */
-    updateUnusedRoutes(unusedRoutes, tombstone) {
-        const blobManagerUnusedRoutes = [];
-        const dataStoreUnusedRoutes = [];
-        for (const route of unusedRoutes) {
-            if (this.isBlobPath(route)) {
-                blobManagerUnusedRoutes.push(route);
-            }
-            else {
-                dataStoreUnusedRoutes.push(route);
-            }
-        }
-        this.blobManager.updateUnusedRoutes(blobManagerUnusedRoutes, tombstone);
-        this.dataStores.updateUnusedRoutes(dataStoreUnusedRoutes, tombstone);
+    updateUnusedRoutes(unusedRoutes) {
+        const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(unusedRoutes);
+        this.blobManager.updateUnusedRoutes(blobManagerRoutes);
+        this.dataStores.updateUnusedRoutes(dataStoreRoutes);
+    }
+    /**
+     * This is called to update objects that are tombstones.
+     * @param tombstonedRoutes - Data store and attachment blob routes that are tombstones in this Container.
+     */
+    updateTombstonedRoutes(tombstonedRoutes) {
+        const { blobManagerRoutes, dataStoreRoutes } = this.getDataStoreAndBlobManagerRoutes(tombstonedRoutes);
+        this.blobManager.updateTombstonedRoutes(blobManagerRoutes);
+        this.dataStores.updateTombstonedRoutes(dataStoreRoutes);
     }
     /**
      * Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
@@ -1300,7 +1320,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     async getGCNodePackagePath(nodePath) {
         switch (this.getNodeType(nodePath)) {
             case GCNodeType.Blob:
-                return ["_blobs"];
+                return [BlobManager.basePath];
             case GCNodeType.DataStore:
             case GCNodeType.SubDataStore:
                 return this.dataStores.getDataStorePackagePath(nodePath);
@@ -1318,6 +1338,25 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         return true;
     }
+    /**
+     * From a given list of routes, separate and return routes that belong to blob manager and data stores.
+     * @param routes - A list of routes that can belong to data stores or blob manager.
+     * @returns - Two route lists - One that contains routes for blob manager and another one that contains routes
+     * for data stores.
+     */
+    getDataStoreAndBlobManagerRoutes(routes) {
+        const blobManagerRoutes = [];
+        const dataStoreRoutes = [];
+        for (const route of routes) {
+            if (this.isBlobPath(route)) {
+                blobManagerRoutes.push(route);
+            }
+            else {
+                dataStoreRoutes.push(route);
+            }
+        }
+        return { blobManagerRoutes, dataStoreRoutes };
+    }
     /**
      * Runs garbage collection and updates the reference / used state of the nodes in the container.
      * @returns the statistics of the garbage collection run; undefined if GC did not run.
@@ -1389,7 +1428,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 if (this.deltaManager.lastSequenceNumber !== summaryRefSeqNum) {
                     return {
                         continue: false,
-                        // eslint-disable-next-line max-len
                         error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
                     };
                 }
@@ -1397,7 +1435,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 if (lastAck !== this.summaryCollection.latestAck) {
                     return {
                         continue: false,
-                        // eslint-disable-next-line max-len
                         error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
                     };
                 }
@@ -1612,7 +1649,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             else if (!this.flushMicroTaskExists) {
                 this.flushMicroTaskExists = true;
                 // Queue a microtask to detect the end of the turn and force a flush.
-                // eslint-disable-next-line @typescript-eslint/no-floating-promises
                 Promise.resolve().then(() => {
                     this.flushMicroTaskExists = false;
                     this.flush();
@@ -1705,13 +1741,32 @@ export class ContainerRuntime extends TypedEventEmitter {
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
         const snapshotTreeFetcher = async () => {
-            const fetchResult = await this.fetchSnapshotFromStorage(null, summaryLogger, {
+            const fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
                 eventName: "RefreshLatestSummaryGetSnapshot",
                 ackHandle,
                 summaryRefSeq,
                 fetchLatest: true,
             });
             const latestSnapshotRefSeq = await seqFromTree(fetchResult.snapshotTree, readAndParseBlob);
+            /**
+             * If the fetched snapshot is older than the one for which the ack was received, close the container.
+             * This should never happen because an ack should be sent after the latest summary is updated in the server.
+             * However, there are couple of scenarios where it's possible:
+             * 1. A file was modified externally resulting in modifying the snapshot's sequence number. This can lead to
+             * the document being unusable and we should not proceed.
+             * 2. The server DB failed after the ack was sent which may delete the corresponding snapshot. Ideally, in
+             * such cases, the file will be rolled back along with the ack and we will eventually reach a consistent
+             * state.
+             */
+            if (latestSnapshotRefSeq < summaryRefSeq) {
+                const error = DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+                    ackHandle,
+                    summaryRefSeq,
+                    latestSnapshotRefSeq,
+                });
+                this.closeFn(error);
+                throw error;
+            }
             summaryLogger.sendTelemetryEvent({
                 eventName: "LatestSummaryRetrieved",
                 ackHandle,
@@ -1725,7 +1780,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         };
         const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, snapshotTreeFetcher, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result, proposalHandle, summaryRefSeq, readAndParseBlob);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -1734,22 +1789,22 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @returns downloaded snapshot's reference sequence number
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
-        const { snapshotTree, versionId } = await this.fetchSnapshotFromStorage(null, summaryLogger, {
+        const { snapshotTree, versionId } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
             eventName: "RefreshLatestSummaryGetSnapshot",
             fetchLatest: true,
-        }, FetchSource.noCache);
+        });
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
         const latestSnapshotRefSeq = await seqFromTree(snapshotTree, readAndParseBlob);
         const result = await this.summarizerNode.refreshLatestSummary(undefined, latestSnapshotRefSeq, async () => snapshotTree, readAndParseBlob, summaryLogger);
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result, undefined, latestSnapshotRefSeq, readAndParseBlob);
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchSnapshotFromStorage(versionId, logger, event, fetchSource) {
+    async fetchLatestSnapshotFromStorage(logger, event) {
         return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
-            const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", fetchSource);
+            const versions = await this.storage.getVersions(null, 1, "refreshLatestSummaryAckFromServer", FetchSource.noCache);
             assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
@@ -1865,6 +1920,7 @@ export class ContainerRuntime extends TypedEventEmitter {
 const waitForSeq = async (deltaManager, targetSeq) => new Promise((resolve, reject) => {
     // TODO: remove cast to any when actual event is determined
     deltaManager.on("closed", reject);
+    deltaManager.on("disposed", reject);
     // If we already reached target sequence number, simply resolve the promise.
     if (deltaManager.lastSequenceNumber >= targetSeq) {
         resolve();