@fluidframework/container-runtime 0.58.3000-61081 → 0.59.1001-62246

package/lib/containerRuntime.js

@@ -7,6 +7,7 @@ import { DataCorruptionError, GenericError, UsageError, extractSafePropertiesFro
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
 import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
 import { addBlobToSummary, addTreeToSummary, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree, calculateStats, } from "@fluidframework/runtime-utils";
+import { GCDataBuilder } from "@fluidframework/garbage-collector";
 import { v4 as uuid } from "uuid";
 import { ContainerFluidHandleContext } from "./containerHandleContext";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry";
@@ -24,7 +25,7 @@ import { OrderedClientCollection, OrderedClientElection } from "./orderedClientE
 import { SummarizerClientElection, summarizerClientType } from "./summarizerClientElection";
 import { formExponentialFn, Throttler } from "./throttler";
 import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
-import { GarbageCollector, gcTreeKey, } from "./garbageCollection";
+import { GarbageCollector, GCNodeType, gcTreeKey, } from "./garbageCollection";
 import { channelToDataStore, isDataStoreAliasMessage, } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
 import { OpTracker } from "./opTelemetry";
@@ -312,7 +313,7 @@ export class ScheduleManager {
             assert(this.batchClientId === undefined, 0x2a2 /* "Batch is interrupted by other client op. Should be caught by trackPending()" */);
             // This could be the beginning of a new batch or an individual message.
             this.emitter.emit("batchBegin", message);
-            this.deltaScheduler.batchBegin();
+            this.deltaScheduler.batchBegin(message);
             const batch = (_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.batch;
             if (batch) {
                 this.batchClientId = message.clientId;
@@ -332,7 +333,7 @@ export class ScheduleManager {
             this.hitError = true;
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", error, message);
-            this.deltaScheduler.batchEnd();
+            this.deltaScheduler.batchEnd(message);
             return;
         }
         const batch = (_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.batch;
@@ -341,7 +342,7 @@ export class ScheduleManager {
         if (this.batchClientId === undefined || batch === false) {
             this.batchClientId = undefined;
             this.emitter.emit("batchEnd", undefined, message);
-            this.deltaScheduler.batchEnd();
+            this.deltaScheduler.batchEnd(message);
             return;
         }
     }
@@ -466,12 +467,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._maxOpSizeInBytes = ((_d = this.mc.config.getNumber(maxOpSizeInBytesKey)) !== null && _d !== void 0 ? _d : defaultMaxOpSizeInBytes);
         this.maxConsecutiveReconnects = (_e = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _e !== void 0 ? _e : this.defaultMaxConsecutiveReconnects;
         this._flushMode = runtimeOptions.flushMode;
-        this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (unusedRoutes) => this.dataStores.deleteUnusedRoutes(unusedRoutes), (nodePath) => this.dataStores.getNodePackagePath(nodePath), 
-        /**
-         * Returns the timestamp of the last message seen by this client. This is used by garbage collector as
-         * the current reference timestamp for tracking unreferenced objects.
-         */
-        () => { var _a, _b, _c; return (_b = (_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp) !== null && _b !== void 0 ? _b : (_c = this.messageAtLastSummary) === null || _c === void 0 ? void 0 : _c.timestamp; }, () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; }, context.baseSnapshot, async (id) => readAndParse(this.storage, id), this.mc.logger, existing, metadata);
+        this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (nodePath) => this.getGCNodePackagePath(nodePath), () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; }, context.baseSnapshot, async (id) => readAndParse(this.storage, id), this.mc.logger, existing, metadata);
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"), 
         // Summarize function to call when summarize is called. Summarizer node always tracks summary state.
@@ -492,7 +488,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (this.context.baseSnapshot) {
             this.summarizerNode.loadBaseSummaryWithoutDifferential(this.context.baseSnapshot);
         }
-        this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getDataStoreBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
+        this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
         this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }), this, this.logger);
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, ChildLogger.create(this.logger, "ScheduleManager"));
         this.deltaSender = this.deltaManager;
@@ -1163,7 +1159,14 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     async createRootDataStoreLegacy(pkg, rootDataStoreId) {
         const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
-        fluidDataStore.bindToContext();
+        // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
+        // older versions, we still have to call bindToContext.
+        if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
+            fluidDataStore.makeVisibleAndAttachGraph();
+        }
+        else {
+            fluidDataStore.bindToContext();
+        }
         return fluidDataStore;
     }
     async createRootDataStore(pkg, rootDataStoreId) {
@@ -1217,7 +1220,14 @@ export class ContainerRuntime extends TypedEventEmitter {
     async _createDataStoreWithPropsLegacy(pkg, props, id = uuid(), isRoot = false) {
         const fluidDataStore = await this.dataStores._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
         if (isRoot) {
-            fluidDataStore.bindToContext();
+            // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
+            // For older versions, we still have to call bindToContext.
+            if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
+                fluidDataStore.makeVisibleAndAttachGraph();
+            }
+            else {
+                fluidDataStore.bindToContext();
+            }
             this.logger.sendTelemetryEvent({
                 eventName: "Root datastore with props",
                 hasProps: props !== undefined,
@@ -1364,7 +1374,12 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param fullGC - true to bypass optimizations and force full generation of GC data.
      */
     async getGCData(fullGC) {
-        return this.dataStores.getGCData(fullGC);
+        const builder = new GCDataBuilder();
+        const dsGCData = await this.dataStores.getGCData(fullGC);
+        builder.addNodes(dsGCData.gcNodes);
+        const blobsGCData = this.blobManager.getGCData(fullGC);
+        builder.addNodes(blobsGCData.gcNodes);
+        return builder.getGCData();
     }
     /**
      * Implementation of IGarbageCollectionRuntime::updateUsedRoutes.
@@ -1378,7 +1393,78 @@ export class ContainerRuntime extends TypedEventEmitter {
         // summarizing is required and asserted by the the summarizer node. We are the root and are
         // always referenced, so the used routes is only self-route (empty string).
         this.summarizerNode.updateUsedRoutes([""]);
-        return this.dataStores.updateUsedRoutes(usedRoutes, gcTimestamp);
+        const dataStoreUsedRoutes = [];
+        for (const route of usedRoutes) {
+            if (route.split("/")[1] !== BlobManager.basePath) {
+                dataStoreUsedRoutes.push(route);
+            }
+        }
+        return this.dataStores.updateUsedRoutes(dataStoreUsedRoutes, gcTimestamp);
+    }
+    /**
+     * When running GC in test mode, this is called to delete objects whose routes are unused. This enables testing
+     * scenarios with accessing deleted content.
+     * @param unusedRoutes - The routes that are unused in all data stores in this Container.
+     */
+    deleteUnusedRoutes(unusedRoutes) {
+        const blobManagerUnusedRoutes = [];
+        const dataStoreUnusedRoutes = [];
+        for (const route of unusedRoutes) {
+            if (this.isBlobPath(route)) {
+                blobManagerUnusedRoutes.push(route);
+            }
+            else {
+                dataStoreUnusedRoutes.push(route);
+            }
+        }
+        this.blobManager.deleteUnusedRoutes(blobManagerUnusedRoutes);
+        this.dataStores.deleteUnusedRoutes(dataStoreUnusedRoutes);
+    }
+    /**
+     * Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
+     */
+    getCurrentReferenceTimestampMs() {
+        var _a, _b, _c;
+        // Use the timestamp of the last message seen by this client as that is server generated. If no messages have
+        // been processed, use the timestamp of the message from the last summary.
+        return (_b = (_a = this.deltaManager.lastMessage) === null || _a === void 0 ? void 0 : _a.timestamp) !== null && _b !== void 0 ? _b : (_c = this.messageAtLastSummary) === null || _c === void 0 ? void 0 : _c.timestamp;
+    }
+    /**
+     * Returns the type of the GC node. Currently, there are nodes that belong to data store and nodes that belong
+     * to the blob manager.
+     */
+    getNodeType(nodePath) {
+        if (this.isBlobPath(nodePath)) {
+            return GCNodeType.Blob;
+        }
+        if (this.dataStores.isDataStoreNode(nodePath)) {
+            return GCNodeType.DataStore;
+        }
+        // Root node ("/") and DDS nodes belong to "Other" node types.
+        return GCNodeType.Other;
+    }
+    /**
+     * Called by GC to retrieve the package path of the node with the given path. The node should belong to a
+     * data store or an attachment blob.
+     */
+    getGCNodePackagePath(nodePath) {
+        // If the node is a blob, return "_blobs" as the package path.
+        if (this.isBlobPath(nodePath)) {
+            return ["_blobs"];
+        }
+        const dataStorePkgPath = this.dataStores.getDataStorePackagePath(nodePath);
+        assert(dataStorePkgPath !== undefined, 0x2d6 /* "Package path requested for unknown node type." */);
+        return dataStorePkgPath;
+    }
+    /**
+     * Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
+     */
+    isBlobPath(path) {
+        const pathParts = path.split("/");
+        if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
+            return false;
+        }
+        return true;
     }
     /**
      * Runs garbage collection and updates the reference / used state of the nodes in the container.