@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.4.1.0.148229

package/lib/dataStoreContext.js

@@ -3,16 +3,15 @@
  * Licensed under the MIT License.
  */
 import { AttachState, } from "@fluidframework/container-definitions";
-import { assert, Deferred, LazyPromise, TypedEventEmitter, } from "@fluidframework/common-utils";
+import { assert, Deferred, LazyPromise, TypedEventEmitter } from "@fluidframework/common-utils";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { BlobTreeEntry } from "@fluidframework/protocol-base";
-import { BindState, channelsTreeName, } from "@fluidframework/runtime-definitions";
+import { channelsTreeName, VisibilityState, } from "@fluidframework/runtime-definitions";
 import { addBlobToSummary, convertSummaryTreeToITree, packagePathToTelemetryProperty, } from "@fluidframework/runtime-utils";
-import { ChildLogger, loggerToMonitoringContext, LoggingError, TelemetryDataTag, ThresholdCounter, } from "@fluidframework/telemetry-utils";
+import { ChildLogger, generateStack, loggerToMonitoringContext, LoggingError, TelemetryDataTag, ThresholdCounter, } from "@fluidframework/telemetry-utils";
 import { DataCorruptionError, DataProcessingError, extractSafePropertiesFromMessage, } from "@fluidframework/container-utils";
-import { dataStoreAttributesBlobName, hasIsolatedChannels, wrapSummaryInChannelsTree, getAttributesFormatVersion, getFluidDataStoreAttributes, } from "./summaryFormat";
-import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
-import { summarizerClientType } from "./summarizerClientElection";
+import { dataStoreAttributesBlobName, hasIsolatedChannels, wrapSummaryInChannelsTree, getAttributesFormatVersion, getFluidDataStoreAttributes, summarizerClientType, } from "./summary";
+import { sendGCUnexpectedUsageEvent, throwOnTombstoneUsageKey } from "./gc";
 function createAttributes(pkg, isRootDataStore) {
     const stringifiedPkg = JSON.stringify(pkg);
     return {
@@ -29,11 +28,10 @@ export function createAttributesBlob(pkg, isRootDataStore) {
  * Represents the context for the store. This context is passed to the store runtime.
  */
 export class FluidDataStoreContext extends TypedEventEmitter {
-    constructor(props, existing, bindState, // Used to assert for state tracking purposes
-    isLocalDataStore, makeLocallyVisibleFn) {
+    constructor(props, existing, isLocalDataStore, makeLocallyVisibleFn) {
+        var _a;
         super();
         this.existing = existing;
-        this.bindState = bindState;
         this.isLocalDataStore = isLocalDataStore;
         this.makeLocallyVisibleFn = makeLocallyVisibleFn;
         this._disposed = false;
@@ -42,6 +40,8 @@ export class FluidDataStoreContext extends TypedEventEmitter {
          * loading.
          */
         this._tombstoned = false;
+        /** If true, this means that this data store context and its children have been removed from the runtime */
+        this.deleted = false;
         this.detachedRuntimeCreation = false;
         this.loaded = false;
         this.pending = [];
@@ -54,23 +54,22 @@ export class FluidDataStoreContext extends TypedEventEmitter {
         // URIs use slashes as delimiters. Handles use URIs.
         // Thus having slashes in types almost guarantees trouble down the road!
         assert(!this.id.includes("/"), 0x13a /* Data store ID contains slash */);
-        this._attachState = this.containerRuntime.attachState !== AttachState.Detached && this.existing ?
-            this.containerRuntime.attachState : AttachState.Detached;
-        this.bindToContext = () => {
-            assert(this.bindState === BindState.NotBound, 0x13b /* "datastore context is already in bound state" */);
-            this.bindState = BindState.Binding;
-            assert(this.channel !== undefined, 0x13c /* "undefined channel on datastore context" */);
-            this.makeLocallyVisible();
-            this.bindState = BindState.Bound;
-        };
+        this._attachState =
+            this.containerRuntime.attachState !== AttachState.Detached && this.existing
+                ? this.containerRuntime.attachState
+                : AttachState.Detached;
         const thisSummarizeInternal = async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext);
-        this.summarizerNode = props.createSummarizerNodeFn(thisSummarizeInternal, async (fullGC) => this.getGCDataInternal(fullGC), async () => this.getBaseGCDetails());
+        this.summarizerNode = props.createSummarizerNodeFn(thisSummarizeInternal, async (fullGC) => this.getGCDataInternal(fullGC));
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "FluidDataStoreContext"));
         this.thresholdOpsCounter = new ThresholdCounter(FluidDataStoreContext.pendingOpsCountThreshold, this.mc.logger);
         // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
         this.throwOnTombstoneUsage =
             this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+                this._containerRuntime.gcTombstoneEnforcementAllowed &&
                 this.clientDetails.type !== summarizerClientType;
+        // By default, a data store can log maximum 10 local changes telemetry in summarizer.
+        this.localChangesTelemetryCount =
+            (_a = this.mc.config.getNumber("Fluid.Telemetry.LocalChangesTelemetryCount")) !== null && _a !== void 0 ? _a : 10;
     }
     get packagePath() {
         assert(this.pkg !== undefined, 0x139 /* "Undefined package path" */);
@@ -100,14 +99,21 @@ export class FluidDataStoreContext extends TypedEventEmitter {
     get containerRuntime() {
         return this._containerRuntime;
     }
+    ensureNoDataModelChanges(callback) {
+        return this._containerRuntime.ensureNoDataModelChanges(callback);
+    }
     get isLoaded() {
         return this.loaded;
     }
     get baseSnapshot() {
         return this._baseSnapshot;
     }
-    get disposed() { return this._disposed; }
-    get tombstoned() { return this._tombstoned; }
+    get disposed() {
+        return this._disposed;
+    }
+    get tombstoned() {
+        return this._tombstoned;
+    }
     get attachState() {
         return this._attachState;
     }
@@ -141,19 +147,32 @@ export class FluidDataStoreContext extends TypedEventEmitter {
         // Dispose any pending runtime after it gets fulfilled
         // Errors are logged where this.channelDeferred is consumed/generated (realizeCore(), bindRuntime())
         if (this.channelDeferred) {
-            this.channelDeferred.promise.then((runtime) => {
+            this.channelDeferred.promise
+                .then((runtime) => {
                 runtime.dispose();
-            }).catch((error) => { });
+            })
+                .catch((error) => { });
         }
     }
+    /**
+     * When delete is called, that means that the data store is permanently removed from the runtime, and will not show up in future summaries
+     * This function is called to prevent ops from being generated from this data store once it has been deleted. Furthermore, this data store
+     * should not receive any ops/signals.
+     */
+    delete() {
+        this.deleted = true;
+    }
     setTombstone(tombstone) {
         if (this.tombstoned === tombstone) {
             return;
         }
         this._tombstoned = tombstone;
     }
-    rejectDeferredRealize(reason, packageName) {
-        throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.CodeArtifact } });
+    rejectDeferredRealize(reason, failedPkgPath, fullPackageName) {
+        throw new LoggingError(reason, {
+            failedPkgPath: { value: failedPkgPath, tag: TelemetryDataTag.CodeArtifact },
+            fullPackageName: packagePathToTelemetryProperty(fullPackageName),
+        });
     }
     async realize() {
         assert(!this.detachedRuntimeCreation, 0x13d /* "Detached runtime creation on realize()" */);
@@ -167,6 +186,7 @@ export class FluidDataStoreContext extends TypedEventEmitter {
                         value: this.id,
                         tag: TelemetryDataTag.CodeArtifact,
                     },
+                    packageName: packagePathToTelemetryProperty(this.pkg),
                 });
                 (_a = this.channelDeferred) === null || _a === void 0 ? void 0 : _a.reject(errorWrapped);
                 this.logger.sendErrorEvent({ eventName: "RealizeError" }, errorWrapped);
@@ -184,18 +204,18 @@ export class FluidDataStoreContext extends TypedEventEmitter {
         let lastPkg;
         for (const pkg of packages) {
             if (!registry) {
-                this.rejectDeferredRealize("No registry for package", lastPkg);
+                this.rejectDeferredRealize("No registry for package", lastPkg, packages);
             }
             lastPkg = pkg;
             entry = await registry.get(pkg);
             if (!entry) {
-                this.rejectDeferredRealize("Registry does not contain entry for the package", pkg);
+                this.rejectDeferredRealize("Registry does not contain entry for the package", pkg, packages);
             }
             registry = entry.IFluidDataStoreRegistry;
         }
         const factory = entry === null || entry === void 0 ? void 0 : entry.IFluidDataStoreFactory;
         if (factory === undefined) {
-            this.rejectDeferredRealize("Can't find factory for package", lastPkg);
+            this.rejectDeferredRealize("Can't find factory for package", lastPkg, packages);
         }
         return { factory, registry };
     }
@@ -365,7 +385,9 @@ export class FluidDataStoreContext extends TypedEventEmitter {
             return;
         }
         // Remove the route to this data store, if it exists.
-        const usedChannelRoutes = this.lastUsedRoutes.filter((id) => { return id !== "/" && id !== ""; });
+        const usedChannelRoutes = this.lastUsedRoutes.filter((id) => {
+            return id !== "/" && id !== "";
+        });
         this.channel.updateUsedRoutes(usedChannelRoutes);
     }
     /**
@@ -382,6 +404,8 @@ export class FluidDataStoreContext extends TypedEventEmitter {
             content,
             type,
         };
+        // Summarizer clients should not submit messages.
+        this.identifyLocalChangeInSummarizer("DataStoreMessageSubmittedInSummarizer", type);
         this._containerRuntime.submitDataStoreOp(this.id, fluidDataStoreContent, localOpMetadata);
     }
     /**
@@ -414,6 +438,11 @@ export class FluidDataStoreContext extends TypedEventEmitter {
      */
     makeLocallyVisible() {
         assert(this.channel !== undefined, 0x2cf /* "undefined channel on datastore context" */);
+        assert(this.channel.visibilityState === VisibilityState.LocallyVisible, 0x590 /* Channel must be locally visible */);
+        this.makeLocallyVisibleFn();
+    }
+    /** @deprecated - To be replaced by calling makeLocallyVisible directly  */
+    bindToContext() {
         this.makeLocallyVisibleFn();
     }
     bindRuntime(channel) {
@@ -452,10 +481,13 @@ export class FluidDataStoreContext extends TypedEventEmitter {
         }
         catch (error) {
             (_a = this.channelDeferred) === null || _a === void 0 ? void 0 : _a.reject(error);
-            this.logger.sendErrorEvent({ eventName: "BindRuntimeError", fluidDataStoreId: {
+            this.logger.sendErrorEvent({
+                eventName: "BindRuntimeError",
+                fluidDataStoreId: {
                     value: this.id,
                     tag: TelemetryDataTag.CodeArtifact,
-                } }, error);
+                },
+            }, error);
         }
     }
     async getAbsoluteUrl(relativeUrl) {
@@ -472,6 +504,12 @@ export class FluidDataStoreContext extends TypedEventEmitter {
     setInMemoryRoot() {
         this._isInMemoryRoot = true;
     }
+    /**
+     * @deprecated - The functionality to get base GC details has been moved to summarizer node.
+     */
+    async getBaseGCDetails() {
+        return {};
+    }
     reSubmit(contents, localOpMetadata) {
         assert(!!this.channel, 0x14b /* "Channel must exist when resubmitting ops" */);
         const innerContents = contents;
@@ -496,30 +534,63 @@ export class FluidDataStoreContext extends TypedEventEmitter {
         return this.channel.applyStashedOp(innerContents.content);
     }
     verifyNotClosed(callSite, checkTombstone = true, safeTelemetryProps = {}) {
+        if (this.deleted) {
+            const messageString = `Context is deleted! Call site [${callSite}]`;
+            const error = new DataCorruptionError(messageString, safeTelemetryProps);
+            this.mc.logger.sendErrorEvent({
+                eventName: "GC_Deleted_DataStore_Changed",
+                callSite,
+            }, error);
+            throw error;
+        }
         if (this._disposed) {
             throw new Error(`Context is closed! Call site [${callSite}]`);
         }
         if (checkTombstone && this.tombstoned) {
             const messageString = `Context is tombstoned! Call site [${callSite}]`;
-            const error = new DataCorruptionError(messageString, Object.assign({ errorMessage: messageString }, safeTelemetryProps));
-            // Always log an error when tombstoned data store is used. However, throw an error only if
-            // throwOnTombstoneUsage is set.
-            this.mc.logger.sendErrorEvent({
+            const error = new DataCorruptionError(messageString, safeTelemetryProps);
+            sendGCUnexpectedUsageEvent(this.mc, {
                 eventName: "GC_Tombstone_DataStore_Changed",
+                category: this.throwOnTombstoneUsage ? "error" : "generic",
+                gcTombstoneEnforcementAllowed: this._containerRuntime.gcTombstoneEnforcementAllowed,
                 callSite,
-                pkg: packagePathToTelemetryProperty(this.pkg),
-            }, error);
-            // Always log an error when tombstoned data store is used. However, throw an error only if
-            // throwOnTombstoneUsage is set and the client is not a summarizer.
+            }, this.pkg, error);
             if (this.throwOnTombstoneUsage) {
                 throw error;
             }
         }
     }
+    /**
+     * Summarizer client should not have local changes. These changes can become part of the summary and can break
+     * eventual consistency. For example, the next summary (say at ref seq# 100) may contain these changes whereas
+     * other clients that are up-to-date till seq# 100 may not have them yet.
+     */
+    identifyLocalChangeInSummarizer(eventName, type) {
+        var _a, _b;
+        if (this.clientDetails.type !== summarizerClientType ||
+            this.localChangesTelemetryCount <= 0) {
+            return;
+        }
+        // Log a telemetry if there are local changes in the summarizer. This will give us data on how often
+        // this is happening and which data stores do this. The eventual goal is to disallow local changes
+        // in the summarizer and the data will help us plan this.
+        this.mc.logger.sendTelemetryEvent({
+            eventName,
+            type,
+            fluidDataStoreId: {
+                value: this.id,
+                tag: TelemetryDataTag.CodeArtifact,
+            },
+            packageName: packagePathToTelemetryProperty(this.pkg),
+            isSummaryInProgress: (_b = (_a = this.summarizerNode).isSummaryInProgress) === null || _b === void 0 ? void 0 : _b.call(_a),
+            stack: generateStack(),
+        });
+        this.localChangesTelemetryCount--;
+    }
     getCreateChildSummarizerNodeFn(id, createParam) {
-        return (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, 
+        return (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, 
         // DDS will not create failure summaries
-        { throwOnFailure: true }, getGCDataFn, getBaseGCDetailsFn);
+        { throwOnFailure: true }, getGCDataFn);
     }
     async uploadBlob(blob) {
         return this.containerRuntime.uploadBlob(blob);
@@ -528,7 +599,7 @@ export class FluidDataStoreContext extends TypedEventEmitter {
 FluidDataStoreContext.pendingOpsCountThreshold = 1000;
 export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
     constructor(props) {
-        super(props, true /* existing */, BindState.Bound, false /* isLocalDataStore */, () => {
+        super(props, true /* existing */, false /* isLocalDataStore */, () => {
             throw new Error("Already attached");
         });
         this.initialSnapshotDetailsP = new LazyPromise(async () => {
@@ -543,9 +614,10 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
                 // For snapshotFormatVersion = "0.1" (1) or above, pkg is jsonified, otherwise it is just a string.
                 const formatVersion = getAttributesFormatVersion(attributes);
                 if (formatVersion < 1) {
-                    pkgFromSnapshot = attributes.pkg.startsWith("[\"") && attributes.pkg.endsWith("\"]")
-                        ? JSON.parse(attributes.pkg)
-                        : [attributes.pkg];
+                    pkgFromSnapshot =
+                        attributes.pkg.startsWith('["') && attributes.pkg.endsWith('"]')
+                            ? JSON.parse(attributes.pkg)
+                            : [attributes.pkg];
                 }
                 else {
                     pkgFromSnapshot = JSON.parse(attributes.pkg);
@@ -570,10 +642,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
             };
         });
         this.initSnapshotValue = props.snapshotTree;
-        this.baseGCDetailsP = new LazyPromise(async () => {
-            var _a;
-            return (_a = (await props.getBaseGCDetails())) !== null && _a !== void 0 ? _a : {};
-        });
         if (props.snapshotTree !== undefined) {
             this.summarizerNode.updateBaseSummaryState(props.snapshotTree);
         }
@@ -581,9 +649,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
     async getInitialSnapshotDetails() {
         return this.initialSnapshotDetailsP;
     }
-    async getBaseGCDetails() {
-        return this.baseGCDetailsP;
-    }
     generateAttachMessage() {
         throw new Error("Cannot attach remote store");
     }
@@ -593,7 +658,9 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
  */
 export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
     constructor(props) {
-        super(props, props.snapshotTree !== undefined ? true : false /* existing */, props.snapshotTree ? BindState.Bound : BindState.NotBound, true /* isLocalDataStore */, props.makeLocallyVisibleFn);
+        super(props, props.snapshotTree !== undefined ? true : false /* existing */, true /* isLocalDataStore */, props.makeLocallyVisibleFn);
+        // Summarizer client should not create local data stores.
+        this.identifyLocalChangeInSummarizer("DataStoreCreatedInSummarizer");
         this.snapshotTree = props.snapshotTree;
         if (props.isRootDataStore === true) {
             this.setInMemoryRoot();
@@ -660,9 +727,22 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
             snapshot,
         };
     }
-    async getBaseGCDetails() {
-        // Local data store does not have initial summary.
-        return {};
+    /**
+     * A context should only be marked as deleted when its a remote context.
+     * Session Expiry at the runtime level should have closed the container creating the local data store context
+     * before delete is even possible. Session Expiry is at 30 days, and sweep is done 36+ days later from the time
+     * it was unreferenced. Thus the sweeping container should have loaded from a snapshot and thus creating a remote
+     * context.
+     */
+    delete() {
+        // TODO: GC:Validation - potentially prevent this from happening or asserting. Maybe throw here.
+        sendGCUnexpectedUsageEvent(this.mc, {
+            eventName: "GC_Deleted_DataStore_Unexpected_Delete",
+            message: "Unexpected deletion of a local data store context",
+            category: "error",
+            gcTombstoneEnforcementAllowed: undefined,
+        }, this.pkg);
+        super.delete();
     }
 }
 /**