@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.1.0.148229

package/dist/gc/garbageCollection.d.ts

@@ -0,0 +1,204 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IRequestHeader } from "@fluidframework/core-interfaces";
+import { IGarbageCollectionDetailsBase, ISummarizeResult, ITelemetryContext } from "@fluidframework/runtime-definitions";
+import { ReadAndParseBlob } from "@fluidframework/runtime-utils";
+import { RefreshSummaryResult } from "../summary";
+import { IGarbageCollector, IGarbageCollectorCreateParams, IGCStats, IGCMetadata } from "./gcDefinitions";
+/**
+ * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
+ * its state across summaries.
+ *
+ * Node - represented as nodeId, it's a node on the GC graph
+ *
+ * Outbound Route - a path from one node to another node, think `nodeA` -\> `nodeB`
+ *
+ * Graph - all nodes with their respective routes
+ *
+ * ```
+ *			 GC Graph
+ *
+ *			   Node
+ *		NodeId = "datastore1"
+ *		   /			 \\
+ *	OutboundRoute   OutboundRoute
+ *		 /				 \\
+ *	   Node			   Node
+ *  NodeId = "dds1"	 NodeId = "dds2"
+ * ```
+ */
+export declare class GarbageCollector implements IGarbageCollector {
+    static create(createParams: IGarbageCollectorCreateParams): IGarbageCollector;
+    private readonly mc;
+    private readonly configs;
+    get shouldRunGC(): boolean;
+    private gcDataFromLastRun;
+    private readonly newReferencesSinceLastRun;
+    private tombstones;
+    private deletedNodes;
+    private readonly baseSnapshotDataP;
+    private readonly initializeGCStateFromBaseSnapshotP;
+    private readonly baseGCDetailsP;
+    private readonly unreferencedNodesState;
+    private sessionExpiryTimer;
+    private readonly loggedUnreferencedEvents;
+    private pendingEventsQueue;
+    private completedRuns;
+    private readonly runtime;
+    private readonly createContainerMetadata;
+    private readonly isSummarizerClient;
+    private readonly summaryStateTracker;
+    /** For a given node path, returns the node's package path. */
+    private readonly getNodePackagePath;
+    /** Returns the timestamp of the last summary generated for this container. */
+    private readonly getLastSummaryTimestampMs;
+    /** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
+    private readonly activeConnection;
+    get summaryStateNeedsReset(): boolean;
+    protected constructor(createParams: IGarbageCollectorCreateParams);
+    /**
+     * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
+     * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
+     */
+    initializeBaseState(): Promise<void>;
+    /**
+     * Update state from the given snapshot data. This is done during load and during refreshing state from a snapshot.
+     * All current tracking is reset and updated from the data in the snapshot.
+     * @param snapshotData - The snapshot data to update state from. If this is undefined, all GC state and tracking
+     * is reset.
+     * @param currentReferenceTimestampMs - The current reference timestamp for marking unreferenced nodes' unreferenced
+     * timestamp.
+     */
+    private updateStateFromSnapshotData;
+    /**
+     * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
+     * to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
+     * @param connected - Whether the runtime connected / disconnected.
+     * @param clientId - The clientId of this runtime.
+     */
+    setConnectionState(connected: boolean, clientId?: string | undefined): void;
+    /**
+     * Runs garbage collection and updates the reference / used state of the nodes in the container.
+     * @returns stats of the GC run or undefined if GC did not run.
+     */
+    collectGarbage(options: {
+        /** Logger to use for logging GC events */
+        logger?: ITelemetryLogger;
+        /** True to run GC sweep phase after the mark phase */
+        runSweep?: boolean;
+        /** True to generate full GC data */
+        fullGC?: boolean;
+    }, telemetryContext?: ITelemetryContext): Promise<IGCStats | undefined>;
+    private runPreGCSteps;
+    private runPostGCSteps;
+    /**
+     * Summarizes the GC data and returns it as a summary tree.
+     * We current write the entire GC state in a single blob. This can be modified later to write multiple
+     * blobs. All the blob keys should start with `gcBlobPrefix`.
+     */
+    summarize(fullTree: boolean, trackState: boolean, telemetryContext?: ITelemetryContext): ISummarizeResult | undefined;
+    getMetadata(): IGCMetadata;
+    /**
+     * Returns a the GC details generated from the base summary. This is used to initialize the GC state of the nodes
+     * in the container.
+     */
+    getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
+    /**
+     * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
+     * is downloaded and should be used to update the state.
+     */
+    refreshLatestSummary(proposalHandle: string | undefined, result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;
+    /**
+     * Called when a node with the given id is updated. If the node is inactive, log an error.
+     * @param nodePath - The id of the node that changed.
+     * @param reason - Whether the node was loaded or changed.
+     * @param timestampMs - The timestamp when the node changed.
+     * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
+     * @param requestHeaders - If the node was loaded via request path, the headers in the request.
+     */
+    nodeUpdated(nodePath: string, reason: "Loaded" | "Changed", timestampMs?: number, packagePath?: readonly string[], requestHeaders?: IRequestHeader): void;
+    /**
+     * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
+     * referenced between summaries so that their unreferenced timestamp can be reset.
+     *
+     * @param fromNodePath - The node from which the reference is added.
+     * @param toNodePath - The node to which the reference is added.
+     */
+    addedOutboundReference(fromNodePath: string, toNodePath: string): void;
+    /**
+     * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
+     * cases where objects are used after they are deleted and throw / log errors accordingly.
+     */
+    isNodeDeleted(nodePath: string): boolean;
+    dispose(): void;
+    /**
+     * Updates the state of the system as per the current GC run. It does the following:
+     * 1. Sets up the current GC state as per the gcData.
+     * 2. Starts tracking for nodes that have become unreferenced in this run.
+     * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
+     * @param gcData - The data representing the reference graph on which GC is run.
+     * @param gcResult - The result of the GC run on the gcData.
+     * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
+     * @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
+     */
+    private updateMarkPhase;
+    /**
+     * Deletes nodes from both the runtime and garbage collection
+     * @param sweepReadyNodes - nodes that are ready to be deleted
+     */
+    private runSweepPhase;
+    /**
+     * @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
+     */
+    private deleteSweptRoutes;
+    /**
+     * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
+     * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
+     * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
+     * these objects while there can be in-memory referenced to it:
+     * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
+     * added, the object may have been accessed and in-memory reference to it added.
+     * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
+     * unreferenced, they could have been accessed and in-memory reference to them added.
+     *
+     * This function identifies nodes that were referenced since the last run.
+     * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
+     *
+     * @returns - a list of all nodes referenced from the last local summary until now.
+     */
+    private findAllNodesReferencedBetweenGCs;
+    /**
+     * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
+     * The principle is that every new reference or outbound route must be notified to GC via the
+     * addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
+     *
+     * In more simple terms:
+     * Missing Explicit References = Current References - Previous References - Explicitly Added References;
+     *
+     * @param currentGCData - The GC data (reference graph) from the current GC run.
+     * @param previousGCData - The GC data (reference graph) from the previous GC run.
+     * @param explicitReferences - New references added explicity between the previous and the current run.
+     * @returns - a list of missing explicit references
+     */
+    private findMissingExplicitReferences;
+    /**
+     * Generates the stats of a garbage collection run from the given results of the run.
+     * @param gcResult - The result of a GC run.
+     * @returns the GC stats of the GC run.
+     */
+    private generateStats;
+    /**
+     * For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
+     * this will give us a view into how much deleted content a container has.
+     */
+    private logSweepEvents;
+    /**
+     * Called when an inactive node is used after. Queue up an event that will be logged next time GC runs.
+     */
+    private inactiveNodeUsed;
+    private logUnreferencedEvents;
+}
+//# sourceMappingURL=garbageCollection.d.ts.map

package/dist/gc/garbageCollection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAGtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAkC,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAWjG,OAAO,EAA4B,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAE5E,OAAO,EAGN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAEX,MAAM,iBAAiB,CAAC;AA2BzB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAI9C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAA0B;IAEnE,OAAO,CAAC,kBAAkB,CAAiC;IAG3D,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAA2B;IACnE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAE5D,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IACrE,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgB;IAEjD,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IA8JjE;;;OAGG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjD;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA4EnC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmBlF;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,gBAAgB,CAAC;QAC1B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;YA0DlB,aAAa;YAOb,cAAc;IAiD5B;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,oBAAoB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,MAAM,EAAE,oBAAoB,EAC5B,gBAAgB,EAAE,gBAAgB,GAChC,OAAO,CAAC,IAAI,CAAC;IA6BhB;;;;;;;OAOG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,cAAc,CAAC,EAAE,cAAc;IAoBhC;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAsCtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;;;;;;OASG;IACH,OAAO,CAAC,eAAe;IAuDvB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAsBrB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IA2ExC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,6BAA6B;IA6CrC;;;;OAIG;IACH,OAAO,CAAC,aAAa;IA0DrB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAoCtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;YAgFV,qBAAqB;CA2CnC"}