@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/lib/blobManager.d.ts

@@ -7,7 +7,9 @@ import { IDocumentStorageService } from "@fluidframework/driver-definitions";
 import { ISequencedDocumentMessage, ISnapshotTree } from "@fluidframework/protocol-definitions";
 import { TypedEventEmitter } from "@fluidframework/common-utils";
 import { IContainerRuntime, IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions";
+import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import { IGarbageCollectionData, ISummaryTreeWithStats, ITelemetryContext } from "@fluidframework/runtime-definitions";
+import { ContainerRuntime } from "./containerRuntime";
 /**
  * This class represents blob (long string)
  * This object is used only when creating (writing) new blob and serialization purposes.
@@ -34,10 +36,12 @@ export interface IBlobManagerLoadInfo {
     ids?: string[];
     redirectTable?: [string, string][];
 }
-export declare type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "connected" | "logger" | "clientDetails"> & TypedEventEmitter<IContainerRuntimeEvents>;
+export declare type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "connected" | "logger" | "clientDetails"> & Pick<ContainerRuntime, "gcTombstoneEnforcementAllowed"> & TypedEventEmitter<IContainerRuntimeEvents>;
 export interface IPendingBlobs {
     [id: string]: {
         blob: string;
+        uploadTime?: number;
+        minTTLInSeconds?: number;
     };
 }
 export interface IBlobManagerEvents {
@@ -46,22 +50,10 @@ export interface IBlobManagerEvents {
 export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
     private readonly routeContext;
     private readonly getStorage;
-    /**
-     * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
-     * deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
-     * the blob as long as it is listed as referenced in future summaries. The summarizing client will know to
-     * include the storage ID in the summary when it sees the op.
-     *
-     * The op will also include a local ID to inform all clients of the relation to the storage ID, without
-     * knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
-     * before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
-     */
-    private readonly sendBlobAttachOp;
     private readonly blobRequested;
-    private readonly addedBlobReference;
     private readonly isBlobDeleted;
     private readonly runtime;
-    private readonly getCurrentReferenceTimestampMs;
+    private readonly closeContainer;
     static readonly basePath = "_blobs";
     private static readonly redirectTableBlobName;
     private readonly mc;
@@ -93,6 +85,7 @@ export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
      * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
      */
     private readonly tombstonedBlobs;
+    private readonly sendBlobAttachOp;
     constructor(routeContext: IFluidHandleContext, snapshot: IBlobManagerLoadInfo, getStorage: () => IDocumentStorageService, 
     /**
      * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
@@ -104,7 +97,7 @@ export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
      * knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
      * before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
      */
-    sendBlobAttachOp: (localId: string, storageId?: string) => void, blobRequested: (blobPath: string) => void, addedBlobReference: (fromNodePath: string, toNodePath: string) => void, isBlobDeleted: (blobPath: string) => boolean, runtime: IBlobManagerRuntime, stashedBlobs: IPendingBlobs | undefined, getCurrentReferenceTimestampMs: () => number | undefined);
+    sendBlobAttachOp: (localId: string, storageId?: string) => void, blobRequested: (blobPath: string) => void, isBlobDeleted: (blobPath: string) => boolean, runtime: IBlobManagerRuntime, stashedBlobs: IPendingBlobs | undefined, closeContainer: (error?: ICriticalContainerError) => void);
     private get pendingOfflineUploads();
     get hasPendingOfflineUploads(): boolean;
     get hasPendingBlobs(): boolean;
@@ -117,12 +110,6 @@ export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
      * again
      */
     private onDisconnected;
-    /**
-     * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`
-     * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
-     * referenced by storing these handles in a referenced DDS.
-     */
-    private getBlobGCNodePath;
     /**
      * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
      * detached or there are no (non-pending) attachment blobs in the document
@@ -148,7 +135,6 @@ export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
      * @param metadata - op metadata containing storage and/or local IDs
      */
     reSubmit(metadata: Record<string, unknown> | undefined): void;
-    private logTimeInfo;
     processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean): void;
     /**
      * Reads blobs needed to load BlobManager from storage.
@@ -161,6 +147,7 @@ export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
      * Load a set of previously attached blob IDs and redirect table from a previous snapshot.
      */
     private load;
+    summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats;
     /**
      * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
      * individually referenced by storing its handle in a referenced DDS. Returns the list of blob ids as GC nodes.
@@ -170,9 +157,28 @@ export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
     getGCData(fullGC?: boolean): IGarbageCollectionData;
     /**
      * This is called to update blobs whose routes are unused. The unused blobs are deleted.
-     * @param unusedRoutes - The routes of the blob nodes that are unused.
+     * @param unusedRoutes - The routes of the blob nodes that are unused. These routes will be based off of local ids.
      */
     updateUnusedRoutes(unusedRoutes: string[]): void;
+    /**
+     * Delete attachment blobs that are sweep ready.
+     * @param sweepReadyBlobRoutes - The routes of blobs that are sweep ready and should be deleted. These routes will
+     * be based off of local ids.
+     * @returns - The routes of blobs that were deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyBlobRoutes: string[]): string[];
+    /**
+     * Delete blobs with the given routes from the redirect table.
+     * The routes are GC nodes paths of format -`/<BlobManager.basePath>/<blobId>`. The blob ids are all local ids.
+     * Deleting the blobs involves 2 steps:
+     * 1. The redirect table entry for the local ids are deleted.
+     * 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
+     * for the storage ids are deleted as well.
+     *
+     * Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
+     * will remove them the next summary. The service would them delete them some time in the future.
+     */
+    private deleteBlobsFromRedirectTable;
     /**
      * This is called to update blobs whose routes are tombstones. Tombstoned blobs enable testing scenarios with
      * accessing deleted content without actually deleting content from summaries.
@@ -184,7 +190,6 @@ export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
      * log an error and throw if necessary.
      */
     private verifyBlobValidity;
-    summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats;
     setRedirectTable(table: Map<string, string>): void;
     getPendingBlobs(): IPendingBlobs;
 }

package/lib/blobManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAEN,yBAAyB,EACzB,aAAa,EACb,MAAM,sCAAsC,CAAC;AAO9C,OAAO,EAKN,iBAAiB,EACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACN,iBAAiB,EACjB,uBAAuB,EACvB,MAAM,+CAA+C,CAAC;AAQvD,OAAO,EACN,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAO7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAc9C,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAf/B,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGpB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAKxB,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGhC;AAmBD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACnC;AAID,oBAAY,mBAAmB,GAAG,IAAI,CACrC,iBAAiB,EACjB,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,eAAe,CACxD,GACA,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;AAwB5C,MAAM,WAAW,aAAa;IAC7B,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB;IAClC,CAAC,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,IAAI,OAAE;CAChD;AAED,qBAAa,WAAY,SAAQ,iBAAiB,CAAC,kBAAkB,CAAC;IA8CpE,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAGjC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAI9B,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAGnC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAExB,OAAO,CAAC,QAAQ,CAAC,8BAA8B;IAvEhD,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAEhE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAO7B;IAEF,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAC/C;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0B;gBAGxC,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB;IAC1D;;;;;;;;;OASG;IACc,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,IAAI,EAG/D,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EAIzC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,IAAI,EAGtE,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,EAC5C,OAAO,EAAE,mBAAmB,EAC7C,YAAY,2BAAoB,EACf,8BAA8B,EAAE,MAAM,MAAM,GAAG,SAAS;IAwB1E,OAAO,KAAK,qBAAqB,GAIhC;IAED,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAED,IAAW,eAAe,IAAI,OAAO,CAKpC;IAED;;OAEG;IACU,WAAW;IAcxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAUtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,KAAK,UAAU,GAerB;IAEY,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAmC9D,OAAO,CAAC,aAAa;YAUP,kBAAkB;IAUnB,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YA4BxE,UAAU;IAYxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,qBAAqB;IAS7B,OAAO,CAAC,eAAe;YAoDT,cAAc;IAkB5B,OAAO,CAAC,mBAAmB;IAkC3B;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAoB7D,OAAO,CAAC,WAAW;IA0BZ,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IA2C7E;;;;;OAKG;WACiB,IAAI,CACvB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GACvD,OAAO,CAAC,oBAAoB,CAAC;IAehC;;OAEG;IACH,OAAO,CAAC,IAAI;IAgBZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IASjE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAcvD;;;;OAIG;IACI,sBAAsB,CAAC,gBAAgB,EAAE,MAAM,EAAE;IA0BxD;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAiDnB,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IA2BtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAoB3C,eAAe,IAAI,aAAa;CAOvC"}
+{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAEN,yBAAyB,EACzB,aAAa,EACb,MAAM,sCAAsC,CAAC;AAO9C,OAAO,EAKN,iBAAiB,EACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACN,iBAAiB,EACjB,uBAAuB,EACvB,MAAM,+CAA+C,CAAC;AACvD,OAAO,EAAe,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAO7F,OAAO,EACN,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAAE,gBAAgB,EAA8B,MAAM,oBAAoB,CAAC;AAKlF;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAc9C,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAf/B,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGpB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAKxB,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGhC;AAmBD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACnC;AAID,oBAAY,mBAAmB,GAAG,IAAI,CACrC,iBAAiB,EACjB,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,eAAe,CACxD,GACA,IAAI,CAAC,gBAAgB,EAAE,+BAA+B,CAAC,GACvD,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;AAuB5C,MAAM,WAAW,aAAa;IAC7B,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9E;AAED,MAAM,WAAW,kBAAkB;IAClC,CAAC,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,IAAI,OAAE;CAChD;AAED,qBAAa,WAAY,SAAQ,iBAAiB,CAAC,kBAAkB,CAAC;IAgDpE,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAc3B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAG9B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAExB,OAAO,CAAC,QAAQ,CAAC,cAAc;IArEhC,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAEhE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAO7B;IAEF,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAC/C;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAE1D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgD;gBAG/D,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB;IAC1D;;;;;;;;;OASG;IACH,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,IAAI,EAG9C,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EAGzC,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,EAC5C,OAAO,EAAE,mBAAmB,EAC7C,YAAY,2BAAoB,EACf,cAAc,EAAE,CAAC,KAAK,CAAC,EAAE,uBAAuB,KAAK,IAAI;IAuE3E,OAAO,KAAK,qBAAqB,GAIhC;IAED,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAED,IAAW,eAAe,IAAI,OAAO,CAKpC;IAED;;OAEG;IACU,WAAW;IAcxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAUtB;;;OAGG;IACH,OAAO,KAAK,UAAU,GAerB;IAEY,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAmC9D,OAAO,CAAC,aAAa;YAUP,kBAAkB;IAUnB,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YA4BxE,UAAU;IAYxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,qBAAqB;IAS7B,OAAO,CAAC,eAAe;YAkDT,cAAc;IAkB5B,OAAO,CAAC,mBAAmB;IAiC3B;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAkBtD,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IA2C7E;;;;;OAKG;WACiB,IAAI,CACvB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GACvD,OAAO,CAAC,oBAAoB,CAAC;IAehC;;OAEG;IACH,OAAO,CAAC,IAAI;IAgBL,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IA2B7E;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAejE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAIvD;;;;;OAKG;IACI,qBAAqB,CAAC,oBAAoB,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAUtE;;;;;;;;;;OAUG;IACH,OAAO,CAAC,4BAA4B;IAwCpC;;;;OAIG;IACI,sBAAsB,CAAC,gBAAgB,EAAE,MAAM,EAAE;IAsBxD;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAiDnB,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAoB3C,eAAe,IAAI,aAAa;CAavC"}

package/lib/blobManager.js

@@ -7,11 +7,11 @@ import { createResponseError, generateHandleContextPath, responseToException, Su
 import { assert, bufferToString, Deferred, stringToBuffer, TypedEventEmitter, } from "@fluidframework/common-utils";
 import { AttachState } from "@fluidframework/container-definitions";
 import { ChildLogger, loggerToMonitoringContext, PerformanceEvent, } from "@fluidframework/telemetry-utils";
+import { GenericError } from "@fluidframework/container-utils";
 import { TombstoneResponseHeaderKey } from "./containerRuntime";
+import { sendGCUnexpectedUsageEvent, sweepAttachmentBlobsKey, throwOnTombstoneLoadKey } from "./gc";
 import { Throttler, formExponentialFn } from "./throttler";
-import { summarizerClientType } from "./summarizerClientElection";
-import { throwOnTombstoneLoadKey } from "./garbageCollectionConstants";
-import { sendGCUnexpectedUsageEvent } from "./garbageCollectionHelpers";
+import { summarizerClientType } from "./summary";
 /**
  * This class represents blob (long string)
  * This object is used only when creating (writing) new blob and serialization purposes.
@@ -81,22 +81,16 @@ export class BlobManager extends TypedEventEmitter {
     // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
     // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
     blobRequested, 
-    // Called when a reference is added to a blob. For instance, when creating a localId / storageId to storageId
-    // mapping in the redirect table.
-    // Node path formats - `/<BlobManager.basePath>/<blobId>`.
-    addedBlobReference, 
     // Called to check if a blob has been deleted by GC.
     // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
-    isBlobDeleted, runtime, stashedBlobs = {}, getCurrentReferenceTimestampMs) {
+    isBlobDeleted, runtime, stashedBlobs = {}, closeContainer) {
         super();
         this.routeContext = routeContext;
         this.getStorage = getStorage;
-        this.sendBlobAttachOp = sendBlobAttachOp;
         this.blobRequested = blobRequested;
-        this.addedBlobReference = addedBlobReference;
         this.isBlobDeleted = isBlobDeleted;
         this.runtime = runtime;
-        this.getCurrentReferenceTimestampMs = getCurrentReferenceTimestampMs;
+        this.closeContainer = closeContainer;
         /**
          * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
          * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
@@ -122,12 +116,28 @@ export class BlobManager extends TypedEventEmitter {
         // Read the feature flag that tells whether to throw when a tombstone blob is requested.
         this.throwOnTombstoneLoad =
             this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
+                this.runtime.gcTombstoneEnforcementAllowed &&
                 this.runtime.clientDetails.type !== summarizerClientType;
         this.runtime.on("disconnected", () => this.onDisconnected());
         this.redirectTable = this.load(snapshot);
         // Begin uploading stashed blobs from previous container instance
         Object.entries(stashedBlobs).forEach(([localId, entry]) => {
             const blob = stringToBuffer(entry.blob, "base64");
+            if (entry.minTTLInSeconds && entry.uploadTime) {
+                const timeLapseSinceLocalUpload = (Date.now() - entry.uploadTime) / 1000;
+                // stashed entries with more than half-life in storage will not be reuploaded
+                if (entry.minTTLInSeconds - timeLapseSinceLocalUpload > entry.minTTLInSeconds / 2) {
+                    this.pendingBlobs.set(localId, {
+                        blob,
+                        status: PendingBlobStatus.OfflinePendingOp,
+                        handleP: new Deferred(),
+                        uploadP: undefined,
+                        uploadTime: entry.uploadTime,
+                        minTTLInSeconds: entry.minTTLInSeconds,
+                    });
+                    return;
+                }
+            }
             this.pendingBlobs.set(localId, {
                 blob,
                 status: PendingBlobStatus.OfflinePendingUpload,
@@ -135,6 +145,30 @@ export class BlobManager extends TypedEventEmitter {
                 uploadP: this.uploadBlob(localId, blob),
             });
         });
+        this.sendBlobAttachOp = (localId, blobId) => {
+            const pendingEntry = this.pendingBlobs.get(localId);
+            if ((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.uploadTime) && (pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.minTTLInSeconds)) {
+                const secondsSinceUpload = (Date.now() - pendingEntry.uploadTime) / 1000;
+                const expired = pendingEntry.minTTLInSeconds - secondsSinceUpload < 0;
+                this.mc.logger.sendTelemetryEvent({
+                    eventName: "sendBlobAttach",
+                    entryStatus: pendingEntry.status,
+                    secondsSinceUpload,
+                    minTTLInSeconds: pendingEntry.minTTLInSeconds,
+                    expired,
+                });
+                if (expired) {
+                    // we want to avoid submitting ops with broken handles
+                    this.closeContainer(new GenericError("Trying to submit a BlobAttach for expired blob", undefined, {
+                        localId,
+                        blobId,
+                        entryStatus: pendingEntry.status,
+                        secondsSinceUpload,
+                    }));
+                }
+            }
+            return sendBlobAttachOp(localId, blobId);
+        };
     }
     get pendingOfflineUploads() {
         return Array.from(this.pendingBlobs.values()).filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);
@@ -170,14 +204,6 @@ export class BlobManager extends TypedEventEmitter {
             }
         }
     }
-    /**
-     * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`
-     * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
-     * referenced by storing these handles in a referenced DDS.
-     */
-    getBlobGCNodePath(blobId) {
-        return `/${BlobManager.basePath}/${blobId}`;
-    }
     /**
      * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
      * detached or there are no (non-pending) attachment blobs in the document
@@ -213,7 +239,7 @@ export class BlobManager extends TypedEventEmitter {
             storageId = attachedStorageId;
         }
         // Let runtime know that the corresponding GC node was requested.
-        this.blobRequested(this.getBlobGCNodePath(blobId));
+        this.blobRequested(getGCNodePathFromBlobId(blobId));
         return PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
             return this.getStorage().readBlob(storageId);
         }, { end: true, cancel: "error" });
@@ -260,11 +286,6 @@ export class BlobManager extends TypedEventEmitter {
      */
     setRedirection(fromId, toId) {
         this.redirectTable.set(fromId, toId);
-        // Notify runtime of a reference added if toId is not undefined. It can be undefined when a blob is uploaded in
-        // detached mode. In this case, the entry will be updated when the blob is updated.
-        if (toId !== undefined) {
-            this.addedBlobReference(this.getBlobGCNodePath(fromId), this.getBlobGCNodePath(toId));
-        }
     }
     deleteAndEmitsIfEmpty(id) {
         if (this.pendingBlobs.has(id)) {
@@ -280,9 +301,8 @@ export class BlobManager extends TypedEventEmitter {
         assert((entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OnlinePendingUpload ||
             (entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OfflinePendingUpload, 0x386 /* Must have pending blob entry for uploaded blob */);
         entry.storageId = response.id;
-        entry.localUploadTime = Date.now();
+        entry.uploadTime = Date.now();
         entry.minTTLInSeconds = response.minTTLInSeconds;
-        entry.serverUploadTime = this.getCurrentReferenceTimestampMs();
         if (this.runtime.connected) {
             if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
                 // Send a blob attach op. This serves two purposes:
@@ -290,7 +310,6 @@ export class BlobManager extends TypedEventEmitter {
                 //    until its storage ID is added to the next summary.
                 // 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
                 //    blob from the server via the storage ID.
-                this.logTimeInfo(entry, "sendBlobAttachResolveTTL");
                 this.sendBlobAttachOp(localId, response.id);
                 if (this.storageIds.has(response.id)) {
                     // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
@@ -354,7 +373,6 @@ export class BlobManager extends TypedEventEmitter {
          * is called on reconnection.
          */
         if (entry.status !== PendingBlobStatus.OnlinePendingOp) {
-            this.logTimeInfo(entry, "sendBlobAttachTransitionOfflineTTL");
             this.sendBlobAttachOp(localId, entry.storageId);
         }
         entry.status =
@@ -373,9 +391,6 @@ export class BlobManager extends TypedEventEmitter {
         const { localId, blobId } = metadata;
         assert(localId !== undefined, 0x50d /* local ID not available on reSubmit */);
         const pendingEntry = this.pendingBlobs.get(localId);
-        if (pendingEntry) {
-            this.logTimeInfo(pendingEntry, "sendBlobAttachResubmitTTL");
-        }
         if (!blobId) {
             // We submitted this op while offline. The blob should have been uploaded by now.
             assert((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.status) === PendingBlobStatus.OfflinePendingOp &&
@@ -384,32 +399,6 @@ export class BlobManager extends TypedEventEmitter {
         }
         return this.sendBlobAttachOp(localId, blobId);
     }
-    logTimeInfo(pendingEntry, eventName) {
-        var _a, _b;
-        let timeLapseSinceLocalUpload = 0;
-        let timeLapseSinceServerUpload = 0;
-        let expiredUsingLocalTime;
-        let expiredUsingServerTime;
-        if (pendingEntry.localUploadTime) {
-            timeLapseSinceLocalUpload = (Date.now() - pendingEntry.localUploadTime) / 1000;
-            expiredUsingLocalTime =
-                ((_a = pendingEntry.minTTLInSeconds) !== null && _a !== void 0 ? _a : 0) - timeLapseSinceLocalUpload < 0 ? true : false;
-        }
-        if (pendingEntry.serverUploadTime) {
-            timeLapseSinceServerUpload = (Date.now() - pendingEntry.serverUploadTime) / 1000;
-            expiredUsingServerTime =
-                ((_b = pendingEntry.minTTLInSeconds) !== null && _b !== void 0 ? _b : 0) - timeLapseSinceServerUpload < 0 ? true : false;
-        }
-        this.mc.logger.sendTelemetryEvent({
-            eventName,
-            entryStatus: pendingEntry.status,
-            timeLapseSinceLocalUpload,
-            timeLapseSinceServerUpload,
-            minTTLInSeconds: pendingEntry.minTTLInSeconds,
-            expiredUsingLocalTime,
-            expiredUsingServerTime,
-        });
-    }
     processBlobAttachOp(message, local) {
         var _a, _b;
         const localId = (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.localId;
@@ -486,6 +475,23 @@ export class BlobManager extends TypedEventEmitter {
         }
         return table;
     }
+    summarize(telemetryContext) {
+        // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
+        const blobIds = this.storageIds.size > 0
+            ? Array.from(this.storageIds)
+            : Array.from(this.redirectTable.keys());
+        const builder = new SummaryTreeBuilder();
+        blobIds.forEach((blobId) => {
+            builder.addAttachment(blobId);
+        });
+        // Any non-identity entries in the table need to be saved in the summary
+        if (this.redirectTable.size > blobIds.length) {
+            builder.addBlob(BlobManager.redirectTableBlobName, 
+            // filter out identity entries
+            JSON.stringify(Array.from(this.redirectTable.entries()).filter(([localId, storageId]) => localId !== storageId)));
+        }
+        return builder.getSummaryTree();
+    }
     /**
      * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
      * individually referenced by storing its handle in a referenced DDS. Returns the list of blob ids as GC nodes.
@@ -496,23 +502,83 @@ export class BlobManager extends TypedEventEmitter {
         const gcData = { gcNodes: {} };
         for (const [localId, storageId] of this.redirectTable) {
             assert(!!storageId, 0x390 /* Must be attached to get GC data */);
-            gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
+            // Only return local ids as GC nodes because a blob can only be referenced via its local id. The storage
+            // id entries have the same key and value, ignore them.
+            // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
+            // by adding its handle to a referenced DDS.
+            if (localId !== storageId) {
+                gcData.gcNodes[getGCNodePathFromBlobId(localId)] = [];
+            }
         }
         return gcData;
     }
     /**
      * This is called to update blobs whose routes are unused. The unused blobs are deleted.
-     * @param unusedRoutes - The routes of the blob nodes that are unused.
+     * @param unusedRoutes - The routes of the blob nodes that are unused. These routes will be based off of local ids.
      */
     updateUnusedRoutes(unusedRoutes) {
-        // The routes or blob node paths are in the same format as returned in getGCData -
-        // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of unusedRoutes) {
-            const pathParts = route.split("/");
-            assert(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x2d5 /* "Invalid blob node id in unused routes." */);
-            const blobId = pathParts[2];
+        this.deleteBlobsFromRedirectTable(unusedRoutes);
+    }
+    /**
+     * Delete attachment blobs that are sweep ready.
+     * @param sweepReadyBlobRoutes - The routes of blobs that are sweep ready and should be deleted. These routes will
+     * be based off of local ids.
+     * @returns - The routes of blobs that were deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyBlobRoutes) {
+        // If sweep for attachment blobs is not enabled, return empty list indicating nothing is deleted.
+        if (this.mc.config.getBoolean(sweepAttachmentBlobsKey) !== true) {
+            return [];
+        }
+        this.deleteBlobsFromRedirectTable(sweepReadyBlobRoutes);
+        return Array.from(sweepReadyBlobRoutes);
+    }
+    /**
+     * Delete blobs with the given routes from the redirect table.
+     * The routes are GC nodes paths of format -`/<BlobManager.basePath>/<blobId>`. The blob ids are all local ids.
+     * Deleting the blobs involves 2 steps:
+     * 1. The redirect table entry for the local ids are deleted.
+     * 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
+     * for the storage ids are deleted as well.
+     *
+     * Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
+     * will remove them the next summary. The service would them delete them some time in the future.
+     */
+    deleteBlobsFromRedirectTable(blobRoutes) {
+        if (blobRoutes.length === 0) {
+            return;
+        }
+        // This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
+        // these storage ids are unused, they will be deleted as well.
+        const maybeUnusedStorageIds = new Set();
+        for (const route of blobRoutes) {
+            const blobId = getBlobIdFromGCNodePath(route);
+            if (!this.redirectTable.has(blobId)) {
+                this.mc.logger.sendErrorEvent({
+                    eventName: "DeletedAttachmentBlobNotFound",
+                    blobId,
+                });
+                continue;
+            }
+            const storageId = this.redirectTable.get(blobId);
+            assert(!!storageId, 0x5bb /* Must be attached to run GC */);
+            maybeUnusedStorageIds.add(storageId);
             this.redirectTable.delete(blobId);
         }
+        // Find out storage ids that are in-use and remove them from maybeUnusedStorageIds. A storage id is in-use if
+        // the redirect table has a local id -> storage id entry for it.
+        for (const [localId, storageId] of this.redirectTable.entries()) {
+            assert(!!storageId, 0x5bc /* Must be attached to run GC */);
+            // For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
+            if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
+                maybeUnusedStorageIds.delete(storageId);
+            }
+        }
+        // For unused storage ids, delete their id -> id entries from the redirect table.
+        // This way they'll be absent from the next summary, and the service is free to delete them from storage.
+        for (const storageId of maybeUnusedStorageIds) {
+            this.redirectTable.delete(storageId);
+        }
     }
     /**
      * This is called to update blobs whose routes are tombstones. Tombstoned blobs enable testing scenarios with
@@ -524,9 +590,8 @@ export class BlobManager extends TypedEventEmitter {
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
         for (const route of tombstonedRoutes) {
-            const pathParts = route.split("/");
-            assert(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x50f /* Invalid blob node id in tombstoned routes. */);
-            tombstonedBlobsSet.add(pathParts[2]);
+            const blobId = getBlobIdFromGCNodePath(route);
+            tombstonedBlobsSet.add(blobId);
         }
         // Remove blobs from the tombstone list that were tombstoned but aren't anymore as per the tombstoneRoutes.
         for (const blobId of this.tombstonedBlobs) {
@@ -551,7 +616,7 @@ export class BlobManager extends TypedEventEmitter {
          * 3. "valid" - It has not been deleted or tombstoned.
          */
         let state = "valid";
-        if (this.isBlobDeleted(this.getBlobGCNodePath(blobId))) {
+        if (this.isBlobDeleted(getGCNodePathFromBlobId(blobId))) {
             state = "deleted";
         }
         else if (this.tombstonedBlobs.has(blobId)) {
@@ -570,29 +635,12 @@ export class BlobManager extends TypedEventEmitter {
                 ? "GC_Tombstone_Blob_Requested"
                 : "GC_Deleted_Blob_Requested",
             category: shouldFail ? "error" : "generic",
-            isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
+            gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
         }, [BlobManager.basePath], error);
         if (shouldFail) {
             throw error;
         }
     }
-    summarize(telemetryContext) {
-        // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
-        const blobIds = this.storageIds.size > 0
-            ? Array.from(this.storageIds)
-            : Array.from(this.redirectTable.keys());
-        const builder = new SummaryTreeBuilder();
-        blobIds.forEach((blobId) => {
-            builder.addAttachment(blobId);
-        });
-        // Any non-identity entries in the table need to be saved in the summary
-        if (this.redirectTable.size > blobIds.length) {
-            builder.addBlob(BlobManager.redirectTableBlobName, 
-            // filter out identity entries
-            JSON.stringify(Array.from(this.redirectTable.entries()).filter(([localId, storageId]) => localId !== storageId)));
-        }
-        return builder.getSummaryTree();
-    }
     setRedirectTable(table) {
         assert(this.runtime.attachState === AttachState.Detached, 0x252 /* "redirect table can only be set in detached container" */);
         assert(this.redirectTable.size === table.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
@@ -606,11 +654,33 @@ export class BlobManager extends TypedEventEmitter {
     getPendingBlobs() {
         const blobs = {};
         for (const [key, entry] of this.pendingBlobs) {
-            blobs[key] = { blob: bufferToString(entry.blob, "base64") };
+            blobs[key] = entry.minTTLInSeconds
+                ? {
+                    blob: bufferToString(entry.blob, "base64"),
+                    uploadTime: entry.uploadTime,
+                    minTTLInSeconds: entry.minTTLInSeconds,
+                }
+                : { blob: bufferToString(entry.blob, "base64") };
         }
         return blobs;
     }
 }
 BlobManager.basePath = "_blobs";
 BlobManager.redirectTableBlobName = ".redirectTable";
+/**
+ * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`.
+ * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
+ * referenced by storing these handles in a referenced DDS.
+ */
+function getGCNodePathFromBlobId(blobId) {
+    return `/${BlobManager.basePath}/${blobId}`;
+}
+/**
+ * For a given GC node path, return the blobId. The node path is of the format `/<BlobManager.basePath>/<blobId>`.
+ */
+function getBlobIdFromGCNodePath(nodePath) {
+    const pathParts = nodePath.split("/");
+    assert(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x5bd /* Invalid blob node path */);
+    return pathParts[2];
+}
 //# sourceMappingURL=blobManager.js.map