@fluidframework/container-runtime 1.3.0 → 2.0.0-dev.1.4.5.105745

package/lib/blobManager.d.ts

@@ -4,9 +4,9 @@
  */
 import { IFluidHandle, IFluidHandleContext } from "@fluidframework/core-interfaces";
 import { IDocumentStorageService } from "@fluidframework/driver-definitions";
-import { ISnapshotTree } from "@fluidframework/protocol-definitions";
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IContainerRuntime } from "@fluidframework/container-runtime-definitions";
+import { ISequencedDocumentMessage, ISnapshotTree } from "@fluidframework/protocol-definitions";
+import { TypedEventEmitter } from "@fluidframework/common-utils";
+import { IContainerRuntime, IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions";
 import { IGarbageCollectionData, ISummaryTreeWithStats, ITelemetryContext } from "@fluidframework/runtime-definitions";
 /**
  * This class represents blob (long string)
@@ -34,47 +34,108 @@ export interface IBlobManagerLoadInfo {
     ids?: string[];
     redirectTable?: [string, string][];
 }
+export declare type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "connected" | "logger"> & TypedEventEmitter<IContainerRuntimeEvents>;
+export interface IPendingBlobs {
+    [id: string]: {
+        blob: string;
+    };
+}
 export declare class BlobManager {
     private readonly routeContext;
     private readonly getStorage;
-    private readonly attachBlobCallback;
+    /**
+     * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
+     * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
+     * will then not delete the blob as long as it is listed as referenced in future summaries.
+     * The summarizing client will know to include the storage ID in the summary when it sees the op.
+     *
+     * The op may also include a local ID to inform all clients of the relation to the storage
+     * ID, without knowledge of which they cannot request the blob from storage. This is also
+     * included in the redirect table in the summary.
+     */
+    private readonly sendBlobAttachOp;
     private readonly gcNodeUpdated;
     private readonly runtime;
-    private readonly logger;
     static readonly basePath = "_blobs";
     private static readonly redirectTableBlobName;
-    private readonly blobIds;
-    private readonly pendingBlobIds;
-    private readonly detachedBlobIds;
-    private redirectTable;
-    constructor(routeContext: IFluidHandleContext, snapshot: IBlobManagerLoadInfo, getStorage: () => IDocumentStorageService, attachBlobCallback: (blobId: string) => void, gcNodeUpdated: (blobPath: string) => void, runtime: IContainerRuntime, logger: ITelemetryLogger);
-    private hasBlob;
+    private readonly logger;
+    /**
+     * Map of local (offline/detached) IDs to storage IDs. Contains identity entries
+     * (id → id) for storage IDs, so all requested IDs should be a key in this map.
+     * Blobs created while the container is detached are stored in IDetachedBlobStorage
+     * which gives local IDs; the storage IDs are filled in at attach time.
+     */
+    private readonly redirectTable;
+    /**
+     * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
+     * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
+     * we must save it. This is true for both the online and offline flow.
+     */
+    private readonly pendingBlobs;
+    /**
+     * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops
+     * don't include local ID in online flow.
+     */
+    private readonly opsInFlight;
+    private readonly retryThrottler;
+    constructor(routeContext: IFluidHandleContext, snapshot: IBlobManagerLoadInfo, getStorage: () => IDocumentStorageService, 
+    /**
+     * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
+     * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
+     * will then not delete the blob as long as it is listed as referenced in future summaries.
+     * The summarizing client will know to include the storage ID in the summary when it sees the op.
+     *
+     * The op may also include a local ID to inform all clients of the relation to the storage
+     * ID, without knowledge of which they cannot request the blob from storage. This is also
+     * included in the redirect table in the summary.
+     */
+    sendBlobAttachOp: (storageId?: string, localId?: string) => void, gcNodeUpdated: (blobPath: string) => void, runtime: IBlobManagerRuntime, stashedBlobs?: IPendingBlobs);
+    private get pendingOfflineUploads();
+    get hasPendingOfflineUploads(): boolean;
+    /**
+     * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
+     */
+    onConnected(): Promise<void>;
+    /**
+     * Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected
+     * again
+     */
+    private onDisconnected;
     /**
      * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`
      * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
      * referenced by storing these handles in a referenced DDS.
      */
     private getBlobGCNodePath;
-    getBlob(blobId: string): Promise<IFluidHandle<ArrayBufferLike>>;
+    /**
+     * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
+     * detached or there are no (non-pending) attachment blobs in the document
+     */
+    private get storageIds();
+    getBlob(blobId: string): Promise<ArrayBufferLike>;
+    private getBlobHandle;
+    private createBlobDetached;
     createBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>>;
-    processBlobAttachOp(blobId: string, local: boolean): void;
+    private uploadBlob;
+    private onUploadResolve;
+    private onUploadReject;
+    private transitionToOffline;
+    /**
+     * Resubmit a BlobAttach op. Used to add storage IDs to ops that were
+     * submitted to runtime while disconnected.
+     * @param metadata - op metadata containing storage and/or local IDs
+     */
+    reSubmit(metadata: Record<string, unknown> | undefined): void;
+    processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean): void;
     /**
      * Reads blobs needed to load BlobManager from storage.
+     * @param blobsTree - Tree containing IDs of previously attached blobs. We
+     * look for the IDs in the blob entries of the tree since the both the r11s
+     * and SPO drivers replace the attachment types returned in snapshot() with blobs.
      */
     static load(blobsTree: ISnapshotTree | undefined, tryFetchBlob: (id: string) => Promise<[string, string][]>): Promise<IBlobManagerLoadInfo>;
     /**
-     * Load a set of previously attached blob IDs from a previous snapshot. Note
-     * that BlobManager tracking and reporting attached blobs is a temporary
-     * solution since storage expects attached blobs to be reported and any that
-     * are not reported as attached may be GCed. In the future attached blob
-     * IDs will be collected at summarization time, and runtime will not care
-     * about the existence or specific formatting of this tree in returned
-     * snapshots.
-     *
-     * @param blobsTree - Tree containing IDs of previously attached blobs. This
-     * corresponds to snapshot() below. We look for the IDs in the blob entries
-     * of the tree since the both the r11s and SPO drivers replace the
-     * attachment types returned in snapshot() with blobs.
+     * Load a set of previously attached blob IDs and redirect table from a previous snapshot.
      */
     private load;
     /**
@@ -91,5 +152,6 @@ export declare class BlobManager {
     deleteUnusedRoutes(unusedRoutes: string[]): void;
     summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats;
     setRedirectTable(table: Map<string, string>): void;
+    getPendingBlobs(): IPendingBlobs;
 }
 //# sourceMappingURL=blobManager.d.ts.map

package/lib/blobManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAGlF,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACpB,MAAM,qCAAqC,CAAC;AAE7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAYxC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAblC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGjB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAK3B,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGnC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACtC;AAED,qBAAa,WAAW;IAchB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAGnC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IArB3B,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IAEjE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;IAGlD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0C;IAEzE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAE1D,OAAO,CAAC,aAAa,CAAkC;gBAGlC,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB,EACzC,kBAAkB,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,EAG5C,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACzC,OAAO,EAAE,iBAAiB,EAC1B,MAAM,EAAE,gBAAgB;IAU7C,OAAO,CAAC,OAAO;IAIf;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;IAqB/D,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;IAgD/E,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IAUzD;;OAEG;WACiB,IAAI,CACpB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GAC1D,OAAO,CAAC,oBAAoB,CAAC;IAchC;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,IAAI;IAeZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAyBjE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAqBhD,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAoBtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;CAWrD"}
+{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAuB,yBAAyB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAGrH,OAAO,EAAoD,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,+CAA+C,CAAC;AAG3G,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACpB,MAAM,qCAAqC,CAAC;AAG7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAYxC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAblC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGjB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAK3B,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGnC;AAmBD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACtC;AAID,oBAAY,mBAAmB,GAC3B,IAAI,CAAC,iBAAiB,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC,GAAG,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;AAmBjH,MAAM,WAAW,aAAa;IAAG,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;KAAE,CAAC;CAAE;AAEnE,qBAAa,WAAW;IAkChB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAGjC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAlD5B,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAE1C;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAEhE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAK5B;gBAGkB,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB;IAC1D;;;;;;;;;OASG;IACc,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,IAAI,EAGhE,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACzC,OAAO,EAAE,mBAAmB,EAC7C,YAAY,GAAE,aAAkB;IAkBpC,OAAO,KAAK,qBAAqB,GAGhC;IAED,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAED;;OAEG;IACU,WAAW;IAWxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAUtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,KAAK,UAAU,GAYrB;IAEY,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA8B9D,OAAO,CAAC,aAAa;YAUP,kBAAkB;IAQnB,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YA0BxE,UAAU;IAYxB,OAAO,CAAC,eAAe;YAoCT,cAAc;IAgB5B,OAAO,CAAC,mBAAmB;IAoB3B;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IActD,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAmC7E;;;;;OAKG;WACiB,IAAI,CACpB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GAC1D,OAAO,CAAC,oBAAoB,CAAC;IAchC;;OAEG;IACH,OAAO,CAAC,IAAI;IAgBZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAyBjE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAoBhD,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAqBtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAa3C,eAAe,IAAI,aAAa;CAO1C"}