@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.2.3.0.115467

package/dist/blobManager.d.ts

@@ -34,7 +34,7 @@ export interface IBlobManagerLoadInfo {
     ids?: string[];
     redirectTable?: [string, string][];
 }
-export declare type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "connected" | "logger"> & TypedEventEmitter<IContainerRuntimeEvents>;
+export declare type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "connected" | "logger" | "clientDetails"> & TypedEventEmitter<IContainerRuntimeEvents>;
 export interface IPendingBlobs {
     [id: string]: {
         blob: string;
@@ -58,7 +58,7 @@ export declare class BlobManager {
     private readonly runtime;
     static readonly basePath = "_blobs";
     private static readonly redirectTableBlobName;
-    private readonly logger;
+    private readonly mc;
     /**
      * Map of local (offline/detached) IDs to storage IDs. Contains identity entries
      * (id → id) for storage IDs, so all requested IDs should be a key in this map.
@@ -78,6 +78,13 @@ export declare class BlobManager {
      */
     private readonly opsInFlight;
     private readonly retryThrottler;
+    /** If true, throw an error when a tombstone attachment blob is retrieved. */
+    private readonly throwOnTombstoneUsage;
+    /**
+     * This stores ides of tombstoned blobs.
+     * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
+     */
+    private readonly tombstonedBlobs;
     constructor(routeContext: IFluidHandleContext, snapshot: IBlobManagerLoadInfo, getStorage: () => IDocumentStorageService, 
     /**
      * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
@@ -146,10 +153,18 @@ export declare class BlobManager {
      */
     getGCData(fullGC?: boolean): IGarbageCollectionData;
     /**
-     * When running GC in test mode, this is called to delete blobs that are unused.
-     * @param unusedRoutes - These are the blob node ids that are unused and should be deleted.
+     * This is called to update blobs whose routes are used. The used blobs are removed from the tombstone list.
+     * @param usedRoutes - The routes of the blob nodes that are used.
+     */
+    updateUsedRoutes(usedRoutes: string[]): void;
+    /**
+     * This is called to update blobs whose routes are unused. The unused blobs are either deleted or marked as
+     * tombstones.
+     * @param unusedRoutes - The routes of the blob nodes that are unused.
+     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
+     * are deleted.
      */
-    deleteUnusedRoutes(unusedRoutes: string[]): void;
+    updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean): void;
     summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats;
     setRedirectTable(table: Map<string, string>): void;
     getPendingBlobs(): IPendingBlobs;

package/dist/blobManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAuB,yBAAyB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAGrH,OAAO,EAAoD,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,+CAA+C,CAAC;AAG3G,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACpB,MAAM,qCAAqC,CAAC;AAG7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAYxC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAblC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGjB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAK3B,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGnC;AAmBD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACtC;AAID,oBAAY,mBAAmB,GAC3B,IAAI,CAAC,iBAAiB,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC,GAAG,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;AAmBjH,MAAM,WAAW,aAAa;IAAG,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;KAAE,CAAC;CAAE;AAEnE,qBAAa,WAAW;IAkChB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAGjC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAlD5B,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAE1C;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAEhE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAK5B;gBAGkB,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB;IAC1D;;;;;;;;;OASG;IACc,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,IAAI,EAGhE,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACzC,OAAO,EAAE,mBAAmB,EAC7C,YAAY,GAAE,aAAkB;IAkBpC,OAAO,KAAK,qBAAqB,GAGhC;IAED,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAED;;OAEG;IACU,WAAW;IAWxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAUtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,KAAK,UAAU,GAYrB;IAEY,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA8B9D,OAAO,CAAC,aAAa;YAUP,kBAAkB;IAQnB,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YA0BxE,UAAU;IAYxB,OAAO,CAAC,eAAe;YAoCT,cAAc;IAgB5B,OAAO,CAAC,mBAAmB;IAoB3B;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IActD,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAmC7E;;;;;OAKG;WACiB,IAAI,CACpB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GAC1D,OAAO,CAAC,oBAAoB,CAAC;IAchC;;OAEG;IACH,OAAO,CAAC,IAAI;IAgBZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAyBjE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAoBhD,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAqBtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAa3C,eAAe,IAAI,aAAa;CAO1C"}
+{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAuB,yBAAyB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAOrH,OAAO,EAAoD,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,+CAA+C,CAAC;AAG3G,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACpB,MAAM,qCAAqC,CAAC;AAK7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAYxC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAblC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGjB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAK3B,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGnC;AAmBD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACtC;AAID,oBAAY,mBAAmB,GAC3B,IAAI,CAAC,iBAAiB,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,eAAe,CAAC,GAAG,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;AAmBnI,MAAM,WAAW,aAAa;IAAG,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;KAAE,CAAC;CAAE;AAEnE,qBAAa,WAAW;IA0ChB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAGjC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IA1D5B,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAEhE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAK5B;IAEH,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAU;IAChD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0B;gBAGrC,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB;IAC1D;;;;;;;;;OASG;IACc,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,IAAI,EAGhE,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACzC,OAAO,EAAE,mBAAmB,EAC7C,YAAY,GAAE,aAAkB;IAuBpC,OAAO,KAAK,qBAAqB,GAGhC;IAED,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAED;;OAEG;IACU,WAAW;IAWxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAUtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,KAAK,UAAU,GAYrB;IAEY,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA0C9D,OAAO,CAAC,aAAa;YAUP,kBAAkB;IAQnB,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YA0BxE,UAAU;IAYxB,OAAO,CAAC,eAAe;YAoCT,cAAc;IAgB5B,OAAO,CAAC,mBAAmB;IAoB3B;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IActD,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAmC7E;;;;;OAKG;WACiB,IAAI,CACpB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GAC1D,OAAO,CAAC,oBAAoB,CAAC;IAchC;;OAEG;IACH,OAAO,CAAC,IAAI;IAgBZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAyBjE;;;OAGG;IACI,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE;IAe5C;;;;;;OAMG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAsBpE,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAqBtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAa3C,eAAe,IAAI,aAAa;CAO1C"}

package/dist/blobManager.js

@@ -11,6 +11,8 @@ const common_utils_1 = require("@fluidframework/common-utils");
 const container_definitions_1 = require("@fluidframework/container-definitions");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const throttler_1 = require("./throttler");
+const summarizerClientElection_1 = require("./summarizerClientElection");
+const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
 /**
  * This class represents blob (long string)
  * This object is used only when creating (writing) new blob and serialization purposes.
@@ -99,7 +101,16 @@ class BlobManager {
         30 * 1000, // 30 sec max delay
         // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
         (0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })));
-        this.logger = telemetry_utils_1.ChildLogger.create(this.runtime.logger, "BlobManager");
+        /**
+         * This stores ides of tombstoned blobs.
+         * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
+         */
+        this.tombstonedBlobs = new Set();
+        this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.runtime.logger, "BlobManager"));
+        // Read the feature flag that tells whether to throw when a tombstone blob is requested.
+        this.throwOnTombstoneUsage =
+            this.mc.config.getBoolean(garbageCollectionConstants_1.throwOnTombstoneUsageKey) === true &&
+                this.runtime.clientDetails.type !== summarizerClientElection_1.summarizerClientType;
         this.runtime.on("disconnected", () => this.onDisconnected());
         this.redirectTable = this.load(snapshot);
         // Begin uploading stashed blobs from previous container instance
@@ -126,7 +137,7 @@ class BlobManager {
     async onConnected() {
         this.retryThrottler.cancel();
         const pendingUploads = this.pendingOfflineUploads.map(async (e) => e.uploadP);
-        await telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, {
+        await telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, {
             eventName: "BlobUploadOnConnected",
             count: pendingUploads.length,
         }, async () => Promise.all(pendingUploads), { start: true, end: true });
@@ -166,6 +177,17 @@ class BlobManager {
         return ids;
     }
     async getBlob(blobId) {
+        const request = { url: blobId };
+        if (this.tombstonedBlobs.has(blobId)) {
+            const error = (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, "Blob removed by gc", request), request);
+            this.mc.logger.sendErrorEvent({
+                eventName: "GC_Tombstone_Blob_Requested",
+                url: request.url,
+            }, error);
+            if (this.throwOnTombstoneUsage) {
+                throw error;
+            }
+        }
         const pending = this.pendingBlobs.get(blobId);
         if (pending) {
             return pending.blob;
@@ -183,7 +205,7 @@ class BlobManager {
         }
         // When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.
         this.gcNodeUpdated(this.getBlobGCNodePath(blobId));
-        return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
+        return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
             return this.getStorage().readBlob(storageId);
         }, { end: true, cancel: "error" });
     }
@@ -204,7 +226,7 @@ class BlobManager {
         }
         if (this.runtime.attachState === container_definitions_1.AttachState.Attaching) {
             // blob upload is not supported in "Attaching" state
-            this.logger.sendTelemetryEvent({ eventName: "CreateBlobWhileAttaching" });
+            this.mc.logger.sendTelemetryEvent({ eventName: "CreateBlobWhileAttaching" });
             await new Promise((resolve) => this.runtime.once("attached", resolve));
         }
         (0, common_utils_1.assert)(this.runtime.attachState === container_definitions_1.AttachState.Attached, 0x385 /* For clarity and paranoid defense against adding future attachment states */);
@@ -221,7 +243,7 @@ class BlobManager {
         return pendingEntry.handleP.promise;
     }
     async uploadBlob(localId, blob) {
-        return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "createBlob" }, async () => this.getStorage().createBlob(blob), { end: true, cancel: this.runtime.connected ? "error" : "generic" }).then((response) => this.onUploadResolve(localId, response), async (err) => this.onUploadReject(localId, err));
+        return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "createBlob" }, async () => this.getStorage().createBlob(blob), { end: true, cancel: this.runtime.connected ? "error" : "generic" }).then((response) => this.onUploadResolve(localId, response), async (err) => this.onUploadReject(localId, err));
     }
     onUploadResolve(localId, response) {
         var _a;
@@ -253,7 +275,7 @@ class BlobManager {
         }
         else {
             // connected to storage but not ordering service?
-            this.logger.sendTelemetryEvent({ eventName: "BlobUploadSuccessWhileDisconnected" });
+            this.mc.logger.sendTelemetryEvent({ eventName: "BlobUploadSuccessWhileDisconnected" });
             if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
                 this.transitionToOffline(localId);
             }
@@ -365,7 +387,7 @@ class BlobManager {
      */
     load(snapshot) {
         var _a, _b, _c;
-        this.logger.sendTelemetryEvent({
+        this.mc.logger.sendTelemetryEvent({
             eventName: "AttachmentBlobsLoaded",
             count: (_b = (_a = snapshot.ids) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0,
             redirectTable: (_c = snapshot.redirectTable) === null || _c === void 0 ? void 0 : _c.length,
@@ -408,22 +430,42 @@ class BlobManager {
         return gcData;
     }
     /**
-     * When running GC in test mode, this is called to delete blobs that are unused.
-     * @param unusedRoutes - These are the blob node ids that are unused and should be deleted.
+     * This is called to update blobs whose routes are used. The used blobs are removed from the tombstone list.
+     * @param usedRoutes - The routes of the blob nodes that are used.
      */
-    deleteUnusedRoutes(unusedRoutes) {
-        var _a;
+    updateUsedRoutes(usedRoutes) {
+        // The routes or blob node paths are in the same format as returned in getGCData -
+        // `/<BlobManager.basePath>/<blobId>`.
+        for (const route of usedRoutes) {
+            const pathParts = route.split("/");
+            (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x4bc /* Invalid blob node id in used routes. */);
+            const blobId = pathParts[2];
+            // Un-tombstone the blob if it was marked tombstone.
+            this.tombstonedBlobs.delete(blobId);
+        }
+    }
+    /**
+     * This is called to update blobs whose routes are unused. The unused blobs are either deleted or marked as
+     * tombstones.
+     * @param unusedRoutes - The routes of the blob nodes that are unused.
+     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
+     * are deleted.
+     */
+    updateUnusedRoutes(unusedRoutes, tombstone) {
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
         for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x2d5 /* "Invalid blob node id in unused routes." */);
             const blobId = pathParts[2];
-            // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The
-            // corresponding storageId may still be used either directly or via other localIds.
-            if ((_a = this.redirectTable) === null || _a === void 0 ? void 0 : _a.has(blobId)) {
+            if (tombstone) {
+                // If tombstone is set, add this blob to the tombstone list.
+                this.tombstonedBlobs.add(blobId);
+            }
+            else {
+                // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The
+                // corresponding storageId may still be used either directly or via other localIds.
                 this.redirectTable.delete(blobId);
-                continue;
             }
         }
     }

package/dist/blobManager.js.map

@@ -1 +1 @@
-{"version":3,"file":"blobManager.js","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,+BAAkC;AAIlC,iEAA8F;AAE9F,+DAAmH;AAEnH,iFAAoE;AACpE,qEAAgF;AAMhF,2CAAuE;AAEvE;;;;;;GAMG;AACH,MAAa,UAAU;IAWnB,YACoB,IAAY,EACZ,YAAiC,EAC1C,GAAuB;QAFd,SAAI,GAAJ,IAAI,CAAQ;QACZ,iBAAY,GAAZ,YAAY,CAAqB;QAC1C,QAAG,GAAH,GAAG,CAAoB;QAb1B,aAAQ,GAAY,KAAK,CAAC;QAe9B,IAAI,CAAC,YAAY,GAAG,IAAA,yCAAyB,EAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3E,CAAC;IAdD,IAAW,YAAY,KAAmB,OAAO,IAAI,CAAC,CAAC,CAAC;IAExD,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAYM,WAAW;QACd,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAEM,IAAI,CAAC,MAAoB;QAC5B,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAClD,CAAC;CACJ;AA1BD,gCA0BC;AAED,MAAM,oBAAoB;IACtB,YAA6B,SAAqB;QAArB,cAAS,GAAT,SAAS,CAAY;QAC1C,YAAO,GAAG,IAAI,uBAAQ,EAAQ,CAAC;IADe,CAAC;IAGhD,KAAK,CAAC,QAAQ;QACjB,OAAO,OAAO,CAAC,IAAI,CAAC;YAChB,IAAI,CAAC,OAAO,CAAC,OAAO;YACpB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;SACjF,CAAC,CAAC;IACP,CAAC;IAEM,MAAM;QACT,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,IAAI,uBAAQ,EAAQ,CAAC;IACxC,CAAC;CACJ;AAeD,qFAAqF;AACrF,uEAAuE;AACvE,IAAK,iBAKJ;AALD,WAAK,iBAAiB;IAClB,uFAAmB,CAAA;IACnB,+EAAe,CAAA;IACf,yFAAoB,CAAA;IACpB,iFAAgB,CAAA;AACpB,CAAC,EALI,iBAAiB,KAAjB,iBAAiB,QAKrB;AAYD,MAAa,WAAW;IAiCpB,YACqB,YAAiC,EAClD,QAA8B,EACb,UAAyC;IAC1D;;;;;;;;;OASG;IACc,gBAAgE;IACjF,0GAA0G;IAC1G,oDAAoD;IACnC,aAAyC,EACzC,OAA4B,EAC7C,eAA8B,EAAE;QAlBf,iBAAY,GAAZ,YAAY,CAAqB;QAEjC,eAAU,GAAV,UAAU,CAA+B;QAWzC,qBAAgB,GAAhB,gBAAgB,CAAgD;QAGhE,kBAAa,GAAb,aAAa,CAA4B;QACzC,YAAO,GAAP,OAAO,CAAqB;QAtCjD;;;;WAIG;QACc,iBAAY,GAA6B,IAAI,GAAG,EAAE,CAAC;QAEpE;;;WAGG;QACc,gBAAW,GAA0B,IAAI,GAAG,EAAE,CAAC;QAE/C,mBAAc,GAAG,IAAI,oBAAoB,CAAC,IAAI,qBAAS,CACpE,EAAE,GAAG,IAAI,EAAE,sBAAsB;QACjC,EAAE,GAAG,IAAI,EAAE,mBAAmB;QAC9B,4EAA4E;QAC5E,IAAA,6BAAiB,EAAC,EAAE,WAAW,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAC1D,CAAC,CAAC;QAuBC,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QACrE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEzC,iEAAiE;QACjE,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE;YACtD,MAAM,IAAI,GAAG,IAAA,6BAAc,EAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAClD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE;gBAC3B,IAAI;gBACJ,MAAM,EAAE,iBAAiB,CAAC,oBAAoB;gBAC9C,OAAO,EAAE,IAAI,uBAAQ,EAAE;gBACvB,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC;aAC1C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAY,qBAAqB;QAC7B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;aACxC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,iBAAiB,CAAC,oBAAoB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAW,wBAAwB;QAC/B,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,WAAW;QACpB,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC9E,MAAM,kCAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE;YAC3C,SAAS,EAAE,uBAAuB;YAClC,KAAK,EAAE,cAAc,CAAC,MAAM;SAC/B,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAC1C,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAC7B,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,cAAc;QAClB,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,eAAe,EAAE;gBACpD,kGAAkG;gBAClG,oCAAoC;gBACpC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;aACrC;SACJ;IACL,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,MAAc;QACpC,OAAO,IAAI,WAAW,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC;IAChD,CAAC;IAED;;;OAGG;IACH,IAAY,UAAU;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAqB,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;QAErE,mEAAmE;QACnE,MAAM,qBAAqB,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEpD,uGAAuG;QACvG,oGAAoG;QACpG,IAAA,qBAAM,EAAC,CAAC,qBAAqB,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAChG,KAAK,CAAC,gGAAgG,CAAC,CAAC;QAE5G,OAAO,GAAkB,CAAC;IAC9B,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,MAAc;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,OAAO,EAAE;YACT,OAAO,OAAO,CAAC,IAAI,CAAC;SACvB;QACD,IAAI,SAAS,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EAAE;YACnD,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAE7E,oFAAoF;YACpF,uFAAuF;YACvF,SAAS,GAAG,MAAM,CAAC;SACtB;aAAM;YACH,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAA,qBAAM,EAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,gCAAgC,CAAC,CAAC;SAC/D;QAED,+GAA+G;QAC/G,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnD,OAAO,kCAAgB,CAAC,cAAc,CAClC,IAAI,CAAC,MAAM,EACX,EAAE,SAAS,EAAE,oBAAoB,EAAE,EAAE,EAAE,SAAS,EAAE,EAClD,KAAK,IAAI,EAAE;YACP,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACjD,CAAC,EACD,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CACjC,CAAC;IACN,CAAC;IAEO,aAAa,CAAC,EAAU;QAC5B,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAC1D,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACpD,OAAO,IAAI,UAAU,CACjB,GAAG,WAAW,CAAC,QAAQ,IAAI,EAAE,EAAE,EAC/B,IAAI,CAAC,YAAY,EACjB,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAC/B,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,IAAqB;QAClD,oFAAoF;QACpF,qFAAqF;QACrF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,IAAqB;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EAAE;YACnD,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;SACxC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,SAAS,EAAE;YACpD,oDAAoD;YACpD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;YAC1E,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;SAChF;QACD,IAAA,qBAAM,EAAC,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EACpD,KAAK,CAAC,8EAA8E,CAAC,CAAC;QAE1F,yFAAyF;QACzF,mGAAmG;QACnG,MAAM,OAAO,GAAG,IAAA,SAAI,GAAE,CAAC;QACvB,MAAM,YAAY,GAAgB;YAC9B,IAAI;YACJ,MAAM,EAAE,iBAAiB,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,uBAAQ,EAAE;YACvB,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC;SAC1C,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAE7C,OAAO,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,OAAe,EAAE,IAAqB;QAC3D,OAAO,kCAAgB,CAAC,cAAc,CAClC,IAAI,CAAC,MAAM,EACX,EAAE,SAAS,EAAE,YAAY,EAAE,EAC3B,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAC9C,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CACtE,CAAC,IAAI,CACF,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,EACrD,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CACnD,CAAC;IACN,CAAC;IAEO,eAAe,CAAC,OAAe,EAAE,QAA6B;;QAClE,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAA,qBAAM,EAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,MAAM,MAAK,iBAAiB,CAAC,mBAAmB;YAC1D,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,MAAM,MAAK,iBAAiB,CAAC,oBAAoB,EACxD,KAAK,CAAC,oDAAoD,CAAC,CAAC;QAChE,KAAK,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACxB,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB,EAAE;gBACxD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;oBAClC,6DAA6D;oBAC7D,+CAA+C;oBAC/C,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;iBACrC;qBAAM;oBACH,iGAAiG;oBACjG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;wBACpC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;qBACtC;oBACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,MAAA,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,mCAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC7F,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,eAAe,CAAC;iBACpD;aACJ;iBAAM,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,oBAAoB,EAAE;gBAChE,8FAA8F;gBAC9F,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;aACrD;SACJ;aAAM;YACH,iDAAiD;YACjD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,oCAAoC,EAAE,CAAC,CAAC;YACpF,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB,EAAE;gBACxD,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;aACrC;YACD,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;SACrD;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,OAAe,EAAE,KAAK;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAA,qBAAM,EAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,kEAAkE,CAAC,CAAC;QAC1F,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB,EAAE;gBACxD,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;aACrC;YACD,2FAA2F;YAC3F,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACtG,OAAO,KAAK,CAAC,OAAO,CAAC;SACxB;aAAM;YACH,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,MAAM,KAAK,CAAC;SACf;IACL,CAAC;IAEO,mBAAmB,CAAC,OAAe;QACvC,IAAA,qBAAM,EAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAChH,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAA,qBAAM,EAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACnD,IAAA,qBAAM,EAAC,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,iBAAiB,CAAC,eAAe,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EACpG,KAAK,CAAC,+DAA+D,CAAC,CAAC;QAE3E,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB;YACjE,CAAC,CAAC,iBAAiB,CAAC,oBAAoB;YACxC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;QAEzC,uGAAuG;QACvG,2GAA2G;QAC3G,yFAAyF;QACzF,4GAA4G;QAC5G,YAAY;QACZ,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAChD,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACI,QAAQ,CAAC,QAA6C;QACzD,IAAA,qBAAM,EAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACnE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAA2C,QAAQ,CAAC;QAC7E,IAAI,CAAC,MAAM,EAAE;YACT,IAAA,qBAAM,EAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,4DAA4D,CAAC,CAAC;YACtF,iFAAiF;YACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpD,IAAA,qBAAM,EAAC,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,MAAM,MAAK,iBAAiB,CAAC,gBAAgB;gBAC9D,CAAC,CAAC,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAA,EAAE,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACpG,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;SACjE;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAEM,mBAAmB,CAAC,OAAkC,EAAE,KAAc;;QACzE,IAAA,qBAAM,EAAC,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,0CAAE,MAAM,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC7E,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE;YACxC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SAC7E;QACD,gCAAgC;QAChC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEzE,IAAI,KAAK,EAAE;YACP,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE;gBACxC,0EAA0E;gBAC1E,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACnE,IAAA,qBAAM,EAAC,CAAC,CAAC,YAAY,EAAE,KAAK,CAAC,qDAAqD,CAAC,CAAC;gBACpF,YAAY,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC7B,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACxD,IAAA,qBAAM,EACF,gBAAgB,KAAK,SAAS,EAC9B,KAAK,CACR,CAAC;oBAEF,2EAA2E;oBAC3E,IAAI,gBAAgB,CAAC,MAAM,KAAK,iBAAiB,CAAC,eAAe,EAAE;wBAC/D,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC9E,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;qBACrC;gBACL,CAAC,CAAC,CAAC;aACN;iBAAM;gBACH,oEAAoE;gBACpE,IAAA,qBAAM,EAAC,CAAA,MAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,0CAAE,MAAM,MAAK,iBAAiB,CAAC,gBAAgB,EACjG,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBAC5D,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACtD;SACJ;IACL,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,KAAK,CAAC,IAAI,CACpB,SAAoC,EACpC,YAAyD;QAEzD,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,EAAE,CAAC;SACb;QACD,IAAI,aAAa,CAAC;QAClB,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAC5D,IAAI,OAAO,EAAE;YACT,aAAa,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;SAC/C;QACD,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC;aACtC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,IAAI,CAAC,QAA8B;;QACvC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;YAC3B,SAAS,EAAE,uBAAuB;YAClC,KAAK,EAAE,MAAA,MAAA,QAAQ,CAAC,GAAG,0CAAE,MAAM,mCAAI,CAAC;YAChC,aAAa,EAAE,MAAA,QAAQ,CAAC,aAAa,0CAAE,MAAM;SAChD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,GAAG,CAA6B,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC1E,IAAI,QAAQ,CAAC,GAAG,EAAE;YACd,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,CAAC;YACnE,yEAAyE;YACzE,6CAA6C;YAC7C,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;SACnF;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACI,SAAS,CAAC,SAAkB,KAAK;QACpC,MAAM,MAAM,GAA2B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACvD;;;YAGI;QACJ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,MAAc,EAAE,EAAE;YACvC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,yFAAyF;QACzF,6GAA6G;QAC7G,kGAAkG;QAClG,4GAA4G;QAC5G,qDAAqD;QACrD,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE;YACnD,IAAA,qBAAM,EAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACjE,oGAAoG;YACpG,kDAAkD;YAClD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC;SACzF;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACI,kBAAkB,CAAC,YAAsB;;QAC5C,kFAAkF;QAClF,sCAAsC;QACtC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE;YAC9B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAA,qBAAM,EACF,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,QAAQ,EAC/D,KAAK,CAAC,8CAA8C,CACvD,CAAC;YACF,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAE5B,mGAAmG;YACnG,mFAAmF;YACnF,IAAI,MAAA,IAAI,CAAC,aAAa,0CAAE,GAAG,CAAC,MAAM,CAAC,EAAE;gBACjC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAClC,SAAS;aACZ;SACJ;IACL,CAAC;IAEM,SAAS,CAAC,gBAAoC;QACjD,gHAAgH;QAChH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/G,MAAM,OAAO,GAAG,IAAI,kCAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACvB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,wEAAwE;QACxE,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE;YAC1C,OAAO,CAAC,OAAO,CACX,WAAW,CAAC,qBAAqB;YACjC,8BAA8B;YAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;iBAClD,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAChE,CAAC;SACL;QAED,OAAO,OAAO,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAEM,gBAAgB,CAAC,KAA0B;QAC9C,IAAA,qBAAM,EAAC,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EACpD,KAAK,CAAC,4DAA4D,CAAC,CAAC;QACxE,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EACzC,KAAK,CAAC,iEAAiE,CAAC,CAAC;QAC7E,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,KAAK,EAAE;YACtC,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACzF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC3C,gCAAgC;YAChC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;SAChD;IACL,CAAC;IAEM,eAAe;QAClB,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE;YAC1C,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAA,6BAAc,EAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;SAC/D;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;;AAneL,kCAoeC;AAne0B,oBAAQ,GAAG,QAAQ,CAAC;AACnB,iCAAqB,GAAG,gBAAgB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { v4 as uuid } from \"uuid\";\nimport { IFluidHandle, IFluidHandleContext } from \"@fluidframework/core-interfaces\";\nimport { IDocumentStorageService } from \"@fluidframework/driver-definitions\";\nimport { ICreateBlobResponse, ISequencedDocumentMessage, ISnapshotTree } from \"@fluidframework/protocol-definitions\";\nimport { generateHandleContextPath, SummaryTreeBuilder } from \"@fluidframework/runtime-utils\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, bufferToString, Deferred, stringToBuffer, TypedEventEmitter } from \"@fluidframework/common-utils\";\nimport { IContainerRuntime, IContainerRuntimeEvents } from \"@fluidframework/container-runtime-definitions\";\nimport { AttachState } from \"@fluidframework/container-definitions\";\nimport { ChildLogger, PerformanceEvent } from \"@fluidframework/telemetry-utils\";\nimport {\n    IGarbageCollectionData,\n    ISummaryTreeWithStats,\n    ITelemetryContext,\n} from \"@fluidframework/runtime-definitions\";\nimport { Throttler, formExponentialFn, IThrottler } from \"./throttler\";\n\n/**\n * This class represents blob (long string)\n * This object is used only when creating (writing) new blob and serialization purposes.\n * De-serialization process goes through FluidObjectHandle and request flow:\n * DataObject.request() recognizes requests in the form of `/blobs/<id>`\n * and loads blob.\n */\nexport class BlobHandle implements IFluidHandle<ArrayBufferLike> {\n    private attached: boolean = false;\n\n    public get IFluidHandle(): IFluidHandle { return this; }\n\n    public get isAttached(): boolean {\n        return this.attached;\n    }\n\n    public readonly absolutePath: string;\n\n    constructor(\n        public readonly path: string,\n        public readonly routeContext: IFluidHandleContext,\n        public get: () => Promise<any>,\n    ) {\n        this.absolutePath = generateHandleContextPath(path, this.routeContext);\n    }\n\n    public attachGraph() {\n        this.attached = true;\n    }\n\n    public bind(handle: IFluidHandle) {\n        throw new Error(\"Cannot bind to blob handle\");\n    }\n}\n\nclass CancellableThrottler {\n    constructor(private readonly throttler: IThrottler) { }\n    private cancelP = new Deferred<void>();\n\n    public async getDelay(): Promise<void> {\n        return Promise.race([\n            this.cancelP.promise,\n            new Promise<void>((resolve) => setTimeout(resolve, this.throttler.getDelay())),\n        ]);\n    }\n\n    public cancel() {\n        this.cancelP.resolve();\n        this.cancelP = new Deferred<void>();\n    }\n}\n\n/**\n * Information from a snapshot needed to load BlobManager\n */\nexport interface IBlobManagerLoadInfo {\n    ids?: string[];\n    redirectTable?: [string, string][];\n}\n\n// Restrict the IContainerRuntime interface to the subset required by BlobManager.  This helps to make\n// the contract explicit and reduces the amount of mocking required for tests.\nexport type IBlobManagerRuntime =\n    Pick<IContainerRuntime, \"attachState\" | \"connected\" | \"logger\"> & TypedEventEmitter<IContainerRuntimeEvents>;\n\n// Note that while offline we \"submit\" an op before uploading the blob, but we always\n// expect blobs to be uploaded before we actually see the op round-trip\nenum PendingBlobStatus {\n    OnlinePendingUpload,\n    OnlinePendingOp,\n    OfflinePendingUpload,\n    OfflinePendingOp,\n}\n\ninterface PendingBlob {\n    blob: ArrayBufferLike;\n    status: PendingBlobStatus;\n    storageId?: string;\n    handleP: Deferred<IFluidHandle<ArrayBufferLike>>;\n    uploadP: Promise<ICreateBlobResponse>;\n}\n\nexport interface IPendingBlobs { [id: string]: { blob: string; }; }\n\nexport class BlobManager {\n    public static readonly basePath = \"_blobs\";\n    private static readonly redirectTableBlobName = \".redirectTable\";\n    private readonly logger: ITelemetryLogger;\n\n    /**\n     * Map of local (offline/detached) IDs to storage IDs. Contains identity entries\n     * (id → id) for storage IDs, so all requested IDs should be a key in this map.\n     * Blobs created while the container is detached are stored in IDetachedBlobStorage\n     * which gives local IDs; the storage IDs are filled in at attach time.\n     */\n    private readonly redirectTable: Map<string, string | undefined>;\n\n    /**\n     * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.\n     * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so\n     * we must save it. This is true for both the online and offline flow.\n     */\n    private readonly pendingBlobs: Map<string, PendingBlob> = new Map();\n\n    /**\n     * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops\n     * don't include local ID in online flow.\n     */\n    private readonly opsInFlight: Map<string, string[]> = new Map();\n\n    private readonly retryThrottler = new CancellableThrottler(new Throttler(\n        60 * 1000, // 60 sec delay window\n        30 * 1000, // 30 sec max delay\n        // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)\n        formExponentialFn({ coefficient: 20, initialDelay: 0 }),\n    ));\n\n    constructor(\n        private readonly routeContext: IFluidHandleContext,\n        snapshot: IBlobManagerLoadInfo,\n        private readonly getStorage: () => IDocumentStorageService,\n        /**\n         * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which\n         * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server\n         * will then not delete the blob as long as it is listed as referenced in future summaries.\n         * The summarizing client will know to include the storage ID in the summary when it sees the op.\n         *\n         * The op may also include a local ID to inform all clients of the relation to the storage\n         * ID, without knowledge of which they cannot request the blob from storage. This is also\n         * included in the redirect table in the summary.\n         */\n        private readonly sendBlobAttachOp: (storageId?: string, localId?: string) => void,\n        // To be called when a blob node is requested. blobPath is the path of the blob's node in GC's graph. It's\n        // of the format `/<BlobManager.basePath>/<blobId>`.\n        private readonly gcNodeUpdated: (blobPath: string) => void,\n        private readonly runtime: IBlobManagerRuntime,\n        stashedBlobs: IPendingBlobs = {},\n    ) {\n        this.logger = ChildLogger.create(this.runtime.logger, \"BlobManager\");\n        this.runtime.on(\"disconnected\", () => this.onDisconnected());\n        this.redirectTable = this.load(snapshot);\n\n        // Begin uploading stashed blobs from previous container instance\n        Object.entries(stashedBlobs).forEach(([localId, entry]) => {\n            const blob = stringToBuffer(entry.blob, \"base64\");\n            this.pendingBlobs.set(localId, {\n                blob,\n                status: PendingBlobStatus.OfflinePendingUpload,\n                handleP: new Deferred(),\n                uploadP: this.uploadBlob(localId, blob),\n            });\n        });\n    }\n\n    private get pendingOfflineUploads() {\n        return Array.from(this.pendingBlobs.values())\n            .filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);\n    }\n\n    public get hasPendingOfflineUploads(): boolean {\n        return this.pendingOfflineUploads.length > 0;\n    }\n\n    /**\n     * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.\n     */\n    public async onConnected() {\n        this.retryThrottler.cancel();\n        const pendingUploads = this.pendingOfflineUploads.map(async (e) => e.uploadP);\n        await PerformanceEvent.timedExecAsync(this.logger, {\n                eventName: \"BlobUploadOnConnected\",\n                count: pendingUploads.length,\n            }, async () => Promise.all(pendingUploads),\n            { start: true, end: true },\n        );\n    }\n\n    /**\n     * Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected\n     * again\n     */\n    private onDisconnected() {\n        for (const [localId, entry] of this.pendingBlobs) {\n            if (entry.status === PendingBlobStatus.OnlinePendingOp) {\n                // This will submit another BlobAttach op for this blob. This is necessary because the one we sent\n                // already didn't have the local ID.\n                this.transitionToOffline(localId);\n            }\n        }\n    }\n\n    /**\n     * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`\n     * This path must match the path of the blob handle returned by the createBlob API because blobs are marked\n     * referenced by storing these handles in a referenced DDS.\n     */\n    private getBlobGCNodePath(blobId: string) {\n        return `/${BlobManager.basePath}/${blobId}`;\n    }\n\n    /**\n     * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is\n     * detached or there are no (non-pending) attachment blobs in the document\n     */\n    private get storageIds(): Set<string> {\n        const ids = new Set<string | undefined>(this.redirectTable.values());\n\n        // If we are detached, we will not have storage IDs, only undefined\n        const undefinedValueInTable = ids.delete(undefined);\n\n        // For a detached container, entries are inserted into the redirect table with an undefined storage ID.\n        // For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.\n        assert(!undefinedValueInTable || this.runtime.attachState === AttachState.Detached && ids.size === 0,\n            0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */);\n\n        return ids as Set<string>;\n    }\n\n    public async getBlob(blobId: string): Promise<ArrayBufferLike> {\n        const pending = this.pendingBlobs.get(blobId);\n        if (pending) {\n            return pending.blob;\n        }\n        let storageId;\n        if (this.runtime.attachState === AttachState.Detached) {\n            assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);\n\n            // Blobs created while the container is detached are stored in IDetachedBlobStorage.\n            // The 'IDocumentStorageService.readBlob()' call below will retrieve these via localId.\n            storageId = blobId;\n        } else {\n            storageId = this.redirectTable.get(blobId);\n            assert(!!storageId, 0x11f /* \"requesting unknown blobs\" */);\n        }\n\n        // When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.\n        this.gcNodeUpdated(this.getBlobGCNodePath(blobId));\n\n        return PerformanceEvent.timedExecAsync(\n            this.logger,\n            { eventName: \"AttachmentReadBlob\", id: storageId },\n            async () => {\n                return this.getStorage().readBlob(storageId);\n            },\n            { end: true, cancel: \"error\" },\n        );\n    }\n\n    private getBlobHandle(id: string): IFluidHandle<ArrayBufferLike> {\n        assert(this.redirectTable.has(id) || this.pendingBlobs.has(id),\n            0x384 /* requesting handle for unknown blob */);\n        return new BlobHandle(\n            `${BlobManager.basePath}/${id}`,\n            this.routeContext,\n            async () => this.getBlob(id),\n        );\n    }\n\n    private async createBlobDetached(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {\n        // Blobs created while the container is detached are stored in IDetachedBlobStorage.\n        // The 'IDocumentStorageService.createBlob()' call below will respond with a localId.\n        const response = await this.getStorage().createBlob(blob);\n        this.redirectTable.set(response.id, undefined);\n        return this.getBlobHandle(response.id);\n    }\n\n    public async createBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {\n        if (this.runtime.attachState === AttachState.Detached) {\n            return this.createBlobDetached(blob);\n        }\n        if (this.runtime.attachState === AttachState.Attaching) {\n            // blob upload is not supported in \"Attaching\" state\n            this.logger.sendTelemetryEvent({ eventName: \"CreateBlobWhileAttaching\" });\n            await new Promise<void>((resolve) => this.runtime.once(\"attached\", resolve));\n        }\n        assert(this.runtime.attachState === AttachState.Attached,\n            0x385 /* For clarity and paranoid defense against adding future attachment states */);\n\n        // Create a local ID for each blob. This is used to support blobs if/when the client goes\n        // offline since we don't have the ID from storage yet. If online flow succeeds this won't be used.\n        const localId = uuid();\n        const pendingEntry: PendingBlob = {\n            blob,\n            status: PendingBlobStatus.OnlinePendingUpload,\n            handleP: new Deferred(),\n            uploadP: this.uploadBlob(localId, blob),\n        };\n        this.pendingBlobs.set(localId, pendingEntry);\n\n        return pendingEntry.handleP.promise;\n    }\n\n    private async uploadBlob(localId: string, blob: ArrayBufferLike): Promise<ICreateBlobResponse> {\n        return PerformanceEvent.timedExecAsync(\n            this.logger,\n            { eventName: \"createBlob\" },\n            async () => this.getStorage().createBlob(blob),\n            { end: true, cancel: this.runtime.connected ? \"error\" : \"generic\" },\n        ).then(\n            (response) => this.onUploadResolve(localId, response),\n            async (err) => this.onUploadReject(localId, err),\n        );\n    }\n\n    private onUploadResolve(localId: string, response: ICreateBlobResponse) {\n        const entry = this.pendingBlobs.get(localId);\n        assert(entry?.status === PendingBlobStatus.OnlinePendingUpload ||\n            entry?.status === PendingBlobStatus.OfflinePendingUpload,\n            0x386 /* Must have pending blob entry for uploaded blob */);\n        entry.storageId = response.id;\n        if (this.runtime.connected) {\n            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {\n                if (this.storageIds.has(response.id)) {\n                    // Storage may dedupe blobs and give us an ID we already know\n                    // no need to submit BlobAttach op in this case\n                    entry.handleP.resolve(this.getBlobHandle(response.id));\n                    this.pendingBlobs.delete(localId);\n                } else {\n                    // Check for still-pending duplicates too; if an op is already in flight we can wait for that one\n                    if (!this.opsInFlight.has(response.id)) {\n                        this.sendBlobAttachOp(response.id);\n                    }\n                    this.opsInFlight.set(response.id, (this.opsInFlight.get(response.id) ?? []).concat(localId));\n                    entry.status = PendingBlobStatus.OnlinePendingOp;\n                }\n            } else if (entry.status === PendingBlobStatus.OfflinePendingUpload) {\n                // We already submitted a BlobAttach op for this blob when it was transitioned to offline flow\n                entry.status = PendingBlobStatus.OfflinePendingOp;\n            }\n        } else {\n            // connected to storage but not ordering service?\n            this.logger.sendTelemetryEvent({ eventName: \"BlobUploadSuccessWhileDisconnected\" });\n            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {\n                this.transitionToOffline(localId);\n            }\n            entry.status = PendingBlobStatus.OfflinePendingOp;\n        }\n        return response;\n    }\n\n    private async onUploadReject(localId: string, error) {\n        const entry = this.pendingBlobs.get(localId);\n        assert(!!entry, 0x387 /* Must have pending blob entry for blob which failed to upload */);\n        if (!this.runtime.connected) {\n            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {\n                this.transitionToOffline(localId);\n            }\n            // we are probably not connected to storage but start another upload request in case we are\n            entry.uploadP = this.retryThrottler.getDelay().then(async () => this.uploadBlob(localId, entry.blob));\n            return entry.uploadP;\n        } else {\n            entry.handleP.reject(error);\n            throw error;\n        }\n    }\n\n    private transitionToOffline(localId: string) {\n        assert(!this.runtime.connected, 0x388 /* Must only transition to offline flow while runtime is disconnected */);\n        const entry = this.pendingBlobs.get(localId);\n        assert(!!entry, 0x389 /* No pending blob entry */);\n        assert([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status),\n            0x38a /* Blob must be in online flow to transition to offline flow */);\n\n        entry.status = entry.status === PendingBlobStatus.OnlinePendingUpload\n            ? PendingBlobStatus.OfflinePendingUpload\n            : PendingBlobStatus.OfflinePendingOp;\n\n        // Submit a BlobAttach op. It's important we submit this op now before returning the blob handle so the\n        // BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise an invalid handle could be\n        // added to the document if the ops are not all successfully submitted upon reconnection.\n        // storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()\n        // is called\n        this.sendBlobAttachOp(entry.storageId, localId);\n        entry.handleP.resolve(this.getBlobHandle(localId));\n    }\n\n    /**\n     * Resubmit a BlobAttach op. Used to add storage IDs to ops that were\n     * submitted to runtime while disconnected.\n     * @param metadata - op metadata containing storage and/or local IDs\n     */\n    public reSubmit(metadata: Record<string, unknown> | undefined) {\n        assert(!!metadata, 0x38b /* Resubmitted ops must have metadata */);\n        const { blobId, localId }: { blobId?: string; localId?: string; } = metadata;\n        if (!blobId) {\n            assert(!!localId, 0x38c /* Submitted BlobAttach ops must have a blobId or localId */);\n            // We submitted this op while offline. The blob should have been uploaded by now.\n            const pendingEntry = this.pendingBlobs.get(localId);\n            assert(pendingEntry?.status === PendingBlobStatus.OfflinePendingOp &&\n                !!pendingEntry?.storageId, 0x38d /* blob must be uploaded before resubmitting BlobAttach op */);\n            return this.sendBlobAttachOp(pendingEntry.storageId, localId);\n        }\n        return this.sendBlobAttachOp(blobId, localId);\n    }\n\n    public processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean) {\n        assert(message?.metadata?.blobId, 0x12a /* \"Missing blob id on metadata\" */);\n        if (message.metadata.localId !== undefined) {\n            this.redirectTable.set(message.metadata.localId, message.metadata.blobId);\n        }\n        // set identity (id -> id) entry\n        this.redirectTable.set(message.metadata.blobId, message.metadata.blobId);\n\n        if (local) {\n            if (message.metadata.localId === undefined) {\n                // Since there is no local ID, we know this op was submitted while online.\n                const waitingBlobs = this.opsInFlight.get(message.metadata.blobId);\n                assert(!!waitingBlobs, 0x38e /* local online BlobAttach op with no pending blob */);\n                waitingBlobs.forEach((localId) => {\n                    const pendingBlobEntry = this.pendingBlobs.get(localId);\n                    assert(\n                        pendingBlobEntry !== undefined,\n                        0x38f, /* local online BlobAttach op with no pending blob entry */\n                    );\n\n                    // It's possible we transitioned to offline flow while waiting for this op.\n                    if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {\n                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(message.metadata.blobId));\n                        this.pendingBlobs.delete(localId);\n                    }\n                });\n            } else {\n                // Each local ID is unique; get the pending blob entry and delete it\n                assert(this.pendingBlobs.get(message.metadata.localId)?.status === PendingBlobStatus.OfflinePendingOp,\n                    0x1f8 /* \"local BlobAttach op with no pending blob\" */);\n                this.pendingBlobs.delete(message.metadata.localId);\n            }\n        }\n    }\n\n    /**\n     * Reads blobs needed to load BlobManager from storage.\n     * @param blobsTree - Tree containing IDs of previously attached blobs. We\n     * look for the IDs in the blob entries of the tree since the both the r11s\n     * and SPO drivers replace the attachment types returned in snapshot() with blobs.\n     */\n    public static async load(\n        blobsTree: ISnapshotTree | undefined,\n        tryFetchBlob: (id: string) => Promise<[string, string][]>,\n    ): Promise<IBlobManagerLoadInfo> {\n        if (!blobsTree) {\n            return {};\n        }\n        let redirectTable;\n        const tableId = blobsTree.blobs[this.redirectTableBlobName];\n        if (tableId) {\n            redirectTable = await tryFetchBlob(tableId);\n        }\n        const ids = Object.entries(blobsTree.blobs)\n            .filter(([k, _]) => k !== this.redirectTableBlobName).map(([_, v]) => v);\n        return { ids, redirectTable };\n    }\n\n    /**\n     * Load a set of previously attached blob IDs and redirect table from a previous snapshot.\n     */\n    private load(snapshot: IBlobManagerLoadInfo): Map<string, string | undefined> {\n        this.logger.sendTelemetryEvent({\n            eventName: \"AttachmentBlobsLoaded\",\n            count: snapshot.ids?.length ?? 0,\n            redirectTable: snapshot.redirectTable?.length,\n        });\n        const table = new Map<string, string | undefined>(snapshot.redirectTable);\n        if (snapshot.ids) {\n            const detached = this.runtime.attachState === AttachState.Detached;\n            // If we are detached, we don't have storage IDs yet, so set to undefined\n            // Otherwise, set identity (id -> id) entries\n            snapshot.ids.forEach((entry) => table.set(entry, detached ? undefined : entry));\n        }\n        return table;\n    }\n\n    /**\n     * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be\n     * individually referenced by storing its handle in a referenced DDS. Returns the list of blob ids as GC nodes.\n     * @param fullGC - true to bypass optimizations and force full generation of GC data. BlobManager doesn't care\n     * about this for now because the data is a simple list of blob ids.\n     */\n    public getGCData(fullGC: boolean = false): IGarbageCollectionData {\n        const gcData: IGarbageCollectionData = { gcNodes: {} };\n        /**\n          * The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned\n          * by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.\n          */\n        this.storageIds.forEach((blobId: string) => {\n            gcData.gcNodes[this.getBlobGCNodePath(blobId)] = [];\n        });\n\n        // For some blobs, the handle returned on creation is based off of the localId. So, these\n        // nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node\n        // must also be marked referenced. So, we add a route from the localId node to the storageId node.\n        // Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or\n        // a blob may be referenced via its storageId handle.\n        for (const [localId, storageId] of this.redirectTable) {\n            assert(!!storageId, 0x390 /* Must be attached to get GC data */);\n            // Add node for the localId and add a route to the storageId node. The storageId node will have been\n            // added above when adding nodes for this.blobIds.\n            gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];\n        }\n\n        return gcData;\n    }\n\n    /**\n     * When running GC in test mode, this is called to delete blobs that are unused.\n     * @param unusedRoutes - These are the blob node ids that are unused and should be deleted.\n     */\n    public deleteUnusedRoutes(unusedRoutes: string[]): void {\n        // The routes or blob node paths are in the same format as returned in getGCData -\n        // `/<BlobManager.basePath>/<blobId>`.\n        for (const route of unusedRoutes) {\n            const pathParts = route.split(\"/\");\n            assert(\n                pathParts.length === 3 && pathParts[1] === BlobManager.basePath,\n                0x2d5 /* \"Invalid blob node id in unused routes.\" */,\n            );\n            const blobId = pathParts[2];\n\n            // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The\n            // corresponding storageId may still be used either directly or via other localIds.\n            if (this.redirectTable?.has(blobId)) {\n                this.redirectTable.delete(blobId);\n                continue;\n            }\n        }\n    }\n\n    public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {\n        // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached\n        const blobIds = this.storageIds.size > 0 ? Array.from(this.storageIds) : Array.from(this.redirectTable.keys());\n        const builder = new SummaryTreeBuilder();\n        blobIds.forEach((blobId) => {\n            builder.addAttachment(blobId);\n        });\n\n        // Any non-identity entries in the table need to be saved in the summary\n        if (this.redirectTable.size > blobIds.length) {\n            builder.addBlob(\n                BlobManager.redirectTableBlobName,\n                // filter out identity entries\n                JSON.stringify(Array.from(this.redirectTable.entries())\n                    .filter(([localId, storageId]) => localId !== storageId)),\n            );\n        }\n\n        return builder.getSummaryTree();\n    }\n\n    public setRedirectTable(table: Map<string, string>) {\n        assert(this.runtime.attachState === AttachState.Detached,\n            0x252 /* \"redirect table can only be set in detached container\" */);\n        assert(this.redirectTable.size === table.size,\n            0x391 /* Redirect table size must match BlobManager's local ID count */);\n        for (const [localId, storageId] of table) {\n            assert(this.redirectTable.has(localId), 0x254 /* \"unrecognized id in redirect table\" */);\n            this.redirectTable.set(localId, storageId);\n            // set identity (id -> id) entry\n            this.redirectTable.set(storageId, storageId);\n        }\n    }\n\n    public getPendingBlobs(): IPendingBlobs {\n        const blobs = {};\n        for (const [key, entry] of this.pendingBlobs) {\n            blobs[key] = { blob: bufferToString(entry.blob, \"base64\") };\n        }\n        return blobs;\n    }\n}\n"]}
+{"version":3,"file":"blobManager.js","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,+BAAkC;AAIlC,iEAKuC;AACvC,+DAAmH;AAEnH,iFAAoE;AACpE,qEAA8H;AAM9H,2CAAuE;AACvE,yEAAkE;AAClE,6EAAwE;AAExE;;;;;;GAMG;AACH,MAAa,UAAU;IAWnB,YACoB,IAAY,EACZ,YAAiC,EAC1C,GAAuB;QAFd,SAAI,GAAJ,IAAI,CAAQ;QACZ,iBAAY,GAAZ,YAAY,CAAqB;QAC1C,QAAG,GAAH,GAAG,CAAoB;QAb1B,aAAQ,GAAY,KAAK,CAAC;QAe9B,IAAI,CAAC,YAAY,GAAG,IAAA,yCAAyB,EAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3E,CAAC;IAdD,IAAW,YAAY,KAAmB,OAAO,IAAI,CAAC,CAAC,CAAC;IAExD,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAYM,WAAW;QACd,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAEM,IAAI,CAAC,MAAoB;QAC5B,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAClD,CAAC;CACJ;AA1BD,gCA0BC;AAED,MAAM,oBAAoB;IACtB,YAA6B,SAAqB;QAArB,cAAS,GAAT,SAAS,CAAY;QAC1C,YAAO,GAAG,IAAI,uBAAQ,EAAQ,CAAC;IADe,CAAC;IAGhD,KAAK,CAAC,QAAQ;QACjB,OAAO,OAAO,CAAC,IAAI,CAAC;YAChB,IAAI,CAAC,OAAO,CAAC,OAAO;YACpB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;SACjF,CAAC,CAAC;IACP,CAAC;IAEM,MAAM;QACT,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,IAAI,uBAAQ,EAAQ,CAAC;IACxC,CAAC;CACJ;AAeD,qFAAqF;AACrF,uEAAuE;AACvE,IAAK,iBAKJ;AALD,WAAK,iBAAiB;IAClB,uFAAmB,CAAA;IACnB,+EAAe,CAAA;IACf,yFAAoB,CAAA;IACpB,iFAAgB,CAAA;AACpB,CAAC,EALI,iBAAiB,KAAjB,iBAAiB,QAKrB;AAYD,MAAa,WAAW;IAyCpB,YACqB,YAAiC,EAClD,QAA8B,EACb,UAAyC;IAC1D;;;;;;;;;OASG;IACc,gBAAgE;IACjF,0GAA0G;IAC1G,oDAAoD;IACnC,aAAyC,EACzC,OAA4B,EAC7C,eAA8B,EAAE;QAlBf,iBAAY,GAAZ,YAAY,CAAqB;QAEjC,eAAU,GAAV,UAAU,CAA+B;QAWzC,qBAAgB,GAAhB,gBAAgB,CAAgD;QAGhE,kBAAa,GAAb,aAAa,CAA4B;QACzC,YAAO,GAAP,OAAO,CAAqB;QA9CjD;;;;WAIG;QACc,iBAAY,GAA6B,IAAI,GAAG,EAAE,CAAC;QAEpE;;;WAGG;QACc,gBAAW,GAA0B,IAAI,GAAG,EAAE,CAAC;QAE/C,mBAAc,GAAG,IAAI,oBAAoB,CAAC,IAAI,qBAAS,CACpE,EAAE,GAAG,IAAI,EAAE,sBAAsB;QACjC,EAAE,GAAG,IAAI,EAAE,mBAAmB;QAC9B,4EAA4E;QAC5E,IAAA,6BAAiB,EAAC,EAAE,WAAW,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAC1D,CAAC,CAAC;QAIH;;;WAGG;QACc,oBAAe,GAAgB,IAAI,GAAG,EAAE,CAAC;QAuBtD,IAAI,CAAC,EAAE,GAAG,IAAA,2CAAyB,EAAC,6BAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;QAC5F,wFAAwF;QACxF,IAAI,CAAC,qBAAqB;YACtB,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,qDAAwB,CAAC,KAAK,IAAI;gBAC5D,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,KAAK,+CAAoB,CAAC;QAE7D,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEzC,iEAAiE;QACjE,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE;YACtD,MAAM,IAAI,GAAG,IAAA,6BAAc,EAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAClD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE;gBAC3B,IAAI;gBACJ,MAAM,EAAE,iBAAiB,CAAC,oBAAoB;gBAC9C,OAAO,EAAE,IAAI,uBAAQ,EAAE;gBACvB,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC;aAC1C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAY,qBAAqB;QAC7B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;aACxC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,iBAAiB,CAAC,oBAAoB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAW,wBAAwB;QAC/B,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,WAAW;QACpB,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC9E,MAAM,kCAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YAC9C,SAAS,EAAE,uBAAuB;YAClC,KAAK,EAAE,cAAc,CAAC,MAAM;SAC/B,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAC1C,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAC7B,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,cAAc;QAClB,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,eAAe,EAAE;gBACpD,kGAAkG;gBAClG,oCAAoC;gBACpC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;aACrC;SACJ;IACL,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,MAAc;QACpC,OAAO,IAAI,WAAW,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC;IAChD,CAAC;IAED;;;OAGG;IACH,IAAY,UAAU;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAqB,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;QAErE,mEAAmE;QACnE,MAAM,qBAAqB,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEpD,uGAAuG;QACvG,oGAAoG;QACpG,IAAA,qBAAM,EAAC,CAAC,qBAAqB,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAChG,KAAK,CAAC,gGAAgG,CAAC,CAAC;QAE5G,OAAO,GAAkB,CAAC;IAC9B,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,MAAc;QAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAG;YACnC,MAAM,KAAK,GAAG,IAAA,mCAAmB,EAAC,IAAA,mCAAmB,EAAC,GAAG,EAAE,oBAAoB,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;YACpG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC;gBAC1B,SAAS,EAAE,6BAA6B;gBACxC,GAAG,EAAE,OAAO,CAAC,GAAG;aACnB,EAAE,KAAK,CAAC,CAAC;YACV,IAAI,IAAI,CAAC,qBAAqB,EAAE;gBAC5B,MAAM,KAAK,CAAC;aACf;SACJ;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,OAAO,EAAE;YACT,OAAO,OAAO,CAAC,IAAI,CAAC;SACvB;QACD,IAAI,SAAS,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EAAE;YACnD,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAE7E,oFAAoF;YACpF,uFAAuF;YACvF,SAAS,GAAG,MAAM,CAAC;SACtB;aAAM;YACH,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAA,qBAAM,EAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,gCAAgC,CAAC,CAAC;SAC/D;QAED,+GAA+G;QAC/G,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnD,OAAO,kCAAgB,CAAC,cAAc,CAClC,IAAI,CAAC,EAAE,CAAC,MAAM,EACd,EAAE,SAAS,EAAE,oBAAoB,EAAE,EAAE,EAAE,SAAS,EAAE,EAClD,KAAK,IAAI,EAAE;YACP,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACjD,CAAC,EACD,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CACjC,CAAC;IACN,CAAC;IAEO,aAAa,CAAC,EAAU;QAC5B,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAC1D,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACpD,OAAO,IAAI,UAAU,CACjB,GAAG,WAAW,CAAC,QAAQ,IAAI,EAAE,EAAE,EAC/B,IAAI,CAAC,YAAY,EACjB,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAC/B,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,IAAqB;QAClD,oFAAoF;QACpF,qFAAqF;QACrF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,IAAqB;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EAAE;YACnD,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;SACxC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,SAAS,EAAE;YACpD,oDAAoD;YACpD,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;YAC7E,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;SAChF;QACD,IAAA,qBAAM,EAAC,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EACpD,KAAK,CAAC,8EAA8E,CAAC,CAAC;QAE1F,yFAAyF;QACzF,mGAAmG;QACnG,MAAM,OAAO,GAAG,IAAA,SAAI,GAAE,CAAC;QACvB,MAAM,YAAY,GAAgB;YAC9B,IAAI;YACJ,MAAM,EAAE,iBAAiB,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,uBAAQ,EAAE;YACvB,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC;SAC1C,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAE7C,OAAO,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,OAAe,EAAE,IAAqB;QAC3D,OAAO,kCAAgB,CAAC,cAAc,CAClC,IAAI,CAAC,EAAE,CAAC,MAAM,EACd,EAAE,SAAS,EAAE,YAAY,EAAE,EAC3B,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAC9C,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CACtE,CAAC,IAAI,CACF,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,EACrD,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CACnD,CAAC;IACN,CAAC;IAEO,eAAe,CAAC,OAAe,EAAE,QAA6B;;QAClE,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAA,qBAAM,EAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,MAAM,MAAK,iBAAiB,CAAC,mBAAmB;YAC1D,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,MAAM,MAAK,iBAAiB,CAAC,oBAAoB,EACxD,KAAK,CAAC,oDAAoD,CAAC,CAAC;QAChE,KAAK,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACxB,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB,EAAE;gBACxD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;oBAClC,6DAA6D;oBAC7D,+CAA+C;oBAC/C,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;iBACrC;qBAAM;oBACH,iGAAiG;oBACjG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;wBACpC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;qBACtC;oBACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,MAAA,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,mCAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC7F,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,eAAe,CAAC;iBACpD;aACJ;iBAAM,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,oBAAoB,EAAE;gBAChE,8FAA8F;gBAC9F,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;aACrD;SACJ;aAAM;YACH,iDAAiD;YACjD,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,oCAAoC,EAAE,CAAC,CAAC;YACvF,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB,EAAE;gBACxD,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;aACrC;YACD,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;SACrD;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,OAAe,EAAE,KAAK;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAA,qBAAM,EAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,kEAAkE,CAAC,CAAC;QAC1F,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB,EAAE;gBACxD,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;aACrC;YACD,2FAA2F;YAC3F,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACtG,OAAO,KAAK,CAAC,OAAO,CAAC;SACxB;aAAM;YACH,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,MAAM,KAAK,CAAC;SACf;IACL,CAAC;IAEO,mBAAmB,CAAC,OAAe;QACvC,IAAA,qBAAM,EAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAChH,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAA,qBAAM,EAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACnD,IAAA,qBAAM,EAAC,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,iBAAiB,CAAC,eAAe,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EACpG,KAAK,CAAC,+DAA+D,CAAC,CAAC;QAE3E,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,iBAAiB,CAAC,mBAAmB;YACjE,CAAC,CAAC,iBAAiB,CAAC,oBAAoB;YACxC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;QAEzC,uGAAuG;QACvG,2GAA2G;QAC3G,yFAAyF;QACzF,4GAA4G;QAC5G,YAAY;QACZ,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAChD,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACI,QAAQ,CAAC,QAA6C;QACzD,IAAA,qBAAM,EAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACnE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAA2C,QAAQ,CAAC;QAC7E,IAAI,CAAC,MAAM,EAAE;YACT,IAAA,qBAAM,EAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,4DAA4D,CAAC,CAAC;YACtF,iFAAiF;YACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpD,IAAA,qBAAM,EAAC,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,MAAM,MAAK,iBAAiB,CAAC,gBAAgB;gBAC9D,CAAC,CAAC,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAA,EAAE,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACpG,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;SACjE;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAEM,mBAAmB,CAAC,OAAkC,EAAE,KAAc;;QACzE,IAAA,qBAAM,EAAC,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,0CAAE,MAAM,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC7E,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE;YACxC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SAC7E;QACD,gCAAgC;QAChC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEzE,IAAI,KAAK,EAAE;YACP,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE;gBACxC,0EAA0E;gBAC1E,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACnE,IAAA,qBAAM,EAAC,CAAC,CAAC,YAAY,EAAE,KAAK,CAAC,qDAAqD,CAAC,CAAC;gBACpF,YAAY,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC7B,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACxD,IAAA,qBAAM,EACF,gBAAgB,KAAK,SAAS,EAC9B,KAAK,CACR,CAAC;oBAEF,2EAA2E;oBAC3E,IAAI,gBAAgB,CAAC,MAAM,KAAK,iBAAiB,CAAC,eAAe,EAAE;wBAC/D,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC9E,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;qBACrC;gBACL,CAAC,CAAC,CAAC;aACN;iBAAM;gBACH,oEAAoE;gBACpE,IAAA,qBAAM,EAAC,CAAA,MAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,0CAAE,MAAM,MAAK,iBAAiB,CAAC,gBAAgB,EACjG,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBAC5D,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACtD;SACJ;IACL,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,KAAK,CAAC,IAAI,CACpB,SAAoC,EACpC,YAAyD;QAEzD,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,EAAE,CAAC;SACb;QACD,IAAI,aAAa,CAAC;QAClB,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAC5D,IAAI,OAAO,EAAE;YACT,aAAa,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;SAC/C;QACD,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC;aACtC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,IAAI,CAAC,QAA8B;;QACvC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC;YAC9B,SAAS,EAAE,uBAAuB;YAClC,KAAK,EAAE,MAAA,MAAA,QAAQ,CAAC,GAAG,0CAAE,MAAM,mCAAI,CAAC;YAChC,aAAa,EAAE,MAAA,QAAQ,CAAC,aAAa,0CAAE,MAAM;SAChD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,GAAG,CAA6B,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC1E,IAAI,QAAQ,CAAC,GAAG,EAAE;YACd,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,CAAC;YACnE,yEAAyE;YACzE,6CAA6C;YAC7C,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;SACnF;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACI,SAAS,CAAC,SAAkB,KAAK;QACpC,MAAM,MAAM,GAA2B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACvD;;;YAGI;QACJ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,MAAc,EAAE,EAAE;YACvC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,yFAAyF;QACzF,6GAA6G;QAC7G,kGAAkG;QAClG,4GAA4G;QAC5G,qDAAqD;QACrD,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE;YACnD,IAAA,qBAAM,EAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACjE,oGAAoG;YACpG,kDAAkD;YAClD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC;SACzF;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACI,gBAAgB,CAAC,UAAoB;QACxC,kFAAkF;QAClF,sCAAsC;QACtC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;YAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAA,qBAAM,EACF,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,QAAQ,EAC/D,KAAK,CAAC,0CAA0C,CACnD,CAAC;YACF,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAC5B,oDAAoD;YACpD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SACvC;IACL,CAAC;IAED;;;;;;OAMG;IACI,kBAAkB,CAAC,YAAsB,EAAE,SAAkB;QAChE,kFAAkF;QAClF,sCAAsC;QACtC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE;YAC9B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAA,qBAAM,EACF,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,QAAQ,EAC/D,KAAK,CAAC,8CAA8C,CACvD,CAAC;YACF,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAE5B,IAAI,SAAS,EAAE;gBACX,4DAA4D;gBAC5D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;aACpC;iBAAM;gBACH,mGAAmG;gBACnG,mFAAmF;gBACnF,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aACrC;SACJ;IACL,CAAC;IAEM,SAAS,CAAC,gBAAoC;QACjD,gHAAgH;QAChH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/G,MAAM,OAAO,GAAG,IAAI,kCAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACvB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,wEAAwE;QACxE,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE;YAC1C,OAAO,CAAC,OAAO,CACX,WAAW,CAAC,qBAAqB;YACjC,8BAA8B;YAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;iBAClD,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAChE,CAAC;SACL;QAED,OAAO,OAAO,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAEM,gBAAgB,CAAC,KAA0B;QAC9C,IAAA,qBAAM,EAAC,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,mCAAW,CAAC,QAAQ,EACpD,KAAK,CAAC,4DAA4D,CAAC,CAAC;QACxE,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EACzC,KAAK,CAAC,iEAAiE,CAAC,CAAC;QAC7E,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,KAAK,EAAE;YACtC,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACzF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC3C,gCAAgC;YAChC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;SAChD;IACL,CAAC;IAEM,eAAe;QAClB,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE;YAC1C,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAA,6BAAc,EAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;SAC/D;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;;AAphBL,kCAqhBC;AAphB0B,oBAAQ,GAAG,QAAQ,CAAC;AACnB,iCAAqB,GAAG,gBAAgB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { v4 as uuid } from \"uuid\";\nimport { IFluidHandle, IFluidHandleContext } from \"@fluidframework/core-interfaces\";\nimport { IDocumentStorageService } from \"@fluidframework/driver-definitions\";\nimport { ICreateBlobResponse, ISequencedDocumentMessage, ISnapshotTree } from \"@fluidframework/protocol-definitions\";\nimport {\n    createResponseError,\n    generateHandleContextPath,\n    responseToException,\n    SummaryTreeBuilder,\n} from \"@fluidframework/runtime-utils\";\nimport { assert, bufferToString, Deferred, stringToBuffer, TypedEventEmitter } from \"@fluidframework/common-utils\";\nimport { IContainerRuntime, IContainerRuntimeEvents } from \"@fluidframework/container-runtime-definitions\";\nimport { AttachState } from \"@fluidframework/container-definitions\";\nimport { ChildLogger, loggerToMonitoringContext, MonitoringContext, PerformanceEvent } from \"@fluidframework/telemetry-utils\";\nimport {\n    IGarbageCollectionData,\n    ISummaryTreeWithStats,\n    ITelemetryContext,\n} from \"@fluidframework/runtime-definitions\";\nimport { Throttler, formExponentialFn, IThrottler } from \"./throttler\";\nimport { summarizerClientType } from \"./summarizerClientElection\";\nimport { throwOnTombstoneUsageKey } from \"./garbageCollectionConstants\";\n\n/**\n * This class represents blob (long string)\n * This object is used only when creating (writing) new blob and serialization purposes.\n * De-serialization process goes through FluidObjectHandle and request flow:\n * DataObject.request() recognizes requests in the form of `/blobs/<id>`\n * and loads blob.\n */\nexport class BlobHandle implements IFluidHandle<ArrayBufferLike> {\n    private attached: boolean = false;\n\n    public get IFluidHandle(): IFluidHandle { return this; }\n\n    public get isAttached(): boolean {\n        return this.attached;\n    }\n\n    public readonly absolutePath: string;\n\n    constructor(\n        public readonly path: string,\n        public readonly routeContext: IFluidHandleContext,\n        public get: () => Promise<any>,\n    ) {\n        this.absolutePath = generateHandleContextPath(path, this.routeContext);\n    }\n\n    public attachGraph() {\n        this.attached = true;\n    }\n\n    public bind(handle: IFluidHandle) {\n        throw new Error(\"Cannot bind to blob handle\");\n    }\n}\n\nclass CancellableThrottler {\n    constructor(private readonly throttler: IThrottler) { }\n    private cancelP = new Deferred<void>();\n\n    public async getDelay(): Promise<void> {\n        return Promise.race([\n            this.cancelP.promise,\n            new Promise<void>((resolve) => setTimeout(resolve, this.throttler.getDelay())),\n        ]);\n    }\n\n    public cancel() {\n        this.cancelP.resolve();\n        this.cancelP = new Deferred<void>();\n    }\n}\n\n/**\n * Information from a snapshot needed to load BlobManager\n */\nexport interface IBlobManagerLoadInfo {\n    ids?: string[];\n    redirectTable?: [string, string][];\n}\n\n// Restrict the IContainerRuntime interface to the subset required by BlobManager.  This helps to make\n// the contract explicit and reduces the amount of mocking required for tests.\nexport type IBlobManagerRuntime =\n    Pick<IContainerRuntime, \"attachState\" | \"connected\" | \"logger\" | \"clientDetails\"> & TypedEventEmitter<IContainerRuntimeEvents>;\n\n// Note that while offline we \"submit\" an op before uploading the blob, but we always\n// expect blobs to be uploaded before we actually see the op round-trip\nenum PendingBlobStatus {\n    OnlinePendingUpload,\n    OnlinePendingOp,\n    OfflinePendingUpload,\n    OfflinePendingOp,\n}\n\ninterface PendingBlob {\n    blob: ArrayBufferLike;\n    status: PendingBlobStatus;\n    storageId?: string;\n    handleP: Deferred<IFluidHandle<ArrayBufferLike>>;\n    uploadP: Promise<ICreateBlobResponse>;\n}\n\nexport interface IPendingBlobs { [id: string]: { blob: string; }; }\n\nexport class BlobManager {\n    public static readonly basePath = \"_blobs\";\n    private static readonly redirectTableBlobName = \".redirectTable\";\n    private readonly mc: MonitoringContext;\n\n    /**\n     * Map of local (offline/detached) IDs to storage IDs. Contains identity entries\n     * (id → id) for storage IDs, so all requested IDs should be a key in this map.\n     * Blobs created while the container is detached are stored in IDetachedBlobStorage\n     * which gives local IDs; the storage IDs are filled in at attach time.\n     */\n    private readonly redirectTable: Map<string, string | undefined>;\n\n    /**\n     * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.\n     * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so\n     * we must save it. This is true for both the online and offline flow.\n     */\n    private readonly pendingBlobs: Map<string, PendingBlob> = new Map();\n\n    /**\n     * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops\n     * don't include local ID in online flow.\n     */\n    private readonly opsInFlight: Map<string, string[]> = new Map();\n\n    private readonly retryThrottler = new CancellableThrottler(new Throttler(\n        60 * 1000, // 60 sec delay window\n        30 * 1000, // 30 sec max delay\n        // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)\n        formExponentialFn({ coefficient: 20, initialDelay: 0 }),\n    ));\n\n    /** If true, throw an error when a tombstone attachment blob is retrieved. */\n    private readonly throwOnTombstoneUsage: boolean;\n    /**\n     * This stores ides of tombstoned blobs.\n     * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.\n     */\n    private readonly tombstonedBlobs: Set<string> = new Set();\n\n    constructor(\n        private readonly routeContext: IFluidHandleContext,\n        snapshot: IBlobManagerLoadInfo,\n        private readonly getStorage: () => IDocumentStorageService,\n        /**\n         * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which\n         * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server\n         * will then not delete the blob as long as it is listed as referenced in future summaries.\n         * The summarizing client will know to include the storage ID in the summary when it sees the op.\n         *\n         * The op may also include a local ID to inform all clients of the relation to the storage\n         * ID, without knowledge of which they cannot request the blob from storage. This is also\n         * included in the redirect table in the summary.\n         */\n        private readonly sendBlobAttachOp: (storageId?: string, localId?: string) => void,\n        // To be called when a blob node is requested. blobPath is the path of the blob's node in GC's graph. It's\n        // of the format `/<BlobManager.basePath>/<blobId>`.\n        private readonly gcNodeUpdated: (blobPath: string) => void,\n        private readonly runtime: IBlobManagerRuntime,\n        stashedBlobs: IPendingBlobs = {},\n    ) {\n        this.mc = loggerToMonitoringContext(ChildLogger.create(this.runtime.logger, \"BlobManager\"));\n        // Read the feature flag that tells whether to throw when a tombstone blob is requested.\n        this.throwOnTombstoneUsage =\n            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&\n            this.runtime.clientDetails.type !== summarizerClientType;\n\n        this.runtime.on(\"disconnected\", () => this.onDisconnected());\n        this.redirectTable = this.load(snapshot);\n\n        // Begin uploading stashed blobs from previous container instance\n        Object.entries(stashedBlobs).forEach(([localId, entry]) => {\n            const blob = stringToBuffer(entry.blob, \"base64\");\n            this.pendingBlobs.set(localId, {\n                blob,\n                status: PendingBlobStatus.OfflinePendingUpload,\n                handleP: new Deferred(),\n                uploadP: this.uploadBlob(localId, blob),\n            });\n        });\n    }\n\n    private get pendingOfflineUploads() {\n        return Array.from(this.pendingBlobs.values())\n            .filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);\n    }\n\n    public get hasPendingOfflineUploads(): boolean {\n        return this.pendingOfflineUploads.length > 0;\n    }\n\n    /**\n     * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.\n     */\n    public async onConnected() {\n        this.retryThrottler.cancel();\n        const pendingUploads = this.pendingOfflineUploads.map(async (e) => e.uploadP);\n        await PerformanceEvent.timedExecAsync(this.mc.logger, {\n                eventName: \"BlobUploadOnConnected\",\n                count: pendingUploads.length,\n            }, async () => Promise.all(pendingUploads),\n            { start: true, end: true },\n        );\n    }\n\n    /**\n     * Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected\n     * again\n     */\n    private onDisconnected() {\n        for (const [localId, entry] of this.pendingBlobs) {\n            if (entry.status === PendingBlobStatus.OnlinePendingOp) {\n                // This will submit another BlobAttach op for this blob. This is necessary because the one we sent\n                // already didn't have the local ID.\n                this.transitionToOffline(localId);\n            }\n        }\n    }\n\n    /**\n     * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`\n     * This path must match the path of the blob handle returned by the createBlob API because blobs are marked\n     * referenced by storing these handles in a referenced DDS.\n     */\n    private getBlobGCNodePath(blobId: string) {\n        return `/${BlobManager.basePath}/${blobId}`;\n    }\n\n    /**\n     * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is\n     * detached or there are no (non-pending) attachment blobs in the document\n     */\n    private get storageIds(): Set<string> {\n        const ids = new Set<string | undefined>(this.redirectTable.values());\n\n        // If we are detached, we will not have storage IDs, only undefined\n        const undefinedValueInTable = ids.delete(undefined);\n\n        // For a detached container, entries are inserted into the redirect table with an undefined storage ID.\n        // For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.\n        assert(!undefinedValueInTable || this.runtime.attachState === AttachState.Detached && ids.size === 0,\n            0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */);\n\n        return ids as Set<string>;\n    }\n\n    public async getBlob(blobId: string): Promise<ArrayBufferLike> {\n        const request = { url: blobId };\n        if (this.tombstonedBlobs.has(blobId) ) {\n            const error = responseToException(createResponseError(404, \"Blob removed by gc\", request), request);\n            this.mc.logger.sendErrorEvent({\n                eventName: \"GC_Tombstone_Blob_Requested\",\n                url: request.url,\n            }, error);\n            if (this.throwOnTombstoneUsage) {\n                throw error;\n            }\n        }\n\n        const pending = this.pendingBlobs.get(blobId);\n        if (pending) {\n            return pending.blob;\n        }\n        let storageId;\n        if (this.runtime.attachState === AttachState.Detached) {\n            assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);\n\n            // Blobs created while the container is detached are stored in IDetachedBlobStorage.\n            // The 'IDocumentStorageService.readBlob()' call below will retrieve these via localId.\n            storageId = blobId;\n        } else {\n            storageId = this.redirectTable.get(blobId);\n            assert(!!storageId, 0x11f /* \"requesting unknown blobs\" */);\n        }\n\n        // When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.\n        this.gcNodeUpdated(this.getBlobGCNodePath(blobId));\n\n        return PerformanceEvent.timedExecAsync(\n            this.mc.logger,\n            { eventName: \"AttachmentReadBlob\", id: storageId },\n            async () => {\n                return this.getStorage().readBlob(storageId);\n            },\n            { end: true, cancel: \"error\" },\n        );\n    }\n\n    private getBlobHandle(id: string): IFluidHandle<ArrayBufferLike> {\n        assert(this.redirectTable.has(id) || this.pendingBlobs.has(id),\n            0x384 /* requesting handle for unknown blob */);\n        return new BlobHandle(\n            `${BlobManager.basePath}/${id}`,\n            this.routeContext,\n            async () => this.getBlob(id),\n        );\n    }\n\n    private async createBlobDetached(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {\n        // Blobs created while the container is detached are stored in IDetachedBlobStorage.\n        // The 'IDocumentStorageService.createBlob()' call below will respond with a localId.\n        const response = await this.getStorage().createBlob(blob);\n        this.redirectTable.set(response.id, undefined);\n        return this.getBlobHandle(response.id);\n    }\n\n    public async createBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {\n        if (this.runtime.attachState === AttachState.Detached) {\n            return this.createBlobDetached(blob);\n        }\n        if (this.runtime.attachState === AttachState.Attaching) {\n            // blob upload is not supported in \"Attaching\" state\n            this.mc.logger.sendTelemetryEvent({ eventName: \"CreateBlobWhileAttaching\" });\n            await new Promise<void>((resolve) => this.runtime.once(\"attached\", resolve));\n        }\n        assert(this.runtime.attachState === AttachState.Attached,\n            0x385 /* For clarity and paranoid defense against adding future attachment states */);\n\n        // Create a local ID for each blob. This is used to support blobs if/when the client goes\n        // offline since we don't have the ID from storage yet. If online flow succeeds this won't be used.\n        const localId = uuid();\n        const pendingEntry: PendingBlob = {\n            blob,\n            status: PendingBlobStatus.OnlinePendingUpload,\n            handleP: new Deferred(),\n            uploadP: this.uploadBlob(localId, blob),\n        };\n        this.pendingBlobs.set(localId, pendingEntry);\n\n        return pendingEntry.handleP.promise;\n    }\n\n    private async uploadBlob(localId: string, blob: ArrayBufferLike): Promise<ICreateBlobResponse> {\n        return PerformanceEvent.timedExecAsync(\n            this.mc.logger,\n            { eventName: \"createBlob\" },\n            async () => this.getStorage().createBlob(blob),\n            { end: true, cancel: this.runtime.connected ? \"error\" : \"generic\" },\n        ).then(\n            (response) => this.onUploadResolve(localId, response),\n            async (err) => this.onUploadReject(localId, err),\n        );\n    }\n\n    private onUploadResolve(localId: string, response: ICreateBlobResponse) {\n        const entry = this.pendingBlobs.get(localId);\n        assert(entry?.status === PendingBlobStatus.OnlinePendingUpload ||\n            entry?.status === PendingBlobStatus.OfflinePendingUpload,\n            0x386 /* Must have pending blob entry for uploaded blob */);\n        entry.storageId = response.id;\n        if (this.runtime.connected) {\n            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {\n                if (this.storageIds.has(response.id)) {\n                    // Storage may dedupe blobs and give us an ID we already know\n                    // no need to submit BlobAttach op in this case\n                    entry.handleP.resolve(this.getBlobHandle(response.id));\n                    this.pendingBlobs.delete(localId);\n                } else {\n                    // Check for still-pending duplicates too; if an op is already in flight we can wait for that one\n                    if (!this.opsInFlight.has(response.id)) {\n                        this.sendBlobAttachOp(response.id);\n                    }\n                    this.opsInFlight.set(response.id, (this.opsInFlight.get(response.id) ?? []).concat(localId));\n                    entry.status = PendingBlobStatus.OnlinePendingOp;\n                }\n            } else if (entry.status === PendingBlobStatus.OfflinePendingUpload) {\n                // We already submitted a BlobAttach op for this blob when it was transitioned to offline flow\n                entry.status = PendingBlobStatus.OfflinePendingOp;\n            }\n        } else {\n            // connected to storage but not ordering service?\n            this.mc.logger.sendTelemetryEvent({ eventName: \"BlobUploadSuccessWhileDisconnected\" });\n            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {\n                this.transitionToOffline(localId);\n            }\n            entry.status = PendingBlobStatus.OfflinePendingOp;\n        }\n        return response;\n    }\n\n    private async onUploadReject(localId: string, error) {\n        const entry = this.pendingBlobs.get(localId);\n        assert(!!entry, 0x387 /* Must have pending blob entry for blob which failed to upload */);\n        if (!this.runtime.connected) {\n            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {\n                this.transitionToOffline(localId);\n            }\n            // we are probably not connected to storage but start another upload request in case we are\n            entry.uploadP = this.retryThrottler.getDelay().then(async () => this.uploadBlob(localId, entry.blob));\n            return entry.uploadP;\n        } else {\n            entry.handleP.reject(error);\n            throw error;\n        }\n    }\n\n    private transitionToOffline(localId: string) {\n        assert(!this.runtime.connected, 0x388 /* Must only transition to offline flow while runtime is disconnected */);\n        const entry = this.pendingBlobs.get(localId);\n        assert(!!entry, 0x389 /* No pending blob entry */);\n        assert([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status),\n            0x38a /* Blob must be in online flow to transition to offline flow */);\n\n        entry.status = entry.status === PendingBlobStatus.OnlinePendingUpload\n            ? PendingBlobStatus.OfflinePendingUpload\n            : PendingBlobStatus.OfflinePendingOp;\n\n        // Submit a BlobAttach op. It's important we submit this op now before returning the blob handle so the\n        // BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise an invalid handle could be\n        // added to the document if the ops are not all successfully submitted upon reconnection.\n        // storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()\n        // is called\n        this.sendBlobAttachOp(entry.storageId, localId);\n        entry.handleP.resolve(this.getBlobHandle(localId));\n    }\n\n    /**\n     * Resubmit a BlobAttach op. Used to add storage IDs to ops that were\n     * submitted to runtime while disconnected.\n     * @param metadata - op metadata containing storage and/or local IDs\n     */\n    public reSubmit(metadata: Record<string, unknown> | undefined) {\n        assert(!!metadata, 0x38b /* Resubmitted ops must have metadata */);\n        const { blobId, localId }: { blobId?: string; localId?: string; } = metadata;\n        if (!blobId) {\n            assert(!!localId, 0x38c /* Submitted BlobAttach ops must have a blobId or localId */);\n            // We submitted this op while offline. The blob should have been uploaded by now.\n            const pendingEntry = this.pendingBlobs.get(localId);\n            assert(pendingEntry?.status === PendingBlobStatus.OfflinePendingOp &&\n                !!pendingEntry?.storageId, 0x38d /* blob must be uploaded before resubmitting BlobAttach op */);\n            return this.sendBlobAttachOp(pendingEntry.storageId, localId);\n        }\n        return this.sendBlobAttachOp(blobId, localId);\n    }\n\n    public processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean) {\n        assert(message?.metadata?.blobId, 0x12a /* \"Missing blob id on metadata\" */);\n        if (message.metadata.localId !== undefined) {\n            this.redirectTable.set(message.metadata.localId, message.metadata.blobId);\n        }\n        // set identity (id -> id) entry\n        this.redirectTable.set(message.metadata.blobId, message.metadata.blobId);\n\n        if (local) {\n            if (message.metadata.localId === undefined) {\n                // Since there is no local ID, we know this op was submitted while online.\n                const waitingBlobs = this.opsInFlight.get(message.metadata.blobId);\n                assert(!!waitingBlobs, 0x38e /* local online BlobAttach op with no pending blob */);\n                waitingBlobs.forEach((localId) => {\n                    const pendingBlobEntry = this.pendingBlobs.get(localId);\n                    assert(\n                        pendingBlobEntry !== undefined,\n                        0x38f, /* local online BlobAttach op with no pending blob entry */\n                    );\n\n                    // It's possible we transitioned to offline flow while waiting for this op.\n                    if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {\n                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(message.metadata.blobId));\n                        this.pendingBlobs.delete(localId);\n                    }\n                });\n            } else {\n                // Each local ID is unique; get the pending blob entry and delete it\n                assert(this.pendingBlobs.get(message.metadata.localId)?.status === PendingBlobStatus.OfflinePendingOp,\n                    0x1f8 /* \"local BlobAttach op with no pending blob\" */);\n                this.pendingBlobs.delete(message.metadata.localId);\n            }\n        }\n    }\n\n    /**\n     * Reads blobs needed to load BlobManager from storage.\n     * @param blobsTree - Tree containing IDs of previously attached blobs. We\n     * look for the IDs in the blob entries of the tree since the both the r11s\n     * and SPO drivers replace the attachment types returned in snapshot() with blobs.\n     */\n    public static async load(\n        blobsTree: ISnapshotTree | undefined,\n        tryFetchBlob: (id: string) => Promise<[string, string][]>,\n    ): Promise<IBlobManagerLoadInfo> {\n        if (!blobsTree) {\n            return {};\n        }\n        let redirectTable;\n        const tableId = blobsTree.blobs[this.redirectTableBlobName];\n        if (tableId) {\n            redirectTable = await tryFetchBlob(tableId);\n        }\n        const ids = Object.entries(blobsTree.blobs)\n            .filter(([k, _]) => k !== this.redirectTableBlobName).map(([_, v]) => v);\n        return { ids, redirectTable };\n    }\n\n    /**\n     * Load a set of previously attached blob IDs and redirect table from a previous snapshot.\n     */\n    private load(snapshot: IBlobManagerLoadInfo): Map<string, string | undefined> {\n        this.mc.logger.sendTelemetryEvent({\n            eventName: \"AttachmentBlobsLoaded\",\n            count: snapshot.ids?.length ?? 0,\n            redirectTable: snapshot.redirectTable?.length,\n        });\n        const table = new Map<string, string | undefined>(snapshot.redirectTable);\n        if (snapshot.ids) {\n            const detached = this.runtime.attachState === AttachState.Detached;\n            // If we are detached, we don't have storage IDs yet, so set to undefined\n            // Otherwise, set identity (id -> id) entries\n            snapshot.ids.forEach((entry) => table.set(entry, detached ? undefined : entry));\n        }\n        return table;\n    }\n\n    /**\n     * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be\n     * individually referenced by storing its handle in a referenced DDS. Returns the list of blob ids as GC nodes.\n     * @param fullGC - true to bypass optimizations and force full generation of GC data. BlobManager doesn't care\n     * about this for now because the data is a simple list of blob ids.\n     */\n    public getGCData(fullGC: boolean = false): IGarbageCollectionData {\n        const gcData: IGarbageCollectionData = { gcNodes: {} };\n        /**\n          * The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned\n          * by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.\n          */\n        this.storageIds.forEach((blobId: string) => {\n            gcData.gcNodes[this.getBlobGCNodePath(blobId)] = [];\n        });\n\n        // For some blobs, the handle returned on creation is based off of the localId. So, these\n        // nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node\n        // must also be marked referenced. So, we add a route from the localId node to the storageId node.\n        // Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or\n        // a blob may be referenced via its storageId handle.\n        for (const [localId, storageId] of this.redirectTable) {\n            assert(!!storageId, 0x390 /* Must be attached to get GC data */);\n            // Add node for the localId and add a route to the storageId node. The storageId node will have been\n            // added above when adding nodes for this.blobIds.\n            gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];\n        }\n\n        return gcData;\n    }\n\n    /**\n     * This is called to update blobs whose routes are used. The used blobs are removed from the tombstone list.\n     * @param usedRoutes - The routes of the blob nodes that are used.\n     */\n    public updateUsedRoutes(usedRoutes: string[]) {\n        // The routes or blob node paths are in the same format as returned in getGCData -\n        // `/<BlobManager.basePath>/<blobId>`.\n        for (const route of usedRoutes) {\n            const pathParts = route.split(\"/\");\n            assert(\n                pathParts.length === 3 && pathParts[1] === BlobManager.basePath,\n                0x4bc /* Invalid blob node id in used routes. */,\n            );\n            const blobId = pathParts[2];\n            // Un-tombstone the blob if it was marked tombstone.\n            this.tombstonedBlobs.delete(blobId);\n        }\n    }\n\n    /**\n     * This is called to update blobs whose routes are unused. The unused blobs are either deleted or marked as\n     * tombstones.\n     * @param unusedRoutes - The routes of the blob nodes that are unused.\n     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they\n     * are deleted.\n     */\n    public updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean): void {\n        // The routes or blob node paths are in the same format as returned in getGCData -\n        // `/<BlobManager.basePath>/<blobId>`.\n        for (const route of unusedRoutes) {\n            const pathParts = route.split(\"/\");\n            assert(\n                pathParts.length === 3 && pathParts[1] === BlobManager.basePath,\n                0x2d5 /* \"Invalid blob node id in unused routes.\" */,\n            );\n            const blobId = pathParts[2];\n\n            if (tombstone) {\n                // If tombstone is set, add this blob to the tombstone list.\n                this.tombstonedBlobs.add(blobId);\n            } else {\n                // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The\n                // corresponding storageId may still be used either directly or via other localIds.\n                this.redirectTable.delete(blobId);\n            }\n        }\n    }\n\n    public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {\n        // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached\n        const blobIds = this.storageIds.size > 0 ? Array.from(this.storageIds) : Array.from(this.redirectTable.keys());\n        const builder = new SummaryTreeBuilder();\n        blobIds.forEach((blobId) => {\n            builder.addAttachment(blobId);\n        });\n\n        // Any non-identity entries in the table need to be saved in the summary\n        if (this.redirectTable.size > blobIds.length) {\n            builder.addBlob(\n                BlobManager.redirectTableBlobName,\n                // filter out identity entries\n                JSON.stringify(Array.from(this.redirectTable.entries())\n                    .filter(([localId, storageId]) => localId !== storageId)),\n            );\n        }\n\n        return builder.getSummaryTree();\n    }\n\n    public setRedirectTable(table: Map<string, string>) {\n        assert(this.runtime.attachState === AttachState.Detached,\n            0x252 /* \"redirect table can only be set in detached container\" */);\n        assert(this.redirectTable.size === table.size,\n            0x391 /* Redirect table size must match BlobManager's local ID count */);\n        for (const [localId, storageId] of table) {\n            assert(this.redirectTable.has(localId), 0x254 /* \"unrecognized id in redirect table\" */);\n            this.redirectTable.set(localId, storageId);\n            // set identity (id -> id) entry\n            this.redirectTable.set(storageId, storageId);\n        }\n    }\n\n    public getPendingBlobs(): IPendingBlobs {\n        const blobs = {};\n        for (const [key, entry] of this.pendingBlobs) {\n            blobs[key] = { blob: bufferToString(entry.blob, \"base64\") };\n        }\n        return blobs;\n    }\n}\n"]}

package/dist/containerRuntime.d.ts

@@ -8,9 +8,11 @@ import { IAudience, IFluidTokenProvider, IContainerContext, IDeltaManager, IRunt
 import { IContainerRuntime, IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions";
 import { TypedEventEmitter } from "@fluidframework/common-utils";
 import { IDocumentStorageService } from "@fluidframework/driver-definitions";
-import { IClientDetails, IDocumentMessage, IQuorumClients, ISequencedDocumentMessage, ISignalMessage, ISummaryTree, MessageType } from "@fluidframework/protocol-definitions";
+import { IClientDetails, IDocumentMessage, IQuorumClients, ISequencedDocumentMessage, ISignalMessage, ISummaryTree } from "@fluidframework/protocol-definitions";
 import { FlushMode, IFluidDataStoreContextDetached, IFluidDataStoreRegistry, IGarbageCollectionData, NamedFluidDataStoreRegistryEntries, ISummaryTreeWithStats, IDataStore, ITelemetryContext } from "@fluidframework/runtime-definitions";
-import { BatchMessage } from "./batchManager";
+import { IBlobManagerLoadInfo } from "./blobManager";
+import { IContainerRuntimeMetadata } from "./summaryFormat";
+import { ISerializedElection } from "./orderedClientElection";
 import { SubmitSummaryResult, ISubmitSummaryOptions, ISummarizer, ISummarizerInternalsProvider, ISummarizerOptions, ISummarizerRuntime, IRefreshSummaryAckOptions } from "./summarizerTypes";
 import { GCNodeType, IGarbageCollectionRuntime, IGCStats } from "./garbageCollection";
 export declare enum ContainerMessageType {
@@ -21,12 +23,6 @@ export declare enum ContainerMessageType {
     Rejoin = "rejoin",
     Alias = "alias"
 }
-export interface IChunkedOp {
-    chunkId: number;
-    totalChunks: number;
-    contents: string;
-    originalType: MessageType | ContainerMessageType;
-}
 export interface ContainerRuntimeMessage {
     contents: any;
     type: ContainerMessageType;
@@ -303,16 +299,6 @@ export declare enum RuntimeMessage {
  * @deprecated - please use version in driver-utils
  */
 export declare function isRuntimeMessage(message: ISequencedDocumentMessage): boolean;
-/**
- * Unpacks runtime messages
- *
- * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
- * @param message - message (as it observed in storage / service)
- * @returns unpacked runtime message
- *
- * @internal
- */
-export declare function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolean;
 /**
  * Legacy ID for the built-in AgentScheduler.  To minimize disruption while removing it, retaining this as a
  * special-case for document dirty state.  Ultimately we should have no special-cases from the
@@ -348,8 +334,10 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * @param requestHandler - Request handlers for the container runtime
      * @param runtimeOptions - Additional options to be passed to the runtime
      * @param existing - (optional) When loading from an existing snapshot. Precedes context.existing if provided
+     * @param containerRuntimeCtor - (optional) Constructor to use to create the ContainerRuntime instance. This
+     * allows mixin classes to leverage this method to define their own async initializer.
      */
-    static load(context: IContainerContext, registryEntries: NamedFluidDataStoreRegistryEntries, requestHandler?: (request: IRequest, runtime: IContainerRuntime) => Promise<IResponse>, runtimeOptions?: IContainerRuntimeOptions, containerScope?: FluidObject, existing?: boolean): Promise<ContainerRuntime>;
+    static load(context: IContainerContext, registryEntries: NamedFluidDataStoreRegistryEntries, requestHandler?: (request: IRequest, runtime: IContainerRuntime) => Promise<IResponse>, runtimeOptions?: IContainerRuntimeOptions, containerScope?: FluidObject, existing?: boolean, containerRuntimeCtor?: typeof ContainerRuntime): Promise<ContainerRuntime>;
     get options(): ILoaderOptions;
     get clientId(): string | undefined;
     get clientDetails(): IClientDetails;
@@ -364,7 +352,6 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
     get IFluidHandleContext(): IFluidHandleContext;
     private readonly handleContext;
     private readonly mc;
-    private readonly opDecompressor;
     private readonly summarizerClientElection?;
     /**
      * summaryManager will only be created if this client is permitted to spawn a summarizing client
@@ -403,18 +390,15 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * It is created only by summarizing container (i.e. one with clientType === "summarizer")
      */
     private readonly _summarizer?;
-    private readonly deltaSender;
     private readonly scheduleManager;
     private readonly blobManager;
     private readonly pendingStateManager;
-    private readonly pendingAttachBatch;
-    private readonly pendingBatch;
+    private readonly outbox;
     private readonly garbageCollector;
-    private readonly chunkMap;
     private readonly dataStores;
+    private readonly remoteMessageProcessor;
     /** The last message processed at the time of the last summary. */
     private messageAtLastSummary;
-    private get emptyBatch();
     private get summarizer();
     private readonly summariesDisabled;
     private isSummariesDisabled;
@@ -432,7 +416,10 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * a summary is generated.
      */
     private nextSummaryNumber;
-    private constructor();
+    /**
+     * @internal
+     */
+    protected constructor(context: IContainerContext, registry: IFluidDataStoreRegistry, metadata: IContainerRuntimeMetadata | undefined, electedSummarizerData: ISerializedElection | undefined, chunks: [string, string[]][], dataStoreAliasMap: [string, string][], runtimeOptions: Readonly<Required<IContainerRuntimeOptions>>, containerScope: FluidObject, logger: ITelemetryLogger, existing: boolean, blobManagerSnapshot: IBlobManagerLoadInfo, _storage: IDocumentStorageService, requestHandler?: ((request: IRequest, runtime: IContainerRuntime) => Promise<IResponse>) | undefined, summaryConfiguration?: ISummaryConfiguration);
     /**
      * Initializes the state from the base snapshot this container runtime loaded from.
      */
@@ -453,7 +440,7 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
     private getDataStoreFromRequest;
     /** Adds the container's metadata to the given summary tree. */
     private addMetadataToSummary;
-    private addContainerStateToSummary;
+    protected addContainerStateToSummary(summaryTree: ISummaryTreeWithStats, fullTree: boolean, trackState: boolean, telemetryContext?: ITelemetryContext): void;
     private shouldContinueReconnecting;
     private resetReconnectCount;
     private replayPendingStates;
@@ -475,8 +462,7 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * This method is expected to be called at the end of a batch.
      */
     private flush;
-    protected flushBatch(batch: BatchMessage[]): void;
-    orderSequentially(callback: () => void): void;
+    orderSequentially<T>(callback: () => T): T;
     createDataStore(pkg: string | string[]): Promise<IDataStore>;
     createDetachedRootDataStore(pkg: Readonly<string[]>, rootDataStoreId: string): IFluidDataStoreContextDetached;
     createDetachedDataStore(pkg: Readonly<string[]>): IFluidDataStoreContextDetached;
@@ -554,7 +540,7 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
     /**
      * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
      * tombstones.
-     * @param unusedRoutes - The routes that are unused in all data stores in this Container.
+     * @param unusedRoutes - The routes that are unused in all data stores and attachment blobs in this Container.
      * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
      * are deleted.
      */
@@ -605,9 +591,6 @@ export declare class ContainerRuntime extends TypedEventEmitter<IContainerRuntim
      * @param options - options controlling how the summary is generated or submitted
      */
     submitSummary(options: ISubmitSummaryOptions): Promise<SubmitSummaryResult>;
-    private processRemoteChunkedMessage;
-    private addChunk;
-    private clearPartialChunks;
     private hasPendingMessages;
     private updateDocumentDirtyState;
     submitDataStoreOp(id: string, contents: any, localOpMetadata?: unknown): void;