@milaboratories/pl-drivers 1.9.0 → 1.10.0

package/dist/drivers/download_blob_url/driver.js

@@ -0,0 +1,148 @@
+import { Computable } from '@milaboratories/computable';
+import { TaskProcessor } from '@milaboratories/ts-helpers';
+import { randomUUID } from 'node:crypto';
+import * as path from 'node:path';
+import { FilesCache } from '../helpers/files_cache.js';
+import { stringifyWithResourceId, resourceIdToString } from '@milaboratories/pl-client';
+import { isFolderURL } from '@milaboratories/pl-model-common';
+import { makeDownloadableBlobSnapshot } from './snapshot.js';
+import { isPlTreeEntry } from '@milaboratories/pl-tree';
+import { rmRFDir, DownloadAndUnarchiveTask } from './task.js';
+import { getPathForFolderURL } from '../urls/url.js';
+import { newId } from './driver_id.js';
+import { nonRecoverableError } from '../download_blob/download_blob_task.js';
+
+/** Downloads .tar, .tar.gz or zip archives,
+ * extracts them into saveDir and gets a url for it. */
+class DownloadBlobToURLDriver {
+    logger;
+    signer;
+    clientDownload;
+    saveDir;
+    opts;
+    idToDownload = new Map();
+    downloadQueue;
+    /** Writes and removes files to a hard drive and holds a counter for every
+     * file that should be kept. */
+    cache;
+    constructor(logger, signer, clientDownload, saveDir, opts = {
+        cacheSoftSizeBytes: 50 * 1024 * 1024,
+        nConcurrentDownloads: 50,
+    }) {
+        this.logger = logger;
+        this.signer = signer;
+        this.clientDownload = clientDownload;
+        this.saveDir = saveDir;
+        this.opts = opts;
+        this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {
+            type: 'exponentialWithMaxDelayBackoff',
+            initialDelay: 10000,
+            maxDelay: 30000,
+            backoffMultiplier: 1.5,
+            jitter: 0.5,
+        });
+        this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);
+    }
+    info() {
+        return {
+            saveDir: this.saveDir,
+            opts: this.opts,
+            idToDownloadSize: this.idToDownload.size,
+            idToDownloadKeys: this.idToDownload.keys(),
+            idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),
+        };
+    }
+    /**
+     * @returns full path to the referenced file
+     */
+    getPathForCustomProtocol(url) {
+        if (isFolderURL(url)) {
+            return getPathForFolderURL(this.signer, url, this.saveDir);
+        }
+        throw new Error(`getPathForCustomProtocol: ${url} is invalid`);
+    }
+    extractArchiveAndGetURL(res, format, ctx) {
+        // wrap result as computable, if we were not given an existing computable context
+        if (ctx === undefined)
+            return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));
+        const rInfo = isPlTreeEntry(res)
+            ? makeDownloadableBlobSnapshot(res, ctx)
+            : res;
+        const callerId = randomUUID();
+        ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));
+        const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);
+        if (result?.url === undefined)
+            ctx.markUnstable(`a path to the downloaded archive might be undefined. The current result: ${result}`);
+        if (result?.error !== undefined)
+            throw result?.error;
+        return result?.url;
+    }
+    extractArchiveAndGetURLNoCtx(rInfo, format, w, callerId) {
+        const task = this.idToDownload.get(newId(rInfo.id, format));
+        if (task != undefined) {
+            task.attach(w, callerId);
+            return task.getURL();
+        }
+        const newTask = this.setNewTask(w, rInfo, format, callerId);
+        this.downloadQueue.push({
+            fn: async () => this.downloadUrl(newTask, callerId),
+            recoverableErrorPredicate: (e) => !nonRecoverableError(e),
+        });
+        return newTask.getURL();
+    }
+    /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
+    async downloadUrl(task, callerId) {
+        await task.download();
+        // Might be undefined if a error happened
+        if (task.getURL()?.url != undefined)
+            this.cache.addCache(task, callerId);
+    }
+    /** Removes a directory and aborts a downloading task when all callers
+     * are not interested in it. */
+    async releasePath(id, format, callerId) {
+        const task = this.idToDownload.get(newId(id, format));
+        if (task == undefined)
+            return;
+        if (this.cache.existsFile(task.path)) {
+            const toDelete = this.cache.removeFile(task.path, callerId);
+            await Promise.all(toDelete.map(async (task) => {
+                await rmRFDir(task.path);
+                this.cache.removeCache(task);
+                this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was removed`
+                    + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`);
+            }));
+        }
+        else {
+            // The task is still in a downloading queue.
+            const deleted = task.counter.dec(callerId);
+            if (deleted)
+                this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was removed from cache`);
+        }
+    }
+    /** Removes all files from a hard drive. */
+    async releaseAll() {
+        this.downloadQueue.stop();
+        await Promise.all(Array.from(this.idToDownload.entries()).map(async ([_, task]) => {
+            await rmRFDir(task.path);
+            this.cache.removeCache(task);
+            this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`);
+        }));
+    }
+    setNewTask(w, rInfo, format, callerId) {
+        const result = new DownloadAndUnarchiveTask(this.logger, this.signer, this.saveDir, this.getFilePath(rInfo.id, format), rInfo, format, this.clientDownload);
+        result.attach(w, callerId);
+        this.idToDownload.set(newId(rInfo.id, format), result);
+        return result;
+    }
+    removeTask(task, reason) {
+        task.abort(reason);
+        task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);
+        this.idToDownload.delete(newId(task.rInfo.id, task.format));
+    }
+    getFilePath(id, format) {
+        return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);
+    }
+}
+
+export { DownloadBlobToURLDriver };
+//# sourceMappingURL=driver.js.map

package/dist/drivers/download_blob_url/driver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.js","sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer } from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport { FilesCache } from '../helpers/files_cache';\nimport type { ResourceId } from '@milaboratories/pl-client';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport { type ArchiveFormat, type BlobToURLDriver, type FolderURL, isFolderURL } from '@milaboratories/pl-model-common';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { makeDownloadableBlobSnapshot } from './snapshot';\nimport type { PlTreeEntry } from '@milaboratories/pl-tree';\nimport { isPlTreeEntry } from '@milaboratories/pl-tree';\nimport { DownloadAndUnarchiveTask, rmRFDir } from './task';\nimport type { ClientDownload } from '../../clients/download';\nimport { getPathForFolderURL } from '../urls/url';\nimport type { Id } from './driver_id';\nimport { newId } from './driver_id';\nimport { nonRecoverableError } from '../download_blob/download_blob_task';\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: 'exponentialWithMaxDelayBackoff',\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined)\n      throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, rInfo: DownloadableBlobSnapshot, format: ArchiveFormat, callerId: string) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n    return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;;;;;AA8BA;AACuD;MAC1C,uBAAuB,CAAA;AASf,IAAA,MAAA;AACA,IAAA,MAAA;AACA,IAAA,cAAA;AACA,IAAA,OAAA;AACA,IAAA,IAAA;AAZX,IAAA,YAAY,GAAsC,IAAI,GAAG,EAAE;AAC3D,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EAChB,MAAc,EACd,cAA8B,EAC9B,OAAe,EACf,IAAA,GAAmC;AAClD,QAAA,kBAAkB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;AACpC,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QAPgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;AAKrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE;AAClF,YAAA,IAAI,EAAE,gCAAgC;AACtC,YAAA,YAAY,EAAE,KAAK;AACnB,YAAA,QAAQ,EAAE,KAAK;AACf,YAAA,iBAAiB,EAAE,GAAG;AACtB,YAAA,MAAM,EAAE,GAAG;AACZ,SAAA,CAAC;AACF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;IAC3D;IAEO,IAAI,GAAA;QACT,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,gBAAgB,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;AACxC,YAAA,gBAAgB,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE;AAC1C,YAAA,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC7F;IACH;AAEA;;AAEG;AACH,IAAA,wBAAwB,CAAC,GAAc,EAAA;AACrC,QAAA,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE;AACpB,YAAA,OAAO,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;QAC5D;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAA,WAAA,CAAa,CAAC;IAChE;AAaA,IAAA,uBAAuB,CACrB,GAA2C,EAC3C,MAAqB,EACrB,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;YACnB,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;AAE7E,QAAA,MAAM,KAAK,GAA6B,aAAa,CAAC,GAAG;AACvD,cAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG;cACrC,GAAG;AAEP,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAE7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAEpE,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AACtF,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,4EAA4E,MAAM,CAAA,CAAE,CACrF;AAEH,QAAA,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS;YAC7B,MAAM,MAAM,EAAE,KAAK;QAErB,OAAO,MAAM,EAAE,GAAG;IACpB;AAEQ,IAAA,4BAA4B,CAClC,KAA+B,EAC/B,MAAqB,EACrB,CAAU,EACV,QAAgB,EAAA;AAEhB,QAAA,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AAE3D,QAAA,IAAI,IAAI,IAAI,SAAS,EAAE;AACrB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;YACnD,yBAAyB,EAAE,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAC1D,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;;AAGA,IAAA,MAAM,WAAW,CAAC,IAA8B,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,IAAI,CAAC,QAAQ,EAAE;;AAErB,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,IAAI,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC1E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,EAAc,EAAE,MAAqB,EAAE,QAAgB,EAAA;AACvE,QAAA,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,IAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAO,IAA8B,KAAI;AACpD,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyB,uBAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,KAAI;AAC9D,YAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,KAA+B,EAAE,MAAqB,EAAE,QAAgB,EAAA;AACrG,QAAA,MAAM,MAAM,GAAG,IAAI,wBAAwB,CACzC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAClC,KAAK,EACL,MAAM,EACN,IAAI,CAAC,cAAc,CACpB;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC;AAEtD,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAA8B,EAAE,MAAc,EAAA;AAC/D,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,SAAA,EAAY,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,aAAa,MAAM,CAAA,CAAE,CAAC;AAC3F,QAAA,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7D;IAEQ,WAAW,CAAC,EAAc,EAAE,MAAqB,EAAA;QACvD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA,EAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAA,CAAE,CAAC;IACnE;AACD;;;;"}

package/dist/drivers/download_blob_url/driver_id.cjs

@@ -0,0 +1,9 @@
+'use strict';
+
+function newId(id, format) {
+    return `id:${String(BigInt(id))}-${format}`;
+}
+// export function
+
+exports.newId = newId;
+//# sourceMappingURL=driver_id.cjs.map

package/dist/drivers/download_blob_url/driver_id.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver_id.cjs","sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { ResourceId } from '@milaboratories/pl-client';\nimport type { ArchiveFormat } from '@milaboratories/pl-model-common';\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: ResourceId, format: ArchiveFormat): Id {\n  return `id:${String(BigInt(id))}-${format}`;\n}\n\n// export function\n"],"names":[],"mappings":";;AAMM,SAAU,KAAK,CAAC,EAAc,EAAE,MAAqB,EAAA;IACzD,OAAO,CAAA,GAAA,EAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAA,CAAA,EAAI,MAAM,CAAA,CAAE;AAC7C;AAEA;;;;"}

package/dist/drivers/download_blob_url/driver_id.d.ts

@@ -3,4 +3,3 @@ import { ArchiveFormat } from '@milaboratories/pl-model-common';
 /** A key in the driver task queue. */
 export type Id = string;
 export declare function newId(id: ResourceId, format: ArchiveFormat): Id;
-//# sourceMappingURL=driver_id.d.ts.map

package/dist/drivers/download_blob_url/driver_id.js

@@ -0,0 +1,7 @@
+function newId(id, format) {
+    return `id:${String(BigInt(id))}-${format}`;
+}
+// export function
+
+export { newId };
+//# sourceMappingURL=driver_id.js.map

package/dist/drivers/download_blob_url/driver_id.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver_id.js","sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { ResourceId } from '@milaboratories/pl-client';\nimport type { ArchiveFormat } from '@milaboratories/pl-model-common';\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: ResourceId, format: ArchiveFormat): Id {\n  return `id:${String(BigInt(id))}-${format}`;\n}\n\n// export function\n"],"names":[],"mappings":"AAMM,SAAU,KAAK,CAAC,EAAc,EAAE,MAAqB,EAAA;IACzD,OAAO,CAAA,GAAA,EAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAA,CAAA,EAAI,MAAM,CAAA,CAAE;AAC7C;AAEA;;;;"}

package/dist/drivers/download_blob_url/snapshot.cjs

@@ -0,0 +1,18 @@
+'use strict';
+
+var plTree = require('@milaboratories/pl-tree');
+
+/** We need only resource type for this driver. */
+const DownloadableBlobSnapshot = plTree.rsSchema({});
+function makeDownloadableBlobSnapshot(entryOrAccessor, ctx) {
+    const node = plTree.isPlTreeEntry(entryOrAccessor)
+        ? ctx.accessor(entryOrAccessor).node()
+        : plTree.isPlTreeEntryAccessor(entryOrAccessor)
+            ? entryOrAccessor.node()
+            : entryOrAccessor;
+    return plTree.makeResourceSnapshot(node, DownloadableBlobSnapshot);
+}
+
+exports.DownloadableBlobSnapshot = DownloadableBlobSnapshot;
+exports.makeDownloadableBlobSnapshot = makeDownloadableBlobSnapshot;
+//# sourceMappingURL=snapshot.cjs.map

package/dist/drivers/download_blob_url/snapshot.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot.cjs","sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from '@milaboratories/computable';\nimport type { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from '@milaboratories/pl-tree';\nimport { isPlTreeEntry, isPlTreeEntryAccessor, makeResourceSnapshot, rsSchema } from '@milaboratories/pl-tree';\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"names":["rsSchema","isPlTreeEntry","isPlTreeEntryAccessor","makeResourceSnapshot"],"mappings":";;;;AAIA;MACa,wBAAwB,GAAGA,eAAQ,CAAC,EAAE;AAG7C,SAAU,4BAA4B,CAC1C,eAAuE,EACvE,GAAkB,EAAA;AAElB,IAAA,MAAM,IAAI,GAAGC,oBAAa,CAAC,eAAe;UACtC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,IAAI;AACpC,UAAEC,4BAAqB,CAAC,eAAe;AACrC,cAAE,eAAe,CAAC,IAAI;cACpB,eAAe;AAErB,IAAA,OAAOC,2BAAoB,CAAC,IAAI,EAAE,wBAAwB,CAAC;AAC7D;;;;;"}

package/dist/drivers/download_blob_url/snapshot.d.ts

@@ -1,7 +1,6 @@
 import { ComputableCtx } from '@milaboratories/computable';
-import { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor, ResourceSnapshotSchema } from '@milaboratories/pl-tree';
+import { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from '@milaboratories/pl-tree';
 /** We need only resource type for this driver. */
-export declare const DownloadableBlobSnapshot: ResourceSnapshotSchema<undefined, undefined, undefined>;
+export declare const DownloadableBlobSnapshot: import('@milaboratories/pl-tree').ResourceSnapshotSchema<undefined, undefined, undefined>;
 export type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;
 export declare function makeDownloadableBlobSnapshot(entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor, ctx: ComputableCtx): DownloadableBlobSnapshot;
-//# sourceMappingURL=snapshot.d.ts.map

package/dist/drivers/download_blob_url/snapshot.js

@@ -0,0 +1,15 @@
+import { rsSchema, isPlTreeEntry, isPlTreeEntryAccessor, makeResourceSnapshot } from '@milaboratories/pl-tree';
+
+/** We need only resource type for this driver. */
+const DownloadableBlobSnapshot = rsSchema({});
+function makeDownloadableBlobSnapshot(entryOrAccessor, ctx) {
+    const node = isPlTreeEntry(entryOrAccessor)
+        ? ctx.accessor(entryOrAccessor).node()
+        : isPlTreeEntryAccessor(entryOrAccessor)
+            ? entryOrAccessor.node()
+            : entryOrAccessor;
+    return makeResourceSnapshot(node, DownloadableBlobSnapshot);
+}
+
+export { DownloadableBlobSnapshot, makeDownloadableBlobSnapshot };
+//# sourceMappingURL=snapshot.js.map

package/dist/drivers/download_blob_url/snapshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot.js","sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from '@milaboratories/computable';\nimport type { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from '@milaboratories/pl-tree';\nimport { isPlTreeEntry, isPlTreeEntryAccessor, makeResourceSnapshot, rsSchema } from '@milaboratories/pl-tree';\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"names":[],"mappings":";;AAIA;MACa,wBAAwB,GAAG,QAAQ,CAAC,EAAE;AAG7C,SAAU,4BAA4B,CAC1C,eAAuE,EACvE,GAAkB,EAAA;AAElB,IAAA,MAAM,IAAI,GAAG,aAAa,CAAC,eAAe;UACtC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,IAAI;AACpC,UAAE,qBAAqB,CAAC,eAAe;AACrC,cAAE,eAAe,CAAC,IAAI;cACpB,eAAe;AAErB,IAAA,OAAO,oBAAoB,CAAC,IAAI,EAAE,wBAAwB,CAAC;AAC7D;;;;"}

package/dist/drivers/download_blob_url/task.cjs

@@ -0,0 +1,209 @@
+'use strict';
+
+var node_stream = require('node:stream');
+var zlib = require('node:zlib');
+var tar = require('tar-fs');
+var path = require('node:path');
+var fs = require('node:fs');
+var fsp = require('node:fs/promises');
+var download = require('../../helpers/download.cjs');
+var computable = require('@milaboratories/computable');
+var tsHelpers = require('@milaboratories/ts-helpers');
+var download$1 = require('../../clients/download.cjs');
+var url = require('../urls/url.cjs');
+var decompress = require('decompress');
+var runtime = require('@protobuf-ts/runtime');
+var plClient = require('@milaboratories/pl-client');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var zlib__namespace = /*#__PURE__*/_interopNamespaceDefault(zlib);
+var tar__namespace = /*#__PURE__*/_interopNamespaceDefault(tar);
+var fsp__namespace = /*#__PURE__*/_interopNamespaceDefault(fsp);
+
+/** Downloads and extracts an archive to a directory. */
+class DownloadAndUnarchiveTask {
+    logger;
+    signer;
+    saveDir;
+    path;
+    rInfo;
+    format;
+    clientDownload;
+    counter = new tsHelpers.CallersCounter();
+    change = new computable.ChangeSource();
+    signalCtl = new AbortController();
+    error;
+    done = false;
+    size = 0;
+    url;
+    state;
+    constructor(logger, signer, saveDir, path, rInfo, format, clientDownload) {
+        this.logger = logger;
+        this.signer = signer;
+        this.saveDir = saveDir;
+        this.path = path;
+        this.rInfo = rInfo;
+        this.format = format;
+        this.clientDownload = clientDownload;
+    }
+    /** A debug info of the task. */
+    info() {
+        return {
+            rInfo: this.rInfo,
+            format: this.format,
+            path: this.path,
+            done: this.done,
+            size: this.size,
+            error: this.error,
+            taskHistory: this.state,
+        };
+    }
+    attach(w, callerId) {
+        this.counter.inc(callerId);
+        if (!this.done)
+            this.change.attachWatcher(w);
+    }
+    async download() {
+        try {
+            const size = await this.downloadAndDecompress(this.signalCtl.signal);
+            this.setDone(size);
+            this.change.markChanged(`download and decompress for ${plClient.resourceIdToString(this.rInfo.id)} finished`);
+            this.logger.info(`blob to URL task is done: ${plClient.stringifyWithResourceId(this.info())}`);
+        }
+        catch (e) {
+            this.logger.warn(`a error was produced: ${e} for blob to URL task: ${plClient.stringifyWithResourceId(this.info())}`);
+            if (nonRecoverableError(e)) {
+                this.setError(e);
+                this.change.markChanged(`download and decompress for ${plClient.resourceIdToString(this.rInfo.id)} failed`);
+                // Just in case we were half-way extracting an archive.
+                await rmRFDir(this.path);
+                return;
+            }
+            throw e;
+        }
+    }
+    /** Does the download part and keeps a state of the process. */
+    async downloadAndDecompress(signal) {
+        this.state = {};
+        this.state.parentDir = path.dirname(this.path);
+        await tsHelpers.ensureDirExists(this.state.parentDir);
+        this.state.fileExisted = await tsHelpers.fileExists(this.path);
+        if (this.state.fileExisted) {
+            return await dirSize(this.path);
+        }
+        const size = await this.clientDownload.withBlobContent(this.rInfo, {}, { signal }, async (content, size) => {
+            this.state.downloaded = true;
+            await tsHelpers.createPathAtomically(this.logger, this.path, async (fPath) => {
+                this.state.tempPath = fPath;
+                this.state.archiveFormat = this.format;
+                switch (this.format) {
+                    case 'tar':
+                        await fsp__namespace.mkdir(fPath); // throws if a directory already exists.
+                        const simpleUntar = node_stream.Writable.toWeb(tar__namespace.extract(fPath));
+                        await content.pipeTo(simpleUntar, { signal });
+                        return;
+                    case 'tgz':
+                        await fsp__namespace.mkdir(fPath); // throws if a directory already exists.
+                        const gunzip = node_stream.Transform.toWeb(zlib__namespace.createGunzip());
+                        const untar = node_stream.Writable.toWeb(tar__namespace.extract(fPath));
+                        await content
+                            .pipeThrough(gunzip, { signal })
+                            .pipeTo(untar, { signal });
+                        return;
+                    case 'zip':
+                        this.state.zipPath = this.path + '.zip';
+                        const f = node_stream.Writable.toWeb(fs.createWriteStream(this.state.zipPath));
+                        await content.pipeTo(f, { signal });
+                        this.state.zipPathCreated = true;
+                        // Without this filter it fails with
+                        // "EISDIR: illegal operation on a directory".
+                        // The workaround is from
+                        // https://github.com/kevva/decompress/issues/46#issuecomment-525048104
+                        await decompress(this.state.zipPath, fPath, {
+                            filter: file => !file.path.endsWith('/'),
+                        });
+                        this.state.zipDecompressed = true;
+                        await fs.promises.rm(this.state.zipPath);
+                        this.state.zipPathDeleted = true;
+                        return;
+                    default:
+                        runtime.assertNever(this.format);
+                }
+            });
+            this.state.pathCreated = true;
+            return size;
+        });
+        return size;
+    }
+    getURL() {
+        if (this.done)
+            return { url: tsHelpers.notEmpty(this.url) };
+        if (this.error)
+            return { error: this.error };
+        return undefined;
+    }
+    setDone(size) {
+        this.done = true;
+        this.size = size;
+        this.url = url.newFolderURL(this.signer, this.saveDir, this.path);
+    }
+    setError(e) {
+        this.error = String(e);
+    }
+    abort(reason) {
+        this.signalCtl.abort(new URLAborted(reason));
+    }
+}
+/** Gets a directory size by calculating sizes recursively. */
+async function dirSize(dir) {
+    const files = await fsp__namespace.readdir(dir, { withFileTypes: true });
+    const sizes = await Promise.all(files.map(async (file) => {
+        const fPath = path.join(dir, file.name);
+        if (file.isDirectory())
+            return await dirSize(fPath);
+        const stat = await fsp__namespace.stat(fPath);
+        return stat.size;
+    }));
+    return sizes.reduce((sum, size) => sum + size, 0);
+}
+/** Do rm -rf on dir. */
+async function rmRFDir(path) {
+    await fsp__namespace.rm(path, { recursive: true, force: true });
+}
+/** Throws when a downloading aborts. */
+class URLAborted extends Error {
+    name = 'URLAborted';
+}
+function nonRecoverableError(e) {
+    return (e instanceof URLAborted
+        || e instanceof download.NetworkError400
+        || e instanceof download$1.UnknownStorageError
+        || e instanceof download$1.WrongLocalFileUrl
+        // file that we downloads from was moved or deleted.
+        || e?.code == 'ENOENT'
+        // A resource was deleted.
+        || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))
+        // wrong archive format
+        || (String(e).includes('incorrect header check')));
+}
+
+exports.DownloadAndUnarchiveTask = DownloadAndUnarchiveTask;
+exports.nonRecoverableError = nonRecoverableError;
+exports.rmRFDir = rmRFDir;
+//# sourceMappingURL=task.cjs.map

package/dist/drivers/download_blob_url/task.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"task.cjs","sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport path from 'path';\nimport fs from 'fs';\nimport * as fsp from 'fs/promises';\nimport { NetworkError400 } from '../../helpers/download';\nimport type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from '@milaboratories/ts-helpers';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { UnknownStorageError, WrongLocalFileUrl, type ClientDownload } from '../../clients/download';\nimport type { ArchiveFormat, FolderURL } from '@milaboratories/pl-model-common';\nimport { newFolderURL } from '../urls/url';\nimport decompress from 'decompress';\nimport { assertNever } from '@protobuf-ts/runtime';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} finished`);\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(`a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`);\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo, \n      {}, \n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case 'tar':\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case 'tgz':\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content\n                .pipeThrough(gunzip, { signal })\n                .pipeTo(untar, { signal });\n              return;\n\n            case 'zip':\n              this.state!.zipPath = this.path + '.zip';\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: file => !file.path.endsWith('/'),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      }\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = 'URLAborted';\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n    // wrong archive format\n    || (String(e).includes('incorrect header check'))\n  );\n}\n"],"names":["CallersCounter","ChangeSource","resourceIdToString","stringifyWithResourceId","ensureDirExists","fileExists","createPathAtomically","fsp","Writable","tar","Transform","zlib","assertNever","notEmpty","newFolderURL","NetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA;MACa,wBAAwB,CAAA;AAWhB,IAAA,MAAA;AACA,IAAA,MAAA;AACR,IAAA,OAAA;AACA,IAAA,IAAA;AACA,IAAA,KAAA;AACA,IAAA,MAAA;AACQ,IAAA,cAAA;AAhBV,IAAA,OAAO,GAAG,IAAIA,wBAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAIC,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;AACA,IAAA,GAAG;AACH,IAAA,KAAK;AAEb,IAAA,WAAA,CACmB,MAAgB,EAChB,MAAc,EACtB,OAAe,EACf,IAAY,EACZ,KAA+B,EAC/B,MAAqB,EACb,cAA8B,EAAA;QAN9B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,MAAM,GAAN,MAAM;QACE,IAAA,CAAA,cAAc,GAAd,cAAc;IAC9B;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACpE,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+BC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;AAEpG,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,0BAAA,EAA6BC,gCAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;QACvF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA,uBAAA,EAA0BA,gCAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;AAE5G,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+BD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAElG,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,qBAAqB,CAAC,MAAmB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;AAEf,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,MAAME,yBAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;AAE3C,QAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,IAAI,CAAC;AACpD,QAAA,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;AAC1B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACpD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAM,CAAC,UAAU,GAAG,IAAI;AAE7B,YAAA,MAAMC,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;AACzE,gBAAA,IAAI,CAAC,KAAM,CAAC,QAAQ,GAAG,KAAK;gBAC5B,IAAI,CAAC,KAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM;AAEvC,gBAAA,QAAQ,IAAI,CAAC,MAAM;AACjB,oBAAA,KAAK,KAAK;wBACR,MAAMC,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,wBAAA,MAAM,WAAW,GAAGC,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACtD,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC7C;AAEF,oBAAA,KAAK,KAAK;wBACR,MAAMF,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAGG,qBAAS,CAAC,KAAK,CAACC,eAAI,CAAC,YAAY,EAAE,CAAC;AACnD,wBAAA,MAAM,KAAK,GAAGH,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEhD,wBAAA,MAAM;AACH,6BAAA,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE;AAC9B,6BAAA,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC5B;AAEF,oBAAA,KAAK,KAAK;wBACR,IAAI,CAAC,KAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM;AAExC,wBAAA,MAAM,CAAC,GAAGD,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC;wBACnE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC;AACnC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;;;;;wBAMjC,MAAM,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,EAAE,KAAK,EAAE;AAC3C,4BAAA,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;AACzC,yBAAA,CAAC;AACF,wBAAA,IAAI,CAAC,KAAM,CAAC,eAAe,GAAG,IAAI;AAElC,wBAAA,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC;AACzC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;wBAEjC;AAEF,oBAAA;AACE,wBAAAI,mBAAW,CAAC,IAAI,CAAC,MAAM,CAAC;;AAE9B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAM,CAAC,WAAW,GAAG,IAAI;AAC9B,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,GAAG,EAAEC,kBAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAEjD,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,GAAG,GAAGC,gBAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IAC/D;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAMP,cAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAMA,cAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAMA,cAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;AAgBA;AACA,MAAM,UAAW,SAAQ,KAAK,CAAA;IAC5B,IAAI,GAAG,YAAY;AACpB;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAYQ;AACb,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;AAEX,YAAC,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC;;YAEtE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;AAErD;;;;;;"}

package/dist/drivers/download_blob_url/task.d.ts

@@ -61,4 +61,3 @@ type DownloadCtx = {
 };
 export declare function nonRecoverableError(e: any): boolean;
 export {};
-//# sourceMappingURL=task.d.ts.map