@milaboratories/pl-drivers 1.16.0 → 1.16.2

package/dist/drivers/download_blob_url/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,MAAI,YAAY,IAAI,CAClB,QAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,QAAkC,cAAc,IAAI,GACtD,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,WAAW,YAAY;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,aAAa,sBAAsB,GAAG;AAC9C,SAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,SAAS,CAAC,GAAG,SAAS"}
+{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAS3C;CACA;CACA;CACA;CACA;CAZnB,+BAA0D,IAAI,IAAI;CAClE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;CACxB,GACA;EARiB,KAAA,SAAA;EACA,KAAA,SAAA;EACA,KAAA,iBAAA;EACA,KAAA,UAAA;EACA,KAAA,OAAA;EAKjB,KAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;EACV,CAAC;EACD,KAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,kBAAkB;CAC1D;CAEA,OAAmB;EACjB,OAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,KAAK;GACzC,cAAc,MAAM,KAAK,KAAK,aAAa,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;EAC7F;CACF;;;;CAKA,yBAAyB,KAAwB;EAC/C,IAAI,YAAY,GAAG,GACjB,OAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,OAAO;EAG3D,MAAM,IAAI,MAAM,6BAA6B,IAAI,YAAY;CAC/D;CAaA,wBACE,KACA,QACA,KAC2D;EAE3D,IAAI,QAAQ,KAAA,GACV,OAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,CAAC,CAAC;EAE5E,MAAM,QAAkC,cAAc,GAAG,IACrD,6BAA6B,KAAK,GAAG,IACrC;EAEJ,MAAM,WAAW,WAAW;EAE5B,IAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,QAAQ,CAAC;EAEnE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,QAAQ;EACrF,IAAI,QAAQ,QAAQ,KAAA,GAClB,IAAI,aACF,4EAA4E,QAC9E;EAEF,IAAI,QAAQ,UAAU,KAAA,GAAW,MAAM,QAAQ;EAE/C,OAAO,QAAQ;CACjB;CAEA,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,MAAM,CAAC;EAE1D,IAAI,QAAQ,KAAA,GAAW;GACrB,KAAK,OAAO,GAAG,QAAQ;GACvB,OAAO,KAAK,OAAO;EACrB;EAEA,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,QAAQ;EAC1D,KAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,QAAQ;GAClD,4BAA4B,MAAM,CAAC,oBAAoB,CAAC;EAC1D,CAAC;EAED,OAAO,QAAQ,OAAO;CACxB;;CAGA,MAAM,YAAY,MAAgC,UAAkB;EAClE,MAAM,KAAK,SAAS;EAEpB,IAAI,KAAK,OAAO,CAAC,EAAE,OAAO,KAAA,GAAW,KAAK,MAAM,SAAS,MAAM,QAAQ;CACzE;;;CAIA,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,MAAM,CAAC;EACpD,IAAI,QAAQ,KAAA,GAAW;EAEvB,IAAI,KAAK,MAAM,WAAW,KAAK,IAAI,GAAG;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ;GAE1D,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;IACrD,MAAM,QAAQ,KAAK,IAAI;IACvB,KAAK,MAAM,YAAY,IAAI;IAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,GAClF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,wBACnD;CAEN;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,MAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;GAC/D,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK,MAAM,YAAY,IAAI;GAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,yCACnD;EACF,CAAC,CACH;CACF;CAEA,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,MAAM,GACjC,OACA,QACA,KAAK,cACP;EACA,OAAO,OAAO,GAAG,QAAQ;EACzB,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM;EAErD,OAAO;CACT;CAEA,WAAmB,MAAgC,QAAgB;EACjE,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,EAAE,EAAE,YAAY,QAAQ;EAC1F,KAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,MAAM,CAAC;CAC5D;CAEA,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,aAAa,sBAAsB,EAAE;EAC7C,OAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,QAAQ,EAAE,GAAG,QAAQ;CAChE;AACF"}

package/dist/drivers/download_blob_url/driver_id.cjs

@@ -1,4 +1,3 @@
-require("../../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 //#region src/drivers/download_blob_url/driver_id.ts
 function newId(id, format) {

package/dist/drivers/download_blob_url/driver_id.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,GAAG;AAC9C,QAAO,MAAM,OAAO,SAAS,CAAC,GAAG"}
+{"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,sBAAA,CAAmC,EAAE;CAC7C,OAAO,MAAM,OAAO,QAAQ,EAAE,GAAG;AACnC"}

package/dist/drivers/download_blob_url/driver_id.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver_id.js","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,aAAa,sBAAsB,GAAG;AAC9C,QAAO,MAAM,OAAO,SAAS,CAAC,GAAG"}
+{"version":3,"file":"driver_id.js","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,aAAa,sBAAsB,EAAE;CAC7C,OAAO,MAAM,OAAO,QAAQ,EAAE,GAAG;AACnC"}

package/dist/drivers/download_blob_url/snapshot.cjs

@@ -1,4 +1,3 @@
-require("../../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 //#region src/drivers/download_blob_url/snapshot.ts
 /** We need only resource type for this driver. */

package/dist/drivers/download_blob_url/snapshot.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;;AAeA,MAAa,4BAAA,GAAA,wBAAA,UAAoC,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,SAAA,GAAA,wBAAA,uBAAA,GAAA,wBAAA,eAN2B,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,IAAA,GAAA,wBAAA,uBACd,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}
+{"version":3,"file":"snapshot.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;AAeA,MAAa,4BAAA,GAAA,wBAAA,SAAA,CAAoC,CAAC,CAAC;AAGnD,SAAgB,6BACd,iBACA,KAC0B;CAO1B,QAAA,GAAA,wBAAA,qBAAA,EAAA,GAAA,wBAAA,cAAA,CAN2B,eAAe,IACtC,IAAI,SAAS,eAAe,CAAC,CAAC,KAAK,KAAA,GAAA,wBAAA,sBAAA,CACb,eAAe,IACnC,gBAAgB,KAAK,IACrB,iBAE4B,wBAAwB;AAC5D"}

package/dist/drivers/download_blob_url/snapshot.d.ts

@@ -1,10 +1,9 @@
 import { ComputableCtx } from "@milaboratories/computable";
-import * as _$_milaboratories_pl_tree0 from "@milaboratories/pl-tree";
 import { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 
 //#region src/drivers/download_blob_url/snapshot.d.ts
 /** We need only resource type for this driver. */
-declare const DownloadableBlobSnapshot: _$_milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, undefined, undefined>;
+declare const DownloadableBlobSnapshot: import("@milaboratories/pl-tree").ResourceSnapshotSchema<undefined, undefined, undefined>;
 type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;
 declare function makeDownloadableBlobSnapshot(entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor, ctx: ComputableCtx): DownloadableBlobSnapshot;
 //#endregion

package/dist/drivers/download_blob_url/snapshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"mappings":";;;;;;cAea,wBAAA,EAAuC,0BAAA,CAAf,sBAAA;AAAA,KACzB,wBAAA,GAA2B,aAAA,QAAqB,wBAAA;AAAA,iBAE5C,4BAAA,CACd,eAAA,EAAiB,WAAA,GAAc,kBAAA,GAAqB,mBAAA,EACpD,GAAA,EAAK,aAAA,GACJ,wBAAA"}
+{"version":3,"file":"snapshot.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"mappings":";;;;;cAea,wBAAA,oCAAwB,sBAAA;AAAA,KACzB,wBAAA,GAA2B,aAAa,QAAQ,wBAAA;AAAA,iBAE5C,4BAAA,CACd,eAAA,EAAiB,WAAA,GAAc,kBAAA,GAAqB,mBAAA,EACpD,GAAA,EAAK,aAAA,GACJ,wBAAA"}

package/dist/drivers/download_blob_url/snapshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.js","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;AAeA,MAAa,2BAA2B,SAAS,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,QAAO,qBANM,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}
+{"version":3,"file":"snapshot.js","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;AAeA,MAAa,2BAA2B,SAAS,CAAC,CAAC;AAGnD,SAAgB,6BACd,iBACA,KAC0B;CAO1B,OAAO,qBANM,cAAc,eAAe,IACtC,IAAI,SAAS,eAAe,CAAC,CAAC,KAAK,IACnC,sBAAsB,eAAe,IACnC,gBAAgB,KAAK,IACrB,iBAE4B,wBAAwB;AAC5D"}

package/dist/drivers/download_blob_url/task.cjs

@@ -8,20 +8,27 @@ let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_stream = require("node:stream");
 let _milaboratories_computable = require("@milaboratories/computable");
 let fs = require("fs");
-fs = require_runtime.__toESM(fs);
+fs = require_runtime.__toESM(fs, 1);
 let node_zlib = require("node:zlib");
-node_zlib = require_runtime.__toESM(node_zlib);
+node_zlib = require_runtime.__toESM(node_zlib, 1);
 let tar_fs = require("tar-fs");
-tar_fs = require_runtime.__toESM(tar_fs);
+tar_fs = require_runtime.__toESM(tar_fs, 1);
 let path = require("path");
-path = require_runtime.__toESM(path);
+path = require_runtime.__toESM(path, 1);
 let fs_promises = require("fs/promises");
-fs_promises = require_runtime.__toESM(fs_promises);
+fs_promises = require_runtime.__toESM(fs_promises, 1);
 let decompress = require("decompress");
-decompress = require_runtime.__toESM(decompress);
+decompress = require_runtime.__toESM(decompress, 1);
 //#region src/drivers/download_blob_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadAndUnarchiveTask = class {
+	logger;
+	signer;
+	saveDir;
+	path;
+	rInfo;
+	format;
+	clientDownload;
 	counter = new _milaboratories_ts_helpers.CallersCounter();
 	change = new _milaboratories_computable.ChangeSource();
 	signalCtl = new AbortController();
@@ -154,6 +161,7 @@ function nonRecoverableError(e) {
 }
 //#endregion
 exports.DownloadAndUnarchiveTask = DownloadAndUnarchiveTask;
+exports.nonRecoverableError = nonRecoverableError;
 exports.rmRFDir = rmRFDir;
 
 //# sourceMappingURL=task.cjs.map

package/dist/drivers/download_blob_url/task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,UAAmB,IAAIA,2BAAAA,gBAAgB;CACvC,SAAkB,IAAIC,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,QACA,OACA,QACA,gBACA;AAPiB,OAAA,SAAA;AACA,OAAA,SAAA;AACR,OAAA,UAAA;AACA,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,SAAA;AACQ,OAAA,iBAAA;;;CAInB,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,oBAAkD,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,8BAAA,GAAA,0BAAA,yBAAqD,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,0BAAA,GAAA,0BAAA,yBAAiD,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,oBAAkD,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAA,QAAK,QAAQ,KAAK,KAAK;AAC9C,SAAA,GAAA,2BAAA,iBAAsB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAME,YAAI,MAAM,MAAM;MACtB,MAAM,cAAcC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAMF,YAAI,MAAM,MAAM;MACtB,MAAM,SAASG,YAAAA,UAAU,MAAMC,UAAK,cAAc,CAAC;MACnD,MAAM,QAAQH,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,YAAAA,SAAS,MAAM,GAAA,QAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,aAAA,GAAA,WAAA,SAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAA,QAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,EAAA,GAAA,qBAAA,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,MAAA,GAAA,2BAAA,UAAc,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAMI,YAAAA,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAA,QAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMA,YAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,QAAc;AAC1C,OAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACbO,wBAAAA,0BAA0B,EAAE,IAC5B,aAAaC,iBAAAA,uBACb,aAAaC,iBAAAA,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
+{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CAWjB;CACA;CACR;CACA;CACA;CACA;CACQ;CAhBnB,UAAmB,IAAIA,2BAAAA,eAAe;CACtC,SAAkB,IAAIC,2BAAAA,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,QACA,OACA,QACA,gBACA;EAPiB,KAAA,SAAA;EACA,KAAA,SAAA;EACR,KAAA,UAAA;EACA,KAAA,OAAA;EACA,KAAA,QAAA;EACA,KAAA,SAAA;EACQ,KAAA,iBAAA;CAChB;;CAGH,OAAc;EACZ,OAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;EACpB;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,WAAW;EACf,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,MAAM;GACnE,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,mBAAA,CAAkD,KAAK,MAAM,EAAE,EAAE,UACnE;GAEA,KAAK,OAAO,KAAK,8BAAA,GAAA,0BAAA,wBAAA,CAAqD,KAAK,KAAK,CAAC,GAAG;EACtF,SAAS,GAAQ;GACf,KAAK,OAAO,KACV,yBAAyB,EAAE,0BAAA,GAAA,0BAAA,wBAAA,CAAiD,KAAK,KAAK,CAAC,GACzF;GAEA,IAAI,oBAAoB,CAAC,GAAG;IAC1B,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,mBAAA,CAAkD,KAAK,MAAM,EAAE,EAAE,QACnE;IAEA,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;;CAGA,MAAc,sBAAsB,QAAsC;EACxE,KAAK,QAAQ,CAAC;EAEd,KAAK,MAAM,YAAY,KAAA,QAAK,QAAQ,KAAK,IAAI;EAC7C,OAAA,GAAA,2BAAA,gBAAA,CAAsB,KAAK,MAAM,SAAS;EAE1C,KAAK,MAAM,cAAc,OAAA,GAAA,2BAAA,WAAA,CAAiB,KAAK,IAAI;EACnD,IAAI,KAAK,MAAM,aACb,OAAO,MAAM,QAAQ,KAAK,IAAI;EA4DhC,OAAO,MAzDY,KAAK,eAAe,gBACrC,KAAK,OACL,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,KAAK,MAAO,aAAa;GAEzB,OAAA,GAAA,2BAAA,qBAAA,CAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;IAC1E,KAAK,MAAO,WAAW;IACvB,KAAK,MAAO,gBAAgB,KAAK;IAEjC,QAAQ,KAAK,QAAb;KACE,KAAK;MACH,MAAME,YAAI,MAAM,KAAK;MACrB,MAAM,cAAcC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,KAAK,CAAC;MACrD,MAAM,QAAQ,OAAO,aAAa,EAAE,OAAO,CAAC;MAC5C;KAEF,KAAK;MACH,MAAMF,YAAI,MAAM,KAAK;MACrB,MAAM,SAASG,YAAAA,UAAU,MAAMC,UAAK,aAAa,CAAC;MAClD,MAAM,QAAQH,YAAAA,SAAS,MAAMC,OAAI,QAAQ,KAAK,CAAC;MAE/C,MAAM,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,EAAE,OAAO,CAAC;MACtE;KAEF,KAAK;MACH,KAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,YAAAA,SAAS,MAAM,GAAA,QAAG,kBAAkB,KAAK,MAAO,OAAO,CAAC;MAClE,MAAM,QAAQ,OAAO,GAAG,EAAE,OAAO,CAAC;MAClC,KAAK,MAAO,iBAAiB;MAM7B,OAAA,GAAA,WAAA,QAAA,CAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,GAAG,EAC3C,CAAC;MACD,KAAK,MAAO,kBAAkB;MAE9B,MAAM,GAAA,QAAG,SAAS,GAAG,KAAK,MAAO,OAAO;MACxC,KAAK,MAAO,iBAAiB;MAE7B;KAEF,SACE,CAAA,GAAA,qBAAA,YAAA,CAAY,KAAK,MAAM;IAC3B;GACF,CAAC;GAED,KAAK,MAAO,cAAc;GAC1B,OAAO;EACT,CACF;CAGF;CAEA,SAAgC;EAC9B,IAAI,KAAK,MAAM,OAAO,EAAE,MAAA,GAAA,2BAAA,SAAA,CAAc,KAAK,GAAG,EAAE;EAEhD,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,MAAMI,YAAAA,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,IAAI;CAC9D;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAA,QAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADYA,YAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,QAAc;CAC1C,MAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD;;AAiBA,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;AACT;AAEA,SAAgB,oBAAoB,GAAQ;CAC1C,OACE,aAAa,cACbO,wBAAAA,0BAA0B,CAAC,KAC3B,aAAaC,iBAAAA,uBACb,aAAaC,iBAAAA,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,CAAC,CAAC,CAAC,SAAS,wBAAwB;AAE/C"}

package/dist/drivers/download_blob_url/task.d.ts

@@ -60,5 +60,5 @@ type DownloadCtx = {
   pathCreated?: boolean;
 };
 //#endregion
-export { DownloadAndUnarchiveTask };
+export { DownloadAndUnarchiveTask, URLResult };
 //# sourceMappingURL=task.d.ts.map

package/dist/drivers/download_blob_url/task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"mappings":";;;;;;;KA6BY,SAAA;EACV,GAAA,GAAM,SAAA;EACN,KAAA;AAAA;;cAIW,wBAAA;EAAA,iBAWQ,MAAA;EAAA,iBACA,MAAA;EAAA,SACR,OAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA,EAAO,wBAAA;EAAA,SACP,MAAA,EAAQ,aAAA;EAAA,iBACA,cAAA;EAAA,SAhBV,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;EAAA,QACQ,GAAA;EAAA,QACA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EAChB,OAAA,UACA,IAAA,UACA,KAAA,EAAO,wBAAA,EACP,MAAA,EAAQ,aAAA,EACA,cAAA,EAAgB,cAAA;;EAI5B,IAAA,CAAA;;;;;;;;;EAYP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAA,GAAQ,OAAA;EAvBI;EAAA,QAoDJ,qBAAA;EAuEd,MAAA,CAAA,GAAU,SAAA;EAAA,QAQF,OAAA;EAAA,QAMA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA;;KA4BH,WAAA;EACH,SAAA;EACA,WAAA;EACA,UAAA;EACA,aAAA,GAAgB,aAAA;EAChB,QAAA;EACA,OAAA;EACA,cAAA;EACA,eAAA;EACA,cAAA;EACA,WAAA;AAAA"}
+{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"mappings":";;;;;;;KA6BY,SAAA;EACV,GAAA,GAAM,SAAS;EACf,KAAA;AAAA;;cAIW,wBAAA;EAAA,iBAWQ,MAAA;EAAA,iBACA,MAAA;EAAA,SACR,OAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA,EAAO,wBAAA;EAAA,SACP,MAAA,EAAQ,aAAA;EAAA,iBACA,cAAA;EAAA,SAhBV,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;EAAA,QACQ,GAAA;EAAA,QACA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EAChB,OAAA,UACA,IAAA,UACA,KAAA,EAAO,wBAAA,EACP,MAAA,EAAQ,aAAA,EACA,cAAA,EAAgB,cAAA;;EAI5B,IAAA;;;;;;;;;EAYP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,IAAQ,OAAA;EAvBI;EAAA,QAoDJ,qBAAA;EAuEd,MAAA,IAAU,SAAA;EAAA,QAQF,OAAA;EAAA,QAMA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA;;KA4BH,WAAA;EACH,SAAA;EACA,WAAA;EACA,UAAA;EACA,aAAA,GAAgB,aAAa;EAC7B,QAAA;EACA,OAAA;EACA,cAAA;EACA,eAAA;EACA,cAAA;EACA,WAAA;AAAA"}

package/dist/drivers/download_blob_url/task.js

@@ -15,6 +15,13 @@ import decompress from "decompress";
 //#region src/drivers/download_blob_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadAndUnarchiveTask = class {
+	logger;
+	signer;
+	saveDir;
+	path;
+	rInfo;
+	format;
+	clientDownload;
 	counter = new CallersCounter();
 	change = new ChangeSource();
 	signalCtl = new AbortController();
@@ -146,6 +153,6 @@ function nonRecoverableError(e) {
 	return e instanceof URLAborted || isDownloadNetworkError400(e) || e instanceof UnknownStorageError || e instanceof WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED") || String(e).includes("incorrect header check");
 }
 //#endregion
-export { DownloadAndUnarchiveTask, rmRFDir };
+export { DownloadAndUnarchiveTask, nonRecoverableError, rmRFDir };
 
 //# sourceMappingURL=task.js.map

package/dist/drivers/download_blob_url/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,MACA,OACA,QACA,gBACA;AAPiB,OAAA,SAAA;AACA,OAAA,SAAA;AACR,OAAA,UAAA;AACA,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,SAAA;AACQ,OAAA,iBAAA;;;CAInB,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
+{"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CAWjB;CACA;CACR;CACA;CACA;CACA;CACQ;CAhBnB,UAAmB,IAAI,eAAe;CACtC,SAAkB,IAAI,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,MACA,OACA,QACA,gBACA;EAPiB,KAAA,SAAA;EACA,KAAA,SAAA;EACR,KAAA,UAAA;EACA,KAAA,OAAA;EACA,KAAA,QAAA;EACA,KAAA,SAAA;EACQ,KAAA,iBAAA;CAChB;;CAGH,OAAc;EACZ,OAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;EACpB;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,WAAW;EACf,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,MAAM;GACnE,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,EAAE,EAAE,UACnE;GAEA,KAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,KAAK,CAAC,GAAG;EACtF,SAAS,GAAQ;GACf,KAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,KAAK,CAAC,GACzF;GAEA,IAAI,oBAAoB,CAAC,GAAG;IAC1B,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,EAAE,EAAE,QACnE;IAEA,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;;CAGA,MAAc,sBAAsB,QAAsC;EACxE,KAAK,QAAQ,CAAC;EAEd,KAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,IAAI;EAC7C,MAAM,gBAAgB,KAAK,MAAM,SAAS;EAE1C,KAAK,MAAM,cAAc,MAAM,WAAW,KAAK,IAAI;EACnD,IAAI,KAAK,MAAM,aACb,OAAO,MAAM,QAAQ,KAAK,IAAI;EA4DhC,OAAO,MAzDY,KAAK,eAAe,gBACrC,KAAK,OACL,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,KAAK,MAAO,aAAa;GAEzB,MAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;IAC1E,KAAK,MAAO,WAAW;IACvB,KAAK,MAAO,gBAAgB,KAAK;IAEjC,QAAQ,KAAK,QAAb;KACE,KAAK;MACH,MAAM,IAAI,MAAM,KAAK;MACrB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,KAAK,CAAC;MACrD,MAAM,QAAQ,OAAO,aAAa,EAAE,OAAO,CAAC;MAC5C;KAEF,KAAK;MACH,MAAM,IAAI,MAAM,KAAK;MACrB,MAAM,SAAS,UAAU,MAAM,KAAK,aAAa,CAAC;MAClD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,KAAK,CAAC;MAE/C,MAAM,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,EAAE,OAAO,CAAC;MACtE;KAEF,KAAK;MACH,KAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,OAAO,CAAC;MAClE,MAAM,QAAQ,OAAO,GAAG,EAAE,OAAO,CAAC;MAClC,KAAK,MAAO,iBAAiB;MAM7B,MAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,GAAG,EAC3C,CAAC;MACD,KAAK,MAAO,kBAAkB;MAE9B,MAAM,GAAG,SAAS,GAAG,KAAK,MAAO,OAAO;MACxC,KAAK,MAAO,iBAAiB;MAE7B;KAEF,SACE,YAAY,KAAK,MAAM;IAC3B;GACF,CAAC;GAED,KAAK,MAAO,cAAc;GAC1B,OAAO;EACT,CACF;CAGF;CAEA,SAAgC;EAC9B,IAAI,KAAK,MAAM,OAAO,EAAE,KAAK,SAAS,KAAK,GAAG,EAAE;EAEhD,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,IAAI;CAC9D;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADY,IAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,MAAc;CAC1C,MAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD;;AAiBA,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;AACT;AAEA,SAAgB,oBAAoB,GAAQ;CAC1C,OACE,aAAa,cACb,0BAA0B,CAAC,KAC3B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,CAAC,CAAC,CAAC,SAAS,wBAAwB;AAE/C"}

package/dist/drivers/download_url/driver.cjs

@@ -7,7 +7,7 @@ const require_task = require("./task.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_path = require("node:path");
-node_path = require_runtime.__toESM(node_path);
+node_path = require_runtime.__toESM(node_path, 1);
 let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let node_crypto = require("node:crypto");
@@ -15,8 +15,12 @@ let node_crypto = require("node:crypto");
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
 var DownloadUrlDriver = class {
+	logger;
+	saveDir;
+	signer;
+	opts;
 	downloadHelper;
-	urlToDownload = /* @__PURE__ */ new Map();
+	taskByKey = /* @__PURE__ */ new Map();
 	downloadQueue;
 	/** Writes and removes files to a hard drive and holds a counter for every
 	* file that should be kept. */
@@ -37,20 +41,37 @@ var DownloadUrlDriver = class {
 	/** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */
 	getUrl(url, ctx) {
 		if (ctx === void 0) return _milaboratories_computable.Computable.make((c) => this.getUrl(url, c));
+		return this.getResult({
+			type: "url",
+			url
+		}, url.toString(), ctx);
+	}
+	/** Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it
+	* into the recycled cache (so a live UI keeps the folder pinned) and returns
+	* a custom-protocol URL. Keyed on `path + ":" + mtime`, so the unpacked UI is
+	* re-derived when the source block changes. */
+	getLocalTgz(path, mtime, ctx) {
+		if (ctx === void 0) return _milaboratories_computable.Computable.make((c) => this.getLocalTgz(path, mtime, c));
+		return this.getResult({
+			type: "local-tgz",
+			path,
+			mtime
+		}, `${path}:${mtime}`, ctx);
+	}
+	getResult(source, key, ctx) {
 		const callerId = (0, node_crypto.randomUUID)();
-		ctx.addOnDestroy(() => this.releasePath(url, callerId));
-		const result = this.getUrlNoCtx(url, ctx.watcher, callerId);
+		ctx.addOnDestroy(() => this.releaseByKey(key, callerId));
+		const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);
 		if (result?.url === void 0) ctx.markUnstable(`a path to the downloaded and untared archive might be undefined. The current result: ${result}`);
 		return result;
 	}
-	getUrlNoCtx(url, w, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	getResultNoCtx(source, key, w, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task !== void 0) {
 			task.attach(w, callerId);
 			return task.getUrl();
 		}
-		const newTask = this.setNewTask(w, url, callerId);
+		const newTask = this.setNewTask(w, source, key, callerId);
 		this.downloadQueue.push({
 			fn: async () => this.downloadUrl(newTask, callerId),
 			recoverableErrorPredicate: (e) => !(e instanceof require_task.URLAborted) && !require_download_errors.isDownloadNetworkError400(e)
@@ -68,9 +89,8 @@ var DownloadUrlDriver = class {
 	}
 	/** Removes a directory and aborts a downloading task when all callers
 	* are not interested in it. */
-	async releasePath(url, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	async releaseByKey(key, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task == void 0) return;
 		if (this.cache.existsFile(task.path)) {
 			const toDelete = this.cache.removeFile(task.path, callerId);
@@ -84,25 +104,27 @@ var DownloadUrlDriver = class {
 	/** Removes all files from a hard drive. */
 	async releaseAll() {
 		this.downloadQueue.stop();
-		await Promise.all(Array.from(this.urlToDownload.entries()).map(async ([, task]) => {
+		await Promise.all(Array.from(this.taskByKey.entries()).map(async ([, task]) => {
 			await require_task.rmRFDir(task.path);
 			this.cache.removeCache(task);
 			this.removeTask(task, `the task ${(0, _milaboratories_pl_client.stringifyWithResourceId)(task.info())} was released when the driver was closed`);
 		}));
 	}
-	setNewTask(w, url, callerId) {
-		const result = new require_task.DownloadByUrlTask(this.logger, this.getFilePath(url), url, this.signer, this.saveDir);
+	setNewTask(w, source, key, callerId) {
+		const result = new require_task.DownloadByUrlTask(this.logger, this.getExtractionPath(key), source, key, this.signer, this.saveDir);
 		result.attach(w, callerId);
-		this.urlToDownload.set(url.toString(), result);
+		this.taskByKey.set(key, result);
 		return result;
 	}
 	removeTask(task, reason) {
 		task.abort(reason);
-		task.change.markChanged(`task for url ${task.url} removed: ${reason}`);
-		this.urlToDownload.delete(task.url.toString());
+		task.change.markChanged(`task for ${task.key} removed: ${reason}`);
+		this.taskByKey.delete(task.key);
 	}
-	getFilePath(url) {
-		const sha256 = (0, node_crypto.createHash)("sha256").update(url.toString()).digest("hex");
+	/** Directory an archive (url or local file) is extracted into; keyed by a
+	* hash of the task key so the same source maps to a stable on-disk path. */
+	getExtractionPath(key) {
+		const sha256 = (0, node_crypto.createHash)("sha256").update(key).digest("hex");
 		return node_path.join(this.saveDir, sha256);
 	}
 };

package/dist/drivers/download_url/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E;CAEA,gCAAwD,IAAI,KAAK;CACjE;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;AATiB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,OAAA;AAMjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAIC,iBAAAA,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,KAAA,EAAW,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,YAAA,GAAA,YAAA,aAAuB;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,KAAA,GAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAaC,aAAAA,eAAe,CAACC,wBAAAA,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,EAAA,GAAA,gCAAA,cAAc,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAOC,YAAAA,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAMC,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WAAmB,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAIC,aAAAA,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,WAAmB,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,YAAoB,KAAkB;EACpC,MAAM,UAAA,GAAA,YAAA,YAAoB,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOC,UAAK,KAAK,KAAK,SAAS,OAAO"}
+{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport type { DownloadSource } from \"./task\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private taskByKey: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n    return this.getResult({ type: \"url\", url }, url.toString(), ctx);\n  }\n\n  getLocalTgz(path: string, mtime: string, ctx: ComputableCtx): UrlResult | undefined;\n  getLocalTgz(path: string, mtime: string): Computable<UrlResult | undefined>;\n\n  /** Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it\n   * into the recycled cache (so a live UI keeps the folder pinned) and returns\n   * a custom-protocol URL. Keyed on `path + \":\" + mtime`, so the unpacked UI is\n   * re-derived when the source block changes. */\n  getLocalTgz(\n    path: string,\n    mtime: string,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    if (ctx === undefined) return Computable.make((c) => this.getLocalTgz(path, mtime, c));\n    return this.getResult({ type: \"local-tgz\", path, mtime }, `${path}:${mtime}`, ctx);\n  }\n\n  private getResult(\n    source: DownloadSource,\n    key: string,\n    ctx: ComputableCtx,\n  ): UrlResult | undefined {\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releaseByKey(key, callerId));\n\n    const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  private getResultNoCtx(source: DownloadSource, key: string, w: Watcher, callerId: string) {\n    const task = this.taskByKey.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, source, key, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releaseByKey(key: string, callerId: string): Promise<void> {\n    const task = this.taskByKey.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.taskByKey.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, source: DownloadSource, key: string, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getExtractionPath(key),\n      source,\n      key,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.taskByKey.set(key, result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${task.key} removed: ${reason}`);\n    this.taskByKey.delete(task.key);\n  }\n\n  /** Directory an archive (url or local file) is extracted into; keyed by a\n   * hash of the task key so the same source maps to a stable on-disk path. */\n  private getExtractionPath(key: string): string {\n    const sha256 = createHash(\"sha256\").update(key).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkDA,IAAa,oBAAb,MAAgF;CAW3D;CAEA;CACA;CACA;CAdnB;CAEA,4BAAoD,IAAI,IAAI;CAC5D;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;CACxB,GACA;EATiB,KAAA,SAAA;EAEA,KAAA,UAAA;EACA,KAAA,SAAA;EACA,KAAA,OAAA;EAMjB,KAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,oBAAoB;EAClF,KAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,kBAAkB;EACxD,KAAK,iBAAiB,IAAIC,iBAAAA,qBAAqB,UAAU;CAC3D;;CASA,OAAO,KAAU,KAAgF;EAE/F,IAAI,QAAQ,KAAA,GAAW,OAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,CAAC,CAAC;EACxE,OAAO,KAAK,UAAU;GAAE,MAAM;GAAO;EAAI,GAAG,IAAI,SAAS,GAAG,GAAG;CACjE;;;;;CASA,YACE,MACA,OACA,KAC2D;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAOA,2BAAAA,WAAW,MAAM,MAAM,KAAK,YAAY,MAAM,OAAO,CAAC,CAAC;EACrF,OAAO,KAAK,UAAU;GAAE,MAAM;GAAa;GAAM;EAAM,GAAG,GAAG,KAAK,GAAG,SAAS,GAAG;CACnF;CAEA,UACE,QACA,KACA,KACuB;EACvB,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAG5B,IAAI,mBAAmB,KAAK,aAAa,KAAK,QAAQ,CAAC;EAEvD,MAAM,SAAS,KAAK,eAAe,QAAQ,KAAK,IAAI,SAAS,QAAQ;EACrE,IAAI,QAAQ,QAAQ,KAAA,GAClB,IAAI,aACF,wFAAwF,QAC1F;EAEF,OAAO;CACT;CAEA,eAAuB,QAAwB,KAAa,GAAY,UAAkB;EACxF,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EAEnC,IAAI,SAAS,KAAA,GAAW;GACtB,KAAK,OAAO,GAAG,QAAQ;GACvB,OAAO,KAAK,OAAO;EACrB;EAEA,MAAM,UAAU,KAAK,WAAW,GAAG,QAAQ,KAAK,QAAQ;EACxD,KAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,QAAQ;GAClD,4BAA4B,MAAM,EAAE,aAAaC,aAAAA,eAAe,CAACC,wBAAAA,0BAA0B,CAAC;EAC9F,CAAC;EAED,OAAO,QAAQ,OAAO;CACxB;CAEA,kBAAkB,KAAqB;EACrC,IAAI,EAAA,GAAA,gCAAA,aAAA,CAAc,GAAG,GACnB,MAAM,IAAI,MAAM,sBAAsB,IAAI,YAAY;EAGxD,OAAOC,YAAAA,qBAAqB,KAAK,QAAQ,KAAK,KAAK,OAAO;CAC5D;;CAGA,MAAM,YAAY,MAAyB,UAAkB;EAC3D,MAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,UAAU;EAE7D,IAAI,KAAK,OAAO,CAAC,EAAE,QAAQ,KAAA,GAAW,KAAK,MAAM,SAAS,MAAM,QAAQ;CAC1E;;;CAIA,MAAM,aAAa,KAAa,UAAiC;EAC/D,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EACnC,IAAI,QAAQ,KAAA,GAAW;EAEvB,IAAI,KAAK,MAAM,WAAW,KAAK,IAAI,GAAG;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ;GAE1D,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;IAC9C,MAAMC,aAAAA,QAAQ,KAAK,IAAI;IACvB,KAAK,MAAM,YAAY,IAAI;IAE3B,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,qCAAA,GAAA,0BAAA,wBAAA,CACE,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,GAClF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,wBACnD;CAEN;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,MAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,UAAU,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;GAC3D,MAAMA,aAAAA,QAAQ,KAAK,IAAI;GACvB,KAAK,MAAM,YAAY,IAAI;GAE3B,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,yCACnD;EACF,CAAC,CACH;CACF;CAEA,WAAmB,GAAY,QAAwB,KAAa,UAAkB;EACpF,MAAM,SAAS,IAAIC,aAAAA,kBACjB,KAAK,QACL,KAAK,kBAAkB,GAAG,GAC1B,QACA,KACA,KAAK,QACL,KAAK,OACP;EACA,OAAO,OAAO,GAAG,QAAQ;EACzB,KAAK,UAAU,IAAI,KAAK,MAAM;EAE9B,OAAO;CACT;CAEA,WAAmB,MAAyB,QAAgB;EAC1D,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,YAAY,KAAK,IAAI,YAAY,QAAQ;EACjE,KAAK,UAAU,OAAO,KAAK,GAAG;CAChC;;;CAIA,kBAA0B,KAAqB;EAC7C,MAAM,UAAA,GAAA,YAAA,WAAA,CAAoB,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,KAAK;EAC5D,OAAOC,UAAK,KAAK,KAAK,SAAS,MAAM;CACvC;AACF"}

package/dist/drivers/download_url/driver.d.ts

@@ -1,7 +1,7 @@
 import { DownloadByUrlTask } from "./task.js";
 import { Dispatcher } from "undici";
 import { MiLogger, Signer } from "@milaboratories/ts-helpers";
-import { Computable, ComputableCtx, Watcher } from "@milaboratories/computable";
+import { Computable, ComputableCtx } from "@milaboratories/computable";
 import { BlockUIURL, FrontendDriver } from "@milaboratories/pl-model-common";
 
 //#region src/drivers/download_url/driver.d.ts
@@ -38,7 +38,7 @@ declare class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver
   private readonly signer;
   private readonly opts;
   private readonly downloadHelper;
-  private urlToDownload;
+  private taskByKey;
   private downloadQueue;
   /** Writes and removes files to a hard drive and holds a counter for every
    * file that should be kept. */
@@ -48,18 +48,23 @@ declare class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver
   getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;
   /** Returns a Computable that do the work */
   getUrl(url: URL): Computable<UrlResult | undefined>;
-  getUrlNoCtx(url: URL, w: Watcher, callerId: string): UrlResult | undefined;
+  getLocalTgz(path: string, mtime: string, ctx: ComputableCtx): UrlResult | undefined;
+  getLocalTgz(path: string, mtime: string): Computable<UrlResult | undefined>;
+  private getResult;
+  private getResultNoCtx;
   getPathForBlockUI(url: string): string;
   /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
   downloadUrl(task: DownloadByUrlTask, callerId: string): Promise<void>;
   /** Removes a directory and aborts a downloading task when all callers
    * are not interested in it. */
-  releasePath(url: URL, callerId: string): Promise<void>;
+  releaseByKey(key: string, callerId: string): Promise<void>;
   /** Removes all files from a hard drive. */
   releaseAll(): Promise<void>;
   private setNewTask;
   private removeTask;
-  private getFilePath;
+  /** Directory an archive (url or local file) is extracted into; keyed by a
+   * hash of the task key so the same source maps to a stable on-disk path. */
+  private getExtractionPath;
 }
 //#endregion
 export { DownloadUrlDriver, DownloadUrlDriverOps, DownloadUrlSyncReader, UrlResult };

package/dist/drivers/download_url/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_url/driver.ts"],"mappings":";;;;;;;UAgBiB,qBAAA;EAAA;;;EAIf,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;AAAA;AAAA,UAGd,SAAA;EAHa;;EAM5B,GAAA,GAAM,UAAA;EANM;EAQZ,KAAA;AAAA;AAAA,KAGU,oBAAA;EAX4B;;AAGxC;;EAaE,kBAAA,UAVgB;EAahB,UAAA;EAbM;;;EAkBN,oBAAA;AAAA;;;cAKW,iBAAA,YAA6B,qBAAA,EAAuB,cAAA;EAAA,iBAW5C,MAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,MAAA;EAAA,iBACA,IAAA;EAAA,iBAdF,cAAA;EAAA,QAET,aAAA;EAAA,QACA,aAAA;;;UAIA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACzB,UAAA,EAAY,UAAA,EACK,OAAA,UACA,MAAA,EAAQ,MAAA,EACR,IAAA,GAAM,oBAAA;EAAA;EAYzB,MAAA,CAAO,GAAA,EAAK,GAAA,EAAK,GAAA,EAAK,aAAA,GAAgB,SAAA;EAAhB;EAGtB,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;EAqB7B,WAAA,CAAY,GAAA,EAAK,GAAA,EAAK,CAAA,EAAG,OAAA,EAAS,QAAA,WAAgB,SAAA;EAkBlD,iBAAA,CAAkB,GAAA;EAvCA;EAgDZ,WAAA,CAAY,IAAA,EAAM,iBAAA,EAAmB,QAAA,WAAgB,OAAA;EA3BlC;;EAmCnB,WAAA,CAAY,GAAA,EAAK,GAAA,EAAK,QAAA,WAAmB,OAAA;EARY;EAwCrD,UAAA,CAAA,GAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAcA,UAAA;EAAA,QAMA,WAAA;AAAA"}
+{"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_url/driver.ts"],"mappings":";;;;;;;UAiBiB,qBAAA;EAAA;;;EAIf,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;AAAA;AAAA,UAGd,SAAA;EAHa;;EAM5B,GAAA,GAAM,UAAU;EANJ;EAQZ,KAAA;AAAA;AAAA,KAGU,oBAAA;EAX4B;AAAA;AAGxC;;EAaE,kBAAA,UAVgB;EAahB,UAAA;EAbM;;;EAkBN,oBAAA;AAAA;;;cAKW,iBAAA,YAA6B,qBAAA,EAAuB,cAAA;EAAA,iBAW5C,MAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,MAAA;EAAA,iBACA,IAAA;EAAA,iBAdF,cAAA;EAAA,QAET,SAAA;EAAA,QACA,aAAA;;;UAIA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACzB,UAAA,EAAY,UAAA,EACK,OAAA,UACA,MAAA,EAAQ,MAAA,EACR,IAAA,GAAM,oBAAA;EAAA;EAYzB,MAAA,CAAO,GAAA,EAAK,GAAA,EAAK,GAAA,EAAK,aAAA,GAAgB,SAAA;EAAhB;EAGtB,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;EAS7B,WAAA,CAAY,IAAA,UAAc,KAAA,UAAe,GAAA,EAAK,aAAA,GAAgB,SAAA;EAC9D,WAAA,CAAY,IAAA,UAAc,KAAA,WAAgB,UAAA,CAAW,SAAA;EAAA,QAe7C,SAAA;EAAA,QAmBA,cAAA;EAiBR,iBAAA,CAAkB,GAAA;EAnDmC;EA4D/C,WAAA,CAAY,IAAA,EAAM,iBAAA,EAAmB,QAAA,WAAgB,OAAA;EAAnC;;EAQlB,YAAA,CAAa,GAAA,UAAa,QAAA,WAAmB,OAAA;EA+BnC;EAAV,UAAA,IAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAeA,UAAA;EA1KqE;;EAAA,QAkLrE,iBAAA;AAAA"}