@milaboratories/pl-drivers 1.11.26 → 1.11.28

package/dist/drivers/download_url/driver.cjs

@@ -5,6 +5,7 @@ var tsHelpers = require('@milaboratories/ts-helpers');
 var node_crypto = require('node:crypto');
 var path = require('node:path');
 var download = require('../../helpers/download.cjs');
+var download_errors = require('../../helpers/download_errors.cjs');
 var files_cache = require('../helpers/files_cache.cjs');
 var plClient = require('@milaboratories/pl-client');
 var plModelCommon = require('@milaboratories/pl-model-common');
@@ -71,15 +72,15 @@ class DownloadUrlDriver {
     }
     getUrlNoCtx(url, w, callerId) {
         const key = url.toString();
-        const task = this.urlToDownload.get(key);
-        if (task !== undefined) {
-            task.attach(w, callerId);
-            return task.getUrl();
+        const task$1 = this.urlToDownload.get(key);
+        if (task$1 !== undefined) {
+            task$1.attach(w, callerId);
+            return task$1.getUrl();
         }
         const newTask = this.setNewTask(w, url, callerId);
         this.downloadQueue.push({
             fn: async () => this.downloadUrl(newTask, callerId),
-            recoverableErrorPredicate: (e) => true,
+            recoverableErrorPredicate: (e) => !(e instanceof task.URLAborted) && !download_errors.isDownloadNetworkError400(e),
         });
         return newTask.getUrl();
     }

package/dist/drivers/download_url/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => true,\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","randomUUID","url","isBlockUIURL","getPathForBlockUIURL","task","rmRFDir","stringifyWithResourceId","DownloadByUrlTask","createHash","path"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAIA,uBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAIC,sBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,IAAI;AACvC,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAACC,KAAW,EAAA;AAC3B,QAAA,IAAI,CAACC,0BAAY,CAACD,KAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsBA,KAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAOE,wBAAoB,CAAC,IAAI,CAAC,MAAM,EAAEF,KAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAMG,MAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAIA,MAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAOA,MAAuB,KAAI;AAC7C,gBAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyBE,gCAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAGF,MAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAEA,MAAI,CAAC,KAAI;AAChE,YAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAIG,sBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAGC,sBAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}
+{"version":3,"file":"driver.cjs","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","randomUUID","task","URLAborted","isDownloadNetworkError400","url","isBlockUIURL","getPathForBlockUIURL","rmRFDir","stringifyWithResourceId","DownloadByUrlTask","createHash","path"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAIA,uBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAIC,sBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAMC,MAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAIA,MAAI,KAAK,SAAS,EAAE;AACtB,YAAAA,MAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAOA,MAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,YAAYC,eAAU,CAAC,IAAI,CAACC,yCAAyB,CAAC,CAAC,CAAC;AAC9F,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAACC,KAAW,EAAA;AAC3B,QAAA,IAAI,CAACC,0BAAY,CAACD,KAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsBA,KAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAOE,wBAAoB,CAAC,IAAI,CAAC,MAAM,EAAEF,KAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAMH,MAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAIA,MAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAOA,MAAuB,KAAI;AAC7C,gBAAA,MAAMM,YAAO,CAACN,MAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYO,gCAAuB,CAACP,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyBO,gCAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAGP,MAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYO,gCAAuB,CAACP,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAEA,MAAI,CAAC,KAAI;AAChE,YAAA,MAAMM,YAAO,CAACN,MAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYO,gCAAuB,CAACP,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAIQ,sBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAGC,sBAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}

package/dist/drivers/download_url/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../../src/drivers/download_url/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,EACV,QAAQ,EACR,MAAM,EACP,MAAM,4BAA4B,CAAC;AAMpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAIzC,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAGlF,OAAO,EAAE,iBAAiB,EAAW,MAAM,QAAQ,CAAC;AAEpD,MAAM,WAAW,qBAAqB;IACpC;;sEAEkE;IAClE,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,SAAS;IACxB;qDACiD;IACjD,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;SAGK;IACL,kBAAkB,EAAE,MAAM,CAAC;IAE3B,mFAAmF;IACnF,UAAU,EAAE,OAAO,CAAC;IAEpB;;SAEK;IACL,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;qCACqC;AACrC,qBAAa,iBAAkB,YAAW,qBAAqB,EAAE,cAAc;IAW3E,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAdvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;IAEtD,OAAO,CAAC,aAAa,CAA6C;IAClE,OAAO,CAAC,aAAa,CAAgB;IAErC;mCAC+B;IAC/B,OAAO,CAAC,KAAK,CAAgC;gBAG1B,MAAM,EAAE,QAAQ,EACjC,UAAU,EAAE,UAAU,EACL,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,oBAItB;IAOH,2DAA2D;IAC3D,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,GAAG,SAAS,GAAG,SAAS;IAE3D,4CAA4C;IAC5C,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;IAwBnD,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAkBlD,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAQtC,wEAAwE;IAClE,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM;IAM3D;mCAC+B;IACzB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B5D,2CAA2C;IACrC,UAAU;IAgBhB,OAAO,CAAC,UAAU;IAclB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,WAAW;CAIpB"}
+{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../../src/drivers/download_url/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,EACV,QAAQ,EACR,MAAM,EACP,MAAM,4BAA4B,CAAC;AAMpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAKzC,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAGlF,OAAO,EAAE,iBAAiB,EAAuB,MAAM,QAAQ,CAAC;AAEhE,MAAM,WAAW,qBAAqB;IACpC;;sEAEkE;IAClE,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,SAAS;IACxB;qDACiD;IACjD,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;SAGK;IACL,kBAAkB,EAAE,MAAM,CAAC;IAE3B,mFAAmF;IACnF,UAAU,EAAE,OAAO,CAAC;IAEpB;;SAEK;IACL,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;qCACqC;AACrC,qBAAa,iBAAkB,YAAW,qBAAqB,EAAE,cAAc;IAW3E,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAdvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;IAEtD,OAAO,CAAC,aAAa,CAA6C;IAClE,OAAO,CAAC,aAAa,CAAgB;IAErC;mCAC+B;IAC/B,OAAO,CAAC,KAAK,CAAgC;gBAG1B,MAAM,EAAE,QAAQ,EACjC,UAAU,EAAE,UAAU,EACL,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,oBAItB;IAOH,2DAA2D;IAC3D,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,GAAG,SAAS,GAAG,SAAS;IAE3D,4CAA4C;IAC5C,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;IAwBnD,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAkBlD,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAQtC,wEAAwE;IAClE,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM;IAM3D;mCAC+B;IACzB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B5D,2CAA2C;IACrC,UAAU;IAgBhB,OAAO,CAAC,UAAU;IAclB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,WAAW;CAIpB"}

package/dist/drivers/download_url/driver.js

@@ -3,11 +3,12 @@ import { TaskProcessor } from '@milaboratories/ts-helpers';
 import { randomUUID, createHash } from 'node:crypto';
 import * as path from 'node:path';
 import { RemoteFileDownloader } from '../../helpers/download.js';
+import { isDownloadNetworkError400 } from '../../helpers/download_errors.js';
 import { FilesCache } from '../helpers/files_cache.js';
 import { stringifyWithResourceId } from '@milaboratories/pl-client';
 import { isBlockUIURL } from '@milaboratories/pl-model-common';
 import { getPathForBlockUIURL } from '../urls/url.js';
-import { rmRFDir, DownloadByUrlTask } from './task.js';
+import { URLAborted, rmRFDir, DownloadByUrlTask } from './task.js';
 
 /** Downloads .tar or .tar.gz archives by given URLs
  * and extracts them into saveDir. */
@@ -58,7 +59,7 @@ class DownloadUrlDriver {
         const newTask = this.setNewTask(w, url, callerId);
         this.downloadQueue.push({
             fn: async () => this.downloadUrl(newTask, callerId),
-            recoverableErrorPredicate: (e) => true,
+            recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),
         });
         return newTask.getUrl();
     }

package/dist/drivers/download_url/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => true,\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;;AAmDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,IAAI;AACvC,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAAC,GAAW,EAAA;AAC3B,QAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAI,IAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAuB,KAAI;AAC7C,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyB,uBAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,KAAI;AAChE,YAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}
+{"version":3,"file":"driver.js","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAoDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,YAAY,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,CAAC;AAC9F,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAAC,GAAW,EAAA;AAC3B,QAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAI,IAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAuB,KAAI;AAC7C,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyB,uBAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,KAAI;AAChE,YAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.11.26",
+  "version": "1.11.28",
   "engines": {
     "node": ">=22"
   },
@@ -32,12 +32,12 @@
     "undici": "~7.16.0",
     "upath": "^2.0.1",
     "zod": "~3.23.8",
-    "@milaboratories/computable": "2.7.4",
-    "@milaboratories/pl-tree": "1.8.20",
-    "@milaboratories/pl-client": "2.16.13",
     "@milaboratories/ts-helpers": "1.5.4",
     "@milaboratories/helpers": "1.12.0",
-    "@milaboratories/pl-model-common": "1.21.10"
+    "@milaboratories/pl-client": "2.16.14",
+    "@milaboratories/pl-model-common": "1.21.10",
+    "@milaboratories/computable": "2.7.4",
+    "@milaboratories/pl-tree": "1.8.21"
   },
   "devDependencies": {
     "@types/decompress": "^4.2.7",
@@ -49,10 +49,10 @@
     "typescript": "~5.6.3",
     "vitest": "^4.0.7",
     "@milaboratories/eslint-config": "1.0.5",
-    "@milaboratories/build-configs": "1.0.9",
-    "@milaboratories/ts-builder": "1.0.6",
-    "@milaboratories/ts-configs": "1.0.6",
-    "@milaboratories/test-helpers": "1.1.1"
+    "@milaboratories/ts-builder": "1.2.0",
+    "@milaboratories/build-configs": "1.2.0",
+    "@milaboratories/test-helpers": "1.1.1",
+    "@milaboratories/ts-configs": "1.2.0"
   },
   "scripts": {
     "type-check": "ts-builder types --target node",

package/src/drivers/download_url/driver.test.ts

@@ -9,7 +9,7 @@ import * as path from 'node:path';
 import { DownloadUrlDriver } from './driver';
 import { test, expect } from 'vitest';
 
-test('should download a tar archive and extracts its content and then deleted', async () => {
+test.skip('should download a tar archive and extracts its content and then deleted', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
@@ -44,31 +44,28 @@ test('should download a tar archive and extracts its content and then deleted',
 }, 45000);
 
 test('should show a error when 404 status code', async () => {
-  try {
-    await TestHelpers.withTempRoot(async (client) => {
-      const logger = new ConsoleLoggerAdapter();
-      const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
-      const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir, genSigner());
-
-      const url = new URL(
-        'https://block.registry.platforma.bio/releases/v1/milaboratory/NOT_FOUND',
-      );
-
-      const c = driver.getUrl(url);
-
-      const url1 = await c.getValue();
-      expect(url1).toBeUndefined();
-
-      await c.awaitChange();
-
-      const url2 = await c.getValue();
-      expect(url2).not.toBeUndefined();
-      expect(url2?.error).not.toBeUndefined();
-    });
-  } catch (e) {
-    console.log('HERE: ', e);
-  }
-});
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
+    const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir, genSigner());
+
+    const url = new URL(
+      'https://block.registry.platforma.bio/releases/v1/milaboratory/NOT_FOUND',
+    );
+
+    const c = driver.getUrl(url);
+
+    const url1 = await c.getValue();
+    expect(url1).toBeUndefined();
+
+    await c.awaitChange();
+
+    const url2 = await c.getValue();
+    expect(url2).not.toBeUndefined();
+    expect(url2?.error).not.toBeUndefined();
+    expect(url2?.url).toBeUndefined();
+  });
+}, 45000);
 
 test('should abort a downloading process when we reset a state of a computable', async () => {
   await TestHelpers.withTempRoot(async (client) => {

package/src/drivers/download_url/driver.ts

@@ -11,12 +11,13 @@ import { createHash, randomUUID } from 'node:crypto';
 import * as path from 'node:path';
 import type { Dispatcher } from 'undici';
 import { RemoteFileDownloader } from '../../helpers/download';
+import { isDownloadNetworkError400 } from '../../helpers/download_errors';
 import { FilesCache } from '../helpers/files_cache';
 import { stringifyWithResourceId } from '@milaboratories/pl-client';
 import type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';
 import { isBlockUIURL } from '@milaboratories/pl-model-common';
 import { getPathForBlockUIURL } from '../urls/url';
-import { DownloadByUrlTask, rmRFDir } from './task';
+import { DownloadByUrlTask, rmRFDir, URLAborted } from './task';
 
 export interface DownloadUrlSyncReader {
   /** Returns a Computable that (when the time will come)
@@ -117,7 +118,7 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver
     const newTask = this.setNewTask(w, url, callerId);
     this.downloadQueue.push({
       fn: async () => this.downloadUrl(newTask, callerId),
-      recoverableErrorPredicate: (e) => true,
+      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),
     });
 
     return newTask.getUrl();