@milaboratories/pl-drivers 1.12.7 → 1.12.9

package/dist/drivers/download_blob_url/task.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"mappings":";;;;;;;KA6BY,SAAA;EACV,GAAA,GAAM,SAAA;EACN,KAAA;AAAA;;cAIW,wBAAA;EAAA,iBAWQ,MAAA;EAAA,iBACA,MAAA;EAAA,SACR,OAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA,EAAO,wBAAA;EAAA,SACP,MAAA,EAAQ,aAAA;EAAA,iBACA,cAAA;EAAA,SAhBV,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;EAAA,QACQ,GAAA;EAAA,QACA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EAChB,OAAA,UACA,IAAA,UACA,KAAA,EAAO,wBAAA,EACP,MAAA,EAAQ,aAAA,EACA,cAAA,EAAgB,cAAA;;EAI5B,IAAA,CAAA;;;;;;;;;EAYP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAA,GAAQ,OAAA;EAvBI;EAAA,QAoDJ,qBAAA;EAuEd,MAAA,CAAA,GAAU,SAAA;EAAA,QAQF,OAAA;EAAA,QAMA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA;;KA4BH,WAAA;EACH,SAAA;EACA,WAAA;EACA,UAAA;EACA,aAAA,GAAgB,aAAA;EAChB,QAAA;EACA,OAAA;EACA,cAAA;EACA,eAAA;EACA,cAAA;EACA,WAAA;AAAA"}

package/dist/drivers/download_blob_url/task.js

@@ -12,7 +12,6 @@ import * as tar from "tar-fs";
 import path from "path";
 import * as fsp from "fs/promises";
 import decompress from "decompress";
-
 //#region src/drivers/download_blob_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadAndUnarchiveTask = class {
@@ -146,7 +145,7 @@ var URLAborted = class extends Error {
 function nonRecoverableError(e) {
 	return e instanceof URLAborted || isDownloadNetworkError400(e) || e instanceof UnknownStorageError || e instanceof WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED") || String(e).includes("incorrect header check");
 }
-
 //#endregion
 export { DownloadAndUnarchiveTask, rmRFDir };
+
 //# sourceMappingURL=task.js.map

package/dist/drivers/download_blob_url/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,AAAS,UAAU,IAAI,gBAAgB;CACvC,AAAS,SAAS,IAAI,cAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP,AAAQ;CACR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAS,SACT,AAAS,MACT,AAAS,OACT,AAAS,QACT,AAAiB,gBACjB;EAPiB;EACA;EACR;EACA;EACA;EACA;EACQ;;;CAInB,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
+{"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,MACA,OACA,QACA,gBACA;AAPiB,OAAA,SAAA;AACA,OAAA,SAAA;AACR,OAAA,UAAA;AACA,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,SAAA;AACQ,OAAA,iBAAA;;;CAInB,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}

package/dist/drivers/download_url/driver.cjs

@@ -1,9 +1,9 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_download_errors = require('../../helpers/download_errors.cjs');
-const require_download = require('../../helpers/download.cjs');
-const require_files_cache = require('../helpers/files_cache.cjs');
-const require_url = require('../urls/url.cjs');
-const require_task = require('./task.cjs');
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_download_errors = require("../../helpers/download_errors.cjs");
+const require_download = require("../../helpers/download.cjs");
+const require_files_cache = require("../helpers/files_cache.cjs");
+const require_url = require("../urls/url.cjs");
+const require_task = require("./task.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_path = require("node:path");
@@ -11,7 +11,6 @@ node_path = require_runtime.__toESM(node_path);
 let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let node_crypto = require("node:crypto");
-
 //#region src/drivers/download_url/driver.ts
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
@@ -107,7 +106,7 @@ var DownloadUrlDriver = class {
 		return node_path.join(this.saveDir, sha256);
 	}
 };
-
 //#endregion
 exports.DownloadUrlDriver = DownloadUrlDriver;
+
 //# sourceMappingURL=driver.cjs.map

package/dist/drivers/download_url/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E,AAAiB;CAEjB,AAAQ,gCAAgD,IAAI,KAAK;CACjE,AAAQ;;;CAIR,AAAQ;CAER,YACE,AAAiB,QACjB,YACA,AAAiB,SACjB,AAAiB,QACjB,AAAiB,OAA6B;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;EATiB;EAEA;EACA;EACA;AAMjB,OAAK,gBAAgB,IAAIA,yCAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAIC,+BAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAIC,sCAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,OAAW,QAAOC,sCAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,wCAAuB;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,OAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,QAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAaC,4BAAe,CAACC,kDAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,mDAAc,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAOC,iCAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,OAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,OAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAMC,qBAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,2FACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAMA,qBAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,AAAQ,WAAW,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAIC,+BACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,AAAQ,WAAW,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,AAAQ,YAAY,KAAkB;EACpC,MAAM,qCAAoB,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOC,UAAK,KAAK,KAAK,SAAS,OAAO"}
+{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E;CAEA,gCAAwD,IAAI,KAAK;CACjE;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;AATiB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,OAAA;AAMjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAIC,iBAAAA,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,KAAA,EAAW,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,YAAA,GAAA,YAAA,aAAuB;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,KAAA,GAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAaC,aAAAA,eAAe,CAACC,wBAAAA,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,EAAA,GAAA,gCAAA,cAAc,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAOC,YAAAA,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAMC,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WAAmB,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAIC,aAAAA,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,WAAmB,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,YAAoB,KAAkB;EACpC,MAAM,UAAA,GAAA,YAAA,YAAoB,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOC,UAAK,KAAK,KAAK,SAAS,OAAO"}

package/dist/drivers/download_url/driver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_url/driver.ts"],"mappings":";;;;;;;UAgBiB,qBAAA;EAAA;;;EAIf,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;AAAA;AAAA,UAGd,SAAA;EAHa;;EAM5B,GAAA,GAAM,UAAA;EANM;EAQZ,KAAA;AAAA;AAAA,KAGU,oBAAA;EAX4B;;AAGxC;;EAaE,kBAAA,UAVgB;EAahB,UAAA;EAbM;;;EAkBN,oBAAA;AAAA;;;cAKW,iBAAA,YAA6B,qBAAA,EAAuB,cAAA;EAAA,iBAW5C,MAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,MAAA;EAAA,iBACA,IAAA;EAAA,iBAdF,cAAA;EAAA,QAET,aAAA;EAAA,QACA,aAAA;;;UAIA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACzB,UAAA,EAAY,UAAA,EACK,OAAA,UACA,MAAA,EAAQ,MAAA,EACR,IAAA,GAAM,oBAAA;EAAA;EAYzB,MAAA,CAAO,GAAA,EAAK,GAAA,EAAK,GAAA,EAAK,aAAA,GAAgB,SAAA;EAAhB;EAGtB,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;EAqB7B,WAAA,CAAY,GAAA,EAAK,GAAA,EAAK,CAAA,EAAG,OAAA,EAAS,QAAA,WAAgB,SAAA;EAkBlD,iBAAA,CAAkB,GAAA;EAvCA;EAgDZ,WAAA,CAAY,IAAA,EAAM,iBAAA,EAAmB,QAAA,WAAgB,OAAA;EA3BlC;;EAmCnB,WAAA,CAAY,GAAA,EAAK,GAAA,EAAK,QAAA,WAAmB,OAAA;EARY;EAwCrD,UAAA,CAAA,GAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAcA,UAAA;EAAA,QAMA,WAAA;AAAA"}

package/dist/drivers/download_url/driver.js

@@ -9,7 +9,6 @@ import * as path$1 from "node:path";
 import { Computable } from "@milaboratories/computable";
 import { isBlockUIURL } from "@milaboratories/pl-model-common";
 import { createHash, randomUUID } from "node:crypto";
-
 //#region src/drivers/download_url/driver.ts
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
@@ -105,7 +104,7 @@ var DownloadUrlDriver = class {
 		return path$1.join(this.saveDir, sha256);
 	}
 };
-
 //#endregion
 export { DownloadUrlDriver };
+
 //# sourceMappingURL=driver.js.map

package/dist/drivers/download_url/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E,AAAiB;CAEjB,AAAQ,gCAAgD,IAAI,KAAK;CACjE,AAAQ;;;CAIR,AAAQ;CAER,YACE,AAAiB,QACjB,YACA,AAAiB,SACjB,AAAiB,QACjB,AAAiB,OAA6B;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;EATiB;EAEA;EACA;EACA;AAMjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAI,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,OAAW,QAAO,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,WAAW,YAAY;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,OAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,QAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAa,eAAe,CAAC,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,CAAC,aAAa,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAO,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,OAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,OAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,AAAQ,WAAW,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAI,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,AAAQ,WAAW,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,AAAQ,YAAY,KAAkB;EACpC,MAAM,SAAS,WAAW,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOA,OAAK,KAAK,KAAK,SAAS,OAAO"}
+{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E;CAEA,gCAAwD,IAAI,KAAK;CACjE;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;AATiB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,OAAA;AAMjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAI,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,KAAA,EAAW,QAAO,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,WAAW,YAAY;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,KAAA,GAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAa,eAAe,CAAC,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,CAAC,aAAa,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAO,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WAAmB,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAI,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,WAAmB,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,YAAoB,KAAkB;EACpC,MAAM,SAAS,WAAW,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOA,OAAK,KAAK,KAAK,SAAS,OAAO"}

package/dist/drivers/download_url/task.cjs

@@ -1,6 +1,6 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_download_errors = require('../../helpers/download_errors.cjs');
-const require_url = require('../urls/url.cjs');
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_download_errors = require("../../helpers/download_errors.cjs");
+const require_url = require("../urls/url.cjs");
 let node_fs_promises = require("node:fs/promises");
 node_fs_promises = require_runtime.__toESM(node_fs_promises);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
@@ -12,7 +12,6 @@ let node_zlib = require("node:zlib");
 node_zlib = require_runtime.__toESM(node_zlib);
 let tar_fs = require("tar-fs");
 tar_fs = require_runtime.__toESM(tar_fs);
-
 //#region src/drivers/download_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadByUrlTask = class {
@@ -109,9 +108,9 @@ async function rmRFDir(path) {
 		force: true
 	});
 }
-
 //#endregion
 exports.DownloadByUrlTask = DownloadByUrlTask;
 exports.URLAborted = URLAborted;
 exports.rmRFDir = rmRFDir;
+
 //# sourceMappingURL=task.cjs.map

package/dist/drivers/download_url/task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (e instanceof URLAborted || isDownloadNetworkError400(e)) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,AAAS,UAAU,IAAIA,2CAAgB;CACvC,AAAS,SAAS,IAAIC,yCAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,AAAiB,QACjB,AAAS,MACT,AAAS,KACT,AAAS,QACT,AAAS,SACT;EALiB;EACR;EACA;EACA;EACA;;CAGX,AAAO,OAAO;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OAAI,aAAa,cAAcC,kDAA0B,EAAE,EAAE;AAC3D,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,wDAAsBC,UAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,iDAAiB,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAASC,sBAAU,MAAMC,UAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,8DAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAMC,iBAAI,MAAM,MAAM;IACtB,MAAM,QAAQC,qBAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAKC,0BAAc,KAAK,QAAQ,KAAK,kDAAkB,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAMH,iBAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQH,UAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMG,iBAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAMA,iBAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}
+{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (e instanceof URLAborted || isDownloadNetworkError400(e)) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAIA,2BAAAA,gBAAgB;CACvC,SAAkB,IAAIC,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OAAI,aAAa,cAAcC,wBAAAA,0BAA0B,EAAE,EAAE;AAC3D,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,SAAA,GAAA,2BAAA,iBAAsBC,UAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAASC,YAAAA,UAAU,MAAMC,UAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAMC,iBAAI,MAAM,MAAM;IACtB,MAAM,QAAQC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAKC,YAAAA,cAAc,KAAK,QAAQ,KAAK,UAAA,GAAA,2BAAA,UAAkB,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAMH,iBAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQH,UAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMG,iBAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAMA,iBAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}

package/dist/drivers/download_url/task.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_url/task.ts"],"mappings":";;;;;;AAqBA;AAAA,cAAa,iBAAA;EAAA,iBASQ,MAAA;EAAA,SACR,IAAA;EAAA,SACA,GAAA,EAAK,GAAA;EAAA,SACL,MAAA,EAAQ,MAAA;EAAA,SACR,OAAA;EAAA,SAZF,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;cAGmB,MAAA,EAAQ,QAAA,EAChB,IAAA,UACA,GAAA,EAAK,GAAA,EACL,MAAA,EAAQ,MAAA,EACR,OAAA;EAGJ,IAAA,CAAA;;;;;;;EAUP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAS,cAAA,EAAgB,oBAAA,EAAsB,UAAA,YAAmB,OAAA;EAAA,QAkB1D,gBAAA;EAmCd,MAAA,CAAA,GAAU,SAAA;EAAA,QAWF,OAAA;EAAA,QAKA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA"}

package/dist/drivers/download_url/task.js

@@ -7,7 +7,6 @@ import { Transform, Writable } from "node:stream";
 import { ChangeSource } from "@milaboratories/computable";
 import * as zlib from "node:zlib";
 import * as tar from "tar-fs";
-
 //#region src/drivers/download_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadByUrlTask = class {
@@ -104,7 +103,7 @@ async function rmRFDir(path) {
 		force: true
 	});
 }
-
 //#endregion
 export { DownloadByUrlTask, URLAborted, rmRFDir };
+
 //# sourceMappingURL=task.js.map

package/dist/drivers/download_url/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (e instanceof URLAborted || isDownloadNetworkError400(e)) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,AAAS,UAAU,IAAI,gBAAgB;CACvC,AAAS,SAAS,IAAI,cAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,AAAiB,QACjB,AAAS,MACT,AAAS,KACT,AAAS,QACT,AAAS,SACT;EALiB;EACR;EACA;EACA;EACA;;CAGX,AAAO,OAAO;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OAAI,aAAa,cAAc,0BAA0B,EAAE,EAAE;AAC3D,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,QAAM,gBAAgBA,OAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,MAAM,WAAW,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAM,IAAI,MAAM,MAAM;IACtB,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAK,cAAc,KAAK,QAAQ,KAAK,SAAS,SAAS,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQA,OAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}
+{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (e instanceof URLAborted || isDownloadNetworkError400(e)) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OAAI,aAAa,cAAc,0BAA0B,EAAE,EAAE;AAC3D,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,QAAM,gBAAgBA,OAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,MAAM,WAAW,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAM,IAAI,MAAM,MAAM;IACtB,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAK,cAAc,KAAK,QAAQ,KAAK,SAAS,SAAS,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQA,OAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}

package/dist/drivers/helpers/download_local_handle.cjs

@@ -1,4 +1,3 @@
-
 //#region src/drivers/helpers/download_local_handle.ts
 const localHandleRegex = /^blob\+local:\/\/download\/(?<path>.*)#(?<signature>.*)$/;
 function newLocalHandle(path, signer) {
@@ -17,9 +16,9 @@ function parseLocalHandle(handle, signer) {
 		signature
 	};
 }
-
 //#endregion
 exports.isLocalBlobHandle = isLocalBlobHandle;
 exports.newLocalHandle = newLocalHandle;
 exports.parseLocalHandle = parseLocalHandle;
+
 //# sourceMappingURL=download_local_handle.cjs.map

package/dist/drivers/helpers/download_local_handle.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_local_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n  return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n  return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n  const parsed = handle.match(localHandleRegex);\n\n  if (parsed === null) {\n    throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { path, signature } = parsed.groups!;\n  signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n  return { path, signature };\n}\n"],"mappings":";;AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;AAC5E,QAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,KAAK;;AAG3D,SAAgB,kBAAkB,QAA2C;AAC3E,QAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;;AAGhD,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,iBAAiB;AAE7C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,SAAS;CAG7E,MAAM,EAAE,MAAM,cAAc,OAAO;AACnC,QAAO,OAAO,MAAM,WAAW,sCAAsC,SAAS;AAE9E,QAAO;EAAE;EAAM;EAAW"}
+{"version":3,"file":"download_local_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n  return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n  return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n  const parsed = handle.match(localHandleRegex);\n\n  if (parsed === null) {\n    throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { path, signature } = parsed.groups!;\n  signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n  return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;AAC5E,QAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,KAAK;;AAG3D,SAAgB,kBAAkB,QAA2C;AAC3E,QAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;;AAGhD,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,iBAAiB;AAE7C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,SAAS;CAG7E,MAAM,EAAE,MAAM,cAAc,OAAO;AACnC,QAAO,OAAO,MAAM,WAAW,sCAAsC,SAAS;AAE9E,QAAO;EAAE;EAAM;EAAW"}

package/dist/drivers/helpers/download_local_handle.js

@@ -16,7 +16,7 @@ function parseLocalHandle(handle, signer) {
 		signature
 	};
 }
-
 //#endregion
 export { isLocalBlobHandle, newLocalHandle, parseLocalHandle };
+
 //# sourceMappingURL=download_local_handle.js.map

package/dist/drivers/helpers/download_remote_handle.cjs

@@ -1,7 +1,6 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_types = require('../types.cjs');
+require("../../_virtual/_rolldown/runtime.cjs");
+const require_types = require("../types.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
-
 //#region src/drivers/helpers/download_remote_handle.ts
 const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<size>\d+?))#(?<signature>.*)$/;
 function newRemoteHandle(rInfo, signer) {
@@ -27,9 +26,9 @@ function parseRemoteHandle(handle, signer) {
 		size: Number(size)
 	};
 }
-
 //#endregion
 exports.isRemoteBlobHandle = isRemoteBlobHandle;
 exports.newRemoteHandle = newRemoteHandle;
 exports.parseRemoteHandle = parseRemoteHandle;
+
 //# sourceMappingURL=download_remote_handle.cjs.map

package/dist/drivers/helpers/download_remote_handle.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport { bigintToResourceId } from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  return {\n    info: {\n      id: bigintToResourceId(BigInt(resourceId)),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;;;AAWA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,OAAO,MAAM,GAAG,CAAC,GAAGA,sBAAQ,MAAM;AAE5F,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,MAAM,cAAc,OAAO;AAEvF,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;AAEhF,QAAO;EACL,MAAM;GACJ,sDAAuB,OAAO,WAAW,CAAC;GAC1C,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}
+{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport { bigintToResourceId } from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  return {\n    info: {\n      id: bigintToResourceId(BigInt(resourceId)),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;;AAWA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,OAAO,MAAM,GAAG,CAAC,GAAGA,cAAAA,QAAQ,MAAM;AAE5F,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,MAAM,cAAc,OAAO;AAEvF,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;AAEhF,QAAO;EACL,MAAM;GACJ,KAAA,GAAA,0BAAA,oBAAuB,OAAO,WAAW,CAAC;GAC1C,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}

package/dist/drivers/helpers/download_remote_handle.js

@@ -1,6 +1,5 @@
 import { getSize } from "../types.js";
 import { bigintToResourceId } from "@milaboratories/pl-client";
-
 //#region src/drivers/helpers/download_remote_handle.ts
 const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<size>\d+?))#(?<signature>.*)$/;
 function newRemoteHandle(rInfo, signer) {
@@ -26,7 +25,7 @@ function parseRemoteHandle(handle, signer) {
 		size: Number(size)
 	};
 }
-
 //#endregion
 export { isRemoteBlobHandle, newRemoteHandle, parseRemoteHandle };
+
 //# sourceMappingURL=download_remote_handle.js.map

package/dist/drivers/helpers/download_remote_handle.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport { bigintToResourceId } from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  return {\n    info: {\n      id: bigintToResourceId(BigInt(resourceId)),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;;AAWA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,OAAO,MAAM,GAAG,CAAC,GAAG,QAAQ,MAAM;AAE5F,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,MAAM,cAAc,OAAO;AAEvF,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;AAEhF,QAAO;EACL,MAAM;GACJ,IAAI,mBAAmB,OAAO,WAAW,CAAC;GAC1C,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}
+{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport { bigintToResourceId } from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  return {\n    info: {\n      id: bigintToResourceId(BigInt(resourceId)),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;AAWA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,OAAO,MAAM,GAAG,CAAC,GAAG,QAAQ,MAAM;AAE5F,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,MAAM,cAAc,OAAO;AAEvF,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;AAEhF,QAAO;EACL,MAAM;GACJ,IAAI,mBAAmB,OAAO,WAAW,CAAC;GAC1C,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}

package/dist/drivers/helpers/files_cache.cjs

@@ -1,6 +1,5 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
+require("../../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
-
 //#region src/drivers/helpers/files_cache.ts
 /** Holds counters of how many callers need the file.
 * If some counters become zero and a cache size exceeds a soft limit,
@@ -51,7 +50,7 @@ var FilesCache = class {
 		this.totalSizeBytes -= file.size;
 	}
 };
-
 //#endregion
 exports.FilesCache = FilesCache;
+
 //# sourceMappingURL=files_cache.cjs.map

package/dist/drivers/helpers/files_cache.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"files_cache.cjs","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n  /** Size in bytes. */\n  size: number;\n  path: PathLike;\n  counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n  private cache: Map<PathLike, T> = new Map();\n  private totalSizeBytes: number = 0;\n\n  constructor(private readonly softSizeBytes: number) {}\n\n  existsFile(path: PathLike): boolean {\n    return this.cache.get(path) != undefined;\n  }\n\n  getFile(path: PathLike, callerId: string): T | undefined {\n    const file = this.cache.get(path);\n    if (file != undefined) {\n      file.counter.inc(callerId);\n    }\n\n    return file;\n  }\n\n  /** Decrements a counter in a cache and if we exceeds\n   * a soft limit, removes files with zero counters. */\n  removeFile(path: PathLike, callerId: string): T[] {\n    mapGet(this.cache, path).counter.dec(callerId);\n    return this.toDelete();\n  }\n\n  /** Returns what results should be deleted to comply with the soft limit. */\n  toDelete(): T[] {\n    if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n    const result: T[] = [];\n    let freedBytes = 0;\n\n    mapEntries(this.cache)\n      .filter(([_, file]: [string, T]) => file.counter.isZero())\n      .forEach(([path, _]) => {\n        if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n          return;\n        }\n\n        const file = mapGet(this.cache, path);\n        freedBytes += file.size;\n        result.push(file);\n      });\n\n    return result;\n  }\n\n  addCache(file: T, callerId: string) {\n    const created = this.cache.get(file.path) == undefined;\n    this.cache.set(file.path, file);\n    file.counter.inc(callerId);\n\n    if (file.size < 0) {\n      throw new Error(`empty sizeBytes: ${file}`);\n    }\n\n    if (created) {\n      this.totalSizeBytes += file.size;\n    }\n  }\n\n  removeCache(file: T) {\n    this.cache.delete(file.path);\n    this.totalSizeBytes -= file.size;\n  }\n}\n"],"mappings":";;;;;;;;AAgBA,IAAa,aAAb,MAA8C;CAC5C,AAAQ,wBAA0B,IAAI,KAAK;CAC3C,AAAQ,iBAAyB;CAEjC,YAAY,AAAiB,eAAuB;EAAvB;;CAE7B,WAAW,MAAyB;AAClC,SAAO,KAAK,MAAM,IAAI,KAAK,IAAI;;CAGjC,QAAQ,MAAgB,UAAiC;EACvD,MAAM,OAAO,KAAK,MAAM,IAAI,KAAK;AACjC,MAAI,QAAQ,OACV,MAAK,QAAQ,IAAI,SAAS;AAG5B,SAAO;;;;CAKT,WAAW,MAAgB,UAAuB;AAChD,yCAAO,KAAK,OAAO,KAAK,CAAC,QAAQ,IAAI,SAAS;AAC9C,SAAO,KAAK,UAAU;;;CAIxB,WAAgB;AACd,MAAI,KAAK,kBAAkB,KAAK,cAAe,QAAO,EAAE;EAExD,MAAM,SAAc,EAAE;EACtB,IAAI,aAAa;AAEjB,6CAAW,KAAK,MAAM,CACnB,QAAQ,CAAC,GAAG,UAAuB,KAAK,QAAQ,QAAQ,CAAC,CACzD,SAAS,CAAC,MAAM,OAAO;AACtB,OAAI,KAAK,iBAAiB,cAAc,KAAK,cAC3C;GAGF,MAAM,8CAAc,KAAK,OAAO,KAAK;AACrC,iBAAc,KAAK;AACnB,UAAO,KAAK,KAAK;IACjB;AAEJ,SAAO;;CAGT,SAAS,MAAS,UAAkB;EAClC,MAAM,UAAU,KAAK,MAAM,IAAI,KAAK,KAAK,IAAI;AAC7C,OAAK,MAAM,IAAI,KAAK,MAAM,KAAK;AAC/B,OAAK,QAAQ,IAAI,SAAS;AAE1B,MAAI,KAAK,OAAO,EACd,OAAM,IAAI,MAAM,oBAAoB,OAAO;AAG7C,MAAI,QACF,MAAK,kBAAkB,KAAK;;CAIhC,YAAY,MAAS;AACnB,OAAK,MAAM,OAAO,KAAK,KAAK;AAC5B,OAAK,kBAAkB,KAAK"}
+{"version":3,"file":"files_cache.cjs","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n  /** Size in bytes. */\n  size: number;\n  path: PathLike;\n  counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n  private cache: Map<PathLike, T> = new Map();\n  private totalSizeBytes: number = 0;\n\n  constructor(private readonly softSizeBytes: number) {}\n\n  existsFile(path: PathLike): boolean {\n    return this.cache.get(path) != undefined;\n  }\n\n  getFile(path: PathLike, callerId: string): T | undefined {\n    const file = this.cache.get(path);\n    if (file != undefined) {\n      file.counter.inc(callerId);\n    }\n\n    return file;\n  }\n\n  /** Decrements a counter in a cache and if we exceeds\n   * a soft limit, removes files with zero counters. */\n  removeFile(path: PathLike, callerId: string): T[] {\n    mapGet(this.cache, path).counter.dec(callerId);\n    return this.toDelete();\n  }\n\n  /** Returns what results should be deleted to comply with the soft limit. */\n  toDelete(): T[] {\n    if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n    const result: T[] = [];\n    let freedBytes = 0;\n\n    mapEntries(this.cache)\n      .filter(([_, file]: [string, T]) => file.counter.isZero())\n      .forEach(([path, _]) => {\n        if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n          return;\n        }\n\n        const file = mapGet(this.cache, path);\n        freedBytes += file.size;\n        result.push(file);\n      });\n\n    return result;\n  }\n\n  addCache(file: T, callerId: string) {\n    const created = this.cache.get(file.path) == undefined;\n    this.cache.set(file.path, file);\n    file.counter.inc(callerId);\n\n    if (file.size < 0) {\n      throw new Error(`empty sizeBytes: ${file}`);\n    }\n\n    if (created) {\n      this.totalSizeBytes += file.size;\n    }\n  }\n\n  removeCache(file: T) {\n    this.cache.delete(file.path);\n    this.totalSizeBytes -= file.size;\n  }\n}\n"],"mappings":";;;;;;;AAgBA,IAAa,aAAb,MAA8C;CAC5C,wBAAkC,IAAI,KAAK;CAC3C,iBAAiC;CAEjC,YAAY,eAAwC;AAAvB,OAAA,gBAAA;;CAE7B,WAAW,MAAyB;AAClC,SAAO,KAAK,MAAM,IAAI,KAAK,IAAI,KAAA;;CAGjC,QAAQ,MAAgB,UAAiC;EACvD,MAAM,OAAO,KAAK,MAAM,IAAI,KAAK;AACjC,MAAI,QAAQ,KAAA,EACV,MAAK,QAAQ,IAAI,SAAS;AAG5B,SAAO;;;;CAKT,WAAW,MAAgB,UAAuB;AAChD,GAAA,GAAA,2BAAA,QAAO,KAAK,OAAO,KAAK,CAAC,QAAQ,IAAI,SAAS;AAC9C,SAAO,KAAK,UAAU;;;CAIxB,WAAgB;AACd,MAAI,KAAK,kBAAkB,KAAK,cAAe,QAAO,EAAE;EAExD,MAAM,SAAc,EAAE;EACtB,IAAI,aAAa;AAEjB,GAAA,GAAA,2BAAA,YAAW,KAAK,MAAM,CACnB,QAAQ,CAAC,GAAG,UAAuB,KAAK,QAAQ,QAAQ,CAAC,CACzD,SAAS,CAAC,MAAM,OAAO;AACtB,OAAI,KAAK,iBAAiB,cAAc,KAAK,cAC3C;GAGF,MAAM,QAAA,GAAA,2BAAA,QAAc,KAAK,OAAO,KAAK;AACrC,iBAAc,KAAK;AACnB,UAAO,KAAK,KAAK;IACjB;AAEJ,SAAO;;CAGT,SAAS,MAAS,UAAkB;EAClC,MAAM,UAAU,KAAK,MAAM,IAAI,KAAK,KAAK,IAAI,KAAA;AAC7C,OAAK,MAAM,IAAI,KAAK,MAAM,KAAK;AAC/B,OAAK,QAAQ,IAAI,SAAS;AAE1B,MAAI,KAAK,OAAO,EACd,OAAM,IAAI,MAAM,oBAAoB,OAAO;AAG7C,MAAI,QACF,MAAK,kBAAkB,KAAK;;CAIhC,YAAY,MAAS;AACnB,OAAK,MAAM,OAAO,KAAK,KAAK;AAC5B,OAAK,kBAAkB,KAAK"}

package/dist/drivers/helpers/files_cache.js

@@ -1,5 +1,4 @@
 import { mapEntries, mapGet } from "@milaboratories/ts-helpers";
-
 //#region src/drivers/helpers/files_cache.ts
 /** Holds counters of how many callers need the file.
 * If some counters become zero and a cache size exceeds a soft limit,
@@ -50,7 +49,7 @@ var FilesCache = class {
 		this.totalSizeBytes -= file.size;
 	}
 };
-
 //#endregion
 export { FilesCache };
+
 //# sourceMappingURL=files_cache.js.map

package/dist/drivers/helpers/files_cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"files_cache.js","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n  /** Size in bytes. */\n  size: number;\n  path: PathLike;\n  counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n  private cache: Map<PathLike, T> = new Map();\n  private totalSizeBytes: number = 0;\n\n  constructor(private readonly softSizeBytes: number) {}\n\n  existsFile(path: PathLike): boolean {\n    return this.cache.get(path) != undefined;\n  }\n\n  getFile(path: PathLike, callerId: string): T | undefined {\n    const file = this.cache.get(path);\n    if (file != undefined) {\n      file.counter.inc(callerId);\n    }\n\n    return file;\n  }\n\n  /** Decrements a counter in a cache and if we exceeds\n   * a soft limit, removes files with zero counters. */\n  removeFile(path: PathLike, callerId: string): T[] {\n    mapGet(this.cache, path).counter.dec(callerId);\n    return this.toDelete();\n  }\n\n  /** Returns what results should be deleted to comply with the soft limit. */\n  toDelete(): T[] {\n    if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n    const result: T[] = [];\n    let freedBytes = 0;\n\n    mapEntries(this.cache)\n      .filter(([_, file]: [string, T]) => file.counter.isZero())\n      .forEach(([path, _]) => {\n        if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n          return;\n        }\n\n        const file = mapGet(this.cache, path);\n        freedBytes += file.size;\n        result.push(file);\n      });\n\n    return result;\n  }\n\n  addCache(file: T, callerId: string) {\n    const created = this.cache.get(file.path) == undefined;\n    this.cache.set(file.path, file);\n    file.counter.inc(callerId);\n\n    if (file.size < 0) {\n      throw new Error(`empty sizeBytes: ${file}`);\n    }\n\n    if (created) {\n      this.totalSizeBytes += file.size;\n    }\n  }\n\n  removeCache(file: T) {\n    this.cache.delete(file.path);\n    this.totalSizeBytes -= file.size;\n  }\n}\n"],"mappings":";;;;;;;AAgBA,IAAa,aAAb,MAA8C;CAC5C,AAAQ,wBAA0B,IAAI,KAAK;CAC3C,AAAQ,iBAAyB;CAEjC,YAAY,AAAiB,eAAuB;EAAvB;;CAE7B,WAAW,MAAyB;AAClC,SAAO,KAAK,MAAM,IAAI,KAAK,IAAI;;CAGjC,QAAQ,MAAgB,UAAiC;EACvD,MAAM,OAAO,KAAK,MAAM,IAAI,KAAK;AACjC,MAAI,QAAQ,OACV,MAAK,QAAQ,IAAI,SAAS;AAG5B,SAAO;;;;CAKT,WAAW,MAAgB,UAAuB;AAChD,SAAO,KAAK,OAAO,KAAK,CAAC,QAAQ,IAAI,SAAS;AAC9C,SAAO,KAAK,UAAU;;;CAIxB,WAAgB;AACd,MAAI,KAAK,kBAAkB,KAAK,cAAe,QAAO,EAAE;EAExD,MAAM,SAAc,EAAE;EACtB,IAAI,aAAa;AAEjB,aAAW,KAAK,MAAM,CACnB,QAAQ,CAAC,GAAG,UAAuB,KAAK,QAAQ,QAAQ,CAAC,CACzD,SAAS,CAAC,MAAM,OAAO;AACtB,OAAI,KAAK,iBAAiB,cAAc,KAAK,cAC3C;GAGF,MAAM,OAAO,OAAO,KAAK,OAAO,KAAK;AACrC,iBAAc,KAAK;AACnB,UAAO,KAAK,KAAK;IACjB;AAEJ,SAAO;;CAGT,SAAS,MAAS,UAAkB;EAClC,MAAM,UAAU,KAAK,MAAM,IAAI,KAAK,KAAK,IAAI;AAC7C,OAAK,MAAM,IAAI,KAAK,MAAM,KAAK;AAC/B,OAAK,QAAQ,IAAI,SAAS;AAE1B,MAAI,KAAK,OAAO,EACd,OAAM,IAAI,MAAM,oBAAoB,OAAO;AAG7C,MAAI,QACF,MAAK,kBAAkB,KAAK;;CAIhC,YAAY,MAAS;AACnB,OAAK,MAAM,OAAO,KAAK,KAAK;AAC5B,OAAK,kBAAkB,KAAK"}
+{"version":3,"file":"files_cache.js","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n  /** Size in bytes. */\n  size: number;\n  path: PathLike;\n  counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n  private cache: Map<PathLike, T> = new Map();\n  private totalSizeBytes: number = 0;\n\n  constructor(private readonly softSizeBytes: number) {}\n\n  existsFile(path: PathLike): boolean {\n    return this.cache.get(path) != undefined;\n  }\n\n  getFile(path: PathLike, callerId: string): T | undefined {\n    const file = this.cache.get(path);\n    if (file != undefined) {\n      file.counter.inc(callerId);\n    }\n\n    return file;\n  }\n\n  /** Decrements a counter in a cache and if we exceeds\n   * a soft limit, removes files with zero counters. */\n  removeFile(path: PathLike, callerId: string): T[] {\n    mapGet(this.cache, path).counter.dec(callerId);\n    return this.toDelete();\n  }\n\n  /** Returns what results should be deleted to comply with the soft limit. */\n  toDelete(): T[] {\n    if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n    const result: T[] = [];\n    let freedBytes = 0;\n\n    mapEntries(this.cache)\n      .filter(([_, file]: [string, T]) => file.counter.isZero())\n      .forEach(([path, _]) => {\n        if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n          return;\n        }\n\n        const file = mapGet(this.cache, path);\n        freedBytes += file.size;\n        result.push(file);\n      });\n\n    return result;\n  }\n\n  addCache(file: T, callerId: string) {\n    const created = this.cache.get(file.path) == undefined;\n    this.cache.set(file.path, file);\n    file.counter.inc(callerId);\n\n    if (file.size < 0) {\n      throw new Error(`empty sizeBytes: ${file}`);\n    }\n\n    if (created) {\n      this.totalSizeBytes += file.size;\n    }\n  }\n\n  removeCache(file: T) {\n    this.cache.delete(file.path);\n    this.totalSizeBytes -= file.size;\n  }\n}\n"],"mappings":";;;;;;AAgBA,IAAa,aAAb,MAA8C;CAC5C,wBAAkC,IAAI,KAAK;CAC3C,iBAAiC;CAEjC,YAAY,eAAwC;AAAvB,OAAA,gBAAA;;CAE7B,WAAW,MAAyB;AAClC,SAAO,KAAK,MAAM,IAAI,KAAK,IAAI,KAAA;;CAGjC,QAAQ,MAAgB,UAAiC;EACvD,MAAM,OAAO,KAAK,MAAM,IAAI,KAAK;AACjC,MAAI,QAAQ,KAAA,EACV,MAAK,QAAQ,IAAI,SAAS;AAG5B,SAAO;;;;CAKT,WAAW,MAAgB,UAAuB;AAChD,SAAO,KAAK,OAAO,KAAK,CAAC,QAAQ,IAAI,SAAS;AAC9C,SAAO,KAAK,UAAU;;;CAIxB,WAAgB;AACd,MAAI,KAAK,kBAAkB,KAAK,cAAe,QAAO,EAAE;EAExD,MAAM,SAAc,EAAE;EACtB,IAAI,aAAa;AAEjB,aAAW,KAAK,MAAM,CACnB,QAAQ,CAAC,GAAG,UAAuB,KAAK,QAAQ,QAAQ,CAAC,CACzD,SAAS,CAAC,MAAM,OAAO;AACtB,OAAI,KAAK,iBAAiB,cAAc,KAAK,cAC3C;GAGF,MAAM,OAAO,OAAO,KAAK,OAAO,KAAK;AACrC,iBAAc,KAAK;AACnB,UAAO,KAAK,KAAK;IACjB;AAEJ,SAAO;;CAGT,SAAS,MAAS,UAAkB;EAClC,MAAM,UAAU,KAAK,MAAM,IAAI,KAAK,KAAK,IAAI,KAAA;AAC7C,OAAK,MAAM,IAAI,KAAK,MAAM,KAAK;AAC/B,OAAK,QAAQ,IAAI,SAAS;AAE1B,MAAI,KAAK,OAAO,EACd,OAAM,IAAI,MAAM,oBAAoB,OAAO;AAG7C,MAAI,QACF,MAAK,kBAAkB,KAAK;;CAIhC,YAAY,MAAS;AACnB,OAAK,MAAM,OAAO,KAAK,KAAK;AAC5B,OAAK,kBAAkB,KAAK"}

package/dist/drivers/helpers/helpers.cjs

@@ -1,4 +1,3 @@
-
 //#region src/drivers/helpers/helpers.ts
 /** Throws when a driver gets a resource with a wrong resource type. */
 var WrongResourceTypeError = class extends Error {
@@ -23,8 +22,8 @@ var Updater = class {
 		})();
 	}
 };
-
 //#endregion
 exports.Updater = Updater;
 exports.WrongResourceTypeError = WrongResourceTypeError;
+
 //# sourceMappingURL=helpers.cjs.map

package/dist/drivers/helpers/helpers.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"helpers.cjs","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n  name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n  private updating: Promise<void> | undefined;\n\n  constructor(private readonly onUpdate: () => Promise<void>) {}\n\n  schedule() {\n    if (this.updating == undefined) {\n      this.updating = (async () => {\n        try {\n          await this.onUpdate();\n        } catch (e) {\n          console.log(`error while updating in Updater: ${e}`);\n        } finally {\n          this.updating = undefined;\n        }\n      })();\n    }\n  }\n}\n"],"mappings":";;;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO;;;;AAKT,IAAa,UAAb,MAAqB;CACnB,AAAQ;CAER,YAAY,AAAiB,UAA+B;EAA/B;;CAE7B,WAAW;AACT,MAAI,KAAK,YAAY,OACnB,MAAK,YAAY,YAAY;AAC3B,OAAI;AACF,UAAM,KAAK,UAAU;YACd,GAAG;AACV,YAAQ,IAAI,oCAAoC,IAAI;aAC5C;AACR,SAAK,WAAW;;MAEhB"}
+{"version":3,"file":"helpers.cjs","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n  name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n  private updating: Promise<void> | undefined;\n\n  constructor(private readonly onUpdate: () => Promise<void>) {}\n\n  schedule() {\n    if (this.updating == undefined) {\n      this.updating = (async () => {\n        try {\n          await this.onUpdate();\n        } catch (e) {\n          console.log(`error while updating in Updater: ${e}`);\n        } finally {\n          this.updating = undefined;\n        }\n      })();\n    }\n  }\n}\n"],"mappings":";;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO;;;;AAKT,IAAa,UAAb,MAAqB;CACnB;CAEA,YAAY,UAAgD;AAA/B,OAAA,WAAA;;CAE7B,WAAW;AACT,MAAI,KAAK,YAAY,KAAA,EACnB,MAAK,YAAY,YAAY;AAC3B,OAAI;AACF,UAAM,KAAK,UAAU;YACd,GAAG;AACV,YAAQ,IAAI,oCAAoC,IAAI;aAC5C;AACR,SAAK,WAAW,KAAA;;MAEhB"}

package/dist/drivers/helpers/helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.d.ts","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"mappings":";;cACa,sBAAA,SAA+B,KAAA;EAC1C,IAAA;AAAA;;;cAKW,OAAA;EAAA,iBAGkB,QAAA;EAAA,QAFrB,QAAA;cAEqB,QAAA,QAAgB,OAAA;EAE7C,QAAA,CAAA;AAAA"}

package/dist/drivers/helpers/helpers.js

@@ -22,7 +22,7 @@ var Updater = class {
 		})();
 	}
 };
-
 //#endregion
 export { Updater, WrongResourceTypeError };
+
 //# sourceMappingURL=helpers.js.map

package/dist/drivers/helpers/helpers.js.map

@@ -1 +1 @@
-{"version":3,"file":"helpers.js","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n  name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n  private updating: Promise<void> | undefined;\n\n  constructor(private readonly onUpdate: () => Promise<void>) {}\n\n  schedule() {\n    if (this.updating == undefined) {\n      this.updating = (async () => {\n        try {\n          await this.onUpdate();\n        } catch (e) {\n          console.log(`error while updating in Updater: ${e}`);\n        } finally {\n          this.updating = undefined;\n        }\n      })();\n    }\n  }\n}\n"],"mappings":";;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO;;;;AAKT,IAAa,UAAb,MAAqB;CACnB,AAAQ;CAER,YAAY,AAAiB,UAA+B;EAA/B;;CAE7B,WAAW;AACT,MAAI,KAAK,YAAY,OACnB,MAAK,YAAY,YAAY;AAC3B,OAAI;AACF,UAAM,KAAK,UAAU;YACd,GAAG;AACV,YAAQ,IAAI,oCAAoC,IAAI;aAC5C;AACR,SAAK,WAAW;;MAEhB"}
+{"version":3,"file":"helpers.js","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n  name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n  private updating: Promise<void> | undefined;\n\n  constructor(private readonly onUpdate: () => Promise<void>) {}\n\n  schedule() {\n    if (this.updating == undefined) {\n      this.updating = (async () => {\n        try {\n          await this.onUpdate();\n        } catch (e) {\n          console.log(`error while updating in Updater: ${e}`);\n        } finally {\n          this.updating = undefined;\n        }\n      })();\n    }\n  }\n}\n"],"mappings":";;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO;;;;AAKT,IAAa,UAAb,MAAqB;CACnB;CAEA,YAAY,UAAgD;AAA/B,OAAA,WAAA;;CAE7B,WAAW;AACT,MAAI,KAAK,YAAY,KAAA,EACnB,MAAK,YAAY,YAAY;AAC3B,OAAI;AACF,UAAM,KAAK,UAAU;YACd,GAAG;AACV,YAAQ,IAAI,oCAAoC,IAAI;aAC5C;AACR,SAAK,WAAW,KAAA;;MAEhB"}