@milaboratories/pl-drivers 1.11.62 → 1.11.64

package/dist/drivers/download_blob_url/task.cjs

@@ -151,7 +151,7 @@ var URLAborted = class extends Error {
 	name = "URLAborted";
 };
 function nonRecoverableError(e) {
-	return e instanceof URLAborted || require_download_errors.isDownloadNetworkError400(e) || e instanceof require_download.UnknownStorageError || e instanceof require_download.WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED") || String(e).includes("incorrect header check");
+	return e instanceof URLAborted || require_download_errors.isDownloadNetworkError400(e) || e instanceof require_download.UnknownStorageError || e instanceof require_download.WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED") || String(e).includes("incorrect header check");
 }
 
 //#endregion

package/dist/drivers/download_blob_url/task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" && (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,AAAS,UAAU,IAAIA,2CAAgB;CACvC,AAAS,SAAS,IAAIC,yCAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP,AAAQ;CACR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAS,SACT,AAASC,QACT,AAAS,OACT,AAAS,QACT,AAAiB,gBACjB;EAPiB;EACA;EACR;EACA;EACA;EACA;EACQ;;;CAInB,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,iFAAkD,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,oFAAqD,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,gFAAiD,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,iFAAkD,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,aAAK,QAAQ,KAAK,KAAK;AAC9C,wDAAsB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,iDAAiB,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,8DAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAMC,YAAI,MAAM,MAAM;MACtB,MAAM,cAAcC,qBAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAMF,YAAI,MAAM,MAAM;MACtB,MAAM,SAASG,sBAAU,MAAMC,UAAK,cAAc,CAAC;MACnD,MAAM,QAAQH,qBAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,qBAAS,MAAM,WAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,oCAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,WAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,uCAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,8CAAc,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAMI,yBAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,aAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMA,YAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,QAAc;AAC1C,OAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACbO,kDAA0B,EAAE,IAC5B,aAAaC,wCACb,aAAaC,sCAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eAAe,EAAE,QAAQ,eAAe,EAAE,QAAQ,cAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
+{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,AAAS,UAAU,IAAIA,2CAAgB;CACvC,AAAS,SAAS,IAAIC,yCAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP,AAAQ;CACR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAS,SACT,AAASC,QACT,AAAS,OACT,AAAS,QACT,AAAiB,gBACjB;EAPiB;EACA;EACR;EACA;EACA;EACA;EACQ;;;CAInB,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,iFAAkD,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,oFAAqD,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,gFAAiD,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,iFAAkD,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,aAAK,QAAQ,KAAK,KAAK;AAC9C,wDAAsB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,iDAAiB,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,8DAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAMC,YAAI,MAAM,MAAM;MACtB,MAAM,cAAcC,qBAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAMF,YAAI,MAAM,MAAM;MACtB,MAAM,SAASG,sBAAU,MAAMC,UAAK,cAAc,CAAC;MACnD,MAAM,QAAQH,qBAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,qBAAS,MAAM,WAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,oCAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,WAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,uCAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,8CAAc,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAMI,yBAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,aAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMA,YAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,QAAc;AAC1C,OAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACbO,kDAA0B,EAAE,IAC5B,aAAaC,wCACb,aAAaC,sCAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}

package/dist/drivers/download_blob_url/task.js

@@ -144,7 +144,7 @@ var URLAborted = class extends Error {
 	name = "URLAborted";
 };
 function nonRecoverableError(e) {
-	return e instanceof URLAborted || isDownloadNetworkError400(e) || e instanceof UnknownStorageError || e instanceof WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED") || String(e).includes("incorrect header check");
+	return e instanceof URLAborted || isDownloadNetworkError400(e) || e instanceof UnknownStorageError || e instanceof WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED") || String(e).includes("incorrect header check");
 }
 
 //#endregion

package/dist/drivers/download_blob_url/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" && (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,AAAS,UAAU,IAAI,gBAAgB;CACvC,AAAS,SAAS,IAAI,cAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP,AAAQ;CACR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAS,SACT,AAAS,MACT,AAAS,OACT,AAAS,QACT,AAAiB,gBACjB;EAPiB;EACA;EACR;EACA;EACA;EACA;EACQ;;;CAInB,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eAAe,EAAE,QAAQ,eAAe,EAAE,QAAQ,cAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
+{"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n  UnknownStorageError,\n  WrongLocalFileUrl,\n  type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(\n        `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n      );\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(\n        `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n      );\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(\n          `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n        );\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case \"tar\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case \"tgz\":\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n              return;\n\n            case \"zip\":\n              this.state!.zipPath = this.path + \".zip\";\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: (file) => !file.path.endsWith(\"/\"),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" &&\n      (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\" || e.code == \"UNIMPLEMENTED\")) ||\n    // wrong archive format\n    String(e).includes(\"incorrect header check\")\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,AAAS,UAAU,IAAI,gBAAgB;CACvC,AAAS,SAAS,IAAI,cAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP,AAAQ;CACR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAS,SACT,AAAS,MACT,AAAS,OACT,AAAS,QACT,AAAiB,gBACjB;EAPiB;EACA;EACR;EACA;EACA;EACA;EACQ;;;CAInB,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.11.62",
+  "version": "1.11.64",
   "description": "Drivers and a low-level clients for log streaming, downloading and uploading files from and to pl",
   "files": [
     "dist/**/*",
@@ -28,12 +28,12 @@
     "tar-fs": "^3.0.9",
     "undici": "~7.16.0",
     "zod": "~3.23.8",
-    "@milaboratories/helpers": "1.13.7",
-    "@milaboratories/pl-model-common": "1.25.2",
     "@milaboratories/computable": "2.8.6",
+    "@milaboratories/helpers": "1.13.7",
+    "@milaboratories/pl-client": "2.17.10",
     "@milaboratories/ts-helpers": "1.7.3",
-    "@milaboratories/pl-tree": "1.8.49",
-    "@milaboratories/pl-client": "2.17.9"
+    "@milaboratories/pl-tree": "1.8.50",
+    "@milaboratories/pl-model-common": "1.25.3"
   },
   "devDependencies": {
     "@types/decompress": "^4.2.7",
@@ -44,9 +44,9 @@
     "typescript": "~5.9.3",
     "vitest": "^4.0.18",
     "@milaboratories/test-helpers": "1.1.10",
-    "@milaboratories/build-configs": "1.5.2",
     "@milaboratories/ts-builder": "1.3.0",
-    "@milaboratories/ts-configs": "1.2.2"
+    "@milaboratories/ts-configs": "1.2.2",
+    "@milaboratories/build-configs": "1.5.2"
   },
   "engines": {
     "node": ">=22"

package/src/drivers/download_blob_url/task.ts

@@ -244,7 +244,8 @@ export function nonRecoverableError(e: any) {
     // file that we downloads from was moved or deleted.
     e?.code == "ENOENT" ||
     // A resource was deleted.
-    (e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED")) ||
+    (e.name == "RpcError" &&
+      (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED")) ||
     // wrong archive format
     String(e).includes("incorrect header check")
   );