@milaboratories/pl-drivers 1.10.16 → 1.10.18

package/dist/clients/download.cjs

@@ -1,6 +1,7 @@
 'use strict';
 
 var plClient = require('@milaboratories/pl-client');
+var helpers = require('@milaboratories/helpers');
 var tsHelpers = require('@milaboratories/ts-helpers');
 var fs = require('node:fs');
 var fsp = require('node:fs/promises');
@@ -61,10 +62,16 @@ class ClientDownload {
     async withBlobContent(info, options, ops, handler) {
         const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
         const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
-        this.logger.info(`download blob ${plClient.stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);
-        return isLocal(downloadUrl)
+        this.logger.info(`blob ${plClient.stringifyWithResourceId(info)} download started, `
+            + `url: ${downloadUrl}, `
+            + `range: ${JSON.stringify(ops.range ?? null)}`);
+        const timer = helpers.PerfTimer.start();
+        const result = isLocal(downloadUrl)
             ? await this.withLocalFileContent(downloadUrl, ops, handler)
             : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
+        this.logger.info(`blob ${plClient.stringifyWithResourceId(info)} download finished, `
+            + `took: ${timer.elapsed()}`);
+        return result;
     }
     async withLocalFileContent(url, ops, handler) {
         const { storageId, relativePath } = parseLocalUrl(url);

package/dist/clients/download.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download.cjs","sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Readable } from 'node:stream';\nimport type { Dispatcher } from 'undici';\nimport type { LocalStorageProjection } from '../drivers/types';\nimport { type ContentHandler, RemoteFileDownloader } from '../helpers/download';\nimport { validateAbsolute } from '../helpers/validate';\nimport type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';\nimport { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';\nimport { type GetContentOptions } from '@milaboratories/pl-model-common';\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly grpcClient: GrpcClientProvider<DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new DownloadClient(transport));\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n    this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);\n\n    return isLocal(downloadUrl)\n      ? await this.withLocalFileContent(downloadUrl, ops, handler)\n      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    return await this.grpcClient.get().getDownloadURL(\n      { resourceId: id, isInternalUse: false },\n      addRTypeToMetadata(type, withAbort),\n    ).response;\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == '')\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === '') return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = 'storage://';\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = 'WrongLocalFileUrl';\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = 'UnknownStorageError';\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== '') {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"names":["ConcurrencyLimitingExecutor","DownloadClient","RemoteFileDownloader","stringifyWithResourceId","fsp","fs","Readable","addRTypeToMetadata","path","validateAbsolute"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmBA;AACqC;MACxB,cAAc,CAAA;AAYP,IAAA,UAAA;AACA,IAAA,MAAA;AAZF,IAAA,UAAU;AACT,IAAA,oBAAoB;;AAGpB,IAAA,qBAAqB;;AAGrB,IAAA,oBAAoB,GAAG,IAAIA,qCAA2B,CAAC,EAAE,CAAC;AAE3E,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB;;IAEhC,gBAA0C,EAAA;QAH1B,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAItB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIC,8BAAc,CAAC,SAAS,CAAC,CAAC;QAClH,IAAI,CAAC,oBAAoB,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;AAChE,QAAA,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,CAAC,gBAAgB,CAAC;IACzE;AAEA,IAAA,KAAK,KAAI;AAET;;;;;AAKG;IACH,MAAM,eAAe,CACnB,IAAkB,EAClB,OAA+B,EAC/B,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC;QAEzF,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,cAAA,EAAiBC,gCAAuB,CAAC,IAAI,CAAC,aAAa,WAAW,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA,CAAE,CAAC;QAEvH,OAAO,OAAO,CAAC,WAAW;cACtB,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO;AAC3D,cAAE,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC;IAC3F;AAEA,IAAA,MAAM,oBAAoB,CACxB,GAAW,EACX,GAAsB,EACtB,OAA0B,EAAA;QAE1B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC;AACtD,QAAA,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC;QAEjF,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAW;AACpD,YAAA,MAAM,OAAO,GAAG;AACd,gBAAA,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI;gBACtB,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,SAAS;gBAC/D,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB;AACD,YAAA,IAAI,MAAiC;YACrC,IAAI,cAAc,GAAG,KAAK;AAE1B,YAAA,IAAI;gBACF,MAAM,IAAI,GAAG,MAAMC,cAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACrC,MAAM,GAAGC,aAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC;gBAC/C,MAAM,SAAS,GAAGC,oBAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC;gBAClD,cAAc,GAAG,IAAI;AACrB,gBAAA,OAAO,MAAM;YACf;YAAE,OAAO,KAAK,EAAE;;gBAEd,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;oBAClD,MAAM,CAAC,OAAO,EAAE;gBAClB;AACA,gBAAA,MAAM,KAAK;YACb;AACF,QAAA,CAAC,CAAC;IACJ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EACpB,MAAoB,EAAA;AAEpB,QAAA,MAAM,SAAS,GAAG,OAAO,IAAI,EAAE;AAC/B,QAAA,SAAS,CAAC,KAAK,GAAG,MAAM;AAExB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CAC/C,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,EACxCC,2BAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CACpC,CAAC,QAAQ;IACZ;AACD;AAEK,SAAU,aAAa,CAAC,GAAW,EAAA;AACvC,IAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC3B,IAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE;AACvB,QAAA,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,GAAG,CAAA,0BAAA,CAA4B,CAAC;IAExF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAI;QACtB,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC3D;AACH;SAEgB,WAAW,CACzB,SAAiB,EACjB,qBAA0C,EAC1C,YAAoB,EAAA;IAEpB,MAAM,IAAI,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS;AAAE,QAAA,MAAM,IAAI,mBAAmB,CAAC,6BAA6B,SAAS,CAAA,CAAE,CAAC;IAE/F,IAAI,IAAI,KAAK,EAAE;AAAE,QAAA,OAAO,YAAY;IAEpC,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC;AACtC;AAEA,MAAM,eAAe,GAAG,YAAY;AACpC,SAAS,OAAO,CAAC,GAAW,EAAA;AAC1B,IAAA,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;AACxC;AAEA;AACM,MAAO,iBAAkB,SAAQ,KAAK,CAAA;IAC1C,IAAI,GAAG,mBAAmB;AAC3B;AAED;AACM,MAAO,mBAAoB,SAAQ,KAAK,CAAA;IAC5C,IAAI,GAAG,qBAAqB;AAC7B;AAEK,SAAU,wBAAwB,CAAC,WAAqC,EAAA;AAC5E,IAAA,MAAM,QAAQ,GAAwB,IAAI,GAAG,EAAE;AAC/C,IAAA,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;;AAE5B,QAAA,IAAI,EAAE,CAAC,SAAS,KAAK,EAAE,EAAE;AACvB,YAAAC,yBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC;QAChC;QACA,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC;IAC1C;AAEA,IAAA,OAAO,QAAQ;AACjB;;;;;;;;;"}
+{"version":3,"file":"download.cjs","sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport { PerfTimer } from '@milaboratories/helpers';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Readable } from 'node:stream';\nimport type { Dispatcher } from 'undici';\nimport type { LocalStorageProjection } from '../drivers/types';\nimport { type ContentHandler, RemoteFileDownloader } from '../helpers/download';\nimport { validateAbsolute } from '../helpers/validate';\nimport type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';\nimport { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';\nimport { type GetContentOptions } from '@milaboratories/pl-model-common';\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly grpcClient: GrpcClientProvider<DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new DownloadClient(transport));\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download started, `\n      + `url: ${downloadUrl}, `\n      + `range: ${JSON.stringify(ops.range ?? null)}`,\n    );\n\n    const timer = PerfTimer.start();\n    const result = isLocal(downloadUrl)\n      ? await this.withLocalFileContent(downloadUrl, ops, handler)\n      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download finished, `\n      + `took: ${timer.elapsed()}`,\n    );\n    return result;\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    return await this.grpcClient.get().getDownloadURL(\n      { resourceId: id, isInternalUse: false },\n      addRTypeToMetadata(type, withAbort),\n    ).response;\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == '')\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === '') return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = 'storage://';\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = 'WrongLocalFileUrl';\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = 'UnknownStorageError';\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== '') {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"names":["ConcurrencyLimitingExecutor","DownloadClient","RemoteFileDownloader","stringifyWithResourceId","PerfTimer","fsp","fs","Readable","addRTypeToMetadata","path","validateAbsolute"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA;AACqC;MACxB,cAAc,CAAA;AAYP,IAAA,UAAA;AACA,IAAA,MAAA;AAZF,IAAA,UAAU;AACT,IAAA,oBAAoB;;AAGpB,IAAA,qBAAqB;;AAGrB,IAAA,oBAAoB,GAAG,IAAIA,qCAA2B,CAAC,EAAE,CAAC;AAE3E,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB;;IAEhC,gBAA0C,EAAA;QAH1B,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAItB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIC,8BAAc,CAAC,SAAS,CAAC,CAAC;QAClH,IAAI,CAAC,oBAAoB,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;AAChE,QAAA,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,CAAC,gBAAgB,CAAC;IACzE;AAEA,IAAA,KAAK,KAAI;AAET;;;;;AAKG;IACH,MAAM,eAAe,CACnB,IAAkB,EAClB,OAA+B,EAC/B,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC;QAEzF,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,QAAQC,gCAAuB,CAAC,IAAI,CAAC,CAAA,mBAAA;AACnC,cAAA,CAAA,KAAA,EAAQ,WAAW,CAAA,EAAA;AACnB,cAAA,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAA,CAAE,CAChD;AAED,QAAA,MAAM,KAAK,GAAGC,iBAAS,CAAC,KAAK,EAAE;AAC/B,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW;cAC9B,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO;AAC3D,cAAE,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC;QAEzF,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,QAAQD,gCAAuB,CAAC,IAAI,CAAC,CAAA,oBAAA;AACnC,cAAA,CAAA,MAAA,EAAS,KAAK,CAAC,OAAO,EAAE,CAAA,CAAE,CAC7B;AACD,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,MAAM,oBAAoB,CACxB,GAAW,EACX,GAAsB,EACtB,OAA0B,EAAA;QAE1B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC;AACtD,QAAA,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC;QAEjF,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAW;AACpD,YAAA,MAAM,OAAO,GAAG;AACd,gBAAA,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI;gBACtB,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,SAAS;gBAC/D,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB;AACD,YAAA,IAAI,MAAiC;YACrC,IAAI,cAAc,GAAG,KAAK;AAE1B,YAAA,IAAI;gBACF,MAAM,IAAI,GAAG,MAAME,cAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACrC,MAAM,GAAGC,aAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC;gBAC/C,MAAM,SAAS,GAAGC,oBAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC;gBAClD,cAAc,GAAG,IAAI;AACrB,gBAAA,OAAO,MAAM;YACf;YAAE,OAAO,KAAK,EAAE;;gBAEd,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;oBAClD,MAAM,CAAC,OAAO,EAAE;gBAClB;AACA,gBAAA,MAAM,KAAK;YACb;AACF,QAAA,CAAC,CAAC;IACJ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EACpB,MAAoB,EAAA;AAEpB,QAAA,MAAM,SAAS,GAAG,OAAO,IAAI,EAAE;AAC/B,QAAA,SAAS,CAAC,KAAK,GAAG,MAAM;AAExB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CAC/C,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,EACxCC,2BAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CACpC,CAAC,QAAQ;IACZ;AACD;AAEK,SAAU,aAAa,CAAC,GAAW,EAAA;AACvC,IAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC3B,IAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE;AACvB,QAAA,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,GAAG,CAAA,0BAAA,CAA4B,CAAC;IAExF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAI;QACtB,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC3D;AACH;SAEgB,WAAW,CACzB,SAAiB,EACjB,qBAA0C,EAC1C,YAAoB,EAAA;IAEpB,MAAM,IAAI,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS;AAAE,QAAA,MAAM,IAAI,mBAAmB,CAAC,6BAA6B,SAAS,CAAA,CAAE,CAAC;IAE/F,IAAI,IAAI,KAAK,EAAE;AAAE,QAAA,OAAO,YAAY;IAEpC,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC;AACtC;AAEA,MAAM,eAAe,GAAG,YAAY;AACpC,SAAS,OAAO,CAAC,GAAW,EAAA;AAC1B,IAAA,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;AACxC;AAEA;AACM,MAAO,iBAAkB,SAAQ,KAAK,CAAA;IAC1C,IAAI,GAAG,mBAAmB;AAC3B;AAED;AACM,MAAO,mBAAoB,SAAQ,KAAK,CAAA;IAC5C,IAAI,GAAG,qBAAqB;AAC7B;AAEK,SAAU,wBAAwB,CAAC,WAAqC,EAAA;AAC5E,IAAA,MAAM,QAAQ,GAAwB,IAAI,GAAG,EAAE;AAC/C,IAAA,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;;AAE5B,QAAA,IAAI,EAAE,CAAC,SAAS,KAAK,EAAE,EAAE;AACvB,YAAAC,yBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC;QAChC;QACA,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC;IAC1C;AAEA,IAAA,OAAO,QAAQ;AACjB;;;;;;;;;"}

package/dist/clients/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/clients/download.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAE/F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAK3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,KAAK,cAAc,EAAwB,MAAM,qBAAqB,CAAC;AAGhF,OAAO,EAAE,cAAc,EAAE,MAAM,yFAAyF,CAAC;AACzH,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE;qCACqC;AACrC,qBAAa,cAAc;aAYP,UAAU,EAAE,UAAU;aACtB,MAAM,EAAE,QAAQ;IAZlC,SAAgB,UAAU,EAAE,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAC/D,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAE5D,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAsB;IAE5D,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuC;gBAG1E,yBAAyB,EAAE,yBAAyB,EACpC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,QAAQ;IAChC,oCAAoC;IACpC,gBAAgB,EAAE,sBAAsB,EAAE;IAO5C,KAAK;IAEL;;;;;OAKG;IACG,eAAe,CAAC,CAAC,EACrB,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,UAAU,GAAG,SAAS,EAC/B,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;IAWP,oBAAoB,CAAC,CAAC,EAC1B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;YA+BC,kBAAkB;CAajC;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM;;;EASxC;AAED,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1C,YAAY,EAAE,MAAM,UAQrB;AAOD,mDAAmD;AACnD,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,IAAI,SAAuB;CAC5B;AAED,+DAA+D;AAC/D,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,IAAI,SAAyB;CAC9B;AAED,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,sBAAsB,EAAE,uBAW7E"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/clients/download.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAE/F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAK3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,KAAK,cAAc,EAAwB,MAAM,qBAAqB,CAAC;AAGhF,OAAO,EAAE,cAAc,EAAE,MAAM,yFAAyF,CAAC;AACzH,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE;qCACqC;AACrC,qBAAa,cAAc;aAYP,UAAU,EAAE,UAAU;aACtB,MAAM,EAAE,QAAQ;IAZlC,SAAgB,UAAU,EAAE,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAC/D,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAE5D,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAsB;IAE5D,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuC;gBAG1E,yBAAyB,EAAE,yBAAyB,EACpC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,QAAQ;IAChC,oCAAoC;IACpC,gBAAgB,EAAE,sBAAsB,EAAE;IAO5C,KAAK;IAEL;;;;;OAKG;IACG,eAAe,CAAC,CAAC,EACrB,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,UAAU,GAAG,SAAS,EAC/B,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;IAsBP,oBAAoB,CAAC,CAAC,EAC1B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;YA+BC,kBAAkB;CAajC;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM;;;EASxC;AAED,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1C,YAAY,EAAE,MAAM,UAQrB;AAOD,mDAAmD;AACnD,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,IAAI,SAAuB;CAC5B;AAED,+DAA+D;AAC/D,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,IAAI,SAAyB;CAC9B;AAED,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,sBAAsB,EAAE,uBAW7E"}

package/dist/clients/download.js

@@ -1,4 +1,5 @@
 import { stringifyWithResourceId, addRTypeToMetadata } from '@milaboratories/pl-client';
+import { PerfTimer } from '@milaboratories/helpers';
 import { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
 import * as fs from 'node:fs';
 import * as fsp from 'node:fs/promises';
@@ -38,10 +39,16 @@ class ClientDownload {
     async withBlobContent(info, options, ops, handler) {
         const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
         const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
-        this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);
-        return isLocal(downloadUrl)
+        this.logger.info(`blob ${stringifyWithResourceId(info)} download started, `
+            + `url: ${downloadUrl}, `
+            + `range: ${JSON.stringify(ops.range ?? null)}`);
+        const timer = PerfTimer.start();
+        const result = isLocal(downloadUrl)
             ? await this.withLocalFileContent(downloadUrl, ops, handler)
             : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
+        this.logger.info(`blob ${stringifyWithResourceId(info)} download finished, `
+            + `took: ${timer.elapsed()}`);
+        return result;
     }
     async withLocalFileContent(url, ops, handler) {
         const { storageId, relativePath } = parseLocalUrl(url);

package/dist/clients/download.js.map

@@ -1 +1 @@
-{"version":3,"file":"download.js","sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Readable } from 'node:stream';\nimport type { Dispatcher } from 'undici';\nimport type { LocalStorageProjection } from '../drivers/types';\nimport { type ContentHandler, RemoteFileDownloader } from '../helpers/download';\nimport { validateAbsolute } from '../helpers/validate';\nimport type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';\nimport { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';\nimport { type GetContentOptions } from '@milaboratories/pl-model-common';\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly grpcClient: GrpcClientProvider<DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new DownloadClient(transport));\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n    this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);\n\n    return isLocal(downloadUrl)\n      ? await this.withLocalFileContent(downloadUrl, ops, handler)\n      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    return await this.grpcClient.get().getDownloadURL(\n      { resourceId: id, isInternalUse: false },\n      addRTypeToMetadata(type, withAbort),\n    ).response;\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == '')\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === '') return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = 'storage://';\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = 'WrongLocalFileUrl';\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = 'UnknownStorageError';\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== '') {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"names":[],"mappings":";;;;;;;;;;AAmBA;AACqC;MACxB,cAAc,CAAA;AAYP,IAAA,UAAA;AACA,IAAA,MAAA;AAZF,IAAA,UAAU;AACT,IAAA,oBAAoB;;AAGpB,IAAA,qBAAqB;;AAGrB,IAAA,oBAAoB,GAAG,IAAI,2BAA2B,CAAC,EAAE,CAAC;AAE3E,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB;;IAEhC,gBAA0C,EAAA;QAH1B,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAItB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC;QAClH,IAAI,CAAC,oBAAoB,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;AAChE,QAAA,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,CAAC,gBAAgB,CAAC;IACzE;AAEA,IAAA,KAAK,KAAI;AAET;;;;;AAKG;IACH,MAAM,eAAe,CACnB,IAAkB,EAClB,OAA+B,EAC/B,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC;QAEzF,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,cAAA,EAAiB,uBAAuB,CAAC,IAAI,CAAC,aAAa,WAAW,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA,CAAE,CAAC;QAEvH,OAAO,OAAO,CAAC,WAAW;cACtB,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO;AAC3D,cAAE,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC;IAC3F;AAEA,IAAA,MAAM,oBAAoB,CACxB,GAAW,EACX,GAAsB,EACtB,OAA0B,EAAA;QAE1B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC;AACtD,QAAA,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC;QAEjF,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAW;AACpD,YAAA,MAAM,OAAO,GAAG;AACd,gBAAA,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI;gBACtB,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,SAAS;gBAC/D,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB;AACD,YAAA,IAAI,MAAiC;YACrC,IAAI,cAAc,GAAG,KAAK;AAE1B,YAAA,IAAI;gBACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACrC,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC;gBAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC;gBAClD,cAAc,GAAG,IAAI;AACrB,gBAAA,OAAO,MAAM;YACf;YAAE,OAAO,KAAK,EAAE;;gBAEd,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;oBAClD,MAAM,CAAC,OAAO,EAAE;gBAClB;AACA,gBAAA,MAAM,KAAK;YACb;AACF,QAAA,CAAC,CAAC;IACJ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EACpB,MAAoB,EAAA;AAEpB,QAAA,MAAM,SAAS,GAAG,OAAO,IAAI,EAAE;AAC/B,QAAA,SAAS,CAAC,KAAK,GAAG,MAAM;AAExB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CAC/C,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,EACxC,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CACpC,CAAC,QAAQ;IACZ;AACD;AAEK,SAAU,aAAa,CAAC,GAAW,EAAA;AACvC,IAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC3B,IAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE;AACvB,QAAA,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,GAAG,CAAA,0BAAA,CAA4B,CAAC;IAExF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAI;QACtB,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC3D;AACH;SAEgB,WAAW,CACzB,SAAiB,EACjB,qBAA0C,EAC1C,YAAoB,EAAA;IAEpB,MAAM,IAAI,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS;AAAE,QAAA,MAAM,IAAI,mBAAmB,CAAC,6BAA6B,SAAS,CAAA,CAAE,CAAC;IAE/F,IAAI,IAAI,KAAK,EAAE;AAAE,QAAA,OAAO,YAAY;IAEpC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC;AACtC;AAEA,MAAM,eAAe,GAAG,YAAY;AACpC,SAAS,OAAO,CAAC,GAAW,EAAA;AAC1B,IAAA,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;AACxC;AAEA;AACM,MAAO,iBAAkB,SAAQ,KAAK,CAAA;IAC1C,IAAI,GAAG,mBAAmB;AAC3B;AAED;AACM,MAAO,mBAAoB,SAAQ,KAAK,CAAA;IAC5C,IAAI,GAAG,qBAAqB;AAC7B;AAEK,SAAU,wBAAwB,CAAC,WAAqC,EAAA;AAC5E,IAAA,MAAM,QAAQ,GAAwB,IAAI,GAAG,EAAE;AAC/C,IAAA,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;;AAE5B,QAAA,IAAI,EAAE,CAAC,SAAS,KAAK,EAAE,EAAE;AACvB,YAAA,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC;QAChC;QACA,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC;IAC1C;AAEA,IAAA,OAAO,QAAQ;AACjB;;;;"}
+{"version":3,"file":"download.js","sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport { PerfTimer } from '@milaboratories/helpers';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Readable } from 'node:stream';\nimport type { Dispatcher } from 'undici';\nimport type { LocalStorageProjection } from '../drivers/types';\nimport { type ContentHandler, RemoteFileDownloader } from '../helpers/download';\nimport { validateAbsolute } from '../helpers/validate';\nimport type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';\nimport { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';\nimport { type GetContentOptions } from '@milaboratories/pl-model-common';\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly grpcClient: GrpcClientProvider<DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new DownloadClient(transport));\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download started, `\n      + `url: ${downloadUrl}, `\n      + `range: ${JSON.stringify(ops.range ?? null)}`,\n    );\n\n    const timer = PerfTimer.start();\n    const result = isLocal(downloadUrl)\n      ? await this.withLocalFileContent(downloadUrl, ops, handler)\n      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download finished, `\n      + `took: ${timer.elapsed()}`,\n    );\n    return result;\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    return await this.grpcClient.get().getDownloadURL(\n      { resourceId: id, isInternalUse: false },\n      addRTypeToMetadata(type, withAbort),\n    ).response;\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == '')\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === '') return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = 'storage://';\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = 'WrongLocalFileUrl';\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = 'UnknownStorageError';\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== '') {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"names":[],"mappings":";;;;;;;;;;;AAoBA;AACqC;MACxB,cAAc,CAAA;AAYP,IAAA,UAAA;AACA,IAAA,MAAA;AAZF,IAAA,UAAU;AACT,IAAA,oBAAoB;;AAGpB,IAAA,qBAAqB;;AAGrB,IAAA,oBAAoB,GAAG,IAAI,2BAA2B,CAAC,EAAE,CAAC;AAE3E,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB;;IAEhC,gBAA0C,EAAA;QAH1B,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAItB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC;QAClH,IAAI,CAAC,oBAAoB,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;AAChE,QAAA,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,CAAC,gBAAgB,CAAC;IACzE;AAEA,IAAA,KAAK,KAAI;AAET;;;;;AAKG;IACH,MAAM,eAAe,CACnB,IAAkB,EAClB,OAA+B,EAC/B,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC;QAEzF,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,QAAQ,uBAAuB,CAAC,IAAI,CAAC,CAAA,mBAAA;AACnC,cAAA,CAAA,KAAA,EAAQ,WAAW,CAAA,EAAA;AACnB,cAAA,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAA,CAAE,CAChD;AAED,QAAA,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,EAAE;AAC/B,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW;cAC9B,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO;AAC3D,cAAE,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC;QAEzF,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,QAAQ,uBAAuB,CAAC,IAAI,CAAC,CAAA,oBAAA;AACnC,cAAA,CAAA,MAAA,EAAS,KAAK,CAAC,OAAO,EAAE,CAAA,CAAE,CAC7B;AACD,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,MAAM,oBAAoB,CACxB,GAAW,EACX,GAAsB,EACtB,OAA0B,EAAA;QAE1B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC;AACtD,QAAA,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC;QAEjF,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAW;AACpD,YAAA,MAAM,OAAO,GAAG;AACd,gBAAA,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI;gBACtB,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,SAAS;gBAC/D,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB;AACD,YAAA,IAAI,MAAiC;YACrC,IAAI,cAAc,GAAG,KAAK;AAE1B,YAAA,IAAI;gBACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACrC,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC;gBAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC;gBAClD,cAAc,GAAG,IAAI;AACrB,gBAAA,OAAO,MAAM;YACf;YAAE,OAAO,KAAK,EAAE;;gBAEd,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;oBAClD,MAAM,CAAC,OAAO,EAAE;gBAClB;AACA,gBAAA,MAAM,KAAK;YACb;AACF,QAAA,CAAC,CAAC;IACJ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EACpB,MAAoB,EAAA;AAEpB,QAAA,MAAM,SAAS,GAAG,OAAO,IAAI,EAAE;AAC/B,QAAA,SAAS,CAAC,KAAK,GAAG,MAAM;AAExB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CAC/C,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,EACxC,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CACpC,CAAC,QAAQ;IACZ;AACD;AAEK,SAAU,aAAa,CAAC,GAAW,EAAA;AACvC,IAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC3B,IAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE;AACvB,QAAA,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,GAAG,CAAA,0BAAA,CAA4B,CAAC;IAExF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAI;QACtB,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC3D;AACH;SAEgB,WAAW,CACzB,SAAiB,EACjB,qBAA0C,EAC1C,YAAoB,EAAA;IAEpB,MAAM,IAAI,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS;AAAE,QAAA,MAAM,IAAI,mBAAmB,CAAC,6BAA6B,SAAS,CAAA,CAAE,CAAC;IAE/F,IAAI,IAAI,KAAK,EAAE;AAAE,QAAA,OAAO,YAAY;IAEpC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC;AACtC;AAEA,MAAM,eAAe,GAAG,YAAY;AACpC,SAAS,OAAO,CAAC,GAAW,EAAA;AAC1B,IAAA,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;AACxC;AAEA;AACM,MAAO,iBAAkB,SAAQ,KAAK,CAAA;IAC1C,IAAI,GAAG,mBAAmB;AAC3B;AAED;AACM,MAAO,mBAAoB,SAAQ,KAAK,CAAA;IAC5C,IAAI,GAAG,qBAAqB;AAC7B;AAEK,SAAU,wBAAwB,CAAC,WAAqC,EAAA;AAC5E,IAAA,MAAM,QAAQ,GAAwB,IAAI,GAAG,EAAE;AAC/C,IAAA,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;;AAE5B,QAAA,IAAI,EAAE,CAAC,SAAS,KAAK,EAAE,EAAE;AACvB,YAAA,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC;QAChC;QACA,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC;IAC1C;AAEA,IAAA,OAAO,QAAQ;AACjB;;;;"}

package/dist/drivers/download_blob/download_blob_task.cjs

@@ -70,13 +70,15 @@ class DownloadBlobTask {
         try {
             const size = await this.ensureDownloaded();
             this.setDone(size);
-            this.change.markChanged(`blob download for ${plClient.resourceIdToString(this.rInfo.id)} finished`);
+            this.change.markChanged(`blob ${plClient.resourceIdToString(this.rInfo.id)} download finished`);
         }
         catch (e) {
-            this.logger.error(`download blob ${plClient.stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);
+            this.logger.error(`blob ${plClient.stringifyWithResourceId(this.rInfo)} download failed, `
+                + `state: ${JSON.stringify(this.state)}, `
+                + `error: ${e}`);
             if (nonRecoverableError(e)) {
                 this.setError(e);
-                this.change.markChanged(`blob download for ${plClient.resourceIdToString(this.rInfo.id)} failed`);
+                this.change.markChanged(`blob ${plClient.resourceIdToString(this.rInfo.id)} download failed`);
                 // Just in case we were half-way extracting an archive.
                 await fsp__namespace.rm(this.path, { force: true });
             }
@@ -94,7 +96,8 @@ class DownloadBlobTask {
         this.signalCtl.signal.throwIfAborted();
         if (alreadyExists) {
             this.state.fileExists = true;
-            this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);
+            this.logger.info(`blob ${plClient.stringifyWithResourceId(this.rInfo)} was already downloaded, `
+                + `path: ${this.state.filePath}`);
             const stat = await fsp__namespace.stat(this.state.filePath);
             this.signalCtl.signal.throwIfAborted();
             this.state.fileSize = stat.size;

package/dist/drivers/download_blob/download_blob_task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob_task.cjs","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} finished`);\n    } catch (e: any) {\n      this.logger.error(`download blob ${stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","fsp","ensureDirExists","path","fileExists","createPathAtomically","Writable","NetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAIA,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAIC,wBAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QAC5F;QAAE,OAAO,CAAM,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiBC,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;AAC5H,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAExF,gBAAA,MAAME,cAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,+BAAA,EAAkC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CAAC;AACzE,YAAA,MAAM,IAAI,GAAG,MAAMH,cAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAMI,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAGC,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;;;"}
+{"version":3,"file":"download_blob_task.cjs","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, `\n        + `state: ${JSON.stringify(this.state)}, `\n        + `error: ${e}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `\n        + `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","fsp","ensureDirExists","path","fileExists","createPathAtomically","Writable","NetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAIA,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAIC,wBAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,kBAAA,CAAoB,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,KAAA,EAAQC,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,kBAAA;kBACzC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,EAAA;kBACpC,CAAA,OAAA,EAAU,CAAC,CAAA,CAAE,CAChB;AACD,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,gBAAA,CAAkB,CAAC;;AAEpF,gBAAA,MAAME,cAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,CAAA,KAAA,EAAQJ,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,yBAAA;AACzC,kBAAA,CAAA,MAAA,EAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CACjC;AACD,YAAA,MAAM,IAAI,GAAG,MAAMC,cAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAMI,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAGC,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;;;"}

package/dist/drivers/download_blob/download_blob_task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob_task.d.ts","sourceRoot":"","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAC/F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EACV,YAAY,EACZ,QAAQ,EACT,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAIL,cAAc,EACf,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAK7D,oEAAoE;AACpE,qBAAa,gBAAgB;IAUzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,QAAQ,CAAC,KAAK,EAAE,gBAAgB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM;IAbvB,QAAQ,CAAC,MAAM,eAAsB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IACnD,SAAgB,OAAO,iBAAwB;IAC/C,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,IAAI,CAAS;IACd,IAAI,SAAK;IAChB,OAAO,CAAC,KAAK,CAAqB;gBAGf,MAAM,EAAE,QAAQ,EAChB,cAAc,EAAE,cAAc,EACtC,KAAK,EAAE,gBAAgB,EACf,MAAM,EAAE,eAAe,EAC/B,IAAI,EAAE,MAAM;IAGvB,+EAA+E;IACxE,IAAI;;;;;;;IAUJ,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAK7B,QAAQ;YAkBP,gBAAgB;IA2CvB,KAAK,CAAC,MAAM,EAAE,MAAM;IAIpB,OAAO,IACV;QAAE,IAAI,EAAE,KAAK,CAAA;KAAE,GACf;QACA,IAAI,EAAE,IAAI,CAAC;QACX,MAAM,EAAE,YAAY,CAAC,sBAAsB,CAAC,CAAC;KAC9C;IASH,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,QAAQ;CAIjB;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,GAAG,WAWzC;AAQD,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,eAAe,GAAG,SAAS,EACnC,IAAI,EAAE,MAAM,EACZ,KAAK,CAAC,EAAE,OAAO,GACd,YAAY,CAAC,sBAAsB,CAAC,CAgBtC;AAED,KAAK,aAAa,GAAG;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB,CAAA"}
+{"version":3,"file":"download_blob_task.d.ts","sourceRoot":"","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAC/F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EACV,YAAY,EACZ,QAAQ,EACT,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAIL,cAAc,EACf,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAK7D,oEAAoE;AACpE,qBAAa,gBAAgB;IAUzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,QAAQ,CAAC,KAAK,EAAE,gBAAgB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM;IAbvB,QAAQ,CAAC,MAAM,eAAsB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IACnD,SAAgB,OAAO,iBAAwB;IAC/C,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,IAAI,CAAS;IACd,IAAI,SAAK;IAChB,OAAO,CAAC,KAAK,CAAqB;gBAGf,MAAM,EAAE,QAAQ,EAChB,cAAc,EAAE,cAAc,EACtC,KAAK,EAAE,gBAAgB,EACf,MAAM,EAAE,eAAe,EAC/B,IAAI,EAAE,MAAM;IAGvB,+EAA+E;IACxE,IAAI;;;;;;;IAUJ,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAK7B,QAAQ;YAsBP,gBAAgB;IA8CvB,KAAK,CAAC,MAAM,EAAE,MAAM;IAIpB,OAAO,IACV;QAAE,IAAI,EAAE,KAAK,CAAA;KAAE,GACf;QACA,IAAI,EAAE,IAAI,CAAC;QACX,MAAM,EAAE,YAAY,CAAC,sBAAsB,CAAC,CAAC;KAC9C;IASH,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,QAAQ;CAIjB;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,GAAG,WAWzC;AAQD,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,eAAe,GAAG,SAAS,EACnC,IAAI,EAAE,MAAM,EACZ,KAAK,CAAC,EAAE,OAAO,GACd,YAAY,CAAC,sBAAsB,CAAC,CAgBtC;AAED,KAAK,aAAa,GAAG;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB,CAAA"}

package/dist/drivers/download_blob/download_blob_task.js

@@ -48,13 +48,15 @@ class DownloadBlobTask {
         try {
             const size = await this.ensureDownloaded();
             this.setDone(size);
-            this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} finished`);
+            this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);
         }
         catch (e) {
-            this.logger.error(`download blob ${stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);
+            this.logger.error(`blob ${stringifyWithResourceId(this.rInfo)} download failed, `
+                + `state: ${JSON.stringify(this.state)}, `
+                + `error: ${e}`);
             if (nonRecoverableError(e)) {
                 this.setError(e);
-                this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} failed`);
+                this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);
                 // Just in case we were half-way extracting an archive.
                 await fsp.rm(this.path, { force: true });
             }
@@ -72,7 +74,8 @@ class DownloadBlobTask {
         this.signalCtl.signal.throwIfAborted();
         if (alreadyExists) {
             this.state.fileExists = true;
-            this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);
+            this.logger.info(`blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `
+                + `path: ${this.state.filePath}`);
             const stat = await fsp.stat(this.state.filePath);
             this.signalCtl.signal.throwIfAborted();
             this.state.fileSize = stat.size;

package/dist/drivers/download_blob/download_blob_task.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob_task.js","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} finished`);\n    } catch (e: any) {\n      this.logger.error(`download blob ${stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["fs"],"mappings":";;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QAC5F;QAAE,OAAO,CAAM,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;AAC5H,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAExF,gBAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,+BAAA,EAAkC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CAAC;AACzE,YAAA,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACA,WAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;"}
+{"version":3,"file":"download_blob_task.js","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, `\n        + `state: ${JSON.stringify(this.state)}, `\n        + `error: ${e}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `\n        + `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["fs"],"mappings":";;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,kBAAA,CAAoB,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,KAAA,EAAQ,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,kBAAA;kBACzC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,EAAA;kBACpC,CAAA,OAAA,EAAU,CAAC,CAAA,CAAE,CAChB;AACD,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,gBAAA,CAAkB,CAAC;;AAEpF,gBAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,CAAA,KAAA,EAAQ,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,yBAAA;AACzC,kBAAA,CAAA,MAAA,EAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CACjC;AACD,YAAA,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACA,WAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;"}

package/dist/helpers/download.cjs

@@ -45,32 +45,42 @@ class RemoteFileDownloader {
         try {
             await checkStatusCodeOk(statusCode, webBody, url);
             ops.signal?.throwIfAborted();
-            // Some backend versions have a bug where they return more data than requested in range.
-            // So we have to manually normalize the stream to the expected size.
-            const size = ops.range ? ops.range.to - ops.range.from : Number(responseHeaders['content-length']);
-            const normalizedStream = webBody.pipeThrough(new (class extends web.TransformStream {
-                constructor(sizeBytes, recordOffByOne) {
-                    super({
-                        transform(chunk, controller) {
-                            const truncatedChunk = chunk.slice(0, sizeBytes);
-                            controller.enqueue(truncatedChunk);
-                            sizeBytes -= truncatedChunk.length;
-                            if (!sizeBytes)
-                                controller.terminate();
-                        },
-                        flush(controller) {
-                            // Some backend versions have a bug where they return 1 less byte than requested in range.
-                            // We cannot always request one more byte because if this end byte is the last byte of the file,
-                            // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
-                            if (sizeBytes === 1) {
-                                recordOffByOne();
-                                controller.error(new OffByOneError());
-                            }
-                        },
-                    });
-                }
-            })(size, () => this.offByOneServers.push(urlOrigin)));
-            const result = await handler(normalizedStream, size);
+            let result = undefined;
+            const contentLength = Number(responseHeaders['content-length']);
+            if (Number.isNaN(contentLength) || contentLength === 0) {
+                // Some backend versions have a bug that they are not returning content-length header.
+                // In this case `content-length` header is returned as 0.
+                // We should not clip the result stream to 0 bytes in such case.
+                result = await handler(webBody, 0);
+            }
+            else {
+                // Some backend versions have a bug where they return more data than requested in range.
+                // So we have to manually normalize the stream to the expected size.
+                const size = ops.range ? ops.range.to - ops.range.from : contentLength;
+                const normalizedStream = webBody.pipeThrough(new (class extends web.TransformStream {
+                    constructor(sizeBytes, recordOffByOne) {
+                        super({
+                            transform(chunk, controller) {
+                                const truncatedChunk = chunk.slice(0, sizeBytes);
+                                controller.enqueue(truncatedChunk);
+                                sizeBytes -= truncatedChunk.length;
+                                if (!sizeBytes)
+                                    controller.terminate();
+                            },
+                            flush(controller) {
+                                // Some backend versions have a bug where they return 1 less byte than requested in range.
+                                // We cannot always request one more byte because if this end byte is the last byte of the file,
+                                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
+                                if (sizeBytes === 1) {
+                                    recordOffByOne();
+                                    controller.error(new OffByOneError());
+                                }
+                            },
+                        });
+                    }
+                })(size, () => this.offByOneServers.push(urlOrigin)));
+                result = await handler(normalizedStream, size);
+            }
             handlerSuccess = true;
             return result;
         }

package/dist/helpers/download.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      headers,\n      signal: ops.signal,\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      // Some backend versions have a bug where they return more data than requested in range.\n      // So we have to manually normalize the stream to the expected size.\n      const size = ops.range ? ops.range.to - ops.range.from : Number(responseHeaders['content-length']);\n      const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n        constructor(sizeBytes: number, recordOffByOne: () => void) {\n          super({\n            transform(chunk: Uint8Array, controller) {\n              const truncatedChunk = chunk.slice(0, sizeBytes);\n              controller.enqueue(truncatedChunk);\n              sizeBytes -= truncatedChunk.length;\n              if (!sizeBytes) controller.terminate();\n            },\n            flush(controller) {\n              // Some backend versions have a bug where they return 1 less byte than requested in range.\n              // We cannot always request one more byte because if this end byte is the last byte of the file,\n              // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n              if (sizeBytes === 1) {\n                recordOffByOne();\n                controller.error(new OffByOneError());\n              }\n            },\n          });\n        }\n      })(size, () => this.offByOneServers.push(urlOrigin)));\n      const result = await handler(normalizedStream, size);\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":["request","Readable","TransformStream","text"],"mappings":";;;;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAMA,cAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAGC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;;;AAI5B,YAAA,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAClG,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAcC,mBAAe,CAAA;gBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,oBAAA,KAAK,CAAC;wBACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;4BACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,4BAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,4BAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,4BAAA,IAAI,CAAC,SAAS;gCAAE,UAAU,CAAC,SAAS,EAAE;wBACxC,CAAC;AACD,wBAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,4BAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,gCAAA,cAAc,EAAE;AAChB,gCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;4BACvC;wBACF,CAAC;AACF,qBAAA,CAAC;gBACJ;AACD,aAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAEpD,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAMC,cAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;;;;"}
+{"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      headers,\n      signal: ops.signal,\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders['content-length']);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n          constructor(sizeBytes: number, recordOffByOne: () => void) {\n            super({\n              transform(chunk: Uint8Array, controller) {\n                const truncatedChunk = chunk.slice(0, sizeBytes);\n                controller.enqueue(truncatedChunk);\n                sizeBytes -= truncatedChunk.length;\n                if (!sizeBytes) controller.terminate();\n              },\n              flush(controller) {\n                // Some backend versions have a bug where they return 1 less byte than requested in range.\n                // We cannot always request one more byte because if this end byte is the last byte of the file,\n                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                if (sizeBytes === 1) {\n                  recordOffByOne();\n                  controller.error(new OffByOneError());\n                }\n              },\n            });\n          }\n        })(size, () => this.offByOneServers.push(urlOrigin)));\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":["request","Readable","TransformStream","text"],"mappings":";;;;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAMA,cAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAGC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAcC,mBAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAMC,cAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;;;;"}

package/dist/helpers/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/helpers/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAEtF,gFAAgF;AAChF,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;;;GAIG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,qBAAa,oBAAoB;aAGH,UAAU,EAAE,UAAU;IAFlD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;gBAEpB,UAAU,EAAE,UAAU;IAE5C,WAAW,CAAC,CAAC,EACjB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;CAgEd"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/helpers/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAEtF,gFAAgF;AAChF,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;;;GAIG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,qBAAa,oBAAoB;aAGH,UAAU,EAAE,UAAU;IAFlD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;gBAEpB,UAAU,EAAE,UAAU;IAE5C,WAAW,CAAC,CAAC,EACjB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;CA0Ed"}

package/dist/helpers/download.js

@@ -43,32 +43,42 @@ class RemoteFileDownloader {
         try {
             await checkStatusCodeOk(statusCode, webBody, url);
             ops.signal?.throwIfAborted();
-            // Some backend versions have a bug where they return more data than requested in range.
-            // So we have to manually normalize the stream to the expected size.
-            const size = ops.range ? ops.range.to - ops.range.from : Number(responseHeaders['content-length']);
-            const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {
-                constructor(sizeBytes, recordOffByOne) {
-                    super({
-                        transform(chunk, controller) {
-                            const truncatedChunk = chunk.slice(0, sizeBytes);
-                            controller.enqueue(truncatedChunk);
-                            sizeBytes -= truncatedChunk.length;
-                            if (!sizeBytes)
-                                controller.terminate();
-                        },
-                        flush(controller) {
-                            // Some backend versions have a bug where they return 1 less byte than requested in range.
-                            // We cannot always request one more byte because if this end byte is the last byte of the file,
-                            // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
-                            if (sizeBytes === 1) {
-                                recordOffByOne();
-                                controller.error(new OffByOneError());
-                            }
-                        },
-                    });
-                }
-            })(size, () => this.offByOneServers.push(urlOrigin)));
-            const result = await handler(normalizedStream, size);
+            let result = undefined;
+            const contentLength = Number(responseHeaders['content-length']);
+            if (Number.isNaN(contentLength) || contentLength === 0) {
+                // Some backend versions have a bug that they are not returning content-length header.
+                // In this case `content-length` header is returned as 0.
+                // We should not clip the result stream to 0 bytes in such case.
+                result = await handler(webBody, 0);
+            }
+            else {
+                // Some backend versions have a bug where they return more data than requested in range.
+                // So we have to manually normalize the stream to the expected size.
+                const size = ops.range ? ops.range.to - ops.range.from : contentLength;
+                const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {
+                    constructor(sizeBytes, recordOffByOne) {
+                        super({
+                            transform(chunk, controller) {
+                                const truncatedChunk = chunk.slice(0, sizeBytes);
+                                controller.enqueue(truncatedChunk);
+                                sizeBytes -= truncatedChunk.length;
+                                if (!sizeBytes)
+                                    controller.terminate();
+                            },
+                            flush(controller) {
+                                // Some backend versions have a bug where they return 1 less byte than requested in range.
+                                // We cannot always request one more byte because if this end byte is the last byte of the file,
+                                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
+                                if (sizeBytes === 1) {
+                                    recordOffByOne();
+                                    controller.error(new OffByOneError());
+                                }
+                            },
+                        });
+                    }
+                })(size, () => this.offByOneServers.push(urlOrigin)));
+                result = await handler(normalizedStream, size);
+            }
             handlerSuccess = true;
             return result;
         }

package/dist/helpers/download.js.map

@@ -1 +1 @@
-{"version":3,"file":"download.js","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      headers,\n      signal: ops.signal,\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      // Some backend versions have a bug where they return more data than requested in range.\n      // So we have to manually normalize the stream to the expected size.\n      const size = ops.range ? ops.range.to - ops.range.from : Number(responseHeaders['content-length']);\n      const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n        constructor(sizeBytes: number, recordOffByOne: () => void) {\n          super({\n            transform(chunk: Uint8Array, controller) {\n              const truncatedChunk = chunk.slice(0, sizeBytes);\n              controller.enqueue(truncatedChunk);\n              sizeBytes -= truncatedChunk.length;\n              if (!sizeBytes) controller.terminate();\n            },\n            flush(controller) {\n              // Some backend versions have a bug where they return 1 less byte than requested in range.\n              // We cannot always request one more byte because if this end byte is the last byte of the file,\n              // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n              if (sizeBytes === 1) {\n                recordOffByOne();\n                controller.error(new OffByOneError());\n              }\n            },\n          });\n        }\n      })(size, () => this.offByOneServers.push(urlOrigin)));\n      const result = await handler(normalizedStream, size);\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":[],"mappings":";;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;;;AAI5B,YAAA,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAClG,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAc,eAAe,CAAA;gBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,oBAAA,KAAK,CAAC;wBACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;4BACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,4BAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,4BAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,4BAAA,IAAI,CAAC,SAAS;gCAAE,UAAU,CAAC,SAAS,EAAE;wBACxC,CAAC;AACD,wBAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,4BAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,gCAAA,cAAc,EAAE;AAChB,gCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;4BACvC;wBACF,CAAC;AACF,qBAAA,CAAC;gBACJ;AACD,aAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAEpD,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;"}
+{"version":3,"file":"download.js","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      headers,\n      signal: ops.signal,\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders['content-length']);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n          constructor(sizeBytes: number, recordOffByOne: () => void) {\n            super({\n              transform(chunk: Uint8Array, controller) {\n                const truncatedChunk = chunk.slice(0, sizeBytes);\n                controller.enqueue(truncatedChunk);\n                sizeBytes -= truncatedChunk.length;\n                if (!sizeBytes) controller.terminate();\n              },\n              flush(controller) {\n                // Some backend versions have a bug where they return 1 less byte than requested in range.\n                // We cannot always request one more byte because if this end byte is the last byte of the file,\n                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                if (sizeBytes === 1) {\n                  recordOffByOne();\n                  controller.error(new OffByOneError());\n                }\n              },\n            });\n          }\n        })(size, () => this.offByOneServers.push(urlOrigin)));\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":[],"mappings":";;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAc,eAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.10.16",
+  "version": "1.10.18",
   "engines": {
     "node": ">=20"
   },
@@ -31,12 +31,12 @@
     "undici": "~7.13.0",
     "upath": "^2.0.1",
     "zod": "~3.23.8",
-    "@milaboratories/ts-helpers": "1.4.7",
-    "@milaboratories/helpers": "1.8.0",
+    "@milaboratories/helpers": "1.8.1",
     "@milaboratories/computable": "2.6.8",
+    "@milaboratories/ts-helpers": "1.4.7",
     "@milaboratories/pl-client": "2.12.2",
-    "@milaboratories/pl-tree": "1.7.13",
-    "@milaboratories/pl-model-common": "1.19.19"
+    "@milaboratories/pl-model-common": "1.19.19",
+    "@milaboratories/pl-tree": "1.7.13"
   },
   "devDependencies": {
     "@types/decompress": "^4.2.7",

package/src/clients/download.ts

@@ -2,6 +2,7 @@
 import type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';
 import { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';
 import type { ResourceInfo } from '@milaboratories/pl-tree';
+import { PerfTimer } from '@milaboratories/helpers';
 import type { MiLogger } from '@milaboratories/ts-helpers';
 import { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
@@ -58,11 +59,22 @@ export class ClientDownload {
     const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
 
     const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
-    this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);
-
-    return isLocal(downloadUrl)
+    this.logger.info(
+      `blob ${stringifyWithResourceId(info)} download started, `
+      + `url: ${downloadUrl}, `
+      + `range: ${JSON.stringify(ops.range ?? null)}`,
+    );
+
+    const timer = PerfTimer.start();
+    const result = isLocal(downloadUrl)
       ? await this.withLocalFileContent(downloadUrl, ops, handler)
       : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
+
+    this.logger.info(
+      `blob ${stringifyWithResourceId(info)} download finished, `
+      + `took: ${timer.elapsed()}`,
+    );
+    return result;
   }
 
   async withLocalFileContent<T>(

package/src/drivers/download_blob/download_blob_task.ts

@@ -59,12 +59,16 @@ export class DownloadBlobTask {
     try {
       const size = await this.ensureDownloaded();
       this.setDone(size);
-      this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} finished`);
+      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);
     } catch (e: any) {
-      this.logger.error(`download blob ${stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);
+      this.logger.error(
+        `blob ${stringifyWithResourceId(this.rInfo)} download failed, `
+        + `state: ${JSON.stringify(this.state)}, `
+        + `error: ${e}`,
+      );
       if (nonRecoverableError(e)) {
         this.setError(e);
-        this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} failed`);
+        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);
         // Just in case we were half-way extracting an archive.
         await fsp.rm(this.path, { force: true });
       }
@@ -86,7 +90,10 @@ export class DownloadBlobTask {
     this.signalCtl.signal.throwIfAborted();
     if (alreadyExists) {
       this.state.fileExists = true;
-      this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);
+      this.logger.info(
+        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `
+        + `path: ${this.state.filePath}`,
+      );
       const stat = await fsp.stat(this.state.filePath);
       this.signalCtl.signal.throwIfAborted();
       this.state.fileSize = stat.size;

package/src/helpers/download.ts

@@ -62,31 +62,41 @@ export class RemoteFileDownloader {
       await checkStatusCodeOk(statusCode, webBody, url);
       ops.signal?.throwIfAborted();
 
-      // Some backend versions have a bug where they return more data than requested in range.
-      // So we have to manually normalize the stream to the expected size.
-      const size = ops.range ? ops.range.to - ops.range.from : Number(responseHeaders['content-length']);
-      const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {
-        constructor(sizeBytes: number, recordOffByOne: () => void) {
-          super({
-            transform(chunk: Uint8Array, controller) {
-              const truncatedChunk = chunk.slice(0, sizeBytes);
-              controller.enqueue(truncatedChunk);
-              sizeBytes -= truncatedChunk.length;
-              if (!sizeBytes) controller.terminate();
-            },
-            flush(controller) {
-              // Some backend versions have a bug where they return 1 less byte than requested in range.
-              // We cannot always request one more byte because if this end byte is the last byte of the file,
-              // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
-              if (sizeBytes === 1) {
-                recordOffByOne();
-                controller.error(new OffByOneError());
-              }
-            },
-          });
-        }
-      })(size, () => this.offByOneServers.push(urlOrigin)));
-      const result = await handler(normalizedStream, size);
+      let result: T | undefined = undefined;
+
+      const contentLength = Number(responseHeaders['content-length']);
+      if (Number.isNaN(contentLength) || contentLength === 0) {
+        // Some backend versions have a bug that they are not returning content-length header.
+        // In this case `content-length` header is returned as 0.
+        // We should not clip the result stream to 0 bytes in such case.
+        result = await handler(webBody, 0);
+      } else {
+        // Some backend versions have a bug where they return more data than requested in range.
+        // So we have to manually normalize the stream to the expected size.
+        const size = ops.range ? ops.range.to - ops.range.from : contentLength;
+        const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {
+          constructor(sizeBytes: number, recordOffByOne: () => void) {
+            super({
+              transform(chunk: Uint8Array, controller) {
+                const truncatedChunk = chunk.slice(0, sizeBytes);
+                controller.enqueue(truncatedChunk);
+                sizeBytes -= truncatedChunk.length;
+                if (!sizeBytes) controller.terminate();
+              },
+              flush(controller) {
+                // Some backend versions have a bug where they return 1 less byte than requested in range.
+                // We cannot always request one more byte because if this end byte is the last byte of the file,
+                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
+                if (sizeBytes === 1) {
+                  recordOffByOne();
+                  controller.error(new OffByOneError());
+                }
+              },
+            });
+          }
+        })(size, () => this.offByOneServers.push(urlOrigin)));
+        result = await handler(normalizedStream, size);
+      }
 
       handlerSuccess = true;
       return result;