@milaboratories/pl-drivers 1.11.1 → 1.11.3

package/dist/clients/upload.cjs

@@ -77,7 +77,7 @@ class ClientUpload {
         if (chunk.length !== contentLength) {
             throw new Error(`Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`);
         }
-        const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));
+        const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));
         try {
             const { body: rawBody, statusCode, headers: responseHeaders, } = await undici.request(info.uploadUrl, {
                 dispatcher: this.httpClient,

package/dist/clients/upload.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["UploadClient","uploadapi_ChecksumAlgorithm","request","addRTypeToMetadata","fs","crc32c"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIA,4BAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAKC,oCAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAExF,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEC,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxFA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACDA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMC,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAGC,aAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;;;;;;"}
+{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["UploadClient","uploadapi_ChecksumAlgorithm","request","addRTypeToMetadata","fs","crc32c"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIA,4BAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAKC,oCAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEC,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxFA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACDA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMC,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAGC,aAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;;;;;;"}

package/dist/clients/upload.js

@@ -56,7 +56,7 @@ class ClientUpload {
         if (chunk.length !== contentLength) {
             throw new Error(`Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`);
         }
-        const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));
+        const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));
         try {
             const { body: rawBody, statusCode, headers: responseHeaders, } = await request(info.uploadUrl, {
                 dispatcher: this.httpClient,

package/dist/clients/upload.js.map

@@ -1 +1 @@
-{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["fs"],"mappings":";;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAK,2BAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAExF,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxF,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;"}
+{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["fs"],"mappings":";;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAK,2BAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxF,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;"}

package/dist/helpers/download.cjs

@@ -36,8 +36,11 @@ class RemoteFileDownloader {
         }
         const { statusCode, body, headers: responseHeaders } = await undici.request(url, {
             dispatcher: this.httpClient,
-            headers,
+            // Undici automatically sets certain headers, so we need to lowercase user-provided headers
+            // to prevent automatic headers from being set and avoid "duplicated headers" error.
+            headers: Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])),
             signal: ops.signal,
+            highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset
         });
         ops.signal?.throwIfAborted();
         const webBody = node_stream.Readable.toWeb(body);

package/dist/helpers/download.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      headers,\n      signal: ops.signal,\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders['content-length']);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n          constructor(sizeBytes: number, recordOffByOne: () => void) {\n            super({\n              transform(chunk: Uint8Array, controller) {\n                const truncatedChunk = chunk.slice(0, sizeBytes);\n                controller.enqueue(truncatedChunk);\n                sizeBytes -= truncatedChunk.length;\n                if (!sizeBytes) controller.terminate();\n              },\n              flush(controller) {\n                // Some backend versions have a bug where they return 1 less byte than requested in range.\n                // We cannot always request one more byte because if this end byte is the last byte of the file,\n                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                if (sizeBytes === 1) {\n                  recordOffByOne();\n                  controller.error(new OffByOneError());\n                }\n              },\n            });\n          }\n        })(size, () => this.offByOneServers.push(urlOrigin)));\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":["request","Readable","TransformStream","text"],"mappings":";;;;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAMA,cAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAGC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAcC,mBAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAMC,cAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;;;;"}
+{"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n      // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n      headers: Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])),\n      signal: ops.signal,\n      highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders['content-length']);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n          constructor(sizeBytes: number, recordOffByOne: () => void) {\n            super({\n              transform(chunk: Uint8Array, controller) {\n                const truncatedChunk = chunk.slice(0, sizeBytes);\n                controller.enqueue(truncatedChunk);\n                sizeBytes -= truncatedChunk.length;\n                if (!sizeBytes) controller.terminate();\n              },\n              flush(controller) {\n                // Some backend versions have a bug where they return 1 less byte than requested in range.\n                // We cannot always request one more byte because if this end byte is the last byte of the file,\n                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                if (sizeBytes === 1) {\n                  recordOffByOne();\n                  controller.error(new OffByOneError());\n                }\n              },\n            });\n          }\n        })(size, () => this.offByOneServers.push(urlOrigin)));\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":["request","Readable","TransformStream","text"],"mappings":";;;;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAMA,cAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;;;AAG3B,YAAA,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;YACtG,MAAM,EAAE,GAAG,CAAC,MAAM;AAClB,YAAA,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;AAC/B,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAGC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAcC,mBAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAMC,cAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;;;;"}

package/dist/helpers/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/helpers/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAEtF,gFAAgF;AAChF,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;;;GAIG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,qBAAa,oBAAoB;aAGH,UAAU,EAAE,UAAU;IAFlD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;gBAEpB,UAAU,EAAE,UAAU;IAE5C,WAAW,CAAC,CAAC,EACjB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;CA0Ed"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/helpers/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAEtF,gFAAgF;AAChF,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;;;GAIG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,qBAAa,oBAAoB;aAGH,UAAU,EAAE,UAAU;IAFlD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;gBAEpB,UAAU,EAAE,UAAU;IAE5C,WAAW,CAAC,CAAC,EACjB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;CA6Ed"}

package/dist/helpers/download.js

@@ -34,8 +34,11 @@ class RemoteFileDownloader {
         }
         const { statusCode, body, headers: responseHeaders } = await request(url, {
             dispatcher: this.httpClient,
-            headers,
+            // Undici automatically sets certain headers, so we need to lowercase user-provided headers
+            // to prevent automatic headers from being set and avoid "duplicated headers" error.
+            headers: Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])),
             signal: ops.signal,
+            highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset
         });
         ops.signal?.throwIfAborted();
         const webBody = Readable.toWeb(body);

package/dist/helpers/download.js.map

@@ -1 +1 @@
-{"version":3,"file":"download.js","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      headers,\n      signal: ops.signal,\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders['content-length']);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n          constructor(sizeBytes: number, recordOffByOne: () => void) {\n            super({\n              transform(chunk: Uint8Array, controller) {\n                const truncatedChunk = chunk.slice(0, sizeBytes);\n                controller.enqueue(truncatedChunk);\n                sizeBytes -= truncatedChunk.length;\n                if (!sizeBytes) controller.terminate();\n              },\n              flush(controller) {\n                // Some backend versions have a bug where they return 1 less byte than requested in range.\n                // We cannot always request one more byte because if this end byte is the last byte of the file,\n                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                if (sizeBytes === 1) {\n                  recordOffByOne();\n                  controller.error(new OffByOneError());\n                }\n              },\n            });\n          }\n        })(size, () => this.offByOneServers.push(urlOrigin)));\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":[],"mappings":";;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAc,eAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;"}
+{"version":3,"file":"download.js","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n  name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const { statusCode, body, headers: responseHeaders } = await request(url, {\n      dispatcher: this.httpClient,\n      // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n      // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n      headers: Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])),\n      signal: ops.signal,\n      highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders['content-length']);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n          constructor(sizeBytes: number, recordOffByOne: () => void) {\n            super({\n              transform(chunk: Uint8Array, controller) {\n                const truncatedChunk = chunk.slice(0, sizeBytes);\n                controller.enqueue(truncatedChunk);\n                sizeBytes -= truncatedChunk.length;\n                if (!sizeBytes) controller.terminate();\n              },\n              flush(controller) {\n                // Some backend versions have a bug where they return 1 less byte than requested in range.\n                // We cannot always request one more byte because if this end byte is the last byte of the file,\n                // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                if (sizeBytes === 1) {\n                  recordOffByOne();\n                  controller.error(new OffByOneError());\n                }\n              },\n            });\n          }\n        })(size, () => this.offByOneServers.push(urlOrigin)));\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n\n    if (400 <= statusCode && statusCode < 500) {\n      throw new NetworkError400(\n        `Http error: statusCode: ${statusCode} `\n        + `url: ${url.toString()}, beginning of body: ${beginning}`);\n    }\n\n    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n  }\n}\n"],"names":[],"mappings":";;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;;;AAG3B,YAAA,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;YACtG,MAAM,EAAE,GAAG,CAAC,MAAM;AAClB,YAAA,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;AAC/B,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAc,eAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.11.1",
+  "version": "1.11.3",
   "engines": {
     "node": ">=22"
   },
@@ -28,14 +28,14 @@
     "decompress": "^4.2.1",
     "denque": "^2.1.0",
     "tar-fs": "^3.0.9",
-    "undici": "~7.13.0",
+    "undici": "~7.16.0",
     "upath": "^2.0.1",
     "zod": "~3.23.8",
+    "@milaboratories/pl-client": "2.13.3",
+    "@milaboratories/ts-helpers": "1.5.1",
+    "@milaboratories/pl-tree": "1.8.3",
     "@milaboratories/helpers": "1.9.0",
-    "@milaboratories/computable": "2.7.0",
-    "@milaboratories/ts-helpers": "1.5.0",
-    "@milaboratories/pl-tree": "1.8.1",
-    "@milaboratories/pl-client": "2.13.1",
+    "@milaboratories/computable": "2.7.1",
     "@milaboratories/pl-model-common": "1.20.0"
   },
   "devDependencies": {
@@ -47,9 +47,9 @@
     "typescript": "~5.6.3",
     "vitest": "^2.1.9",
     "@milaboratories/ts-builder": "1.0.5",
+    "@milaboratories/ts-configs": "1.0.6",
     "@milaboratories/build-configs": "1.0.8",
-    "@milaboratories/eslint-config": "1.0.4",
-    "@milaboratories/ts-configs": "1.0.6"
+    "@milaboratories/eslint-config": "1.0.4"
   },
   "scripts": {
     "type-check": "ts-builder types --target node",

package/src/clients/upload.ts

@@ -99,7 +99,7 @@ export class ClientUpload {
       );
     }
 
-    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));
+    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));
 
     try {
       const {

package/src/helpers/download.ts

@@ -50,8 +50,11 @@ export class RemoteFileDownloader {
 
     const { statusCode, body, headers: responseHeaders } = await request(url, {
       dispatcher: this.httpClient,
-      headers,
+      // Undici automatically sets certain headers, so we need to lowercase user-provided headers
+      // to prevent automatic headers from being set and avoid "duplicated headers" error.
+      headers: Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])),
       signal: ops.signal,
+      highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset
     });
     ops.signal?.throwIfAborted();