@milaboratories/pl-drivers 1.9.1 → 1.10.0

package/dist/clients/upload.cjs

@@ -3,6 +3,8 @@
 var plClient = require('@milaboratories/pl-client');
 var fsp = require('node:fs/promises');
 var undici = require('undici');
+var crc32 = require('@node-rs/crc32');
+var protocol = require('../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs');
 var protocol_client = require('../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs');
 
 function _interopNamespaceDefault(e) {
@@ -37,6 +39,9 @@ class NetworkError extends Error {
 class NoFileForUploading extends Error {
     name = 'NoFileForUploading';
 }
+class BadRequestError extends Error {
+    name = 'BadRequestError';
+}
 /** Low-level client for grpc uploadapi.
  * The user should pass here a concrete BlobUpload/<storageId> resource,
  * it can be got from handle field of BlobUpload. */
@@ -55,26 +60,24 @@ class ClientUpload {
         return {
             overall: init.partsCount,
             toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
+            checksumAlgorithm: init.checksumAlgorithm,
+            checksumHeader: init.checksumHeader,
         };
     }
-    async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, options) {
+    async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, checksumAlgorithm, checksumHeader, options) {
         const info = await this.grpcGetPartUrl({ id, type }, partNumber, 0n, // we update progress as a separate call later.
         options);
         const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
         await checkExpectedMTime(path, expectedMTimeUnix);
+        const crc32cChecksum = calculateCrc32cChecksum(chunk);
+        if (checksumAlgorithm === protocol.uploadapi_ChecksumAlgorithm.CRC32C) {
+            info.headers.push({ name: checksumHeader, value: crc32cChecksum });
+        }
         const contentLength = Number(info.chunkEnd - info.chunkStart);
         if (chunk.length !== contentLength) {
             throw new Error(`Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`);
         }
         const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));
-        const contentLengthKey = Object.keys(headers).find((key) => key.toLowerCase() === 'content-length');
-        if (contentLengthKey) {
-            const existingContentLength = Number(headers[contentLengthKey]);
-            if (existingContentLength !== contentLength) {
-                throw new Error(`Content-Length mismatch: expected ${contentLength}, but got ${existingContentLength} in headers`);
-            }
-        }
-        // content length will be automatically added by undici, so we don't need to set it here
         try {
             const { body: rawBody, statusCode, headers: responseHeaders, } = await undici.request(info.uploadUrl, {
                 dispatcher: this.httpClient,
@@ -99,6 +102,8 @@ class ClientUpload {
         catch (e) {
             if (e instanceof NetworkError)
                 throw e;
+            if (e instanceof BadRequestError)
+                throw e;
             throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);
         }
         await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);
@@ -122,7 +127,8 @@ class ClientUpload {
             .response;
     }
     async grpcGetPartUrl({ id, type }, partNumber, uploadedPartSize, options) {
-        return await this.grpcClient.get().getPartURL({ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false }, plClient.addRTypeToMetadata(type, options)).response;
+        // partChecksum isn't used for now but protoc requires it to be set
+        return await this.grpcClient.get().getPartURL({ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' }, plClient.addRTypeToMetadata(type, options)).response;
     }
     async grpcUpdateProgress({ id, type }, bytesProcessed, options) {
         await this.grpcClient.get().updateProgress({
@@ -174,12 +180,27 @@ async function checkExpectedMTime(path, expectedMTimeUnix) {
     }
 }
 function checkStatusCodeOk(statusCode, body, headers, info) {
+    if (statusCode == 400) {
+        throw new BadRequestError(`response is not ok, status code: ${statusCode},`
+            + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);
+    }
     if (statusCode != 200) {
         throw new NetworkError(`response is not ok, status code: ${statusCode},`
             + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);
     }
 }
+/** Calculate CRC32C checksum of a buffer and return as base64 string */
+function calculateCrc32cChecksum(data) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+    const checksum = crc32.crc32c(data);
+    // Convert to unsigned 32-bit integer and then to base64
+    const buffer = Buffer.alloc(4);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+    buffer.writeUInt32BE(checksum, 0);
+    return buffer.toString('base64');
+}
 
+exports.BadRequestError = BadRequestError;
 exports.ClientUpload = ClientUpload;
 exports.MTimeError = MTimeError;
 exports.NetworkError = NetworkError;

package/dist/clients/upload.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport type { uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));\n\n    const contentLengthKey = Object.keys(headers).find((key) => key.toLowerCase() === 'content-length');\n    if (contentLengthKey) {\n      const existingContentLength = Number(headers[contentLengthKey]);\n      if (existingContentLength !== contentLength) {\n        throw new Error(\n          `Content-Length mismatch: expected ${contentLength}, but got ${existingContentLength} in headers`,\n        );\n      }\n    }\n\n    // content length will be automatically added by undici, so we don't need to set it here\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n"],"names":["UploadClient","request","addRTypeToMetadata","fs"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAaM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIA,4BAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAKpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;SAClE;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAExF,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,WAAW,EAAE,KAAK,gBAAgB,CAAC;QACnG,IAAI,gBAAgB,EAAE;YACpB,MAAM,qBAAqB,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAC/D,YAAA,IAAI,qBAAqB,KAAK,aAAa,EAAE;gBAC3C,MAAM,IAAI,KAAK,CACb,CAAA,kCAAA,EAAqC,aAAa,CAAA,UAAA,EAAa,qBAAqB,CAAA,WAAA,CAAa,CAClG;YACH;QACF;;AAIA,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEC,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;AAEpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,EACtEA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACDA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMC,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;;;;;;;;"}
+{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { crc32c } from '@node-rs/crc32';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["UploadClient","uploadapi_ChecksumAlgorithm","request","addRTypeToMetadata","fs","crc32c"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIA,4BAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAKC,oCAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAExF,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEC,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxFA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACDA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMC,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;;AAE3C,IAAA,MAAM,QAAQ,GAAGC,YAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;;;;;;"}

package/dist/clients/upload.d.ts

@@ -3,6 +3,7 @@ import { ResourceInfo } from '@milaboratories/pl-tree';
 import { MiLogger } from '@milaboratories/ts-helpers';
 import { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import { Dispatcher } from 'undici';
+import { uploadapi_ChecksumAlgorithm } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
 export declare class MTimeError extends Error {
     name: string;
 }
@@ -16,6 +17,9 @@ export declare class NetworkError extends Error {
 export declare class NoFileForUploading extends Error {
     name: string;
 }
+export declare class BadRequestError extends Error {
+    name: string;
+}
 /** Low-level client for grpc uploadapi.
  * The user should pass here a concrete BlobUpload/<storageId> resource,
  * it can be got from handle field of BlobUpload. */
@@ -28,8 +32,10 @@ export declare class ClientUpload {
     initUpload({ id, type }: ResourceInfo, options?: RpcOptions): Promise<{
         overall: bigint;
         toUpload: bigint[];
+        checksumAlgorithm: uploadapi_ChecksumAlgorithm;
+        checksumHeader: string;
     }>;
-    partUpload({ id, type }: ResourceInfo, path: string, expectedMTimeUnix: bigint, partNumber: bigint, options?: RpcOptions): Promise<void>;
+    partUpload({ id, type }: ResourceInfo, path: string, expectedMTimeUnix: bigint, partNumber: bigint, checksumAlgorithm: uploadapi_ChecksumAlgorithm, checksumHeader: string, options?: RpcOptions): Promise<void>;
     finalize(info: ResourceInfo, options?: RpcOptions): Promise<import('../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol').uploadapi_Finalize_Response>;
     /** Calculates parts that need to be uploaded from the parts that were
      * already uploaded. */

package/dist/clients/upload.js

@@ -1,6 +1,8 @@
 import { addRTypeToMetadata } from '@milaboratories/pl-client';
 import * as fsp from 'node:fs/promises';
 import { request } from 'undici';
+import { crc32c } from '@node-rs/crc32';
+import { uploadapi_ChecksumAlgorithm } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js';
 import { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js';
 
 class MTimeError extends Error {
@@ -16,6 +18,9 @@ class NetworkError extends Error {
 class NoFileForUploading extends Error {
     name = 'NoFileForUploading';
 }
+class BadRequestError extends Error {
+    name = 'BadRequestError';
+}
 /** Low-level client for grpc uploadapi.
  * The user should pass here a concrete BlobUpload/<storageId> resource,
  * it can be got from handle field of BlobUpload. */
@@ -34,26 +39,24 @@ class ClientUpload {
         return {
             overall: init.partsCount,
             toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
+            checksumAlgorithm: init.checksumAlgorithm,
+            checksumHeader: init.checksumHeader,
         };
     }
-    async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, options) {
+    async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, checksumAlgorithm, checksumHeader, options) {
         const info = await this.grpcGetPartUrl({ id, type }, partNumber, 0n, // we update progress as a separate call later.
         options);
         const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
         await checkExpectedMTime(path, expectedMTimeUnix);
+        const crc32cChecksum = calculateCrc32cChecksum(chunk);
+        if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {
+            info.headers.push({ name: checksumHeader, value: crc32cChecksum });
+        }
         const contentLength = Number(info.chunkEnd - info.chunkStart);
         if (chunk.length !== contentLength) {
             throw new Error(`Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`);
         }
         const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));
-        const contentLengthKey = Object.keys(headers).find((key) => key.toLowerCase() === 'content-length');
-        if (contentLengthKey) {
-            const existingContentLength = Number(headers[contentLengthKey]);
-            if (existingContentLength !== contentLength) {
-                throw new Error(`Content-Length mismatch: expected ${contentLength}, but got ${existingContentLength} in headers`);
-            }
-        }
-        // content length will be automatically added by undici, so we don't need to set it here
         try {
             const { body: rawBody, statusCode, headers: responseHeaders, } = await request(info.uploadUrl, {
                 dispatcher: this.httpClient,
@@ -78,6 +81,8 @@ class ClientUpload {
         catch (e) {
             if (e instanceof NetworkError)
                 throw e;
+            if (e instanceof BadRequestError)
+                throw e;
             throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);
         }
         await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);
@@ -101,7 +106,8 @@ class ClientUpload {
             .response;
     }
     async grpcGetPartUrl({ id, type }, partNumber, uploadedPartSize, options) {
-        return await this.grpcClient.get().getPartURL({ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false }, addRTypeToMetadata(type, options)).response;
+        // partChecksum isn't used for now but protoc requires it to be set
+        return await this.grpcClient.get().getPartURL({ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' }, addRTypeToMetadata(type, options)).response;
     }
     async grpcUpdateProgress({ id, type }, bytesProcessed, options) {
         await this.grpcClient.get().updateProgress({
@@ -153,11 +159,25 @@ async function checkExpectedMTime(path, expectedMTimeUnix) {
     }
 }
 function checkStatusCodeOk(statusCode, body, headers, info) {
+    if (statusCode == 400) {
+        throw new BadRequestError(`response is not ok, status code: ${statusCode},`
+            + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);
+    }
     if (statusCode != 200) {
         throw new NetworkError(`response is not ok, status code: ${statusCode},`
             + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);
     }
 }
+/** Calculate CRC32C checksum of a buffer and return as base64 string */
+function calculateCrc32cChecksum(data) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+    const checksum = crc32c(data);
+    // Convert to unsigned 32-bit integer and then to base64
+    const buffer = Buffer.alloc(4);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+    buffer.writeUInt32BE(checksum, 0);
+    return buffer.toString('base64');
+}
 
-export { ClientUpload, MTimeError, NetworkError, NoFileForUploading, UnexpectedEOF };
+export { BadRequestError, ClientUpload, MTimeError, NetworkError, NoFileForUploading, UnexpectedEOF };
 //# sourceMappingURL=upload.js.map

package/dist/clients/upload.js.map

@@ -1 +1 @@
-{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport type { uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));\n\n    const contentLengthKey = Object.keys(headers).find((key) => key.toLowerCase() === 'content-length');\n    if (contentLengthKey) {\n      const existingContentLength = Number(headers[contentLengthKey]);\n      if (existingContentLength !== contentLength) {\n        throw new Error(\n          `Content-Length mismatch: expected ${contentLength}, but got ${existingContentLength} in headers`,\n        );\n      }\n    }\n\n    // content length will be automatically added by undici, so we don't need to set it here\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n"],"names":["fs"],"mappings":";;;;;AAaM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAKpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;SAClE;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAExF,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,WAAW,EAAE,KAAK,gBAAgB,CAAC;QACnG,IAAI,gBAAgB,EAAE;YACpB,MAAM,qBAAqB,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAC/D,YAAA,IAAI,qBAAqB,KAAK,aAAa,EAAE;gBAC3C,MAAM,IAAI,KAAK,CACb,CAAA,kCAAA,EAAqC,aAAa,CAAA,UAAA,EAAa,qBAAqB,CAAA,WAAA,CAAa,CAClG;YACH;QACF;;AAIA,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;AAEpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,EACtE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;;;;"}
+{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { crc32c } from '@node-rs/crc32';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["fs"],"mappings":";;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAK,2BAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAExF,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxF,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;;AAE3C,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;"}

package/dist/drivers/upload_task.cjs

@@ -63,8 +63,8 @@ class UploadTask {
             this.change.markChanged(`blob upload for ${plClient.resourceIdToString(this.res.id)} finished`);
         }
         catch (e) {
-            this.setRetriableError(e);
             if (isResourceWasDeletedError(e)) {
+                this.setRetriableError(e);
                 this.logger.warn(`resource was deleted while uploading a blob: ${e}`);
                 this.change.markChanged(`blob upload for ${plClient.resourceIdToString(this.res.id)} aborted, resource was deleted`);
                 this.setDone(true);
@@ -76,6 +76,7 @@ class UploadTask {
                 this.setTerminalError(e);
                 return;
             }
+            this.setRetriableError(e);
             if (isHeadersTimeoutError(e)) {
                 // we probably have a slow internet, we need to slow things a bit.
                 this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);
@@ -160,7 +161,7 @@ async function uploadBlob(logger, clientBlob, res, uploadData, isDoneFn, speed)
     const partUploadFn = (part) => async (controller) => {
         if (isDoneFn())
             return;
-        await clientBlob.partUpload(res, uploadData.localPath, BigInt(uploadData.modificationTime), part, { timeout });
+        await clientBlob.partUpload(res, uploadData.localPath, BigInt(uploadData.modificationTime), part, parts.checksumAlgorithm, parts.checksumHeader, { timeout });
         logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);
         // if we had a network freeze, it will be increased slowly.
         speed.nPartsWithThisUploadSpeed++;
@@ -262,7 +263,10 @@ function isResourceWasDeletedError(e) {
         && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS'));
 }
 function nonRecoverableError(e) {
-    return e instanceof upload.MTimeError || e instanceof upload.UnexpectedEOF || e instanceof upload.NoFileForUploading;
+    return e instanceof upload.MTimeError ||
+        e instanceof upload.UnexpectedEOF ||
+        e instanceof upload.NoFileForUploading ||
+        e instanceof upload.BadRequestError;
 }
 function isHeadersTimeoutError(e) {
     return e?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);

package/dist/drivers/upload_task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"upload_task.cjs","sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientProgress, ProgressStatus } from '../clients/progress';\nimport type { ClientUpload } from '../clients/upload';\nimport { MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';\nimport type { ImportResourceSnapshot } from './types';\nimport { ImportFileHandleUploadData } from './types';\nimport assert from 'node:assert';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n  private nMaxUploads: number;\n  private nPartsWithThisUploadSpeed = 0;\n  private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n  /** If this is upload progress this field will be defined */\n  private uploadData?: ImportFileHandleUploadData;\n  public progress: sdk.ImportProgress;\n\n  /** If failed, then getting a progress is terminally failed. */\n  public failed?: boolean;\n\n  /** True if the blob was existed.\n   * At this case, the task will show progress == 1.0. */\n  private alreadyExisted = false;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly maxNConcurrentPartsUpload: number,\n    signer: Signer,\n    public readonly res: ImportResourceSnapshot,\n  ) {\n    this.nMaxUploads = this.maxNConcurrentPartsUpload;\n    const { uploadData, progress } = newProgress(res, signer);\n    this.uploadData = uploadData;\n    this.progress = progress;\n  }\n\n  public getProgress(w: Watcher, callerId: string) {\n    this.incCounter(w, callerId);\n\n    if (this.failed) {\n      this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n      throw new Error(this.progress.lastError);\n    }\n\n    return cloneProgress(this.progress);\n  }\n\n  public shouldScheduleUpload(): boolean {\n    return isMyUpload(this.progress);\n  }\n\n  /** Uploads a blob if it's not BlobIndex. */\n  public async uploadBlobTask() {\n    try {\n      await uploadBlob(\n        this.logger,\n        this.clientBlob,\n        this.res,\n        this.uploadData!,\n        this.isComputableDone.bind(this),\n        {\n          nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n          nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n          currentSpeed: this.nMaxUploads,\n          maxSpeed: this.maxNConcurrentPartsUpload,\n        },\n      );\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n    } catch (e: any) {\n      this.setRetriableError(e);\n\n      if (isResourceWasDeletedError(e)) {\n        this.logger.warn(`resource was deleted while uploading a blob: ${e}`);\n        this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);\n        this.setDone(true);\n\n        return;\n      }\n\n      this.logger.error(`error while uploading a blob: ${e}`);\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n      if (nonRecoverableError(e)) {\n        this.setTerminalError(e);\n        return;\n      }\n\n      if (isHeadersTimeoutError(e)) {\n        // we probably have a slow internet, we need to slow things a bit.\n        this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n      }\n\n      throw e;\n    }\n  }\n\n  public async updateStatus() {\n    try {\n      // we do it with timeout in case we have slow internet.\n      const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n      const oldStatus = this.progress.status;\n      const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n      this.progress.status = newStatus;\n      this.setDone(status.done);\n\n      if (status.done || this.progress.status.progress != oldStatus?.progress) {\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n      }\n    } catch (e: any) {\n      this.setRetriableError(e);\n\n      if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {\n        this.logger.warn(`deadline exceeded while getting a status of BlobImport`);\n        return;\n      }\n\n      if (isResourceWasDeletedError(e)) {\n        this.logger.warn(\n          `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,\n        );\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed, resource not found`);\n        this.setDone(true);\n        return;\n      }\n\n      this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n      // It was a terminal error, but when a connection drops,\n      // this will stop the whole task, so we make it retryable.\n      // It was like that:\n      // this.change.markChanged();\n      // this.setTerminalError(e);\n    }\n  }\n\n  /** Set non-terminal error, that task can be retried. */\n  private setRetriableError(e: unknown) {\n    this.progress.lastError = String(e);\n  }\n\n  /** Set a terminal error, the task will throw a error instead of a progress. */\n  private setTerminalError(e: unknown) {\n    this.progress.lastError = String(e);\n    this.progress.done = false;\n    this.failed = true;\n  }\n\n  public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n    if (isImportResourceOutputSet(res)) {\n      this.setDone(true);\n      this.alreadyExisted = true;\n    }\n  }\n\n  private setDone(done: boolean) {\n    this.progress.done = done;\n    if (done) this.progress.lastError = undefined;\n  }\n\n  public incCounter(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  public decCounter(callerId: string) {\n    return this.counter.dec(callerId);\n  }\n\n  private isComputableDone() {\n    return this.counter.isZero();\n  }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n  logger: MiLogger,\n  clientBlob: ClientUpload,\n  res: ResourceInfo,\n  uploadData: ImportFileHandleUploadData,\n  isDoneFn: () => boolean,\n  speed: {\n    nPartsWithThisUploadSpeed: number,\n    nPartsToIncreaseUpload: number,\n    currentSpeed: number,\n    maxSpeed: number,\n  },\n) {\n  assert(isUpload(res), 'the upload operation can be done only for BlobUploads');\n  const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n  if (isDoneFn()) return;\n  const parts = await clientBlob.initUpload(res, { timeout });\n  logger.info(\n    `started to upload blob ${res.id},`\n    + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`\n    + ` number of concurrent uploads: ${speed.currentSpeed}`,\n  );\n\n  const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n    if (isDoneFn()) return;\n    await clientBlob.partUpload(\n      res,\n      uploadData.localPath,\n      BigInt(uploadData.modificationTime),\n      part,\n      { timeout }\n    );\n    logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n    // if we had a network freeze, it will be increased slowly.\n    speed.nPartsWithThisUploadSpeed++;\n    if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n      speed.nPartsWithThisUploadSpeed = 0;\n      speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n      controller.setConcurrency(speed.currentSpeed);\n    }\n  };\n\n  await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n  if (isDoneFn()) return;\n  await clientBlob.finalize(res, { timeout });\n\n  logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n  let isUploadSignMatch: boolean | undefined;\n  let uploadData: ImportFileHandleUploadData | undefined;\n  if (isUpload(res)) {\n    uploadData = ImportFileHandleUploadData.parse(res.data);\n    isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n  }\n\n  return {\n    uploadData: uploadData,\n    progress: {\n      done: false,\n      status: undefined,\n      isUpload: isUpload(res),\n      isUploadSignMatch: isUploadSignMatch,\n      lastError: undefined,\n    } satisfies sdk.ImportProgress,\n  };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n  return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n  const cloned: sdk.ImportProgress = {\n    done: progress.done,\n    isUpload: progress.isUpload,\n    isUploadSignMatch: progress.isUploadSignMatch,\n    lastError: progress.lastError,\n  };\n\n  if (progress.status)\n    cloned.status = {\n      progress: progress.status.progress,\n      bytesProcessed: progress.status.bytesProcessed,\n      bytesTotal: progress.status.bytesTotal,\n    };\n\n  return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n  return 'blob' in res.fields\n    ? res.fields.blob !== undefined\n    : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n  return res.type.name.startsWith('BlobUpload');\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n  try {\n    signer.verify(path, signature);\n    return true;\n  } catch (e) {\n    return false;\n  }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n  return {\n    progress: proto.progress ?? 0,\n    bytesProcessed: Number(proto.bytesProcessed),\n    bytesTotal: Number(proto.bytesTotal),\n  };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n  * to backend because the result was already there,\n  * the backend does show us a status that nothing were uploaded,\n  * but we need to show the client that everything is OK.\n  * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n  if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n    return {\n      progress: 1.0,\n      bytesProcessed: Number(status.bytesTotal),\n      bytesTotal: Number(status.bytesTotal),\n    };\n  }\n\n  return status;\n}\n\nexport function isResourceWasDeletedError(e: any) {\n  return (\n    e.name == 'RpcError'\n    && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')\n  );\n}\n\nexport function nonRecoverableError(e: any) {\n  return e instanceof MTimeError || e instanceof UnexpectedEOF || e instanceof NoFileForUploading;\n}\n\nfunction isHeadersTimeoutError(e: any) {\n  return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n  const newConcurrency = Math.min(current + 2, max);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`)\n\n  return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n  const newConcurrency = Math.max(Math.round(current / 2), min);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`)\n\n  return newConcurrency;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","asyncPool","ImportFileHandleUploadData","MTimeError","UnexpectedEOF","NoFileForUploading"],"mappings":";;;;;;;;;AAcA;;AAEgC;MACnB,UAAU,CAAA;AAmBF,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,cAAA;AACA,IAAA,yBAAA;AAED,IAAA,GAAA;AAvBD,IAAA,MAAM,GAAiB,IAAIA,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;AACvD,IAAA,WAAW;IACX,yBAAyB,GAAG,CAAC;AAC7B,IAAA,sBAAsB,GAAG,EAAE,CAAC;;AAG5B,IAAA,UAAU;AACX,IAAA,QAAQ;;AAGR,IAAA,MAAM;AAEb;AACuD;IAC/C,cAAc,GAAG,KAAK;IAE9B,WAAA,CACmB,MAAgB,EAChB,UAAwB,EACxB,cAA8B,EAC9B,yBAAiC,EAClD,MAAc,EACE,GAA2B,EAAA;QAL1B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,yBAAyB,GAAzB,yBAAyB;QAE1B,IAAA,CAAA,GAAG,GAAH,GAAG;AAEnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,yBAAyB;AACjD,QAAA,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;AACzD,QAAA,IAAI,CAAC,UAAU,GAAG,UAAU;AAC5B,QAAA,IAAI,CAAC,QAAQ,GAAG,QAAQ;IAC1B;IAEO,WAAW,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC7C,QAAA,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC;AAE5B,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,6BAAA,EAAgC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAA,CAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC1C;AAEA,QAAA,OAAO,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;IACrC;IAEO,oBAAoB,GAAA;AACzB,QAAA,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClC;;AAGO,IAAA,MAAM,cAAc,GAAA;AACzB,QAAA,IAAI;YACF,MAAM,UAAU,CACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,UAAW,EAChB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAChC;gBACE,yBAAyB,EAAE,IAAI,CAAC,yBAAyB;gBACzD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;gBACnD,YAAY,EAAE,IAAI,CAAC,WAAW;gBAC9B,QAAQ,EAAE,IAAI,CAAC,yBAAyB;AACzC,aAAA,CACF;AACD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBC,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;AAEzB,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;gBAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,6CAAA,EAAgD,CAAC,CAAA,CAAE,CAAC;AACrE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,8BAAA,CAAgC,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAElB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,8BAAA,EAAiC,CAAC,CAAA,CAAE,CAAC;AACvD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;AAEpF,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACxB;YACF;AAEA,YAAA,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE;;AAE5B,gBAAA,IAAI,CAAC,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1E;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEO,IAAA,MAAM,YAAY,GAAA;AACvB,QAAA,IAAI;;AAEF,YAAA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAEhF,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;AACtC,YAAA,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;AACnF,YAAA,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS;AAChC,YAAA,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;AAEzB,YAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,QAAQ,EAAE;AACvE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;YACzF;QACF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEzB,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,mBAAmB,KAAK,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,mBAAmB,CAAC,EAAE;AACxG,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,sDAAA,CAAwD,CAAC;gBAC1E;YACF;AAEA,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iEAAiE,CAAC,CAAA,EAAA,EAAKC,gCAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA,CAAE,CAC3G;AACD,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBD,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,4BAAA,CAA8B,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,uDAAA,EAA0D,CAAC,CAAA,CAAE,CAAC;;;;;;QAMlF;IACF;;AAGQ,IAAA,iBAAiB,CAAC,CAAU,EAAA;QAClC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IACrC;;AAGQ,IAAA,gBAAgB,CAAC,CAAU,EAAA;QACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;AACnC,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK;AAC1B,QAAA,IAAI,CAAC,MAAM,GAAG,IAAI;IACpB;AAEO,IAAA,kBAAkB,CAAC,GAA2B,EAAA;AACnD,QAAA,IAAI,yBAAyB,CAAC,GAAG,CAAC,EAAE;AAClC,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,cAAc,GAAG,IAAI;QAC5B;IACF;AAEQ,IAAA,OAAO,CAAC,IAAa,EAAA;AAC3B,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI;AACzB,QAAA,IAAI,IAAI;AAAE,YAAA,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,SAAS;IAC/C;IAEO,UAAU,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEO,IAAA,UAAU,CAAC,QAAgB,EAAA;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;IAEQ,gBAAgB,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;IAC9B;AACD;AAED;AACO,eAAe,UAAU,CAC9B,MAAgB,EAChB,UAAwB,EACxB,GAAiB,EACjB,UAAsC,EACtC,QAAuB,EACvB,KAKC,EAAA;IAED,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,uDAAuD,CAAC;AAC9E,IAAA,MAAM,OAAO,GAAG,KAAK,CAAC;AAEtB,IAAA,IAAI,QAAQ,EAAE;QAAE;AAChB,IAAA,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;AAC3D,IAAA,MAAM,CAAC,IAAI,CACT,0BAA0B,GAAG,CAAC,EAAE,CAAA,CAAA;UAC9B,CAAA,gBAAA,EAAmB,KAAK,CAAC,OAAO,CAAA,kBAAA,EAAqB,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAA,CAAA;AAC1E,UAAA,CAAA,+BAAA,EAAkC,KAAK,CAAC,YAAY,CAAA,CAAE,CACzD;IAED,MAAM,YAAY,GAAG,CAAC,IAAY,KAAK,OAAO,UAA+B,KAAI;AAC/E,QAAA,IAAI,QAAQ,EAAE;YAAE;QAChB,MAAM,UAAU,CAAC,UAAU,CACzB,GAAG,EACH,UAAU,CAAC,SAAS,EACpB,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,EACnC,IAAI,EACJ,EAAE,OAAO,EAAE,CACZ;AACD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,OAAO,iBAAiB,GAAG,CAAC,EAAE,CAAA,CAAE,CAAC;;QAG7E,KAAK,CAAC,yBAAyB,EAAE;QACjC,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC,sBAAsB,EAAE;AACnE,YAAA,KAAK,CAAC,yBAAyB,GAAG,CAAC;AACnC,YAAA,KAAK,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC;AACpF,YAAA,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC;QAC/C;AACF,IAAA,CAAC;AAED,IAAA,MAAME,mBAAS,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAErE,IAAA,IAAI,QAAQ,EAAE;QAAE;IAChB,MAAM,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,CAAA,sBAAA,EAAyB,GAAG,CAAC,EAAE,CAAA,UAAA,CAAY,CAAC;AAC1D;AAEA,SAAS,WAAW,CAAC,GAA2B,EAAE,MAAc,EAAA;AAC9D,IAAA,IAAI,iBAAsC;AAC1C,IAAA,IAAI,UAAkD;AACtD,IAAA,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE;QACjB,UAAU,GAAGC,gCAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACvD,QAAA,iBAAiB,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,aAAa,CAAC;IACzF;IAEA,OAAO;AACL,QAAA,UAAU,EAAE,UAAU;AACtB,QAAA,QAAQ,EAAE;AACR,YAAA,IAAI,EAAE,KAAK;AACX,YAAA,MAAM,EAAE,SAAS;AACjB,YAAA,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvB,YAAA,iBAAiB,EAAE,iBAAiB;AACpC,YAAA,SAAS,EAAE,SAAS;AACQ,SAAA;KAC/B;AACH;AAEA;AACM,SAAU,UAAU,CAAC,CAAqB,EAAA;IAC9C,OAAO,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC;AACrD;AAEA;;;;AAIqD;AACrD,SAAS,aAAa,CAAC,QAA4B,EAAA;AACjD,KAAmC;QACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,SAAS,EAAE,QAAQ,CAAC,SAAS;;IAG/B,IAAI,QAAQ,CAAC,MAAM;SACD;AACd,YAAA,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;AAClC,YAAA,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc;AAC9C,YAAA,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;UACvC;AAEH,IAAA,OAAO,QAAQ;AACjB;AAEA,SAAS,yBAAyB,CAAC,GAA2B,EAAA;AAC5D,IAAA,OAAO,MAAM,IAAI,GAAG,CAAC;AACnB,UAAE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK;UACpB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;AAC1C;AAEM,SAAU,QAAQ,CAAC,GAAiB,EAAA;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;AAC/C;SAEgB,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAA;AACzE,IAAA,IAAI;AACF,QAAA,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC;AAC9B,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,CAAC,EAAE;AACV,QAAA,OAAO,KAAK;IACd;AACF;AAEA,SAAS,aAAa,CAAC,KAAqB,EAAA;IAC1C,OAAO;AACL,QAAA,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;AAC7B,QAAA,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;AAC5C,QAAA,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;KACrC;AACH;AAEA;;;;AAI+E;AAC/E,SAAS,qBAAqB,CAAC,cAAuB,EAAE,MAAwB,EAAA;AAC9E,IAAA,IAAI,cAAc,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,EAAE;QAC1E,OAAO;AACL,YAAA,QAAQ,EAAE,GAAG;AACb,YAAA,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;AACzC,YAAA,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;SACtC;IACH;AAEA,IAAA,OAAO,MAAM;AACf;AAEM,SAAU,yBAAyB,CAAC,CAAM,EAAA;AAC9C,IAAA,QACE,CAAC,CAAC,IAAI,IAAI;AACP,YAAC,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,gBAAgB,CAAC;AAEnF;AAEM,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,OAAO,CAAC,YAAYC,iBAAU,IAAI,CAAC,YAAYC,oBAAa,IAAI,CAAC,YAAYC,yBAAkB;AACjG;AAEA,SAAS,qBAAqB,CAAC,CAAM,EAAA;IACnC,OAAQ,CAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA,uBAAA,CAAyB,CAAC;AAClE;AAEA;AACA,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC;IACjD,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;AAEA;AACuC;AACvC,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC;IAC7D,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;;;;;;;;;;"}
+{"version":3,"file":"upload_task.cjs","sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientProgress, ProgressStatus } from '../clients/progress';\nimport type { ClientUpload } from '../clients/upload';\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';\nimport type { ImportResourceSnapshot } from './types';\nimport { ImportFileHandleUploadData } from './types';\nimport assert from 'node:assert';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n  private nMaxUploads: number;\n  private nPartsWithThisUploadSpeed = 0;\n  private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n  /** If this is upload progress this field will be defined */\n  private uploadData?: ImportFileHandleUploadData;\n  public progress: sdk.ImportProgress;\n\n  /** If failed, then getting a progress is terminally failed. */\n  public failed?: boolean;\n\n  /** True if the blob was existed.\n   * At this case, the task will show progress == 1.0. */\n  private alreadyExisted = false;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly maxNConcurrentPartsUpload: number,\n    signer: Signer,\n    public readonly res: ImportResourceSnapshot,\n  ) {\n    this.nMaxUploads = this.maxNConcurrentPartsUpload;\n    const { uploadData, progress } = newProgress(res, signer);\n    this.uploadData = uploadData;\n    this.progress = progress;\n  }\n\n  public getProgress(w: Watcher, callerId: string) {\n    this.incCounter(w, callerId);\n\n    if (this.failed) {\n      this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n      throw new Error(this.progress.lastError);\n    }\n\n    return cloneProgress(this.progress);\n  }\n\n  public shouldScheduleUpload(): boolean {\n    return isMyUpload(this.progress);\n  }\n\n  /** Uploads a blob if it's not BlobIndex. */\n  public async uploadBlobTask() {\n    try {\n      await uploadBlob(\n        this.logger,\n        this.clientBlob,\n        this.res,\n        this.uploadData!,\n        this.isComputableDone.bind(this),\n        {\n          nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n          nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n          currentSpeed: this.nMaxUploads,\n          maxSpeed: this.maxNConcurrentPartsUpload,\n        },\n      );\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n    } catch (e: any) {\n\n      if (isResourceWasDeletedError(e)) {\n        this.setRetriableError(e);\n        this.logger.warn(`resource was deleted while uploading a blob: ${e}`);\n        this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);\n        this.setDone(true);\n\n        return;\n      }\n\n      this.logger.error(`error while uploading a blob: ${e}`);\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n      if (nonRecoverableError(e)) {\n        this.setTerminalError(e);\n        return;\n      }\n\n      this.setRetriableError(e);\n\n      if (isHeadersTimeoutError(e)) {\n        // we probably have a slow internet, we need to slow things a bit.\n        this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n      }\n\n      throw e;\n    }\n  }\n\n  public async updateStatus() {\n    try {\n      // we do it with timeout in case we have slow internet.\n      const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n      const oldStatus = this.progress.status;\n      const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n      this.progress.status = newStatus;\n      this.setDone(status.done);\n\n      if (status.done || this.progress.status.progress != oldStatus?.progress) {\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n      }\n    } catch (e: any) {\n      this.setRetriableError(e);\n\n      if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {\n        this.logger.warn(`deadline exceeded while getting a status of BlobImport`);\n        return;\n      }\n\n      if (isResourceWasDeletedError(e)) {\n        this.logger.warn(\n          `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,\n        );\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed, resource not found`);\n        this.setDone(true);\n        return;\n      }\n\n      this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n      // It was a terminal error, but when a connection drops,\n      // this will stop the whole task, so we make it retryable.\n      // It was like that:\n      // this.change.markChanged();\n      // this.setTerminalError(e);\n    }\n  }\n\n  /** Set non-terminal error, that task can be retried. */\n  private setRetriableError(e: unknown) {\n    this.progress.lastError = String(e);\n  }\n\n  /** Set a terminal error, the task will throw a error instead of a progress. */\n  private setTerminalError(e: unknown) {\n    this.progress.lastError = String(e);\n    this.progress.done = false;\n    this.failed = true;\n  }\n\n  public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n    if (isImportResourceOutputSet(res)) {\n      this.setDone(true);\n      this.alreadyExisted = true;\n    }\n  }\n\n  private setDone(done: boolean) {\n    this.progress.done = done;\n    if (done) this.progress.lastError = undefined;\n  }\n\n  public incCounter(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  public decCounter(callerId: string) {\n    return this.counter.dec(callerId);\n  }\n\n  private isComputableDone() {\n    return this.counter.isZero();\n  }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n  logger: MiLogger,\n  clientBlob: ClientUpload,\n  res: ResourceInfo,\n  uploadData: ImportFileHandleUploadData,\n  isDoneFn: () => boolean,\n  speed: {\n    nPartsWithThisUploadSpeed: number,\n    nPartsToIncreaseUpload: number,\n    currentSpeed: number,\n    maxSpeed: number,\n  },\n) {\n  assert(isUpload(res), 'the upload operation can be done only for BlobUploads');\n  const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n  if (isDoneFn()) return;\n  const parts = await clientBlob.initUpload(res, { timeout });\n  logger.info(\n    `started to upload blob ${res.id},`\n    + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`\n    + ` number of concurrent uploads: ${speed.currentSpeed}`,\n  );\n\n  const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n    if (isDoneFn()) return;\n    await clientBlob.partUpload(\n      res,\n      uploadData.localPath,\n      BigInt(uploadData.modificationTime),\n      part,\n      parts.checksumAlgorithm,\n      parts.checksumHeader,\n      { timeout }\n    );\n    logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n    // if we had a network freeze, it will be increased slowly.\n    speed.nPartsWithThisUploadSpeed++;\n    if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n      speed.nPartsWithThisUploadSpeed = 0;\n      speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n      controller.setConcurrency(speed.currentSpeed);\n    }\n  };\n\n  await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n  if (isDoneFn()) return;\n  await clientBlob.finalize(res, { timeout });\n\n  logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n  let isUploadSignMatch: boolean | undefined;\n  let uploadData: ImportFileHandleUploadData | undefined;\n  if (isUpload(res)) {\n    uploadData = ImportFileHandleUploadData.parse(res.data);\n    isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n  }\n\n  return {\n    uploadData: uploadData,\n    progress: {\n      done: false,\n      status: undefined,\n      isUpload: isUpload(res),\n      isUploadSignMatch: isUploadSignMatch,\n      lastError: undefined,\n    } satisfies sdk.ImportProgress,\n  };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n  return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n  const cloned: sdk.ImportProgress = {\n    done: progress.done,\n    isUpload: progress.isUpload,\n    isUploadSignMatch: progress.isUploadSignMatch,\n    lastError: progress.lastError,\n  };\n\n  if (progress.status)\n    cloned.status = {\n      progress: progress.status.progress,\n      bytesProcessed: progress.status.bytesProcessed,\n      bytesTotal: progress.status.bytesTotal,\n    };\n\n  return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n  return 'blob' in res.fields\n    ? res.fields.blob !== undefined\n    : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n  return res.type.name.startsWith('BlobUpload');\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n  try {\n    signer.verify(path, signature);\n    return true;\n  } catch (e) {\n    return false;\n  }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n  return {\n    progress: proto.progress ?? 0,\n    bytesProcessed: Number(proto.bytesProcessed),\n    bytesTotal: Number(proto.bytesTotal),\n  };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n  * to backend because the result was already there,\n  * the backend does show us a status that nothing were uploaded,\n  * but we need to show the client that everything is OK.\n  * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n  if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n    return {\n      progress: 1.0,\n      bytesProcessed: Number(status.bytesTotal),\n      bytesTotal: Number(status.bytesTotal),\n    };\n  }\n\n  return status;\n}\n\nexport function isResourceWasDeletedError(e: any) {\n  return (\n    e.name == 'RpcError'\n    && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')\n  );\n}\n\nexport function nonRecoverableError(e: any) {\n  return e instanceof MTimeError ||\n    e instanceof UnexpectedEOF ||\n    e instanceof NoFileForUploading ||\n    e instanceof BadRequestError;\n}\n\nfunction isHeadersTimeoutError(e: any) {\n  return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n  const newConcurrency = Math.min(current + 2, max);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`)\n\n  return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n  const newConcurrency = Math.max(Math.round(current / 2), min);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`)\n\n  return newConcurrency;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","asyncPool","ImportFileHandleUploadData","MTimeError","UnexpectedEOF","NoFileForUploading","BadRequestError"],"mappings":";;;;;;;;;AAcA;;AAEgC;MACnB,UAAU,CAAA;AAmBF,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,cAAA;AACA,IAAA,yBAAA;AAED,IAAA,GAAA;AAvBD,IAAA,MAAM,GAAiB,IAAIA,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;AACvD,IAAA,WAAW;IACX,yBAAyB,GAAG,CAAC;AAC7B,IAAA,sBAAsB,GAAG,EAAE,CAAC;;AAG5B,IAAA,UAAU;AACX,IAAA,QAAQ;;AAGR,IAAA,MAAM;AAEb;AACuD;IAC/C,cAAc,GAAG,KAAK;IAE9B,WAAA,CACmB,MAAgB,EAChB,UAAwB,EACxB,cAA8B,EAC9B,yBAAiC,EAClD,MAAc,EACE,GAA2B,EAAA;QAL1B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,yBAAyB,GAAzB,yBAAyB;QAE1B,IAAA,CAAA,GAAG,GAAH,GAAG;AAEnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,yBAAyB;AACjD,QAAA,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;AACzD,QAAA,IAAI,CAAC,UAAU,GAAG,UAAU;AAC5B,QAAA,IAAI,CAAC,QAAQ,GAAG,QAAQ;IAC1B;IAEO,WAAW,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC7C,QAAA,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC;AAE5B,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,6BAAA,EAAgC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAA,CAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC1C;AAEA,QAAA,OAAO,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;IACrC;IAEO,oBAAoB,GAAA;AACzB,QAAA,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClC;;AAGO,IAAA,MAAM,cAAc,GAAA;AACzB,QAAA,IAAI;YACF,MAAM,UAAU,CACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,UAAW,EAChB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAChC;gBACE,yBAAyB,EAAE,IAAI,CAAC,yBAAyB;gBACzD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;gBACnD,YAAY,EAAE,IAAI,CAAC,WAAW;gBAC9B,QAAQ,EAAE,IAAI,CAAC,yBAAyB;AACzC,aAAA,CACF;AACD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBC,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,6CAAA,EAAgD,CAAC,CAAA,CAAE,CAAC;AACrE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,8BAAA,CAAgC,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAElB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,8BAAA,EAAiC,CAAC,CAAA,CAAE,CAAC;AACvD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;AAEpF,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACxB;YACF;AAEA,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;AAEzB,YAAA,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE;;AAE5B,gBAAA,IAAI,CAAC,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1E;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEO,IAAA,MAAM,YAAY,GAAA;AACvB,QAAA,IAAI;;AAEF,YAAA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAEhF,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;AACtC,YAAA,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;AACnF,YAAA,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS;AAChC,YAAA,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;AAEzB,YAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,QAAQ,EAAE;AACvE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;YACzF;QACF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEzB,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,mBAAmB,KAAK,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,mBAAmB,CAAC,EAAE;AACxG,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,sDAAA,CAAwD,CAAC;gBAC1E;YACF;AAEA,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iEAAiE,CAAC,CAAA,EAAA,EAAKC,gCAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA,CAAE,CAC3G;AACD,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBD,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,4BAAA,CAA8B,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,uDAAA,EAA0D,CAAC,CAAA,CAAE,CAAC;;;;;;QAMlF;IACF;;AAGQ,IAAA,iBAAiB,CAAC,CAAU,EAAA;QAClC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IACrC;;AAGQ,IAAA,gBAAgB,CAAC,CAAU,EAAA;QACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;AACnC,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK;AAC1B,QAAA,IAAI,CAAC,MAAM,GAAG,IAAI;IACpB;AAEO,IAAA,kBAAkB,CAAC,GAA2B,EAAA;AACnD,QAAA,IAAI,yBAAyB,CAAC,GAAG,CAAC,EAAE;AAClC,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,cAAc,GAAG,IAAI;QAC5B;IACF;AAEQ,IAAA,OAAO,CAAC,IAAa,EAAA;AAC3B,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI;AACzB,QAAA,IAAI,IAAI;AAAE,YAAA,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,SAAS;IAC/C;IAEO,UAAU,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEO,IAAA,UAAU,CAAC,QAAgB,EAAA;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;IAEQ,gBAAgB,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;IAC9B;AACD;AAED;AACO,eAAe,UAAU,CAC9B,MAAgB,EAChB,UAAwB,EACxB,GAAiB,EACjB,UAAsC,EACtC,QAAuB,EACvB,KAKC,EAAA;IAED,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,uDAAuD,CAAC;AAC9E,IAAA,MAAM,OAAO,GAAG,KAAK,CAAC;AAEtB,IAAA,IAAI,QAAQ,EAAE;QAAE;AAChB,IAAA,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;AAC3D,IAAA,MAAM,CAAC,IAAI,CACT,0BAA0B,GAAG,CAAC,EAAE,CAAA,CAAA;UAC9B,CAAA,gBAAA,EAAmB,KAAK,CAAC,OAAO,CAAA,kBAAA,EAAqB,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAA,CAAA;AAC1E,UAAA,CAAA,+BAAA,EAAkC,KAAK,CAAC,YAAY,CAAA,CAAE,CACzD;IAED,MAAM,YAAY,GAAG,CAAC,IAAY,KAAK,OAAO,UAA+B,KAAI;AAC/E,QAAA,IAAI,QAAQ,EAAE;YAAE;AAChB,QAAA,MAAM,UAAU,CAAC,UAAU,CACzB,GAAG,EACH,UAAU,CAAC,SAAS,EACpB,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,EACnC,IAAI,EACJ,KAAK,CAAC,iBAAiB,EACvB,KAAK,CAAC,cAAc,EACpB,EAAE,OAAO,EAAE,CACZ;AACD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,OAAO,iBAAiB,GAAG,CAAC,EAAE,CAAA,CAAE,CAAC;;QAG7E,KAAK,CAAC,yBAAyB,EAAE;QACjC,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC,sBAAsB,EAAE;AACnE,YAAA,KAAK,CAAC,yBAAyB,GAAG,CAAC;AACnC,YAAA,KAAK,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC;AACpF,YAAA,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC;QAC/C;AACF,IAAA,CAAC;AAED,IAAA,MAAME,mBAAS,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAErE,IAAA,IAAI,QAAQ,EAAE;QAAE;IAChB,MAAM,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,CAAA,sBAAA,EAAyB,GAAG,CAAC,EAAE,CAAA,UAAA,CAAY,CAAC;AAC1D;AAEA,SAAS,WAAW,CAAC,GAA2B,EAAE,MAAc,EAAA;AAC9D,IAAA,IAAI,iBAAsC;AAC1C,IAAA,IAAI,UAAkD;AACtD,IAAA,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE;QACjB,UAAU,GAAGC,gCAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACvD,QAAA,iBAAiB,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,aAAa,CAAC;IACzF;IAEA,OAAO;AACL,QAAA,UAAU,EAAE,UAAU;AACtB,QAAA,QAAQ,EAAE;AACR,YAAA,IAAI,EAAE,KAAK;AACX,YAAA,MAAM,EAAE,SAAS;AACjB,YAAA,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvB,YAAA,iBAAiB,EAAE,iBAAiB;AACpC,YAAA,SAAS,EAAE,SAAS;AACQ,SAAA;KAC/B;AACH;AAEA;AACM,SAAU,UAAU,CAAC,CAAqB,EAAA;IAC9C,OAAO,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC;AACrD;AAEA;;;;AAIqD;AACrD,SAAS,aAAa,CAAC,QAA4B,EAAA;AACjD,KAAmC;QACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,SAAS,EAAE,QAAQ,CAAC,SAAS;;IAG/B,IAAI,QAAQ,CAAC,MAAM;SACD;AACd,YAAA,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;AAClC,YAAA,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc;AAC9C,YAAA,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;UACvC;AAEH,IAAA,OAAO,QAAQ;AACjB;AAEA,SAAS,yBAAyB,CAAC,GAA2B,EAAA;AAC5D,IAAA,OAAO,MAAM,IAAI,GAAG,CAAC;AACnB,UAAE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK;UACpB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;AAC1C;AAEM,SAAU,QAAQ,CAAC,GAAiB,EAAA;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;AAC/C;SAEgB,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAA;AACzE,IAAA,IAAI;AACF,QAAA,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC;AAC9B,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,CAAC,EAAE;AACV,QAAA,OAAO,KAAK;IACd;AACF;AAEA,SAAS,aAAa,CAAC,KAAqB,EAAA;IAC1C,OAAO;AACL,QAAA,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;AAC7B,QAAA,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;AAC5C,QAAA,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;KACrC;AACH;AAEA;;;;AAI+E;AAC/E,SAAS,qBAAqB,CAAC,cAAuB,EAAE,MAAwB,EAAA;AAC9E,IAAA,IAAI,cAAc,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,EAAE;QAC1E,OAAO;AACL,YAAA,QAAQ,EAAE,GAAG;AACb,YAAA,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;AACzC,YAAA,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;SACtC;IACH;AAEA,IAAA,OAAO,MAAM;AACf;AAEM,SAAU,yBAAyB,CAAC,CAAM,EAAA;AAC9C,IAAA,QACE,CAAC,CAAC,IAAI,IAAI;AACP,YAAC,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,gBAAgB,CAAC;AAEnF;AAEM,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,OAAO,CAAC,YAAYC,iBAAU;AAC5B,QAAA,CAAC,YAAYC,oBAAa;AAC1B,QAAA,CAAC,YAAYC,yBAAkB;QAC/B,CAAC,YAAYC,sBAAe;AAChC;AAEA,SAAS,qBAAqB,CAAC,CAAM,EAAA;IACnC,OAAQ,CAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA,uBAAA,CAAyB,CAAC;AAClE;AAEA;AACA,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC;IACjD,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;AAEA;AACuC;AACvC,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC;IAC7D,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;;;;;;;;;;"}

package/dist/drivers/upload_task.d.ts

@@ -1,7 +1,7 @@
 import { Watcher } from '@milaboratories/computable';
 import { MiLogger, Signer } from '@milaboratories/ts-helpers';
 import { ClientProgress } from '../clients/progress';
-import { ClientUpload, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
+import { ClientUpload, BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
 import { ImportResourceSnapshot, ImportFileHandleUploadData } from './types';
 import { ResourceInfo } from '@milaboratories/pl-tree';
 import type * as sdk from '@milaboratories/pl-model-common';
@@ -55,4 +55,4 @@ export declare function isMyUpload(p: sdk.ImportProgress): boolean;
 export declare function isUpload(res: ResourceInfo): boolean;
 export declare function isSignMatch(signer: Signer, path: string, signature: string): boolean;
 export declare function isResourceWasDeletedError(e: any): boolean;
-export declare function nonRecoverableError(e: any): e is MTimeError | UnexpectedEOF | NoFileForUploading;
+export declare function nonRecoverableError(e: any): e is MTimeError | UnexpectedEOF | NoFileForUploading | BadRequestError;

package/dist/drivers/upload_task.js

@@ -1,7 +1,7 @@
 import { ChangeSource } from '@milaboratories/computable';
 import { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';
 import { CallersCounter, asyncPool } from '@milaboratories/ts-helpers';
-import { MTimeError, UnexpectedEOF, NoFileForUploading } from '../clients/upload.js';
+import { MTimeError, UnexpectedEOF, NoFileForUploading, BadRequestError } from '../clients/upload.js';
 import { ImportFileHandleUploadData } from './types.js';
 import assert from 'node:assert';
 
@@ -61,8 +61,8 @@ class UploadTask {
             this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);
         }
         catch (e) {
-            this.setRetriableError(e);
             if (isResourceWasDeletedError(e)) {
+                this.setRetriableError(e);
                 this.logger.warn(`resource was deleted while uploading a blob: ${e}`);
                 this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);
                 this.setDone(true);
@@ -74,6 +74,7 @@ class UploadTask {
                 this.setTerminalError(e);
                 return;
             }
+            this.setRetriableError(e);
             if (isHeadersTimeoutError(e)) {
                 // we probably have a slow internet, we need to slow things a bit.
                 this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);
@@ -158,7 +159,7 @@ async function uploadBlob(logger, clientBlob, res, uploadData, isDoneFn, speed)
     const partUploadFn = (part) => async (controller) => {
         if (isDoneFn())
             return;
-        await clientBlob.partUpload(res, uploadData.localPath, BigInt(uploadData.modificationTime), part, { timeout });
+        await clientBlob.partUpload(res, uploadData.localPath, BigInt(uploadData.modificationTime), part, parts.checksumAlgorithm, parts.checksumHeader, { timeout });
         logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);
         // if we had a network freeze, it will be increased slowly.
         speed.nPartsWithThisUploadSpeed++;
@@ -260,7 +261,10 @@ function isResourceWasDeletedError(e) {
         && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS'));
 }
 function nonRecoverableError(e) {
-    return e instanceof MTimeError || e instanceof UnexpectedEOF || e instanceof NoFileForUploading;
+    return e instanceof MTimeError ||
+        e instanceof UnexpectedEOF ||
+        e instanceof NoFileForUploading ||
+        e instanceof BadRequestError;
 }
 function isHeadersTimeoutError(e) {
     return e?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);