@milaboratories/pl-drivers 1.11.19 → 1.11.21

package/dist/clients/upload.cjs

@@ -3,8 +3,8 @@
 var plClient = require('@milaboratories/pl-client');
 var fsp = require('node:fs/promises');
 var undici = require('undici');
-var protocol = require('../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs');
-var protocol_client = require('../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs');
+var protocol = require('../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs');
+var protocol_client = require('../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs');
 var crc32c = require('./crc32c.cjs');
 
 function _interopNamespaceDefault(e) {
@@ -48,29 +48,84 @@ class BadRequestError extends Error {
 class ClientUpload {
     httpClient;
     logger;
-    grpcClient;
-    constructor(grpcClientProviderFactory, httpClient, _, logger) {
+    wire;
+    constructor(wireClientProviderFactory, httpClient, _, logger) {
         this.httpClient = httpClient;
         this.logger = logger;
-        this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new protocol_client.UploadClient(transport));
+        this.wire = wireClientProviderFactory.createWireClientProvider((wire) => {
+            if (wire.type === 'grpc') {
+                return new protocol_client.UploadClient(wire.Transport);
+            }
+            return plClient.RestAPI.createClient({
+                hostAndPort: wire.Config.hostAndPort,
+                ssl: wire.Config.ssl,
+                dispatcher: wire.Dispatcher,
+                middlewares: wire.Middlewares,
+            });
+        });
     }
     close() { }
     async initUpload({ id, type }, options) {
-        const init = await this.grpcInit(id, type, options);
+        const client = this.wire.get();
+        if (client instanceof protocol_client.UploadClient) {
+            const init = (await client.init({ resourceId: id }, plClient.addRTypeToMetadata(type, options)))
+                .response;
+            return {
+                overall: init.partsCount,
+                toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
+                checksumAlgorithm: init.checksumAlgorithm,
+                checksumHeader: init.checksumHeader,
+            };
+        }
+        const init = (await client.POST('/v1/upload/init', {
+            body: {
+                resourceId: id.toString(),
+            },
+            headers: { ...plClient.createRTypeRoutingHeader(type) },
+        })).data;
         return {
-            overall: init.partsCount,
-            toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
+            overall: BigInt(init.partsCount),
+            toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),
             checksumAlgorithm: init.checksumAlgorithm,
             checksumHeader: init.checksumHeader,
         };
     }
     async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, checksumAlgorithm, checksumHeader, options) {
-        const info = await this.grpcGetPartUrl({ id, type }, partNumber, 0n, // we update progress as a separate call later.
-        options);
+        const client = this.wire.get();
+        let info;
+        if (client instanceof protocol_client.UploadClient) {
+            // partChecksum isn't used for now but protoc requires it to be set
+            info = (await client.getPartURL({
+                resourceId: id,
+                partNumber,
+                uploadedPartSize: 0n,
+                isInternalUse: false,
+                partChecksum: '',
+            }, plClient.addRTypeToMetadata(type, options))).response;
+        }
+        else {
+            const resp = (await client.POST('/v1/upload/get-part-url', {
+                body: {
+                    resourceId: id.toString(),
+                    partNumber: partNumber.toString(),
+                    uploadedPartSize: '0',
+                    isInternalUse: false,
+                    partChecksum: '',
+                },
+                headers: { ...plClient.createRTypeRoutingHeader(type) },
+            })).data;
+            info = {
+                uploadUrl: resp.uploadUrl,
+                method: resp.method,
+                headers: resp.headers,
+                chunkStart: BigInt(resp.chunkStart),
+                chunkEnd: BigInt(resp.chunkEnd),
+            };
+        }
         const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
         await checkExpectedMTime(path, expectedMTimeUnix);
         const crc32cChecksum = calculateCrc32cChecksum(chunk);
-        if (checksumAlgorithm === protocol.uploadapi_ChecksumAlgorithm.CRC32C) {
+        if (checksumAlgorithm === protocol.UploadAPI_ChecksumAlgorithm.CRC32C) {
             info.headers.push({ name: checksumHeader, value: crc32cChecksum });
         }
         const contentLength = Number(info.chunkEnd - info.chunkStart);
@@ -106,10 +161,21 @@ class ClientUpload {
                 throw e;
             throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);
         }
-        await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);
+        await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);
     }
     async finalize(info, options) {
-        return await this.grpcFinalize(info, options);
+        const client = this.wire.get();
+        if (client instanceof protocol_client.UploadClient) {
+            await client.finalize({ resourceId: info.id }, plClient.addRTypeToMetadata(info.type, options));
+        }
+        else {
+            await client.POST('/v1/upload/finalize', {
+                body: {
+                    resourceId: info.id.toString(),
+                },
+                headers: { ...plClient.createRTypeRoutingHeader(info.type) },
+            });
+        }
     }
     /** Calculates parts that need to be uploaded from the parts that were
      * already uploaded. */
@@ -122,23 +188,23 @@ class ClientUpload {
         }
         return toUpload;
     }
-    async grpcInit(id, type, options) {
-        return await this.grpcClient.get().init({ resourceId: id }, plClient.addRTypeToMetadata(type, options))
-            .response;
-    }
-    async grpcGetPartUrl({ id, type }, partNumber, uploadedPartSize, options) {
-        // partChecksum isn't used for now but protoc requires it to be set
-        return await this.grpcClient.get().getPartURL({ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' }, plClient.addRTypeToMetadata(type, options)).response;
-    }
-    async grpcUpdateProgress({ id, type }, bytesProcessed, options) {
-        await this.grpcClient.get().updateProgress({
-            resourceId: id,
-            bytesProcessed,
-        }, plClient.addRTypeToMetadata(type, options)).response;
-    }
-    async grpcFinalize({ id, type }, options) {
-        return await this.grpcClient.get().finalize({ resourceId: id }, plClient.addRTypeToMetadata(type, options))
-            .response;
+    async updateProgress({ id, type }, bytesProcessed, options) {
+        const client = this.wire.get();
+        if (client instanceof protocol_client.UploadClient) {
+            await client.updateProgress({
+                resourceId: id,
+                bytesProcessed,
+            }, plClient.addRTypeToMetadata(type, options)).response;
+            return;
+        }
+        await client.POST('/v1/upload/update-progress', {
+            body: {
+                resourceId: id.toString(),
+                bytesProcessed: bytesProcessed.toString(),
+            },
+            headers: { ...plClient.createRTypeRoutingHeader(type) },
+        });
+        return;
     }
 }
 async function readFileChunk(path, chunkStart, chunkEnd) {

package/dist/clients/upload.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["UploadClient","uploadapi_ChecksumAlgorithm","request","addRTypeToMetadata","fs","crc32c"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIA,4BAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAKC,oCAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEC,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxFA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACDA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEA,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMC,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAGC,aAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;;;;;;"}
+{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { WireClientProvider, WireClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { UploadAPI_ChecksumAlgorithm, type UploadAPI_GetPartURL_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport type { UploadApiPaths, UploadRestClientType } from '../proto-rest';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly wire: WireClientProvider<UploadRestClientType | UploadClient>;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider(\n      (wire) => {\n        if (wire.type === 'grpc') {\n          return new UploadClient(wire.Transport);\n        }\n\n        return RestAPI.createClient<UploadApiPaths>({\n          hostAndPort: wire.Config.hostAndPort,\n          ssl: wire.Config.ssl,\n          dispatcher: wire.Dispatcher,\n          middlewares: wire.Middlewares,\n        });\n      },\n    );\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const client = this.wire.get();\n\n    if (client instanceof UploadClient) {\n      const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n        .response;\n\n      return {\n        overall: init.partsCount,\n        toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n        checksumAlgorithm: init.checksumAlgorithm,\n        checksumHeader: init.checksumHeader,\n      };\n    }\n\n    const init = (await client.POST('/v1/upload/init', {\n      body: {\n        resourceId: id.toString(),\n      },\n      headers: { ...createRTypeRoutingHeader(type) },\n    })).data!;\n\n    return {\n      overall: BigInt(init.partsCount),\n      toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const client = this.wire.get();\n\n    let info: UploadAPI_GetPartURL_Response;\n    if (client instanceof UploadClient) {\n      // partChecksum isn't used for now but protoc requires it to be set\n      info = (await client.getPartURL(\n        {\n          resourceId: id,\n          partNumber,\n          uploadedPartSize: 0n,\n          isInternalUse: false,\n          partChecksum: '',\n        },\n        addRTypeToMetadata(type, options),\n      )).response;\n    } else {\n      const resp = (await client.POST('/v1/upload/get-part-url', {\n        body: {\n          resourceId: id.toString(),\n          partNumber: partNumber.toString(),\n          uploadedPartSize: '0',\n          isInternalUse: false,\n          partChecksum: '',\n        },\n        headers: { ...createRTypeRoutingHeader(type) },\n      })).data!;\n      info = {\n        uploadUrl: resp.uploadUrl,\n        method: resp.method,\n        headers: resp.headers,\n        chunkStart: BigInt(resp.chunkStart),\n        chunkEnd: BigInt(resp.chunkEnd),\n      };\n    }\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    const client = this.wire.get();\n\n    if (client instanceof UploadClient) {\n      await client.finalize({ resourceId: info.id }, addRTypeToMetadata(info.type, options));\n    } else {\n      await client.POST('/v1/upload/finalize', {\n        body: {\n          resourceId: info.id.toString(),\n        },\n        headers: { ...createRTypeRoutingHeader(info.type) },\n      });\n    }\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async updateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ): Promise<void> {\n    const client = this.wire.get();\n\n    if (client instanceof UploadClient) {\n      await client.updateProgress(\n        {\n          resourceId: id,\n          bytesProcessed,\n        },\n        addRTypeToMetadata(type, options),\n      ).response;\n      return;\n    }\n\n    await client.POST('/v1/upload/update-progress', {\n      body: {\n        resourceId: id.toString(),\n        bytesProcessed: bytesProcessed.toString(),\n      },\n      headers: { ...createRTypeRoutingHeader(type) },\n    });\n    return;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: UploadAPI_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["UploadClient","RestAPI","addRTypeToMetadata","createRTypeRoutingHeader","UploadAPI_ChecksumAlgorithm","request","fs","crc32c"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAeM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,IAAI;AAErB,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;QAEtB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAIA,4BAAY,CAAC,IAAI,CAAC,SAAS,CAAC;YACzC;YAEA,OAAOC,gBAAO,CAAC,YAAY,CAAiB;AAC1C,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;QAOpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYD,4BAAY,EAAE;YAClC,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACnF,iBAAA,QAAQ;YAEX,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,gBAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;gBACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;QACH;QAEA,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjD,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AAC1B,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;SAC/C,CAAC,EAAE,IAAK;QAET,OAAO;AACL,YAAA,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrF,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,IAAmC;AACvC,QAAA,IAAI,MAAM,YAAYH,4BAAY,EAAE;;AAElC,YAAA,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAC7B;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,UAAU;AACV,gBAAA,gBAAgB,EAAE,EAAE;AACpB,gBAAA,aAAa,EAAE,KAAK;AACpB,gBAAA,YAAY,EAAE,EAAE;aACjB,EACDE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,EAAE,QAAQ;QACb;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;AACzD,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,oBAAA,UAAU,EAAE,UAAU,CAAC,QAAQ,EAAE;AACjC,oBAAA,gBAAgB,EAAE,GAAG;AACrB,oBAAA,aAAa,EAAE,KAAK;AACpB,oBAAA,YAAY,EAAE,EAAE;AACjB,iBAAA;AACD,gBAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;aAC/C,CAAC,EAAE,IAAK;AACT,YAAA,IAAI,GAAG;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;AACnC,gBAAA,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aAChC;QACH;AAEA,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAKC,oCAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC3F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYL,4BAAY,EAAE;YAClC,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,EAAEE,2BAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxF;aAAO;AACL,YAAA,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;AACvC,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpD,aAAA,CAAC;QACJ;IACF;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;IAEQ,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYH,4BAAY,EAAE;YAClC,MAAM,MAAM,CAAC,cAAc,CACzB;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,cAAc;aACf,EACDE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;YACV;QACF;AAEA,QAAA,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;AAC9C,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,gBAAA,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;AAC1C,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;AAC/C,SAAA,CAAC;QACF;IACF;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMG,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAGC,aAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;;;;;;"}

package/dist/clients/upload.d.ts

@@ -1,9 +1,9 @@
-import type { GrpcClientProviderFactory, PlClient } from '@milaboratories/pl-client';
+import type { WireClientProviderFactory, PlClient } from '@milaboratories/pl-client';
 import type { ResourceInfo } from '@milaboratories/pl-tree';
 import type { MiLogger } from '@milaboratories/ts-helpers';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import type { Dispatcher } from 'undici';
-import { uploadapi_ChecksumAlgorithm } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
+import { UploadAPI_ChecksumAlgorithm } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
 export declare class MTimeError extends Error {
     name: string;
 }
@@ -26,23 +26,20 @@ export declare class BadRequestError extends Error {
 export declare class ClientUpload {
     readonly httpClient: Dispatcher;
     readonly logger: MiLogger;
-    private readonly grpcClient;
-    constructor(grpcClientProviderFactory: GrpcClientProviderFactory, httpClient: Dispatcher, _: PlClient, logger: MiLogger);
+    private readonly wire;
+    constructor(wireClientProviderFactory: WireClientProviderFactory, httpClient: Dispatcher, _: PlClient, logger: MiLogger);
     close(): void;
     initUpload({ id, type }: ResourceInfo, options?: RpcOptions): Promise<{
         overall: bigint;
         toUpload: bigint[];
-        checksumAlgorithm: uploadapi_ChecksumAlgorithm;
+        checksumAlgorithm: UploadAPI_ChecksumAlgorithm;
         checksumHeader: string;
     }>;
-    partUpload({ id, type }: ResourceInfo, path: string, expectedMTimeUnix: bigint, partNumber: bigint, checksumAlgorithm: uploadapi_ChecksumAlgorithm, checksumHeader: string, options?: RpcOptions): Promise<void>;
-    finalize(info: ResourceInfo, options?: RpcOptions): Promise<import("../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol").uploadapi_Finalize_Response>;
+    partUpload({ id, type }: ResourceInfo, path: string, expectedMTimeUnix: bigint, partNumber: bigint, checksumAlgorithm: UploadAPI_ChecksumAlgorithm, checksumHeader: string, options?: RpcOptions): Promise<void>;
+    finalize(info: ResourceInfo, options?: RpcOptions): Promise<void>;
     /** Calculates parts that need to be uploaded from the parts that were
      * already uploaded. */
     private partsToUpload;
-    private grpcInit;
-    private grpcGetPartUrl;
-    private grpcUpdateProgress;
-    private grpcFinalize;
+    private updateProgress;
 }
 //# sourceMappingURL=upload.d.ts.map

package/dist/clients/upload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/clients/upload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAsB,yBAAyB,EAAE,QAAQ,EAA4B,MAAM,2BAA2B,CAAC;AAEnI,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEzC,OAAO,EAAE,2BAA2B,EAAsC,MAAM,gFAAgF,CAAC;AAMjK,qBAAa,UAAW,SAAQ,KAAK;IACnC,IAAI,SAAgB;CACrB;AAED,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,IAAI,SAAkB;CACvB;AAED,0CAA0C;AAC1C,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,IAAI,SAAwB;CAC7B;AAED,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;oDAEoD;AACpD,qBAAa,YAAY;aAKL,UAAU,EAAE,UAAU;aAEtB,MAAM,EAAE,QAAQ;IANlC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmC;gBAG5D,yBAAyB,EAAE,yBAAyB,EACpC,UAAU,EAAE,UAAU,EACtC,CAAC,EAAE,QAAQ,EACK,MAAM,EAAE,QAAQ;IAKlC,KAAK;IAEQ,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC;QACP,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,iBAAiB,EAAE,2BAA2B,CAAC;QAC/C,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IAUS,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,iBAAiB,EAAE,2BAA2B,EAC9C,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,UAAU;IAgET,QAAQ,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,UAAU;IAI9D;2BACuB;IACvB,OAAO,CAAC,aAAa;YAWP,QAAQ;YAKR,cAAc;YAad,kBAAkB;YAclB,YAAY;CAI3B"}
+{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/clients/upload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAsB,yBAAyB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEzG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEzC,OAAO,EAAE,2BAA2B,EAAsC,MAAM,qFAAqF,CAAC;AAOtK,qBAAa,UAAW,SAAQ,KAAK;IACnC,IAAI,SAAgB;CACrB;AAED,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,IAAI,SAAkB;CACvB;AAED,0CAA0C;AAC1C,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,IAAI,SAAwB;CAC7B;AAED,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;oDAEoD;AACpD,qBAAa,YAAY;aAKL,UAAU,EAAE,UAAU;aAEtB,MAAM,EAAE,QAAQ;IANlC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0D;gBAG7E,yBAAyB,EAAE,yBAAyB,EACpC,UAAU,EAAE,UAAU,EACtC,CAAC,EAAE,QAAQ,EACK,MAAM,EAAE,QAAQ;IAkBlC,KAAK;IAEQ,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC;QACP,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,iBAAiB,EAAE,2BAA2B,CAAC;QAC/C,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IA8BS,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,iBAAiB,EAAE,2BAA2B,EAC9C,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,UAAU;IA4FT,QAAQ,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,UAAU;IAe9D;2BACuB;IACvB,OAAO,CAAC,aAAa;YAWP,cAAc;CA2B7B"}

package/dist/clients/upload.js

@@ -1,8 +1,8 @@
-import { addRTypeToMetadata } from '@milaboratories/pl-client';
+import { RestAPI, addRTypeToMetadata, createRTypeRoutingHeader } from '@milaboratories/pl-client';
 import * as fsp from 'node:fs/promises';
 import { request } from 'undici';
-import { uploadapi_ChecksumAlgorithm } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js';
-import { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js';
+import { UploadAPI_ChecksumAlgorithm } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js';
+import { UploadClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js';
 import { crc32c } from './crc32c.js';
 
 class MTimeError extends Error {
@@ -27,29 +27,84 @@ class BadRequestError extends Error {
 class ClientUpload {
     httpClient;
     logger;
-    grpcClient;
-    constructor(grpcClientProviderFactory, httpClient, _, logger) {
+    wire;
+    constructor(wireClientProviderFactory, httpClient, _, logger) {
         this.httpClient = httpClient;
         this.logger = logger;
-        this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));
+        this.wire = wireClientProviderFactory.createWireClientProvider((wire) => {
+            if (wire.type === 'grpc') {
+                return new UploadClient(wire.Transport);
+            }
+            return RestAPI.createClient({
+                hostAndPort: wire.Config.hostAndPort,
+                ssl: wire.Config.ssl,
+                dispatcher: wire.Dispatcher,
+                middlewares: wire.Middlewares,
+            });
+        });
     }
     close() { }
     async initUpload({ id, type }, options) {
-        const init = await this.grpcInit(id, type, options);
+        const client = this.wire.get();
+        if (client instanceof UploadClient) {
+            const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))
+                .response;
+            return {
+                overall: init.partsCount,
+                toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
+                checksumAlgorithm: init.checksumAlgorithm,
+                checksumHeader: init.checksumHeader,
+            };
+        }
+        const init = (await client.POST('/v1/upload/init', {
+            body: {
+                resourceId: id.toString(),
+            },
+            headers: { ...createRTypeRoutingHeader(type) },
+        })).data;
         return {
-            overall: init.partsCount,
-            toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
+            overall: BigInt(init.partsCount),
+            toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),
             checksumAlgorithm: init.checksumAlgorithm,
             checksumHeader: init.checksumHeader,
         };
     }
     async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, checksumAlgorithm, checksumHeader, options) {
-        const info = await this.grpcGetPartUrl({ id, type }, partNumber, 0n, // we update progress as a separate call later.
-        options);
+        const client = this.wire.get();
+        let info;
+        if (client instanceof UploadClient) {
+            // partChecksum isn't used for now but protoc requires it to be set
+            info = (await client.getPartURL({
+                resourceId: id,
+                partNumber,
+                uploadedPartSize: 0n,
+                isInternalUse: false,
+                partChecksum: '',
+            }, addRTypeToMetadata(type, options))).response;
+        }
+        else {
+            const resp = (await client.POST('/v1/upload/get-part-url', {
+                body: {
+                    resourceId: id.toString(),
+                    partNumber: partNumber.toString(),
+                    uploadedPartSize: '0',
+                    isInternalUse: false,
+                    partChecksum: '',
+                },
+                headers: { ...createRTypeRoutingHeader(type) },
+            })).data;
+            info = {
+                uploadUrl: resp.uploadUrl,
+                method: resp.method,
+                headers: resp.headers,
+                chunkStart: BigInt(resp.chunkStart),
+                chunkEnd: BigInt(resp.chunkEnd),
+            };
+        }
         const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
         await checkExpectedMTime(path, expectedMTimeUnix);
         const crc32cChecksum = calculateCrc32cChecksum(chunk);
-        if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {
+        if (checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {
             info.headers.push({ name: checksumHeader, value: crc32cChecksum });
         }
         const contentLength = Number(info.chunkEnd - info.chunkStart);
@@ -85,10 +140,21 @@ class ClientUpload {
                 throw e;
             throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);
         }
-        await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);
+        await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);
     }
     async finalize(info, options) {
-        return await this.grpcFinalize(info, options);
+        const client = this.wire.get();
+        if (client instanceof UploadClient) {
+            await client.finalize({ resourceId: info.id }, addRTypeToMetadata(info.type, options));
+        }
+        else {
+            await client.POST('/v1/upload/finalize', {
+                body: {
+                    resourceId: info.id.toString(),
+                },
+                headers: { ...createRTypeRoutingHeader(info.type) },
+            });
+        }
     }
     /** Calculates parts that need to be uploaded from the parts that were
      * already uploaded. */
@@ -101,23 +167,23 @@ class ClientUpload {
         }
         return toUpload;
     }
-    async grpcInit(id, type, options) {
-        return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))
-            .response;
-    }
-    async grpcGetPartUrl({ id, type }, partNumber, uploadedPartSize, options) {
-        // partChecksum isn't used for now but protoc requires it to be set
-        return await this.grpcClient.get().getPartURL({ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' }, addRTypeToMetadata(type, options)).response;
-    }
-    async grpcUpdateProgress({ id, type }, bytesProcessed, options) {
-        await this.grpcClient.get().updateProgress({
-            resourceId: id,
-            bytesProcessed,
-        }, addRTypeToMetadata(type, options)).response;
-    }
-    async grpcFinalize({ id, type }, options) {
-        return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))
-            .response;
+    async updateProgress({ id, type }, bytesProcessed, options) {
+        const client = this.wire.get();
+        if (client instanceof UploadClient) {
+            await client.updateProgress({
+                resourceId: id,
+                bytesProcessed,
+            }, addRTypeToMetadata(type, options)).response;
+            return;
+        }
+        await client.POST('/v1/upload/update-progress', {
+            body: {
+                resourceId: id.toString(),
+                bytesProcessed: bytesProcessed.toString(),
+            },
+            headers: { ...createRTypeRoutingHeader(type) },
+        });
+        return;
     }
 }
 async function readFileChunk(path, chunkStart, chunkEnd) {

package/dist/clients/upload.js.map

@@ -1 +1 @@
-{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { GrpcClientProvider, GrpcClientProviderFactory, PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly grpcClient: GrpcClientProvider<UploadClient>;\n\n  constructor(\n    grpcClientProviderFactory: GrpcClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new UploadClient(transport));\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: uploadapi_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const init = await this.grpcInit(id, type, options);\n    return {\n      overall: init.partsCount,\n      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: uploadapi_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const info = await this.grpcGetPartUrl(\n      { id, type },\n      partNumber,\n      0n, // we update progress as a separate call later.\n      options,\n    );\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcFinalize(info, options);\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async grpcInit(id: ResourceId, type: ResourceType, options?: RpcOptions) {\n    return await this.grpcClient.get().init({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n\n  private async grpcGetPartUrl(\n    { id, type }: ResourceInfo,\n    partNumber: bigint,\n    uploadedPartSize: bigint,\n    options?: RpcOptions,\n  ) {\n    // partChecksum isn't used for now but protoc requires it to be set\n    return await this.grpcClient.get().getPartURL(\n      { resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcUpdateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ) {\n    await this.grpcClient.get().updateProgress(\n      {\n        resourceId: id,\n        bytesProcessed,\n      },\n      addRTypeToMetadata(type, options),\n    ).response;\n  }\n\n  private async grpcFinalize({ id, type }: ResourceInfo, options?: RpcOptions) {\n    return await this.grpcClient.get().finalize({ resourceId: id }, addRTypeToMetadata(type, options))\n      .response;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: uploadapi_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["fs"],"mappings":";;;;;;;AAcM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,UAAU;AAE3B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IAClH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;AAOpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,YAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;YACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;AAEpB,QAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,EAAE,EAAE,EAAE,IAAI,EAAE,EACZ,UAAU,EACV,EAAE;AACF,QAAA,OAAO,CACR;AAED,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAK,2BAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC/F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC;IAC/C;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;AAEQ,IAAA,MAAM,QAAQ,CAAC,EAAc,EAAE,IAAkB,EAAE,OAAoB,EAAA;QAC7E,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC1F,aAAA,QAAQ;IACb;AAEQ,IAAA,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,UAAkB,EAClB,gBAAwB,EACxB,OAAoB,EAAA;;AAGpB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAC3C,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,EACxF,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CACxC;AACE,YAAA,UAAU,EAAE,EAAE;YACd,cAAc;SACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;IACZ;IAEQ,MAAM,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,EAAgB,EAAE,OAAoB,EAAA;QACzE,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;AAC9F,aAAA,QAAQ;IACb;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;"}
+{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { WireClientProvider, WireClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { UploadAPI_ChecksumAlgorithm, type UploadAPI_GetPartURL_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport type { UploadApiPaths, UploadRestClientType } from '../proto-rest';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n  name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n  name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n  name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n  name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n  name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n  private readonly wire: WireClientProvider<UploadRestClientType | UploadClient>;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    _: PlClient,\n    public readonly logger: MiLogger,\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider(\n      (wire) => {\n        if (wire.type === 'grpc') {\n          return new UploadClient(wire.Transport);\n        }\n\n        return RestAPI.createClient<UploadApiPaths>({\n          hostAndPort: wire.Config.hostAndPort,\n          ssl: wire.Config.ssl,\n          dispatcher: wire.Dispatcher,\n          middlewares: wire.Middlewares,\n        });\n      },\n    );\n  }\n\n  close() {}\n\n  public async initUpload(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n  ): Promise<{\n      overall: bigint;\n      toUpload: bigint[];\n      checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n      checksumHeader: string;\n    }> {\n    const client = this.wire.get();\n\n    if (client instanceof UploadClient) {\n      const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n        .response;\n\n      return {\n        overall: init.partsCount,\n        toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n        checksumAlgorithm: init.checksumAlgorithm,\n        checksumHeader: init.checksumHeader,\n      };\n    }\n\n    const init = (await client.POST('/v1/upload/init', {\n      body: {\n        resourceId: id.toString(),\n      },\n      headers: { ...createRTypeRoutingHeader(type) },\n    })).data!;\n\n    return {\n      overall: BigInt(init.partsCount),\n      toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n      checksumAlgorithm: init.checksumAlgorithm,\n      checksumHeader: init.checksumHeader,\n    };\n  }\n\n  public async partUpload(\n    { id, type }: ResourceInfo,\n    path: string,\n    expectedMTimeUnix: bigint,\n    partNumber: bigint,\n    checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n    checksumHeader: string,\n    options?: RpcOptions,\n  ) {\n    const client = this.wire.get();\n\n    let info: UploadAPI_GetPartURL_Response;\n    if (client instanceof UploadClient) {\n      // partChecksum isn't used for now but protoc requires it to be set\n      info = (await client.getPartURL(\n        {\n          resourceId: id,\n          partNumber,\n          uploadedPartSize: 0n,\n          isInternalUse: false,\n          partChecksum: '',\n        },\n        addRTypeToMetadata(type, options),\n      )).response;\n    } else {\n      const resp = (await client.POST('/v1/upload/get-part-url', {\n        body: {\n          resourceId: id.toString(),\n          partNumber: partNumber.toString(),\n          uploadedPartSize: '0',\n          isInternalUse: false,\n          partChecksum: '',\n        },\n        headers: { ...createRTypeRoutingHeader(type) },\n      })).data!;\n      info = {\n        uploadUrl: resp.uploadUrl,\n        method: resp.method,\n        headers: resp.headers,\n        chunkStart: BigInt(resp.chunkStart),\n        chunkEnd: BigInt(resp.chunkEnd),\n      };\n    }\n\n    const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n    await checkExpectedMTime(path, expectedMTimeUnix);\n\n    const crc32cChecksum = calculateCrc32cChecksum(chunk);\n    if (checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n      info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n    }\n\n    const contentLength = Number(info.chunkEnd - info.chunkStart);\n    if (chunk.length !== contentLength) {\n      throw new Error(\n        `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n      );\n    }\n\n    const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n    try {\n      const {\n        body: rawBody,\n        statusCode,\n        headers: responseHeaders,\n      } = await request(info.uploadUrl, {\n        dispatcher: this.httpClient,\n        body: chunk,\n        // We got headers only after we send\n        // the whole body (in case of S3 PUT requests it's 5 MB).\n        // It might be slow with a slow connection (or with SSH),\n        // that's why we got big timeout here.\n        headersTimeout: 60000,\n        bodyTimeout: 60000,\n        // Prevent connection reuse by setting \"Connection: close\" header.\n        // This works around an issue with the backend's built-in S3 implementation\n        // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n        reset: true,\n        headers,\n        method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n      });\n\n      // always read the body for resources to be garbage collected.\n      const body = await rawBody.text();\n      checkStatusCodeOk(statusCode, body, responseHeaders, info);\n    } catch (e: unknown) {\n      if (e instanceof NetworkError)\n        throw e;\n\n      if (e instanceof BadRequestError)\n        throw e;\n\n      throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n    }\n\n    await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n  }\n\n  public async finalize(info: ResourceInfo, options?: RpcOptions) {\n    const client = this.wire.get();\n\n    if (client instanceof UploadClient) {\n      await client.finalize({ resourceId: info.id }, addRTypeToMetadata(info.type, options));\n    } else {\n      await client.POST('/v1/upload/finalize', {\n        body: {\n          resourceId: info.id.toString(),\n        },\n        headers: { ...createRTypeRoutingHeader(info.type) },\n      });\n    }\n  }\n\n  /** Calculates parts that need to be uploaded from the parts that were\n   * already uploaded. */\n  private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n    const toUpload: bigint[] = [];\n    const uploaded = new Set(uploadedParts);\n\n    for (let i = 1n; i <= partsCount; i++) {\n      if (!uploaded.has(i)) toUpload.push(i);\n    }\n\n    return toUpload;\n  }\n\n  private async updateProgress(\n    { id, type }: ResourceInfo,\n    bytesProcessed: bigint,\n    options?: RpcOptions,\n  ): Promise<void> {\n    const client = this.wire.get();\n\n    if (client instanceof UploadClient) {\n      await client.updateProgress(\n        {\n          resourceId: id,\n          bytesProcessed,\n        },\n        addRTypeToMetadata(type, options),\n      ).response;\n      return;\n    }\n\n    await client.POST('/v1/upload/update-progress', {\n      body: {\n        resourceId: id.toString(),\n        bytesProcessed: bytesProcessed.toString(),\n      },\n      headers: { ...createRTypeRoutingHeader(type) },\n    });\n    return;\n  }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n  let f: fs.FileHandle | undefined;\n  try {\n    f = await fs.open(path);\n    const len = Number(chunkEnd - chunkStart);\n    const pos = Number(chunkStart);\n    const b = Buffer.alloc(len);\n    const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n    return b.subarray(0, bytesRead);\n  } catch (e: unknown) {\n    if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n    throw e;\n  } finally {\n    await f?.close();\n  }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n  let bytesReadTotal = 0;\n  while (bytesReadTotal < len) {\n    const { bytesRead } = await f.read(\n      b,\n      bytesReadTotal,\n      len - bytesReadTotal,\n      position + bytesReadTotal,\n    );\n    if (bytesRead === 0) {\n      throw new UnexpectedEOF('file ended earlier than expected.');\n    }\n    bytesReadTotal += bytesRead;\n  }\n\n  return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n  const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n  if (mTime > expectedMTimeUnix) {\n    throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n  }\n}\n\nfunction checkStatusCodeOk(\n  statusCode: number,\n  body: string,\n  headers: IncomingHttpHeaders,\n  info: UploadAPI_GetPartURL_Response,\n) {\n  if (statusCode == 400) {\n    throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n  }\n\n  if (statusCode != 200) {\n    throw new NetworkError(\n      `response is not ok, status code: ${statusCode},`\n      + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n    );\n  }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n  const checksum = crc32c(data);\n  // Convert to unsigned 32-bit integer and then to base64\n  const buffer = Buffer.alloc(4);\n\n  buffer.writeUInt32BE(checksum, 0);\n  return buffer.toString('base64');\n}\n"],"names":["fs"],"mappings":";;;;;;;AAeM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,IAAI;AAErB,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;QAEtB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;YACzC;YAEA,OAAO,OAAO,CAAC,YAAY,CAAiB;AAC1C,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;QAOpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACnF,iBAAA,QAAQ;YAEX,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,gBAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;gBACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;QACH;QAEA,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjD,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AAC1B,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;SAC/C,CAAC,EAAE,IAAK;QAET,OAAO;AACL,YAAA,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrF,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,IAAmC;AACvC,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;;AAElC,YAAA,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAC7B;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,UAAU;AACV,gBAAA,gBAAgB,EAAE,EAAE;AACpB,gBAAA,aAAa,EAAE,KAAK;AACpB,gBAAA,YAAY,EAAE,EAAE;aACjB,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,EAAE,QAAQ;QACb;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;AACzD,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,oBAAA,UAAU,EAAE,UAAU,CAAC,QAAQ,EAAE;AACjC,oBAAA,gBAAgB,EAAE,GAAG;AACrB,oBAAA,aAAa,EAAE,KAAK;AACpB,oBAAA,YAAY,EAAE,EAAE;AACjB,iBAAA;AACD,gBAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;aAC/C,CAAC,EAAE,IAAK;AACT,YAAA,IAAI,GAAG;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;AACnC,gBAAA,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aAChC;QACH;AAEA,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAK,2BAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC3F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxF;aAAO;AACL,YAAA,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;AACvC,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpD,aAAA,CAAC;QACJ;IACF;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;IAEQ,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,MAAM,CAAC,cAAc,CACzB;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,cAAc;aACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;YACV;QACF;AAEA,QAAA,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;AAC9C,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,gBAAA,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;AAC1C,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;AAC/C,SAAA,CAAC;QACF;IACF;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;"}

package/dist/drivers/download_blob/download_blob.cjs

@@ -23,7 +23,7 @@ var blob_key = require('./blob_key.cjs');
 var download_blob_task = require('./download_blob_task.cjs');
 var files_cache = require('../helpers/files_cache.cjs');
 var cache = require('./sparse_cache/cache.cjs');
-var download = require('../../helpers/download.cjs');
+var download_errors = require('../../helpers/download_errors.cjs');
 
 function _interopNamespaceDefault(e) {
     var n = Object.create(null);
@@ -194,7 +194,7 @@ class DownloadDriver {
             return await request();
         }
         catch (error) {
-            if (download.isOffByOneError(error)) {
+            if (download_errors.isOffByOneError(error)) {
                 return await request();
             }
             throw error;