@milaboratories/pl-drivers 1.11.43 → 1.11.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/clients/upload.cjs
CHANGED
|
@@ -124,9 +124,8 @@ class ClientUpload {
|
|
|
124
124
|
}
|
|
125
125
|
const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
|
|
126
126
|
await checkExpectedMTime(path, expectedMTimeUnix);
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
info.headers.push({ name: checksumHeader, value: crc32cChecksum });
|
|
127
|
+
if (checksumHeader && checksumAlgorithm === protocol.UploadAPI_ChecksumAlgorithm.CRC32C) {
|
|
128
|
+
info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });
|
|
130
129
|
}
|
|
131
130
|
const contentLength = Number(info.chunkEnd - info.chunkStart);
|
|
132
131
|
if (chunk.length !== contentLength) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { WireClientProvider, WireClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { UploadAPI_ChecksumAlgorithm, type UploadAPI_GetPartURL_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport type { UploadApiPaths, UploadRestClientType } from '../proto-rest';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType | UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider(\n (wire) => {\n if (wire.type === 'grpc') {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n },\n );\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n .response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (await client.POST('/v1/upload/init', {\n body: {\n resourceId: id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (await client.getPartURL(\n {\n resourceId: id,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: '',\n },\n addRTypeToMetadata(type, options),\n )).response;\n } else {\n const resp = (await client.POST('/v1/upload/get-part-url', {\n body: {\n resourceId: id.toString(),\n partNumber: partNumber.toString(),\n uploadedPartSize: '0',\n isInternalUse: false,\n partChecksum: '',\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n const crc32cChecksum = calculateCrc32cChecksum(chunk);\n if (checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError)\n throw e;\n\n if (e instanceof BadRequestError)\n throw e;\n\n throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize({ resourceId: info.id }, addRTypeToMetadata(info.type, options));\n } else {\n await client.POST('/v1/upload/finalize', {\n body: {\n resourceId: info.id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /** Calculates parts that need to be uploaded from the parts that were\n * already uploaded. */\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: id,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST('/v1/upload/update-progress', {\n body: {\n resourceId: id.toString(),\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle | undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF('file ended earlier than expected.');\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString('base64');\n}\n"],"names":["UploadClient","RestAPI","addRTypeToMetadata","createRTypeRoutingHeader","UploadAPI_ChecksumAlgorithm","request","fs","crc32c"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAeM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,IAAI;AAErB,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;QAEtB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAIA,4BAAY,CAAC,IAAI,CAAC,SAAS,CAAC;YACzC;YAEA,OAAOC,gBAAO,CAAC,YAAY,CAAiB;AAC1C,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;QAOpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYD,4BAAY,EAAE;YAClC,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACnF,iBAAA,QAAQ;YAEX,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,gBAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;gBACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;QACH;QAEA,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjD,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AAC1B,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;SAC/C,CAAC,EAAE,IAAK;QAET,OAAO;AACL,YAAA,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrF,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,IAAmC;AACvC,QAAA,IAAI,MAAM,YAAYH,4BAAY,EAAE;;AAElC,YAAA,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAC7B;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,UAAU;AACV,gBAAA,gBAAgB,EAAE,EAAE;AACpB,gBAAA,aAAa,EAAE,KAAK;AACpB,gBAAA,YAAY,EAAE,EAAE;aACjB,EACDE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,EAAE,QAAQ;QACb;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;AACzD,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,oBAAA,UAAU,EAAE,UAAU,CAAC,QAAQ,EAAE;AACjC,oBAAA,gBAAgB,EAAE,GAAG;AACrB,oBAAA,aAAa,EAAE,KAAK;AACpB,oBAAA,YAAY,EAAE,EAAE;AACjB,iBAAA;AACD,gBAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;aAC/C,CAAC,EAAE,IAAK;AACT,YAAA,IAAI,GAAG;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;AACnC,gBAAA,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aAChC;QACH;AAEA,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAKC,oCAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC3F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYL,4BAAY,EAAE;YAClC,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,EAAEE,2BAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxF;aAAO;AACL,YAAA,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;AACvC,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpD,aAAA,CAAC;QACJ;IACF;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;IAEQ,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYH,4BAAY,EAAE;YAClC,MAAM,MAAM,CAAC,cAAc,CACzB;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,cAAc;aACf,EACDE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;YACV;QACF;AAEA,QAAA,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;AAC9C,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,gBAAA,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;AAC1C,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;AAC/C,SAAA,CAAC;QACF;IACF;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMG,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAGC,aAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;;;;;;"}
|
|
1
|
+
{"version":3,"file":"upload.cjs","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { WireClientProvider, WireClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { UploadAPI_ChecksumAlgorithm, type UploadAPI_GetPartURL_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport type { UploadApiPaths, UploadRestClientType } from '../proto-rest';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType | UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider(\n (wire) => {\n if (wire.type === 'grpc') {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n },\n );\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n .response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (await client.POST('/v1/upload/init', {\n body: {\n resourceId: id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (await client.getPartURL(\n {\n resourceId: id,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: '',\n },\n addRTypeToMetadata(type, options),\n )).response;\n } else {\n const resp = (await client.POST('/v1/upload/get-part-url', {\n body: {\n resourceId: id.toString(),\n partNumber: partNumber.toString(),\n uploadedPartSize: '0',\n isInternalUse: false,\n partChecksum: '',\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError)\n throw e;\n\n if (e instanceof BadRequestError)\n throw e;\n\n throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize({ resourceId: info.id }, addRTypeToMetadata(info.type, options));\n } else {\n await client.POST('/v1/upload/finalize', {\n body: {\n resourceId: info.id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /** Calculates parts that need to be uploaded from the parts that were\n * already uploaded. */\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: id,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST('/v1/upload/update-progress', {\n body: {\n resourceId: id.toString(),\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle | undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF('file ended earlier than expected.');\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString('base64');\n}\n"],"names":["UploadClient","RestAPI","addRTypeToMetadata","createRTypeRoutingHeader","UploadAPI_ChecksumAlgorithm","request","fs","crc32c"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAeM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,IAAI;AAErB,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;QAEtB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAIA,4BAAY,CAAC,IAAI,CAAC,SAAS,CAAC;YACzC;YAEA,OAAOC,gBAAO,CAAC,YAAY,CAAiB;AAC1C,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;QAOpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYD,4BAAY,EAAE;YAClC,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAEE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACnF,iBAAA,QAAQ;YAEX,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,gBAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;gBACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;QACH;QAEA,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjD,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AAC1B,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;SAC/C,CAAC,EAAE,IAAK;QAET,OAAO;AACL,YAAA,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrF,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,IAAmC;AACvC,QAAA,IAAI,MAAM,YAAYH,4BAAY,EAAE;;AAElC,YAAA,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAC7B;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,UAAU;AACV,gBAAA,gBAAgB,EAAE,EAAE;AACpB,gBAAA,aAAa,EAAE,KAAK;AACpB,gBAAA,YAAY,EAAE,EAAE;aACjB,EACDE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,EAAE,QAAQ;QACb;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;AACzD,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,oBAAA,UAAU,EAAE,UAAU,CAAC,QAAQ,EAAE;AACjC,oBAAA,gBAAgB,EAAE,GAAG;AACrB,oBAAA,aAAa,EAAE,KAAK;AACpB,oBAAA,YAAY,EAAE,EAAE;AACjB,iBAAA;AACD,gBAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;aAC/C,CAAC,EAAE,IAAK;AACT,YAAA,IAAI,GAAG;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;AACnC,gBAAA,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aAChC;QACH;AAEA,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;QAEjD,IAAI,cAAc,IAAI,iBAAiB,KAAKC,oCAA2B,CAAC,MAAM,EAAE;AAC9E,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC;QACpF;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAMC,cAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC3F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYL,4BAAY,EAAE;YAClC,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,EAAEE,2BAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxF;aAAO;AACL,YAAA,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;AACvC,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpD,aAAA,CAAC;QACJ;IACF;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;IAEQ,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYH,4BAAY,EAAE;YAClC,MAAM,MAAM,CAAC,cAAc,CACzB;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,cAAc;aACf,EACDE,2BAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;YACV;QACF;AAEA,QAAA,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;AAC9C,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,gBAAA,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;AAC1C,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,IAAI,CAAC,EAAE;AAC/C,SAAA,CAAC;QACF;IACF;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMG,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAGC,aAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;;;;;;"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/clients/upload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAsB,yBAAyB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEzG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEzC,OAAO,EAAE,2BAA2B,EAAsC,MAAM,qFAAqF,CAAC;AAOtK,qBAAa,UAAW,SAAQ,KAAK;IACnC,IAAI,SAAgB;CACrB;AAED,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,IAAI,SAAkB;CACvB;AAED,0CAA0C;AAC1C,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,IAAI,SAAwB;CAC7B;AAED,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;oDAEoD;AACpD,qBAAa,YAAY;aAKL,UAAU,EAAE,UAAU;aAEtB,MAAM,EAAE,QAAQ;IANlC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0D;gBAG7E,yBAAyB,EAAE,yBAAyB,EACpC,UAAU,EAAE,UAAU,EACtC,CAAC,EAAE,QAAQ,EACK,MAAM,EAAE,QAAQ;IAkBlC,KAAK;IAEQ,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC;QACP,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,iBAAiB,EAAE,2BAA2B,CAAC;QAC/C,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IA8BS,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,iBAAiB,EAAE,2BAA2B,EAC9C,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,UAAU;
|
|
1
|
+
{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/clients/upload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAsB,yBAAyB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEzG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEzC,OAAO,EAAE,2BAA2B,EAAsC,MAAM,qFAAqF,CAAC;AAOtK,qBAAa,UAAW,SAAQ,KAAK;IACnC,IAAI,SAAgB;CACrB;AAED,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,IAAI,SAAkB;CACvB;AAED,0CAA0C;AAC1C,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,IAAI,SAAwB;CAC7B;AAED,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;oDAEoD;AACpD,qBAAa,YAAY;aAKL,UAAU,EAAE,UAAU;aAEtB,MAAM,EAAE,QAAQ;IANlC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0D;gBAG7E,yBAAyB,EAAE,yBAAyB,EACpC,UAAU,EAAE,UAAU,EACtC,CAAC,EAAE,QAAQ,EACK,MAAM,EAAE,QAAQ;IAkBlC,KAAK;IAEQ,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC;QACP,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,iBAAiB,EAAE,2BAA2B,CAAC;QAC/C,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IA8BS,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,iBAAiB,EAAE,2BAA2B,EAC9C,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,UAAU;IA2FT,QAAQ,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,UAAU;IAe9D;2BACuB;IACvB,OAAO,CAAC,aAAa;YAWP,cAAc;CA2B7B"}
|
package/dist/clients/upload.js
CHANGED
|
@@ -103,9 +103,8 @@ class ClientUpload {
|
|
|
103
103
|
}
|
|
104
104
|
const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
|
|
105
105
|
await checkExpectedMTime(path, expectedMTimeUnix);
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
info.headers.push({ name: checksumHeader, value: crc32cChecksum });
|
|
106
|
+
if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {
|
|
107
|
+
info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });
|
|
109
108
|
}
|
|
110
109
|
const contentLength = Number(info.chunkEnd - info.chunkStart);
|
|
111
110
|
if (chunk.length !== contentLength) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { WireClientProvider, WireClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { UploadAPI_ChecksumAlgorithm, type UploadAPI_GetPartURL_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport type { UploadApiPaths, UploadRestClientType } from '../proto-rest';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType | UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider(\n (wire) => {\n if (wire.type === 'grpc') {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n },\n );\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n .response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (await client.POST('/v1/upload/init', {\n body: {\n resourceId: id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (await client.getPartURL(\n {\n resourceId: id,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: '',\n },\n addRTypeToMetadata(type, options),\n )).response;\n } else {\n const resp = (await client.POST('/v1/upload/get-part-url', {\n body: {\n resourceId: id.toString(),\n partNumber: partNumber.toString(),\n uploadedPartSize: '0',\n isInternalUse: false,\n partChecksum: '',\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n const crc32cChecksum = calculateCrc32cChecksum(chunk);\n if (checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: crc32cChecksum });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError)\n throw e;\n\n if (e instanceof BadRequestError)\n throw e;\n\n throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize({ resourceId: info.id }, addRTypeToMetadata(info.type, options));\n } else {\n await client.POST('/v1/upload/finalize', {\n body: {\n resourceId: info.id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /** Calculates parts that need to be uploaded from the parts that were\n * already uploaded. */\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: id,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST('/v1/upload/update-progress', {\n body: {\n resourceId: id.toString(),\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle | undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF('file ended earlier than expected.');\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString('base64');\n}\n"],"names":["fs"],"mappings":";;;;;;;AAeM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,IAAI;AAErB,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;QAEtB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;YACzC;YAEA,OAAO,OAAO,CAAC,YAAY,CAAiB;AAC1C,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;QAOpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACnF,iBAAA,QAAQ;YAEX,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,gBAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;gBACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;QACH;QAEA,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjD,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AAC1B,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;SAC/C,CAAC,EAAE,IAAK;QAET,OAAO;AACL,YAAA,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrF,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,IAAmC;AACvC,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;;AAElC,YAAA,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAC7B;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,UAAU;AACV,gBAAA,gBAAgB,EAAE,EAAE;AACpB,gBAAA,aAAa,EAAE,KAAK;AACpB,gBAAA,YAAY,EAAE,EAAE;aACjB,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,EAAE,QAAQ;QACb;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;AACzD,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,oBAAA,UAAU,EAAE,UAAU,CAAC,QAAQ,EAAE;AACjC,oBAAA,gBAAgB,EAAE,GAAG;AACrB,oBAAA,aAAa,EAAE,KAAK;AACpB,oBAAA,YAAY,EAAE,EAAE;AACjB,iBAAA;AACD,gBAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;aAC/C,CAAC,EAAE,IAAK;AACT,YAAA,IAAI,GAAG;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;AACnC,gBAAA,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aAChC;QACH;AAEA,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;AAEjD,QAAA,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC;AACrD,QAAA,IAAI,iBAAiB,KAAK,2BAA2B,CAAC,MAAM,EAAE;AAC5D,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACpE;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC3F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxF;aAAO;AACL,YAAA,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;AACvC,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpD,aAAA,CAAC;QACJ;IACF;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;IAEQ,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,MAAM,CAAC,cAAc,CACzB;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,cAAc;aACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;YACV;QACF;AAEA,QAAA,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;AAC9C,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,gBAAA,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;AAC1C,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;AAC/C,SAAA,CAAC;QACF;IACF;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;"}
|
|
1
|
+
{"version":3,"file":"upload.js","sources":["../../src/clients/upload.ts"],"sourcesContent":["import type { WireClientProvider, WireClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs/promises';\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { UploadAPI_ChecksumAlgorithm, type UploadAPI_GetPartURL_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';\nimport { UploadClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';\nimport type { UploadApiPaths, UploadRestClientType } from '../proto-rest';\nimport { crc32c } from './crc32c';\n\nimport type { IncomingHttpHeaders } from 'undici/types/header';\n\nexport class MTimeError extends Error {\n name = 'MTimeError';\n}\n\nexport class UnexpectedEOF extends Error {\n name = 'UnexpectedEOF';\n}\n\nexport class NetworkError extends Error {\n name = 'NetworkError';\n}\n\n/** Happens when the file doesn't exist */\nexport class NoFileForUploading extends Error {\n name = 'NoFileForUploading';\n}\n\nexport class BadRequestError extends Error {\n name = 'BadRequestError';\n}\n\n/** Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. */\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType | UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider(\n (wire) => {\n if (wire.type === 'grpc') {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n },\n );\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n .response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (await client.POST('/v1/upload/init', {\n body: {\n resourceId: id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (await client.getPartURL(\n {\n resourceId: id,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: '',\n },\n addRTypeToMetadata(type, options),\n )).response;\n } else {\n const resp = (await client.POST('/v1/upload/get-part-url', {\n body: {\n resourceId: id.toString(),\n partNumber: partNumber.toString(),\n uploadedPartSize: '0',\n isInternalUse: false,\n partChecksum: '',\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name.toLowerCase(), value]));\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError)\n throw e;\n\n if (e instanceof BadRequestError)\n throw e;\n\n throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize({ resourceId: info.id }, addRTypeToMetadata(info.type, options));\n } else {\n await client.POST('/v1/upload/finalize', {\n body: {\n resourceId: info.id.toString(),\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /** Calculates parts that need to be uploaded from the parts that were\n * already uploaded. */\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: id,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST('/v1/upload/update-progress', {\n body: {\n resourceId: id.toString(),\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle | undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === 'object' && ('code' in e) && e.code == 'ENOENT') throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/** Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. */\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF('file ended earlier than expected.');\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n throw new BadRequestError(`response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},`\n + ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/** Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString('base64');\n}\n"],"names":["fs"],"mappings":";;;;;;;AAeM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAEK,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,MAAO,YAAa,SAAQ,KAAK,CAAA;IACrC,IAAI,GAAG,cAAc;AACtB;AAED;AACM,MAAO,kBAAmB,SAAQ,KAAK,CAAA;IAC3C,IAAI,GAAG,oBAAoB;AAC5B;AAEK,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;AAEoD;MACvC,YAAY,CAAA;AAKL,IAAA,UAAA;AAEA,IAAA,MAAA;AAND,IAAA,IAAI;AAErB,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtC,CAAW,EACK,MAAgB,EAAA;QAFhB,IAAA,CAAA,UAAU,GAAV,UAAU;QAEV,IAAA,CAAA,MAAM,GAAN,MAAM;QAEtB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;YACzC;YAEA,OAAO,OAAO,CAAC,YAAY,CAAiB;AAC1C,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;IAEF,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EAAA;QAOpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACnF,iBAAA,QAAQ;YAEX,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,UAAU;AACxB,gBAAA,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;gBACjE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;QACH;QAEA,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjD,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AAC1B,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;SAC/C,CAAC,EAAE,IAAK;QAET,OAAO;AACL,YAAA,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrF,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC;IACH;AAEO,IAAA,MAAM,UAAU,CACrB,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,EAClB,iBAA8C,EAC9C,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,IAAmC;AACvC,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;;AAElC,YAAA,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAC7B;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,UAAU;AACV,gBAAA,gBAAgB,EAAE,EAAE;AACpB,gBAAA,aAAa,EAAE,KAAK;AACpB,gBAAA,YAAY,EAAE,EAAE;aACjB,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,EAAE,QAAQ;QACb;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;AACzD,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,oBAAA,UAAU,EAAE,UAAU,CAAC,QAAQ,EAAE;AACjC,oBAAA,gBAAgB,EAAE,GAAG;AACrB,oBAAA,aAAa,EAAE,KAAK;AACpB,oBAAA,YAAY,EAAE,EAAE;AACjB,iBAAA;AACD,gBAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;aAC/C,CAAC,EAAE,IAAK;AACT,YAAA,IAAI,GAAG;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;AACnC,gBAAA,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aAChC;QACH;AAEA,QAAA,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;AACvE,QAAA,MAAM,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,CAAC;QAEjD,IAAI,cAAc,IAAI,iBAAiB,KAAK,2BAA2B,CAAC,MAAM,EAAE;AAC9E,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC;QACpF;AAEA,QAAA,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;AAC7D,QAAA,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa,EAAE;YAClC,MAAM,IAAI,KAAK,CACb,CAAA,8BAAA,EAAiC,aAAa,CAAA,iBAAA,EAAoB,KAAK,CAAC,MAAM,CAAA,gBAAA,CAAkB,CACjG;QACH;AAEA,QAAA,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtG,QAAA,IAAI;AACF,YAAA,MAAM,EACJ,IAAI,EAAE,OAAO,EACb,UAAU,EACV,OAAO,EAAE,eAAe,GACzB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;AAC3B,gBAAA,IAAI,EAAE,KAAK;;;;;AAKX,gBAAA,cAAc,EAAE,KAAK;AACrB,gBAAA,WAAW,EAAE,KAAK;;;;AAIlB,gBAAA,KAAK,EAAE,IAAI;gBACX,OAAO;AACP,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAA2B;AAC3D,aAAA,CAAC;;AAGF,YAAA,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;YACjC,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC;QAC5D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,YAAY;AAC3B,gBAAA,MAAM,CAAC;YAET,IAAI,CAAC,YAAY,eAAe;AAC9B,gBAAA,MAAM,CAAC;YAET,MAAM,IAAI,KAAK,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,oDAAA,EAAuD,IAAI,CAAC,SAAS,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAE,CAAC;QAC1K;QAEA,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC3F;AAEO,IAAA,MAAM,QAAQ,CAAC,IAAkB,EAAE,OAAoB,EAAA;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxF;aAAO;AACL,YAAA,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;AACvC,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpD,aAAA,CAAC;QACJ;IACF;AAEA;AACuB;IACf,aAAa,CAAC,UAAkB,EAAE,aAAuB,EAAA;QAC/D,MAAM,QAAQ,GAAa,EAAE;AAC7B,QAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC;AAEvC,QAAA,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,EAAE,EAAE;AACrC,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAE,gBAAA,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC;AAEA,QAAA,OAAO,QAAQ;IACjB;IAEQ,MAAM,cAAc,CAC1B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,cAAsB,EACtB,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,YAAY,EAAE;YAClC,MAAM,MAAM,CAAC,cAAc,CACzB;AACE,gBAAA,UAAU,EAAE,EAAE;gBACd,cAAc;aACf,EACD,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAClC,CAAC,QAAQ;YACV;QACF;AAEA,QAAA,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;AAC9C,YAAA,IAAI,EAAE;AACJ,gBAAA,UAAU,EAAE,EAAE,CAAC,QAAQ,EAAE;AACzB,gBAAA,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;AAC1C,aAAA;AACD,YAAA,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,IAAI,CAAC,EAAE;AAC/C,SAAA,CAAC;QACF;IACF;AACD;AAED,eAAe,aAAa,CAAC,IAAY,EAAE,UAAkB,EAAE,QAAgB,EAAA;AAC7E,IAAA,IAAI,CAA4B;AAChC,IAAA,IAAI;QACF,CAAC,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;AACzC,QAAA,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AAC3B,QAAA,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;QAE7D,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC;IACjC;IAAE,OAAO,CAAU,EAAE;AACnB,QAAA,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ;AAAE,YAAA,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,IAAI,CAAA,cAAA,CAAgB,CAAC;AAC7I,QAAA,MAAM,CAAC;IACT;YAAU;AACR,QAAA,MAAM,CAAC,EAAE,KAAK,EAAE;IAClB;AACF;AAEA;AACsE;AACtE,eAAe,qBAAqB,CAAC,CAAgB,EAAE,CAAS,EAAE,GAAW,EAAE,QAAgB,EAAA;IAC7F,IAAI,cAAc,GAAG,CAAC;AACtB,IAAA,OAAO,cAAc,GAAG,GAAG,EAAE;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAChC,CAAC,EACD,cAAc,EACd,GAAG,GAAG,cAAc,EACpB,QAAQ,GAAG,cAAc,CAC1B;AACD,QAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,YAAA,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC;QAC9D;QACA,cAAc,IAAI,SAAS;IAC7B;AAEA,IAAA,OAAO,cAAc;AACvB;AAEA,eAAe,kBAAkB,CAAC,IAAY,EAAE,iBAAyB,EAAA;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;AACtE,IAAA,IAAI,KAAK,GAAG,iBAAiB,EAAE;QAC7B,MAAM,IAAI,UAAU,CAAC,CAAA,mCAAA,EAAsC,iBAAiB,CAAA,OAAA,EAAU,KAAK,CAAA,CAAA,CAAG,CAAC;IACjG;AACF;AAEA,SAAS,iBAAiB,CACxB,UAAkB,EAClB,IAAY,EACZ,OAA4B,EAC5B,IAAmC,EAAA;AAEnC,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,eAAe,CAAC,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AACpE,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAAC;IACpF;AAEA,IAAA,IAAI,UAAU,IAAI,GAAG,EAAE;AACrB,QAAA,MAAM,IAAI,YAAY,CACpB,CAAA,iCAAA,EAAoC,UAAU,CAAA,CAAA;AAC5C,cAAA,CAAA,OAAA,EAAU,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAA,CAAE,CAChF;IACH;AACF;AAEA;AACA,SAAS,uBAAuB,CAAC,IAAY,EAAA;AAC3C,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;IAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAE9B,IAAA,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjC,IAAA,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAClC;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-drivers",
|
|
3
|
-
"version": "1.11.
|
|
3
|
+
"version": "1.11.45",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=22"
|
|
6
6
|
},
|
|
@@ -32,12 +32,12 @@
|
|
|
32
32
|
"undici": "~7.16.0",
|
|
33
33
|
"upath": "^2.0.1",
|
|
34
34
|
"zod": "~3.23.8",
|
|
35
|
+
"@milaboratories/pl-client": "2.16.26",
|
|
36
|
+
"@milaboratories/pl-tree": "1.8.34",
|
|
37
|
+
"@milaboratories/computable": "2.8.2",
|
|
35
38
|
"@milaboratories/ts-helpers": "1.7.0",
|
|
36
|
-
"@milaboratories/pl-tree": "1.8.33",
|
|
37
39
|
"@milaboratories/helpers": "1.13.0",
|
|
38
|
-
"@milaboratories/
|
|
39
|
-
"@milaboratories/pl-client": "2.16.25",
|
|
40
|
-
"@milaboratories/pl-model-common": "1.24.2"
|
|
40
|
+
"@milaboratories/pl-model-common": "1.24.3"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/decompress": "^4.2.7",
|
|
@@ -48,9 +48,9 @@
|
|
|
48
48
|
"eslint": "^9.25.1",
|
|
49
49
|
"typescript": "~5.6.3",
|
|
50
50
|
"vitest": "^4.0.16",
|
|
51
|
-
"@milaboratories/ts-builder": "1.2.4",
|
|
52
51
|
"@milaboratories/build-configs": "1.4.0",
|
|
53
52
|
"@milaboratories/eslint-config": "1.0.5",
|
|
53
|
+
"@milaboratories/ts-builder": "1.2.4",
|
|
54
54
|
"@milaboratories/ts-configs": "1.2.0",
|
|
55
55
|
"@milaboratories/test-helpers": "1.1.3"
|
|
56
56
|
},
|
package/src/clients/upload.ts
CHANGED
|
@@ -149,9 +149,8 @@ export class ClientUpload {
|
|
|
149
149
|
const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
|
|
150
150
|
await checkExpectedMTime(path, expectedMTimeUnix);
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
info.headers.push({ name: checksumHeader, value: crc32cChecksum });
|
|
152
|
+
if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {
|
|
153
|
+
info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });
|
|
155
154
|
}
|
|
156
155
|
|
|
157
156
|
const contentLength = Number(info.chunkEnd - info.chunkStart);
|