npm - @milaboratories/pl-drivers - Versions diffs - 1.13.0 → 1.14.0 - Mend

@milaboratories/pl-drivers 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (141) hide show

package/dist/clients/upload.cjs CHANGED Viewed

@@ -43,9 +43,13 @@ var ClientUpload = class {
 	}
 	close() {}
 	async initUpload({ id, type }, options) {
+		const { globalId, signature } = (0, _milaboratories_pl_client.parseSignedResourceId)(id);
 		const client = this.wire.get();
 		if (client instanceof require_protocol_client.UploadClient) {
-			const init = (await client.init({ resourceId: id }, (0, _milaboratories_pl_client.addRTypeToMetadata)(type, options))).response;
+			const init = (await client.init({
+				resourceId: globalId,
+				resourceSignature: signature
+			}, (0, _milaboratories_pl_client.addRTypeToMetadata)(type, options))).response;
 			return {
 				overall: init.partsCount,
 				toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
@@ -55,8 +59,8 @@ var ClientUpload = class {
 		}
 		const init = (await client.POST("/v1/upload/init", {
 			body: {
-				resourceId: id.toString(),
-				resourceSignature: ""
+				resourceId: globalId.toString(),
+				resourceSignature: (0, _milaboratories_pl_client.signatureToBase64Url)(signature)
 			},
 			headers: { ...(0, _milaboratories_pl_client.createRTypeRoutingHeader)(type) }
 		})).data;
@@ -68,10 +72,12 @@ var ClientUpload = class {
 		};
 	}
 	async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, checksumAlgorithm, checksumHeader, options) {
+		const { globalId, signature } = (0, _milaboratories_pl_client.parseSignedResourceId)(id);
 		const client = this.wire.get();
 		let info;
 		if (client instanceof require_protocol_client.UploadClient) info = (await client.getPartURL({
-			resourceId: id,
+			resourceId: globalId,
+			resourceSignature: signature,
 			partNumber,
 			uploadedPartSize: 0n,
 			isInternalUse: false,
@@ -80,8 +86,8 @@ var ClientUpload = class {
 		else {
 			const resp = (await client.POST("/v1/upload/get-part-url", {
 				body: {
-					resourceId: id.toString(),
-					resourceSignature: "",
+					resourceId: globalId.toString(),
+					resourceSignature: (0, _milaboratories_pl_client.signatureToBase64Url)(signature),
 					partNumber: partNumber.toString(),
 					uploadedPartSize: "0",
 					isInternalUse: false,
@@ -128,16 +134,18 @@ var ClientUpload = class {
 		}, BigInt(info.chunkEnd - info.chunkStart), options);
 	}
 	async finalize(info, options) {
+		const { globalId, signature } = (0, _milaboratories_pl_client.parseSignedResourceId)(info.id);
 		const client = this.wire.get();
 		if (client instanceof require_protocol_client.UploadClient) await client.finalize({
-			resourceId: info.id,
+			resourceId: globalId,
+			resourceSignature: signature,
 			checksumAlgorithm: require_protocol.UploadAPI_ChecksumAlgorithm.UNSPECIFIED,
 			checksum: new Uint8Array(0)
 		}, (0, _milaboratories_pl_client.addRTypeToMetadata)(info.type, options));
 		else await client.POST("/v1/upload/finalize", {
 			body: {
-				resourceId: info.id.toString(),
-				resourceSignature: "",
+				resourceId: globalId.toString(),
+				resourceSignature: (0, _milaboratories_pl_client.signatureToBase64Url)(signature),
 				checksumAlgorithm: 0,
 				checksum: ""
 			},
@@ -153,18 +161,20 @@ var ClientUpload = class {
 		return toUpload;
 	}
 	async updateProgress({ id, type }, bytesProcessed, options) {
+		const { globalId, signature } = (0, _milaboratories_pl_client.parseSignedResourceId)(id);
 		const client = this.wire.get();
 		if (client instanceof require_protocol_client.UploadClient) {
 			await client.updateProgress({
-				resourceId: id,
+				resourceId: globalId,
+				resourceSignature: signature,
 				bytesProcessed
 			}, (0, _milaboratories_pl_client.addRTypeToMetadata)(type, options)).response;
 			return;
 		}
 		await client.POST("/v1/upload/update-progress", {
 			body: {
-				resourceId: id.toString(),
-				resourceSignature: "",
+				resourceId: globalId.toString(),
+				resourceSignature: (0, _milaboratories_pl_client.signatureToBase64Url)(signature),
 				bytesProcessed: bytesProcessed.toString()
 			},
 			headers: { ...(0, _milaboratories_pl_client.createRTypeRoutingHeader)(type) }

package/dist/clients/upload.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"upload.cjs","names":["UploadClient","RestAPI","UploadAPI_ChecksumAlgorithm","fs","crc32c"],"sources":["../../src/clients/upload.ts"],"sourcesContent":["import type {\n WireClientProvider,\n WireClientProviderFactory,\n PlClient,\n} from \"@milaboratories/pl-client\";\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs/promises\";\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport {\n UploadAPI_ChecksumAlgorithm,\n type UploadAPI_GetPartURL_Response,\n} from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol\";\nimport { UploadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client\";\nimport type { UploadApiPaths, UploadRestClientType } from \"../proto-rest\";\nimport { crc32c } from \"./crc32c\";\n\nimport type { IncomingHttpHeaders } from \"undici/types/header\";\n\nexport class MTimeError extends Error {\n name = \"MTimeError\";\n}\n\nexport class UnexpectedEOF extends Error {\n name = \"UnexpectedEOF\";\n}\n\nexport class NetworkError extends Error {\n name = \"NetworkError\";\n}\n\n/** Happens when the file doesn't exist /\nexport class NoFileForUploading extends Error {\n name = \"NoFileForUploading\";\n}\n\nexport class BadRequestError extends Error {\n name = \"BadRequestError\";\n}\n\n/* Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. /\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType \| UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider((wire) => {\n if (wire.type === \"grpc\") {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n });\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n .response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (\n await client.POST(\"/v1/upload/init\", {\n body: {\n resourceId: id.toString(),\n resourceSignature: \"\",\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (\n await client.getPartURL(\n {\n resourceId: id,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: \"\",\n },\n addRTypeToMetadata(type, options),\n )\n ).response;\n } else {\n const resp = (\n await client.POST(\"/v1/upload/get-part-url\", {\n body: {\n resourceId: id.toString(),\n resourceSignature: \"\",\n partNumber: partNumber.toString(),\n uploadedPartSize: \"0\",\n isInternalUse: false,\n partChecksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(\n info.headers.map(({ name, value }) => [name.toLowerCase(), value]),\n );\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError) throw e;\n\n if (e instanceof BadRequestError) throw e;\n\n throw new Error(\n `partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`,\n );\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize(\n {\n resourceId: info.id,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm.UNSPECIFIED,\n checksum: new Uint8Array(0),\n },\n addRTypeToMetadata(info.type, options),\n );\n } else {\n await client.POST(\"/v1/upload/finalize\", {\n body: {\n resourceId: info.id.toString(),\n resourceSignature: \"\",\n checksumAlgorithm: 0,\n checksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /* Calculates parts that need to be uploaded from the parts that were\n * already uploaded. /\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: id,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST(\"/v1/upload/update-progress\", {\n body: {\n resourceId: id.toString(),\n resourceSignature: \"\",\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle \| undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === \"object\" && \"code\" in e && e.code == \"ENOENT\")\n throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/* Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. /\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF(\"file ended earlier than expected.\");\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\n/* S3 error codes that indicate expired/invalid temporary credentials.\n * These are transient: a retry with a fresh presigned URL will succeed\n * once the backend refreshes its STS credentials. /\nconst TRANSIENT_S3_ERROR_CODES = [\"ExpiredToken\", \"RequestExpired\", \"TokenRefreshRequired\"];\n\nfunction isTransientS3Error(body: string): boolean {\n return TRANSIENT_S3_ERROR_CODES.some((code) => body.includes(`<Code>${code}</Code>`));\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n if (isTransientS3Error(body)) {\n throw new NetworkError(\n `transient S3 credential error, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n throw new BadRequestError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/* Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString(\"base64\");\n}\n"],"mappings":";;;;;;;;;AAsBA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;AAGT,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAO;;AAGT,IAAa,eAAb,cAAkC,MAAM;CACtC,OAAO;;;AAIT,IAAa,qBAAb,cAAwC,MAAM;CAC5C,OAAO;;AAGT,IAAa,kBAAb,cAAqC,MAAM;CACzC,OAAO;;;;;AAMT,IAAa,eAAb,MAA0B;CACxB;CAEA,YACE,2BACA,YACA,GACA,QACA;AAHgB,OAAA,aAAA;AAEA,OAAA,SAAA;AAEhB,OAAK,OAAO,0BAA0B,0BAA0B,SAAS;AACvE,OAAI,KAAK,SAAS,OAChB,QAAO,IAAIA,wBAAAA,aAAa,KAAK,UAAU;AAGzC,UAAOC,0BAAAA,QAAQ,aAA6B;IAC1C,aAAa,KAAK,OAAO;IACzB,KAAK,KAAK,OAAO;IACjB,YAAY,KAAK;IACjB,aAAa,KAAK;IACnB,CAAC;IACF;;CAGJ,QAAQ;CAER,MAAa,WACX,EAAE,IAAI,QACN,SAMC;EACD,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkBD,wBAAAA,cAAc;GAClC,MAAM,QAAQ,MAAM,OAAO,KAAK,EAAE,YAAY,IAAI,GAAA,GAAA,0BAAA,oBAAqB,MAAM,QAAQ,CAAC,EACnF;AAEH,UAAO;IACL,SAAS,KAAK;IACd,UAAU,KAAK,cAAc,KAAK,YAAY,KAAK,cAAc;IACjE,mBAAmB,KAAK;IACxB,gBAAgB,KAAK;IACtB;;EAGH,MAAM,QACJ,MAAM,OAAO,KAAK,mBAAmB;GACnC,MAAM;IACJ,YAAY,GAAG,UAAU;IACzB,mBAAmB;IACpB;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;GAC/C,CAAC,EACF;AAEF,SAAO;GACL,SAAS,OAAO,KAAK,WAAW;GAChC,UAAU,KAAK,cAAc,OAAO,KAAK,WAAW,EAAE,KAAK,cAAc,IAAI,OAAO,CAAC;GACrF,mBAAmB,KAAK;GACxB,gBAAgB,KAAK;GACtB;;CAGH,MAAa,WACX,EAAE,IAAI,QACN,MACA,mBACA,YACA,mBACA,gBACA,SACA;EACA,MAAM,SAAS,KAAK,KAAK,KAAK;EAE9B,IAAI;AACJ,MAAI,kBAAkBA,wBAAAA,aAEpB,SACE,MAAM,OAAO,WACX;GACE,YAAY;GACZ;GACA,kBAAkB;GAClB,eAAe;GACf,cAAc;GACf,GAAA,GAAA,0BAAA,oBACkB,MAAM,QAAQ,CAClC,EACD;OACG;GACL,MAAM,QACJ,MAAM,OAAO,KAAK,2BAA2B;IAC3C,MAAM;KACJ,YAAY,GAAG,UAAU;KACzB,mBAAmB;KACnB,YAAY,WAAW,UAAU;KACjC,kBAAkB;KAClB,eAAe;KACf,cAAc;KACf;IACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;IAC/C,CAAC,EACF;AACF,UAAO;IACL,WAAW,KAAK;IAChB,QAAQ,KAAK;IACb,SAAS,KAAK;IACd,YAAY,OAAO,KAAK,WAAW;IACnC,UAAU,OAAO,KAAK,SAAS;IAChC;;EAGH,MAAM,QAAQ,MAAM,cAAc,MAAM,KAAK,YAAY,KAAK,SAAS;AACvE,QAAM,mBAAmB,MAAM,kBAAkB;AAEjD,MAAI,kBAAkB,sBAAsBE,iBAAAA,4BAA4B,OACtE,MAAK,QAAQ,KAAK;GAAE,MAAM;GAAgB,OAAO,wBAAwB,MAAM;GAAE,CAAC;EAGpF,MAAM,gBAAgB,OAAO,KAAK,WAAW,KAAK,WAAW;AAC7D,MAAI,MAAM,WAAW,cACnB,OAAM,IAAI,MACR,iCAAiC,cAAc,mBAAmB,MAAM,OAAO,kBAChF;EAGH,MAAM,UAAU,OAAO,YACrB,KAAK,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,KAAK,aAAa,EAAE,MAAM,CAAC,CACnE;AAED,MAAI;GACF,MAAM,EACJ,MAAM,SACN,YACA,SAAS,oBACP,OAAA,GAAA,OAAA,SAAc,KAAK,WAAW;IAChC,YAAY,KAAK;IACjB,MAAM;IAKN,gBAAgB;IAChB,aAAa;IAIb,OAAO;IACP;IACA,QAAQ,KAAK,OAAO,aAAa;IAClC,CAAC;AAIF,qBAAkB,YADL,MAAM,QAAQ,MAAM,EACG,iBAAiB,KAAK;WACnD,GAAY;AACnB,OAAI,aAAa,aAAc,OAAM;AAErC,OAAI,aAAa,gBAAiB,OAAM;AAExC,SAAM,IAAI,MACR,qBAAqB,KAAK,UAAU,EAAE,CAAC,sDAAsD,KAAK,UAAU,aAAa,KAAK,UAAU,KAAK,QAAQ,GACtJ;;AAGH,QAAM,KAAK,eAAe;GAAE;GAAI;GAAM,EAAE,OAAO,KAAK,WAAW,KAAK,WAAW,EAAE,QAAQ;;CAG3F,MAAa,SAAS,MAAoB,SAAsB;EAC9D,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkBF,wBAAAA,aACpB,OAAM,OAAO,SACX;GACE,YAAY,KAAK;GACjB,mBAAmBE,iBAAAA,4BAA4B;GAC/C,UAAU,IAAI,WAAW,EAAE;GAC5B,GAAA,GAAA,0BAAA,oBACkB,KAAK,MAAM,QAAQ,CACvC;MAED,OAAM,OAAO,KAAK,uBAAuB;GACvC,MAAM;IACJ,YAAY,KAAK,GAAG,UAAU;IAC9B,mBAAmB;IACnB,mBAAmB;IACnB,UAAU;IACX;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,KAAK,EAAE;GACpD,CAAC;;;;CAMN,cAAsB,YAAoB,eAAmC;EAC3E,MAAM,WAAqB,EAAE;EAC7B,MAAM,WAAW,IAAI,IAAI,cAAc;AAEvC,OAAK,IAAI,IAAI,IAAI,KAAK,YAAY,IAChC,KAAI,CAAC,SAAS,IAAI,EAAE,CAAE,UAAS,KAAK,EAAE;AAGxC,SAAO;;CAGT,MAAc,eACZ,EAAE,IAAI,QACN,gBACA,SACe;EACf,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkBF,wBAAAA,cAAc;AAClC,SAAM,OAAO,eACX;IACE,YAAY;IACZ;IACD,GAAA,GAAA,0BAAA,oBACkB,MAAM,QAAQ,CAClC,CAAC;AACF;;AAGF,QAAM,OAAO,KAAK,8BAA8B;GAC9C,MAAM;IACJ,YAAY,GAAG,UAAU;IACzB,mBAAmB;IACnB,gBAAgB,eAAe,UAAU;IAC1C;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;GAC/C,CAAC;;;AAKN,eAAe,cAAc,MAAc,YAAoB,UAAmC;CAChG,IAAI;AACJ,KAAI;AACF,MAAI,MAAMG,iBAAG,KAAK,KAAK;EACvB,MAAM,MAAM,OAAO,WAAW,WAAW;EACzC,MAAM,MAAM,OAAO,WAAW;EAC9B,MAAM,IAAI,OAAO,MAAM,IAAI;EAC3B,MAAM,YAAY,MAAM,sBAAsB,GAAG,GAAG,KAAK,IAAI;AAE7D,SAAO,EAAE,SAAS,GAAG,UAAU;UACxB,GAAY;AACnB,MAAI,KAAK,OAAO,MAAM,YAAY,UAAU,KAAK,EAAE,QAAQ,SACzD,OAAM,IAAI,mBAAmB,oBAAoB,KAAK,gBAAgB;AACxE,QAAM;WACE;AACR,QAAM,GAAG,OAAO;;;;;AAMpB,eAAe,sBAAsB,GAAkB,GAAW,KAAa,UAAkB;CAC/F,IAAI,iBAAiB;AACrB,QAAO,iBAAiB,KAAK;EAC3B,MAAM,EAAE,cAAc,MAAM,EAAE,KAC5B,GACA,gBACA,MAAM,gBACN,WAAW,eACZ;AACD,MAAI,cAAc,EAChB,OAAM,IAAI,cAAc,oCAAoC;AAE9D,oBAAkB;;AAGpB,QAAO;;AAGT,eAAe,mBAAmB,MAAc,mBAA2B;CACzE,MAAM,QAAQ,OAAO,KAAK,OAAO,MAAMA,iBAAG,KAAK,KAAK,EAAE,UAAU,IAAK,CAAC;AACtE,KAAI,QAAQ,kBACV,OAAM,IAAI,WAAW,sCAAsC,kBAAkB,SAAS,MAAM,GAAG;;;;;AAOnG,MAAM,2BAA2B;CAAC;CAAgB;CAAkB;CAAuB;AAE3F,SAAS,mBAAmB,MAAuB;AACjD,QAAO,yBAAyB,MAAM,SAAS,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;;AAGvF,SAAS,kBACP,YACA,MACA,SACA,MACA;AACA,KAAI,cAAc,KAAK;AACrB,MAAI,mBAAmB,KAAK,CAC1B,OAAM,IAAI,aACR,+CAA+C,WAAW,UAC9C,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;AAEH,QAAM,IAAI,gBACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;AAGH,KAAI,cAAc,IAChB,OAAM,IAAI,aACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;;AAKL,SAAS,wBAAwB,MAAsB;CACrD,MAAM,WAAWC,eAAAA,OAAO,KAAK;CAE7B,MAAM,SAAS,OAAO,MAAM,EAAE;AAE9B,QAAO,cAAc,UAAU,EAAE;AACjC,QAAO,OAAO,SAAS,SAAS"}
1	+ {"version":3,"file":"upload.cjs","names":["UploadClient","RestAPI","UploadAPI_ChecksumAlgorithm","fs","crc32c"],"sources":["../../src/clients/upload.ts"],"sourcesContent":["import type {\n WireClientProvider,\n WireClientProviderFactory,\n PlClient,\n} from \"@milaboratories/pl-client\";\nimport {\n addRTypeToMetadata,\n createRTypeRoutingHeader,\n RestAPI,\n parseSignedResourceId,\n signatureToBase64Url,\n} from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs/promises\";\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport {\n UploadAPI_ChecksumAlgorithm,\n type UploadAPI_GetPartURL_Response,\n} from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol\";\nimport { UploadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client\";\nimport type { UploadApiPaths, UploadRestClientType } from \"../proto-rest\";\nimport { crc32c } from \"./crc32c\";\n\nimport type { IncomingHttpHeaders } from \"undici/types/header\";\n\nexport class MTimeError extends Error {\n name = \"MTimeError\";\n}\n\nexport class UnexpectedEOF extends Error {\n name = \"UnexpectedEOF\";\n}\n\nexport class NetworkError extends Error {\n name = \"NetworkError\";\n}\n\n/** Happens when the file doesn't exist /\nexport class NoFileForUploading extends Error {\n name = \"NoFileForUploading\";\n}\n\nexport class BadRequestError extends Error {\n name = \"BadRequestError\";\n}\n\n/* Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. /\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType \| UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider((wire) => {\n if (wire.type === \"grpc\") {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n });\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const { globalId, signature } = parseSignedResourceId(id);\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (\n await client.init(\n { resourceId: globalId, resourceSignature: signature },\n addRTypeToMetadata(type, options),\n )\n ).response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (\n await client.POST(\"/v1/upload/init\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const { globalId, signature } = parseSignedResourceId(id);\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (\n await client.getPartURL(\n {\n resourceId: globalId,\n resourceSignature: signature,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: \"\",\n },\n addRTypeToMetadata(type, options),\n )\n ).response;\n } else {\n const resp = (\n await client.POST(\"/v1/upload/get-part-url\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n partNumber: partNumber.toString(),\n uploadedPartSize: \"0\",\n isInternalUse: false,\n partChecksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(\n info.headers.map(({ name, value }) => [name.toLowerCase(), value]),\n );\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError) throw e;\n\n if (e instanceof BadRequestError) throw e;\n\n throw new Error(\n `partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`,\n );\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const { globalId, signature } = parseSignedResourceId(info.id);\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize(\n {\n resourceId: globalId,\n resourceSignature: signature,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm.UNSPECIFIED,\n checksum: new Uint8Array(0),\n },\n addRTypeToMetadata(info.type, options),\n );\n } else {\n await client.POST(\"/v1/upload/finalize\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n checksumAlgorithm: 0,\n checksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /* Calculates parts that need to be uploaded from the parts that were\n * already uploaded. /\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const { globalId, signature } = parseSignedResourceId(id);\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: globalId,\n resourceSignature: signature,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST(\"/v1/upload/update-progress\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle \| undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === \"object\" && \"code\" in e && e.code == \"ENOENT\")\n throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/* Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. /\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF(\"file ended earlier than expected.\");\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\n/* S3 error codes that indicate expired/invalid temporary credentials.\n * These are transient: a retry with a fresh presigned URL will succeed\n * once the backend refreshes its STS credentials. /\nconst TRANSIENT_S3_ERROR_CODES = [\"ExpiredToken\", \"RequestExpired\", \"TokenRefreshRequired\"];\n\nfunction isTransientS3Error(body: string): boolean {\n return TRANSIENT_S3_ERROR_CODES.some((code) => body.includes(`<Code>${code}</Code>`));\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n if (isTransientS3Error(body)) {\n throw new NetworkError(\n `transient S3 credential error, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n throw new BadRequestError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/* Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString(\"base64\");\n}\n"],"mappings":";;;;;;;;;AA4BA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;AAGT,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAO;;AAGT,IAAa,eAAb,cAAkC,MAAM;CACtC,OAAO;;;AAIT,IAAa,qBAAb,cAAwC,MAAM;CAC5C,OAAO;;AAGT,IAAa,kBAAb,cAAqC,MAAM;CACzC,OAAO;;;;;AAMT,IAAa,eAAb,MAA0B;CACxB;CAEA,YACE,2BACA,YACA,GACA,QACA;AAHgB,OAAA,aAAA;AAEA,OAAA,SAAA;AAEhB,OAAK,OAAO,0BAA0B,0BAA0B,SAAS;AACvE,OAAI,KAAK,SAAS,OAChB,QAAO,IAAIA,wBAAAA,aAAa,KAAK,UAAU;AAGzC,UAAOC,0BAAAA,QAAQ,aAA6B;IAC1C,aAAa,KAAK,OAAO;IACzB,KAAK,KAAK,OAAO;IACjB,YAAY,KAAK;IACjB,aAAa,KAAK;IACnB,CAAC;IACF;;CAGJ,QAAQ;CAER,MAAa,WACX,EAAE,IAAI,QACN,SAMC;EACD,MAAM,EAAE,UAAU,eAAA,GAAA,0BAAA,uBAAoC,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkBD,wBAAAA,cAAc;GAClC,MAAM,QACJ,MAAM,OAAO,KACX;IAAE,YAAY;IAAU,mBAAmB;IAAW,GAAA,GAAA,0BAAA,oBACnC,MAAM,QAAQ,CAClC,EACD;AAEF,UAAO;IACL,SAAS,KAAK;IACd,UAAU,KAAK,cAAc,KAAK,YAAY,KAAK,cAAc;IACjE,mBAAmB,KAAK;IACxB,gBAAgB,KAAK;IACtB;;EAGH,MAAM,QACJ,MAAM,OAAO,KAAK,mBAAmB;GACnC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,oBAAA,GAAA,0BAAA,sBAAwC,UAAU;IACnD;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;GAC/C,CAAC,EACF;AAEF,SAAO;GACL,SAAS,OAAO,KAAK,WAAW;GAChC,UAAU,KAAK,cAAc,OAAO,KAAK,WAAW,EAAE,KAAK,cAAc,IAAI,OAAO,CAAC;GACrF,mBAAmB,KAAK;GACxB,gBAAgB,KAAK;GACtB;;CAGH,MAAa,WACX,EAAE,IAAI,QACN,MACA,mBACA,YACA,mBACA,gBACA,SACA;EACA,MAAM,EAAE,UAAU,eAAA,GAAA,0BAAA,uBAAoC,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;EAE9B,IAAI;AACJ,MAAI,kBAAkBA,wBAAAA,aAEpB,SACE,MAAM,OAAO,WACX;GACE,YAAY;GACZ,mBAAmB;GACnB;GACA,kBAAkB;GAClB,eAAe;GACf,cAAc;GACf,GAAA,GAAA,0BAAA,oBACkB,MAAM,QAAQ,CAClC,EACD;OACG;GACL,MAAM,QACJ,MAAM,OAAO,KAAK,2BAA2B;IAC3C,MAAM;KACJ,YAAY,SAAS,UAAU;KAC/B,oBAAA,GAAA,0BAAA,sBAAwC,UAAU;KAClD,YAAY,WAAW,UAAU;KACjC,kBAAkB;KAClB,eAAe;KACf,cAAc;KACf;IACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;IAC/C,CAAC,EACF;AACF,UAAO;IACL,WAAW,KAAK;IAChB,QAAQ,KAAK;IACb,SAAS,KAAK;IACd,YAAY,OAAO,KAAK,WAAW;IACnC,UAAU,OAAO,KAAK,SAAS;IAChC;;EAGH,MAAM,QAAQ,MAAM,cAAc,MAAM,KAAK,YAAY,KAAK,SAAS;AACvE,QAAM,mBAAmB,MAAM,kBAAkB;AAEjD,MAAI,kBAAkB,sBAAsBE,iBAAAA,4BAA4B,OACtE,MAAK,QAAQ,KAAK;GAAE,MAAM;GAAgB,OAAO,wBAAwB,MAAM;GAAE,CAAC;EAGpF,MAAM,gBAAgB,OAAO,KAAK,WAAW,KAAK,WAAW;AAC7D,MAAI,MAAM,WAAW,cACnB,OAAM,IAAI,MACR,iCAAiC,cAAc,mBAAmB,MAAM,OAAO,kBAChF;EAGH,MAAM,UAAU,OAAO,YACrB,KAAK,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,KAAK,aAAa,EAAE,MAAM,CAAC,CACnE;AAED,MAAI;GACF,MAAM,EACJ,MAAM,SACN,YACA,SAAS,oBACP,OAAA,GAAA,OAAA,SAAc,KAAK,WAAW;IAChC,YAAY,KAAK;IACjB,MAAM;IAKN,gBAAgB;IAChB,aAAa;IAIb,OAAO;IACP;IACA,QAAQ,KAAK,OAAO,aAAa;IAClC,CAAC;AAIF,qBAAkB,YADL,MAAM,QAAQ,MAAM,EACG,iBAAiB,KAAK;WACnD,GAAY;AACnB,OAAI,aAAa,aAAc,OAAM;AAErC,OAAI,aAAa,gBAAiB,OAAM;AAExC,SAAM,IAAI,MACR,qBAAqB,KAAK,UAAU,EAAE,CAAC,sDAAsD,KAAK,UAAU,aAAa,KAAK,UAAU,KAAK,QAAQ,GACtJ;;AAGH,QAAM,KAAK,eAAe;GAAE;GAAI;GAAM,EAAE,OAAO,KAAK,WAAW,KAAK,WAAW,EAAE,QAAQ;;CAG3F,MAAa,SAAS,MAAoB,SAAsB;EAC9D,MAAM,EAAE,UAAU,eAAA,GAAA,0BAAA,uBAAoC,KAAK,GAAG;EAC9D,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkBF,wBAAAA,aACpB,OAAM,OAAO,SACX;GACE,YAAY;GACZ,mBAAmB;GACnB,mBAAmBE,iBAAAA,4BAA4B;GAC/C,UAAU,IAAI,WAAW,EAAE;GAC5B,GAAA,GAAA,0BAAA,oBACkB,KAAK,MAAM,QAAQ,CACvC;MAED,OAAM,OAAO,KAAK,uBAAuB;GACvC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,oBAAA,GAAA,0BAAA,sBAAwC,UAAU;IAClD,mBAAmB;IACnB,UAAU;IACX;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,KAAK,EAAE;GACpD,CAAC;;;;CAMN,cAAsB,YAAoB,eAAmC;EAC3E,MAAM,WAAqB,EAAE;EAC7B,MAAM,WAAW,IAAI,IAAI,cAAc;AAEvC,OAAK,IAAI,IAAI,IAAI,KAAK,YAAY,IAChC,KAAI,CAAC,SAAS,IAAI,EAAE,CAAE,UAAS,KAAK,EAAE;AAGxC,SAAO;;CAGT,MAAc,eACZ,EAAE,IAAI,QACN,gBACA,SACe;EACf,MAAM,EAAE,UAAU,eAAA,GAAA,0BAAA,uBAAoC,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkBF,wBAAAA,cAAc;AAClC,SAAM,OAAO,eACX;IACE,YAAY;IACZ,mBAAmB;IACnB;IACD,GAAA,GAAA,0BAAA,oBACkB,MAAM,QAAQ,CAClC,CAAC;AACF;;AAGF,QAAM,OAAO,KAAK,8BAA8B;GAC9C,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,oBAAA,GAAA,0BAAA,sBAAwC,UAAU;IAClD,gBAAgB,eAAe,UAAU;IAC1C;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;GAC/C,CAAC;;;AAKN,eAAe,cAAc,MAAc,YAAoB,UAAmC;CAChG,IAAI;AACJ,KAAI;AACF,MAAI,MAAMG,iBAAG,KAAK,KAAK;EACvB,MAAM,MAAM,OAAO,WAAW,WAAW;EACzC,MAAM,MAAM,OAAO,WAAW;EAC9B,MAAM,IAAI,OAAO,MAAM,IAAI;EAC3B,MAAM,YAAY,MAAM,sBAAsB,GAAG,GAAG,KAAK,IAAI;AAE7D,SAAO,EAAE,SAAS,GAAG,UAAU;UACxB,GAAY;AACnB,MAAI,KAAK,OAAO,MAAM,YAAY,UAAU,KAAK,EAAE,QAAQ,SACzD,OAAM,IAAI,mBAAmB,oBAAoB,KAAK,gBAAgB;AACxE,QAAM;WACE;AACR,QAAM,GAAG,OAAO;;;;;AAMpB,eAAe,sBAAsB,GAAkB,GAAW,KAAa,UAAkB;CAC/F,IAAI,iBAAiB;AACrB,QAAO,iBAAiB,KAAK;EAC3B,MAAM,EAAE,cAAc,MAAM,EAAE,KAC5B,GACA,gBACA,MAAM,gBACN,WAAW,eACZ;AACD,MAAI,cAAc,EAChB,OAAM,IAAI,cAAc,oCAAoC;AAE9D,oBAAkB;;AAGpB,QAAO;;AAGT,eAAe,mBAAmB,MAAc,mBAA2B;CACzE,MAAM,QAAQ,OAAO,KAAK,OAAO,MAAMA,iBAAG,KAAK,KAAK,EAAE,UAAU,IAAK,CAAC;AACtE,KAAI,QAAQ,kBACV,OAAM,IAAI,WAAW,sCAAsC,kBAAkB,SAAS,MAAM,GAAG;;;;;AAOnG,MAAM,2BAA2B;CAAC;CAAgB;CAAkB;CAAuB;AAE3F,SAAS,mBAAmB,MAAuB;AACjD,QAAO,yBAAyB,MAAM,SAAS,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;;AAGvF,SAAS,kBACP,YACA,MACA,SACA,MACA;AACA,KAAI,cAAc,KAAK;AACrB,MAAI,mBAAmB,KAAK,CAC1B,OAAM,IAAI,aACR,+CAA+C,WAAW,UAC9C,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;AAEH,QAAM,IAAI,gBACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;AAGH,KAAI,cAAc,IAChB,OAAM,IAAI,aACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;;AAKL,SAAS,wBAAwB,MAAsB;CACrD,MAAM,WAAWC,eAAAA,OAAO,KAAK;CAE7B,MAAM,SAAS,OAAO,MAAM,EAAE;AAE9B,QAAO,cAAc,UAAU,EAAE;AACjC,QAAO,OAAO,SAAS,SAAS"}

package/dist/clients/upload.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"upload.d.ts","names":[],"sources":["../../src/clients/upload.ts"],"mappings":";;;;;;;;~~cAsBa~~,UAAA,SAAmB,KAAA;EAC9B,IAAA;AAAA;AAAA,cAGW,aAAA,SAAsB,KAAA;EACjC,IAAA;AAAA;AAAA,cAGW,YAAA,SAAqB,KAAA;EAChC,IAAA;AAAA;;cAIW,kBAAA,SAA2B,KAAA;EACtC,IAAA;AAAA;AAAA,cAGW,eAAA,SAAwB,KAAA;EACnC,IAAA;AAAA;;AALF;;cAWa,YAAA;EAAA,SAKO,UAAA,EAAY,UAAA;EAAA,SAEZ,MAAA,EAAQ,QAAA;EAAA,iBANT,IAAA;cAGf,yBAAA,EAA2B,yBAAA,EACX,UAAA,EAAY,UAAA,EAC5B,CAAA,EAAG,QAAA,EACa,MAAA,EAAQ,QAAA;EAgB1B,KAAA,CAAA;EAEa,UAAA,CAAA;IACT,EAAA;IAAI;EAAA,GAAQ,YAAA,EACd,OAAA,GAAU,UAAA,GACT,OAAA;IACD,OAAA;IACA,QAAA;IACA,iBAAA,EAAmB,2BAAA;IACnB,cAAA;EAAA;~~EAkCW~~,UAAA,CAAA;IACT,EAAA;IAAI;EAAA,GAAQ,YAAA,EACd,IAAA,UACA,iBAAA,UACA,UAAA,UACA,iBAAA,EAAmB,2BAAA,EACnB,cAAA,UACA,OAAA,GAAU,UAAA,GAAU,OAAA;~~EAkGT~~,QAAA,CAAS,IAAA,EAAM,YAAA,EAAc,OAAA,GAAU,UAAA,GAAU,OAAA;~~EAtKhC~~;;EAAA,~~QAiMtB~~,aAAA;EAAA,QAWM,cAAA;AAAA"}
1	+ {"version":3,"file":"upload.d.ts","names":[],"sources":["../../src/clients/upload.ts"],"mappings":";;;;;;;;cA4Ba,UAAA,SAAmB,KAAA;EAC9B,IAAA;AAAA;AAAA,cAGW,aAAA,SAAsB,KAAA;EACjC,IAAA;AAAA;AAAA,cAGW,YAAA,SAAqB,KAAA;EAChC,IAAA;AAAA;;cAIW,kBAAA,SAA2B,KAAA;EACtC,IAAA;AAAA;AAAA,cAGW,eAAA,SAAwB,KAAA;EACnC,IAAA;AAAA;;AALF;;cAWa,YAAA;EAAA,SAKO,UAAA,EAAY,UAAA;EAAA,SAEZ,MAAA,EAAQ,QAAA;EAAA,iBANT,IAAA;cAGf,yBAAA,EAA2B,yBAAA,EACX,UAAA,EAAY,UAAA,EAC5B,CAAA,EAAG,QAAA,EACa,MAAA,EAAQ,QAAA;EAgB1B,KAAA,CAAA;EAEa,UAAA,CAAA;IACT,EAAA;IAAI;EAAA,GAAQ,YAAA,EACd,OAAA,GAAU,UAAA,GACT,OAAA;IACD,OAAA;IACA,QAAA;IACA,iBAAA,EAAmB,2BAAA;IACnB,cAAA;EAAA;EAuCW,UAAA,CAAA;IACT,EAAA;IAAI;EAAA,GAAQ,YAAA,EACd,IAAA,UACA,iBAAA,UACA,UAAA,UACA,iBAAA,EAAmB,2BAAA,EACnB,cAAA,UACA,OAAA,GAAU,UAAA,GAAU,OAAA;EAoGT,QAAA,CAAS,IAAA,EAAM,YAAA,EAAc,OAAA,GAAU,UAAA,GAAU,OAAA;EA7KhC;;EAAA,QA0MtB,aAAA;EAAA,QAWM,cAAA;AAAA"}

package/dist/clients/upload.js CHANGED Viewed

@@ -1,7 +1,7 @@
 import { UploadAPI_ChecksumAlgorithm } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js";
 import { UploadClient } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js";
 import { crc32c } from "./crc32c.js";
-import { RestAPI, addRTypeToMetadata, createRTypeRoutingHeader } from "@milaboratories/pl-client";
+import { RestAPI, addRTypeToMetadata, createRTypeRoutingHeader, parseSignedResourceId, signatureToBase64Url } from "@milaboratories/pl-client";
 import * as fs from "node:fs/promises";
 import { request } from "undici";
 //#region src/clients/upload.ts
@@ -41,9 +41,13 @@ var ClientUpload = class {
 	}
 	close() {}
 	async initUpload({ id, type }, options) {
+		const { globalId, signature } = parseSignedResourceId(id);
 		const client = this.wire.get();
 		if (client instanceof UploadClient) {
-			const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options))).response;
+			const init = (await client.init({
+				resourceId: globalId,
+				resourceSignature: signature
+			}, addRTypeToMetadata(type, options))).response;
 			return {
 				overall: init.partsCount,
 				toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
@@ -53,8 +57,8 @@ var ClientUpload = class {
 		}
 		const init = (await client.POST("/v1/upload/init", {
 			body: {
-				resourceId: id.toString(),
-				resourceSignature: ""
+				resourceId: globalId.toString(),
+				resourceSignature: signatureToBase64Url(signature)
 			},
 			headers: { ...createRTypeRoutingHeader(type) }
 		})).data;
@@ -66,10 +70,12 @@ var ClientUpload = class {
 		};
 	}
 	async partUpload({ id, type }, path, expectedMTimeUnix, partNumber, checksumAlgorithm, checksumHeader, options) {
+		const { globalId, signature } = parseSignedResourceId(id);
 		const client = this.wire.get();
 		let info;
 		if (client instanceof UploadClient) info = (await client.getPartURL({
-			resourceId: id,
+			resourceId: globalId,
+			resourceSignature: signature,
 			partNumber,
 			uploadedPartSize: 0n,
 			isInternalUse: false,
@@ -78,8 +84,8 @@ var ClientUpload = class {
 		else {
 			const resp = (await client.POST("/v1/upload/get-part-url", {
 				body: {
-					resourceId: id.toString(),
-					resourceSignature: "",
+					resourceId: globalId.toString(),
+					resourceSignature: signatureToBase64Url(signature),
 					partNumber: partNumber.toString(),
 					uploadedPartSize: "0",
 					isInternalUse: false,
@@ -126,16 +132,18 @@ var ClientUpload = class {
 		}, BigInt(info.chunkEnd - info.chunkStart), options);
 	}
 	async finalize(info, options) {
+		const { globalId, signature } = parseSignedResourceId(info.id);
 		const client = this.wire.get();
 		if (client instanceof UploadClient) await client.finalize({
-			resourceId: info.id,
+			resourceId: globalId,
+			resourceSignature: signature,
 			checksumAlgorithm: UploadAPI_ChecksumAlgorithm.UNSPECIFIED,
 			checksum: new Uint8Array(0)
 		}, addRTypeToMetadata(info.type, options));
 		else await client.POST("/v1/upload/finalize", {
 			body: {
-				resourceId: info.id.toString(),
-				resourceSignature: "",
+				resourceId: globalId.toString(),
+				resourceSignature: signatureToBase64Url(signature),
 				checksumAlgorithm: 0,
 				checksum: ""
 			},
@@ -151,18 +159,20 @@ var ClientUpload = class {
 		return toUpload;
 	}
 	async updateProgress({ id, type }, bytesProcessed, options) {
+		const { globalId, signature } = parseSignedResourceId(id);
 		const client = this.wire.get();
 		if (client instanceof UploadClient) {
 			await client.updateProgress({
-				resourceId: id,
+				resourceId: globalId,
+				resourceSignature: signature,
 				bytesProcessed
 			}, addRTypeToMetadata(type, options)).response;
 			return;
 		}
 		await client.POST("/v1/upload/update-progress", {
 			body: {
-				resourceId: id.toString(),
-				resourceSignature: "",
+				resourceId: globalId.toString(),
+				resourceSignature: signatureToBase64Url(signature),
 				bytesProcessed: bytesProcessed.toString()
 			},
 			headers: { ...createRTypeRoutingHeader(type) }

package/dist/clients/upload.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"upload.js","names":[],"sources":["../../src/clients/upload.ts"],"sourcesContent":["import type {\n WireClientProvider,\n WireClientProviderFactory,\n PlClient,\n} from \"@milaboratories/pl-client\";\nimport { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs/promises\";\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport {\n UploadAPI_ChecksumAlgorithm,\n type UploadAPI_GetPartURL_Response,\n} from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol\";\nimport { UploadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client\";\nimport type { UploadApiPaths, UploadRestClientType } from \"../proto-rest\";\nimport { crc32c } from \"./crc32c\";\n\nimport type { IncomingHttpHeaders } from \"undici/types/header\";\n\nexport class MTimeError extends Error {\n name = \"MTimeError\";\n}\n\nexport class UnexpectedEOF extends Error {\n name = \"UnexpectedEOF\";\n}\n\nexport class NetworkError extends Error {\n name = \"NetworkError\";\n}\n\n/** Happens when the file doesn't exist /\nexport class NoFileForUploading extends Error {\n name = \"NoFileForUploading\";\n}\n\nexport class BadRequestError extends Error {\n name = \"BadRequestError\";\n}\n\n/* Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. /\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType \| UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider((wire) => {\n if (wire.type === \"grpc\") {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n });\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options)))\n .response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (\n await client.POST(\"/v1/upload/init\", {\n body: {\n resourceId: id.toString(),\n resourceSignature: \"\",\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (\n await client.getPartURL(\n {\n resourceId: id,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: \"\",\n },\n addRTypeToMetadata(type, options),\n )\n ).response;\n } else {\n const resp = (\n await client.POST(\"/v1/upload/get-part-url\", {\n body: {\n resourceId: id.toString(),\n resourceSignature: \"\",\n partNumber: partNumber.toString(),\n uploadedPartSize: \"0\",\n isInternalUse: false,\n partChecksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(\n info.headers.map(({ name, value }) => [name.toLowerCase(), value]),\n );\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError) throw e;\n\n if (e instanceof BadRequestError) throw e;\n\n throw new Error(\n `partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`,\n );\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize(\n {\n resourceId: info.id,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm.UNSPECIFIED,\n checksum: new Uint8Array(0),\n },\n addRTypeToMetadata(info.type, options),\n );\n } else {\n await client.POST(\"/v1/upload/finalize\", {\n body: {\n resourceId: info.id.toString(),\n resourceSignature: \"\",\n checksumAlgorithm: 0,\n checksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /* Calculates parts that need to be uploaded from the parts that were\n * already uploaded. /\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: id,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST(\"/v1/upload/update-progress\", {\n body: {\n resourceId: id.toString(),\n resourceSignature: \"\",\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle \| undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === \"object\" && \"code\" in e && e.code == \"ENOENT\")\n throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/* Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. /\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF(\"file ended earlier than expected.\");\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\n/* S3 error codes that indicate expired/invalid temporary credentials.\n * These are transient: a retry with a fresh presigned URL will succeed\n * once the backend refreshes its STS credentials. /\nconst TRANSIENT_S3_ERROR_CODES = [\"ExpiredToken\", \"RequestExpired\", \"TokenRefreshRequired\"];\n\nfunction isTransientS3Error(body: string): boolean {\n return TRANSIENT_S3_ERROR_CODES.some((code) => body.includes(`<Code>${code}</Code>`));\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n if (isTransientS3Error(body)) {\n throw new NetworkError(\n `transient S3 credential error, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n throw new BadRequestError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/* Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString(\"base64\");\n}\n"],"mappings":";;;;;;;AAsBA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;AAGT,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAO;;AAGT,IAAa,eAAb,cAAkC,MAAM;CACtC,OAAO;;;AAIT,IAAa,qBAAb,cAAwC,MAAM;CAC5C,OAAO;;AAGT,IAAa,kBAAb,cAAqC,MAAM;CACzC,OAAO;;;;;AAMT,IAAa,eAAb,MAA0B;CACxB;CAEA,YACE,2BACA,YACA,GACA,QACA;AAHgB,OAAA,aAAA;AAEA,OAAA,SAAA;AAEhB,OAAK,OAAO,0BAA0B,0BAA0B,SAAS;AACvE,OAAI,KAAK,SAAS,OAChB,QAAO,IAAI,aAAa,KAAK,UAAU;AAGzC,UAAO,QAAQ,aAA6B;IAC1C,aAAa,KAAK,OAAO;IACzB,KAAK,KAAK,OAAO;IACjB,YAAY,KAAK;IACjB,aAAa,KAAK;IACnB,CAAC;IACF;;CAGJ,QAAQ;CAER,MAAa,WACX,EAAE,IAAI,QACN,SAMC;EACD,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkB,cAAc;GAClC,MAAM,QAAQ,MAAM,OAAO,KAAK,EAAE,YAAY,IAAI,EAAE,mBAAmB,MAAM,QAAQ,CAAC,EACnF;AAEH,UAAO;IACL,SAAS,KAAK;IACd,UAAU,KAAK,cAAc,KAAK,YAAY,KAAK,cAAc;IACjE,mBAAmB,KAAK;IACxB,gBAAgB,KAAK;IACtB;;EAGH,MAAM,QACJ,MAAM,OAAO,KAAK,mBAAmB;GACnC,MAAM;IACJ,YAAY,GAAG,UAAU;IACzB,mBAAmB;IACpB;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;GAC/C,CAAC,EACF;AAEF,SAAO;GACL,SAAS,OAAO,KAAK,WAAW;GAChC,UAAU,KAAK,cAAc,OAAO,KAAK,WAAW,EAAE,KAAK,cAAc,IAAI,OAAO,CAAC;GACrF,mBAAmB,KAAK;GACxB,gBAAgB,KAAK;GACtB;;CAGH,MAAa,WACX,EAAE,IAAI,QACN,MACA,mBACA,YACA,mBACA,gBACA,SACA;EACA,MAAM,SAAS,KAAK,KAAK,KAAK;EAE9B,IAAI;AACJ,MAAI,kBAAkB,aAEpB,SACE,MAAM,OAAO,WACX;GACE,YAAY;GACZ;GACA,kBAAkB;GAClB,eAAe;GACf,cAAc;GACf,EACD,mBAAmB,MAAM,QAAQ,CAClC,EACD;OACG;GACL,MAAM,QACJ,MAAM,OAAO,KAAK,2BAA2B;IAC3C,MAAM;KACJ,YAAY,GAAG,UAAU;KACzB,mBAAmB;KACnB,YAAY,WAAW,UAAU;KACjC,kBAAkB;KAClB,eAAe;KACf,cAAc;KACf;IACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;IAC/C,CAAC,EACF;AACF,UAAO;IACL,WAAW,KAAK;IAChB,QAAQ,KAAK;IACb,SAAS,KAAK;IACd,YAAY,OAAO,KAAK,WAAW;IACnC,UAAU,OAAO,KAAK,SAAS;IAChC;;EAGH,MAAM,QAAQ,MAAM,cAAc,MAAM,KAAK,YAAY,KAAK,SAAS;AACvE,QAAM,mBAAmB,MAAM,kBAAkB;AAEjD,MAAI,kBAAkB,sBAAsB,4BAA4B,OACtE,MAAK,QAAQ,KAAK;GAAE,MAAM;GAAgB,OAAO,wBAAwB,MAAM;GAAE,CAAC;EAGpF,MAAM,gBAAgB,OAAO,KAAK,WAAW,KAAK,WAAW;AAC7D,MAAI,MAAM,WAAW,cACnB,OAAM,IAAI,MACR,iCAAiC,cAAc,mBAAmB,MAAM,OAAO,kBAChF;EAGH,MAAM,UAAU,OAAO,YACrB,KAAK,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,KAAK,aAAa,EAAE,MAAM,CAAC,CACnE;AAED,MAAI;GACF,MAAM,EACJ,MAAM,SACN,YACA,SAAS,oBACP,MAAM,QAAQ,KAAK,WAAW;IAChC,YAAY,KAAK;IACjB,MAAM;IAKN,gBAAgB;IAChB,aAAa;IAIb,OAAO;IACP;IACA,QAAQ,KAAK,OAAO,aAAa;IAClC,CAAC;AAIF,qBAAkB,YADL,MAAM,QAAQ,MAAM,EACG,iBAAiB,KAAK;WACnD,GAAY;AACnB,OAAI,aAAa,aAAc,OAAM;AAErC,OAAI,aAAa,gBAAiB,OAAM;AAExC,SAAM,IAAI,MACR,qBAAqB,KAAK,UAAU,EAAE,CAAC,sDAAsD,KAAK,UAAU,aAAa,KAAK,UAAU,KAAK,QAAQ,GACtJ;;AAGH,QAAM,KAAK,eAAe;GAAE;GAAI;GAAM,EAAE,OAAO,KAAK,WAAW,KAAK,WAAW,EAAE,QAAQ;;CAG3F,MAAa,SAAS,MAAoB,SAAsB;EAC9D,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkB,aACpB,OAAM,OAAO,SACX;GACE,YAAY,KAAK;GACjB,mBAAmB,4BAA4B;GAC/C,UAAU,IAAI,WAAW,EAAE;GAC5B,EACD,mBAAmB,KAAK,MAAM,QAAQ,CACvC;MAED,OAAM,OAAO,KAAK,uBAAuB;GACvC,MAAM;IACJ,YAAY,KAAK,GAAG,UAAU;IAC9B,mBAAmB;IACnB,mBAAmB;IACnB,UAAU;IACX;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,KAAK,EAAE;GACpD,CAAC;;;;CAMN,cAAsB,YAAoB,eAAmC;EAC3E,MAAM,WAAqB,EAAE;EAC7B,MAAM,WAAW,IAAI,IAAI,cAAc;AAEvC,OAAK,IAAI,IAAI,IAAI,KAAK,YAAY,IAChC,KAAI,CAAC,SAAS,IAAI,EAAE,CAAE,UAAS,KAAK,EAAE;AAGxC,SAAO;;CAGT,MAAc,eACZ,EAAE,IAAI,QACN,gBACA,SACe;EACf,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkB,cAAc;AAClC,SAAM,OAAO,eACX;IACE,YAAY;IACZ;IACD,EACD,mBAAmB,MAAM,QAAQ,CAClC,CAAC;AACF;;AAGF,QAAM,OAAO,KAAK,8BAA8B;GAC9C,MAAM;IACJ,YAAY,GAAG,UAAU;IACzB,mBAAmB;IACnB,gBAAgB,eAAe,UAAU;IAC1C;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;GAC/C,CAAC;;;AAKN,eAAe,cAAc,MAAc,YAAoB,UAAmC;CAChG,IAAI;AACJ,KAAI;AACF,MAAI,MAAM,GAAG,KAAK,KAAK;EACvB,MAAM,MAAM,OAAO,WAAW,WAAW;EACzC,MAAM,MAAM,OAAO,WAAW;EAC9B,MAAM,IAAI,OAAO,MAAM,IAAI;EAC3B,MAAM,YAAY,MAAM,sBAAsB,GAAG,GAAG,KAAK,IAAI;AAE7D,SAAO,EAAE,SAAS,GAAG,UAAU;UACxB,GAAY;AACnB,MAAI,KAAK,OAAO,MAAM,YAAY,UAAU,KAAK,EAAE,QAAQ,SACzD,OAAM,IAAI,mBAAmB,oBAAoB,KAAK,gBAAgB;AACxE,QAAM;WACE;AACR,QAAM,GAAG,OAAO;;;;;AAMpB,eAAe,sBAAsB,GAAkB,GAAW,KAAa,UAAkB;CAC/F,IAAI,iBAAiB;AACrB,QAAO,iBAAiB,KAAK;EAC3B,MAAM,EAAE,cAAc,MAAM,EAAE,KAC5B,GACA,gBACA,MAAM,gBACN,WAAW,eACZ;AACD,MAAI,cAAc,EAChB,OAAM,IAAI,cAAc,oCAAoC;AAE9D,oBAAkB;;AAGpB,QAAO;;AAGT,eAAe,mBAAmB,MAAc,mBAA2B;CACzE,MAAM,QAAQ,OAAO,KAAK,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,UAAU,IAAK,CAAC;AACtE,KAAI,QAAQ,kBACV,OAAM,IAAI,WAAW,sCAAsC,kBAAkB,SAAS,MAAM,GAAG;;;;;AAOnG,MAAM,2BAA2B;CAAC;CAAgB;CAAkB;CAAuB;AAE3F,SAAS,mBAAmB,MAAuB;AACjD,QAAO,yBAAyB,MAAM,SAAS,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;;AAGvF,SAAS,kBACP,YACA,MACA,SACA,MACA;AACA,KAAI,cAAc,KAAK;AACrB,MAAI,mBAAmB,KAAK,CAC1B,OAAM,IAAI,aACR,+CAA+C,WAAW,UAC9C,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;AAEH,QAAM,IAAI,gBACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;AAGH,KAAI,cAAc,IAChB,OAAM,IAAI,aACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;;AAKL,SAAS,wBAAwB,MAAsB;CACrD,MAAM,WAAW,OAAO,KAAK;CAE7B,MAAM,SAAS,OAAO,MAAM,EAAE;AAE9B,QAAO,cAAc,UAAU,EAAE;AACjC,QAAO,OAAO,SAAS,SAAS"}
1	+ {"version":3,"file":"upload.js","names":[],"sources":["../../src/clients/upload.ts"],"sourcesContent":["import type {\n WireClientProvider,\n WireClientProviderFactory,\n PlClient,\n} from \"@milaboratories/pl-client\";\nimport {\n addRTypeToMetadata,\n createRTypeRoutingHeader,\n RestAPI,\n parseSignedResourceId,\n signatureToBase64Url,\n} from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs/promises\";\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport {\n UploadAPI_ChecksumAlgorithm,\n type UploadAPI_GetPartURL_Response,\n} from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol\";\nimport { UploadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client\";\nimport type { UploadApiPaths, UploadRestClientType } from \"../proto-rest\";\nimport { crc32c } from \"./crc32c\";\n\nimport type { IncomingHttpHeaders } from \"undici/types/header\";\n\nexport class MTimeError extends Error {\n name = \"MTimeError\";\n}\n\nexport class UnexpectedEOF extends Error {\n name = \"UnexpectedEOF\";\n}\n\nexport class NetworkError extends Error {\n name = \"NetworkError\";\n}\n\n/** Happens when the file doesn't exist /\nexport class NoFileForUploading extends Error {\n name = \"NoFileForUploading\";\n}\n\nexport class BadRequestError extends Error {\n name = \"BadRequestError\";\n}\n\n/* Low-level client for grpc uploadapi.\n * The user should pass here a concrete BlobUpload/<storageId> resource,\n * it can be got from handle field of BlobUpload. /\nexport class ClientUpload {\n private readonly wire: WireClientProvider<UploadRestClientType \| UploadClient>;\n\n constructor(\n wireClientProviderFactory: WireClientProviderFactory,\n public readonly httpClient: Dispatcher,\n _: PlClient,\n public readonly logger: MiLogger,\n ) {\n this.wire = wireClientProviderFactory.createWireClientProvider((wire) => {\n if (wire.type === \"grpc\") {\n return new UploadClient(wire.Transport);\n }\n\n return RestAPI.createClient<UploadApiPaths>({\n hostAndPort: wire.Config.hostAndPort,\n ssl: wire.Config.ssl,\n dispatcher: wire.Dispatcher,\n middlewares: wire.Middlewares,\n });\n });\n }\n\n close() {}\n\n public async initUpload(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n ): Promise<{\n overall: bigint;\n toUpload: bigint[];\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm;\n checksumHeader: string;\n }> {\n const { globalId, signature } = parseSignedResourceId(id);\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n const init = (\n await client.init(\n { resourceId: globalId, resourceSignature: signature },\n addRTypeToMetadata(type, options),\n )\n ).response;\n\n return {\n overall: init.partsCount,\n toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n const init = (\n await client.POST(\"/v1/upload/init\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n\n return {\n overall: BigInt(init.partsCount),\n toUpload: this.partsToUpload(BigInt(init.partsCount), init.uploadedParts.map(BigInt)),\n checksumAlgorithm: init.checksumAlgorithm,\n checksumHeader: init.checksumHeader,\n };\n }\n\n public async partUpload(\n { id, type }: ResourceInfo,\n path: string,\n expectedMTimeUnix: bigint,\n partNumber: bigint,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm,\n checksumHeader: string,\n options?: RpcOptions,\n ) {\n const { globalId, signature } = parseSignedResourceId(id);\n const client = this.wire.get();\n\n let info: UploadAPI_GetPartURL_Response;\n if (client instanceof UploadClient) {\n // partChecksum isn't used for now but protoc requires it to be set\n info = (\n await client.getPartURL(\n {\n resourceId: globalId,\n resourceSignature: signature,\n partNumber,\n uploadedPartSize: 0n,\n isInternalUse: false,\n partChecksum: \"\",\n },\n addRTypeToMetadata(type, options),\n )\n ).response;\n } else {\n const resp = (\n await client.POST(\"/v1/upload/get-part-url\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n partNumber: partNumber.toString(),\n uploadedPartSize: \"0\",\n isInternalUse: false,\n partChecksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(type) },\n })\n ).data!;\n info = {\n uploadUrl: resp.uploadUrl,\n method: resp.method,\n headers: resp.headers,\n chunkStart: BigInt(resp.chunkStart),\n chunkEnd: BigInt(resp.chunkEnd),\n };\n }\n\n const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);\n await checkExpectedMTime(path, expectedMTimeUnix);\n\n if (checksumHeader && checksumAlgorithm === UploadAPI_ChecksumAlgorithm.CRC32C) {\n info.headers.push({ name: checksumHeader, value: calculateCrc32cChecksum(chunk) });\n }\n\n const contentLength = Number(info.chunkEnd - info.chunkStart);\n if (chunk.length !== contentLength) {\n throw new Error(\n `Chunk size mismatch: expected ${contentLength} bytes, but read ${chunk.length} bytes from file`,\n );\n }\n\n const headers = Object.fromEntries(\n info.headers.map(({ name, value }) => [name.toLowerCase(), value]),\n );\n\n try {\n const {\n body: rawBody,\n statusCode,\n headers: responseHeaders,\n } = await request(info.uploadUrl, {\n dispatcher: this.httpClient,\n body: chunk,\n // We got headers only after we send\n // the whole body (in case of S3 PUT requests it's 5 MB).\n // It might be slow with a slow connection (or with SSH),\n // that's why we got big timeout here.\n headersTimeout: 60000,\n bodyTimeout: 60000,\n // Prevent connection reuse by setting \"Connection: close\" header.\n // This works around an issue with the backend's built-in S3 implementation\n // that caused HTTP/1.1 protocol lines to be included in the uploaded file content.\n reset: true,\n headers,\n method: info.method.toUpperCase() as Dispatcher.HttpMethod,\n });\n\n // always read the body for resources to be garbage collected.\n const body = await rawBody.text();\n checkStatusCodeOk(statusCode, body, responseHeaders, info);\n } catch (e: unknown) {\n if (e instanceof NetworkError) throw e;\n\n if (e instanceof BadRequestError) throw e;\n\n throw new Error(\n `partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`,\n );\n }\n\n await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);\n }\n\n public async finalize(info: ResourceInfo, options?: RpcOptions) {\n const { globalId, signature } = parseSignedResourceId(info.id);\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.finalize(\n {\n resourceId: globalId,\n resourceSignature: signature,\n checksumAlgorithm: UploadAPI_ChecksumAlgorithm.UNSPECIFIED,\n checksum: new Uint8Array(0),\n },\n addRTypeToMetadata(info.type, options),\n );\n } else {\n await client.POST(\"/v1/upload/finalize\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n checksumAlgorithm: 0,\n checksum: \"\",\n },\n headers: { ...createRTypeRoutingHeader(info.type) },\n });\n }\n }\n\n /* Calculates parts that need to be uploaded from the parts that were\n * already uploaded. /\n private partsToUpload(partsCount: bigint, uploadedParts: bigint[]): bigint[] {\n const toUpload: bigint[] = [];\n const uploaded = new Set(uploadedParts);\n\n for (let i = 1n; i <= partsCount; i++) {\n if (!uploaded.has(i)) toUpload.push(i);\n }\n\n return toUpload;\n }\n\n private async updateProgress(\n { id, type }: ResourceInfo,\n bytesProcessed: bigint,\n options?: RpcOptions,\n ): Promise<void> {\n const { globalId, signature } = parseSignedResourceId(id);\n const client = this.wire.get();\n\n if (client instanceof UploadClient) {\n await client.updateProgress(\n {\n resourceId: globalId,\n resourceSignature: signature,\n bytesProcessed,\n },\n addRTypeToMetadata(type, options),\n ).response;\n return;\n }\n\n await client.POST(\"/v1/upload/update-progress\", {\n body: {\n resourceId: globalId.toString(),\n resourceSignature: signatureToBase64Url(signature),\n bytesProcessed: bytesProcessed.toString(),\n },\n headers: { ...createRTypeRoutingHeader(type) },\n });\n return;\n }\n}\n\nasync function readFileChunk(path: string, chunkStart: bigint, chunkEnd: bigint): Promise<Buffer> {\n let f: fs.FileHandle \| undefined;\n try {\n f = await fs.open(path);\n const len = Number(chunkEnd - chunkStart);\n const pos = Number(chunkStart);\n const b = Buffer.alloc(len);\n const bytesRead = await readBytesFromPosition(f, b, len, pos);\n\n return b.subarray(0, bytesRead);\n } catch (e: unknown) {\n if (e && typeof e === \"object\" && \"code\" in e && e.code == \"ENOENT\")\n throw new NoFileForUploading(`there is no file ${path} for uploading`);\n throw e;\n } finally {\n await f?.close();\n }\n}\n\n/* Read len bytes from a given position.\n * Without this, `FileHandle.read` can read less bytes than needed. /\nasync function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {\n let bytesReadTotal = 0;\n while (bytesReadTotal < len) {\n const { bytesRead } = await f.read(\n b,\n bytesReadTotal,\n len - bytesReadTotal,\n position + bytesReadTotal,\n );\n if (bytesRead === 0) {\n throw new UnexpectedEOF(\"file ended earlier than expected.\");\n }\n bytesReadTotal += bytesRead;\n }\n\n return bytesReadTotal;\n}\n\nasync function checkExpectedMTime(path: string, expectedMTimeUnix: bigint) {\n const mTime = BigInt(Math.floor((await fs.stat(path)).mtimeMs / 1000));\n if (mTime > expectedMTimeUnix) {\n throw new MTimeError(`file was modified, expected mtime: ${expectedMTimeUnix}, got: ${mTime}.`);\n }\n}\n\n/* S3 error codes that indicate expired/invalid temporary credentials.\n * These are transient: a retry with a fresh presigned URL will succeed\n * once the backend refreshes its STS credentials. /\nconst TRANSIENT_S3_ERROR_CODES = [\"ExpiredToken\", \"RequestExpired\", \"TokenRefreshRequired\"];\n\nfunction isTransientS3Error(body: string): boolean {\n return TRANSIENT_S3_ERROR_CODES.some((code) => body.includes(`<Code>${code}</Code>`));\n}\n\nfunction checkStatusCodeOk(\n statusCode: number,\n body: string,\n headers: IncomingHttpHeaders,\n info: UploadAPI_GetPartURL_Response,\n) {\n if (statusCode == 400) {\n if (isTransientS3Error(body)) {\n throw new NetworkError(\n `transient S3 credential error, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n throw new BadRequestError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n\n if (statusCode != 200) {\n throw new NetworkError(\n `response is not ok, status code: ${statusCode},` +\n ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`,\n );\n }\n}\n\n/* Calculate CRC32C checksum of a buffer and return as base64 string */\nfunction calculateCrc32cChecksum(data: Buffer): string {\n const checksum = crc32c(data);\n // Convert to unsigned 32-bit integer and then to base64\n const buffer = Buffer.alloc(4);\n\n buffer.writeUInt32BE(checksum, 0);\n return buffer.toString(\"base64\");\n}\n"],"mappings":";;;;;;;AA4BA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;AAGT,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAO;;AAGT,IAAa,eAAb,cAAkC,MAAM;CACtC,OAAO;;;AAIT,IAAa,qBAAb,cAAwC,MAAM;CAC5C,OAAO;;AAGT,IAAa,kBAAb,cAAqC,MAAM;CACzC,OAAO;;;;;AAMT,IAAa,eAAb,MAA0B;CACxB;CAEA,YACE,2BACA,YACA,GACA,QACA;AAHgB,OAAA,aAAA;AAEA,OAAA,SAAA;AAEhB,OAAK,OAAO,0BAA0B,0BAA0B,SAAS;AACvE,OAAI,KAAK,SAAS,OAChB,QAAO,IAAI,aAAa,KAAK,UAAU;AAGzC,UAAO,QAAQ,aAA6B;IAC1C,aAAa,KAAK,OAAO;IACzB,KAAK,KAAK,OAAO;IACjB,YAAY,KAAK;IACjB,aAAa,KAAK;IACnB,CAAC;IACF;;CAGJ,QAAQ;CAER,MAAa,WACX,EAAE,IAAI,QACN,SAMC;EACD,MAAM,EAAE,UAAU,cAAc,sBAAsB,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkB,cAAc;GAClC,MAAM,QACJ,MAAM,OAAO,KACX;IAAE,YAAY;IAAU,mBAAmB;IAAW,EACtD,mBAAmB,MAAM,QAAQ,CAClC,EACD;AAEF,UAAO;IACL,SAAS,KAAK;IACd,UAAU,KAAK,cAAc,KAAK,YAAY,KAAK,cAAc;IACjE,mBAAmB,KAAK;IACxB,gBAAgB,KAAK;IACtB;;EAGH,MAAM,QACJ,MAAM,OAAO,KAAK,mBAAmB;GACnC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,mBAAmB,qBAAqB,UAAU;IACnD;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;GAC/C,CAAC,EACF;AAEF,SAAO;GACL,SAAS,OAAO,KAAK,WAAW;GAChC,UAAU,KAAK,cAAc,OAAO,KAAK,WAAW,EAAE,KAAK,cAAc,IAAI,OAAO,CAAC;GACrF,mBAAmB,KAAK;GACxB,gBAAgB,KAAK;GACtB;;CAGH,MAAa,WACX,EAAE,IAAI,QACN,MACA,mBACA,YACA,mBACA,gBACA,SACA;EACA,MAAM,EAAE,UAAU,cAAc,sBAAsB,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;EAE9B,IAAI;AACJ,MAAI,kBAAkB,aAEpB,SACE,MAAM,OAAO,WACX;GACE,YAAY;GACZ,mBAAmB;GACnB;GACA,kBAAkB;GAClB,eAAe;GACf,cAAc;GACf,EACD,mBAAmB,MAAM,QAAQ,CAClC,EACD;OACG;GACL,MAAM,QACJ,MAAM,OAAO,KAAK,2BAA2B;IAC3C,MAAM;KACJ,YAAY,SAAS,UAAU;KAC/B,mBAAmB,qBAAqB,UAAU;KAClD,YAAY,WAAW,UAAU;KACjC,kBAAkB;KAClB,eAAe;KACf,cAAc;KACf;IACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;IAC/C,CAAC,EACF;AACF,UAAO;IACL,WAAW,KAAK;IAChB,QAAQ,KAAK;IACb,SAAS,KAAK;IACd,YAAY,OAAO,KAAK,WAAW;IACnC,UAAU,OAAO,KAAK,SAAS;IAChC;;EAGH,MAAM,QAAQ,MAAM,cAAc,MAAM,KAAK,YAAY,KAAK,SAAS;AACvE,QAAM,mBAAmB,MAAM,kBAAkB;AAEjD,MAAI,kBAAkB,sBAAsB,4BAA4B,OACtE,MAAK,QAAQ,KAAK;GAAE,MAAM;GAAgB,OAAO,wBAAwB,MAAM;GAAE,CAAC;EAGpF,MAAM,gBAAgB,OAAO,KAAK,WAAW,KAAK,WAAW;AAC7D,MAAI,MAAM,WAAW,cACnB,OAAM,IAAI,MACR,iCAAiC,cAAc,mBAAmB,MAAM,OAAO,kBAChF;EAGH,MAAM,UAAU,OAAO,YACrB,KAAK,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,KAAK,aAAa,EAAE,MAAM,CAAC,CACnE;AAED,MAAI;GACF,MAAM,EACJ,MAAM,SACN,YACA,SAAS,oBACP,MAAM,QAAQ,KAAK,WAAW;IAChC,YAAY,KAAK;IACjB,MAAM;IAKN,gBAAgB;IAChB,aAAa;IAIb,OAAO;IACP;IACA,QAAQ,KAAK,OAAO,aAAa;IAClC,CAAC;AAIF,qBAAkB,YADL,MAAM,QAAQ,MAAM,EACG,iBAAiB,KAAK;WACnD,GAAY;AACnB,OAAI,aAAa,aAAc,OAAM;AAErC,OAAI,aAAa,gBAAiB,OAAM;AAExC,SAAM,IAAI,MACR,qBAAqB,KAAK,UAAU,EAAE,CAAC,sDAAsD,KAAK,UAAU,aAAa,KAAK,UAAU,KAAK,QAAQ,GACtJ;;AAGH,QAAM,KAAK,eAAe;GAAE;GAAI;GAAM,EAAE,OAAO,KAAK,WAAW,KAAK,WAAW,EAAE,QAAQ;;CAG3F,MAAa,SAAS,MAAoB,SAAsB;EAC9D,MAAM,EAAE,UAAU,cAAc,sBAAsB,KAAK,GAAG;EAC9D,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkB,aACpB,OAAM,OAAO,SACX;GACE,YAAY;GACZ,mBAAmB;GACnB,mBAAmB,4BAA4B;GAC/C,UAAU,IAAI,WAAW,EAAE;GAC5B,EACD,mBAAmB,KAAK,MAAM,QAAQ,CACvC;MAED,OAAM,OAAO,KAAK,uBAAuB;GACvC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,mBAAmB,qBAAqB,UAAU;IAClD,mBAAmB;IACnB,UAAU;IACX;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,KAAK,EAAE;GACpD,CAAC;;;;CAMN,cAAsB,YAAoB,eAAmC;EAC3E,MAAM,WAAqB,EAAE;EAC7B,MAAM,WAAW,IAAI,IAAI,cAAc;AAEvC,OAAK,IAAI,IAAI,IAAI,KAAK,YAAY,IAChC,KAAI,CAAC,SAAS,IAAI,EAAE,CAAE,UAAS,KAAK,EAAE;AAGxC,SAAO;;CAGT,MAAc,eACZ,EAAE,IAAI,QACN,gBACA,SACe;EACf,MAAM,EAAE,UAAU,cAAc,sBAAsB,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAE9B,MAAI,kBAAkB,cAAc;AAClC,SAAM,OAAO,eACX;IACE,YAAY;IACZ,mBAAmB;IACnB;IACD,EACD,mBAAmB,MAAM,QAAQ,CAClC,CAAC;AACF;;AAGF,QAAM,OAAO,KAAK,8BAA8B;GAC9C,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,mBAAmB,qBAAqB,UAAU;IAClD,gBAAgB,eAAe,UAAU;IAC1C;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;GAC/C,CAAC;;;AAKN,eAAe,cAAc,MAAc,YAAoB,UAAmC;CAChG,IAAI;AACJ,KAAI;AACF,MAAI,MAAM,GAAG,KAAK,KAAK;EACvB,MAAM,MAAM,OAAO,WAAW,WAAW;EACzC,MAAM,MAAM,OAAO,WAAW;EAC9B,MAAM,IAAI,OAAO,MAAM,IAAI;EAC3B,MAAM,YAAY,MAAM,sBAAsB,GAAG,GAAG,KAAK,IAAI;AAE7D,SAAO,EAAE,SAAS,GAAG,UAAU;UACxB,GAAY;AACnB,MAAI,KAAK,OAAO,MAAM,YAAY,UAAU,KAAK,EAAE,QAAQ,SACzD,OAAM,IAAI,mBAAmB,oBAAoB,KAAK,gBAAgB;AACxE,QAAM;WACE;AACR,QAAM,GAAG,OAAO;;;;;AAMpB,eAAe,sBAAsB,GAAkB,GAAW,KAAa,UAAkB;CAC/F,IAAI,iBAAiB;AACrB,QAAO,iBAAiB,KAAK;EAC3B,MAAM,EAAE,cAAc,MAAM,EAAE,KAC5B,GACA,gBACA,MAAM,gBACN,WAAW,eACZ;AACD,MAAI,cAAc,EAChB,OAAM,IAAI,cAAc,oCAAoC;AAE9D,oBAAkB;;AAGpB,QAAO;;AAGT,eAAe,mBAAmB,MAAc,mBAA2B;CACzE,MAAM,QAAQ,OAAO,KAAK,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,UAAU,IAAK,CAAC;AACtE,KAAI,QAAQ,kBACV,OAAM,IAAI,WAAW,sCAAsC,kBAAkB,SAAS,MAAM,GAAG;;;;;AAOnG,MAAM,2BAA2B;CAAC;CAAgB;CAAkB;CAAuB;AAE3F,SAAS,mBAAmB,MAAuB;AACjD,QAAO,yBAAyB,MAAM,SAAS,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;;AAGvF,SAAS,kBACP,YACA,MACA,SACA,MACA;AACA,KAAI,cAAc,KAAK;AACrB,MAAI,mBAAmB,KAAK,CAC1B,OAAM,IAAI,aACR,+CAA+C,WAAW,UAC9C,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;AAEH,QAAM,IAAI,gBACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;AAGH,KAAI,cAAc,IAChB,OAAM,IAAI,aACR,oCAAoC,WAAW,UACnC,KAAK,aAAa,KAAK,UAAU,QAAQ,CAAC,SAAS,KAAK,YACrE;;;AAKL,SAAS,wBAAwB,MAAsB;CACrD,MAAM,WAAW,OAAO,KAAK;CAE7B,MAAM,SAAS,OAAO,MAAM,EAAE;AAE9B,QAAO,cAAc,UAAU,EAAE;AACjC,QAAO,OAAO,SAAS,SAAS"}

package/dist/drivers/download_blob/blob_key.cjs CHANGED Viewed

@@ -1,10 +1,11 @@
 const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
-require("@milaboratories/pl-client");
+let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let node_path = require("node:path");
 node_path = require_runtime.__toESM(node_path);
 //#region src/drivers/download_blob/blob_key.ts
 function blobKey(rId) {
-	return `${BigInt(rId)}`;
+	const { globalId } = (0, _milaboratories_pl_client.parseSignedResourceId)(rId);
+	return `${globalId}`;
 }
 function pathToKey(fPath) {
 	return node_path.basename(fPath);

package/dist/drivers/download_blob/blob_key.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"blob_key.cjs","names":["path"],"sources":["../../../src/drivers/download_blob/blob_key.ts"],"sourcesContent":["import { ~~bigintToResourceId~~, ~~ResourceId~~ } from \"@milaboratories/pl-client\";\nimport * as path from \"node:path\";\n\nexport function blobKey(rId: ~~ResourceId~~): string {\n ~~return~~ `${~~BigInt(rId)~~}~~`;\n}\n\nexport~~ ~~function~~ ~~pathToKey~~(~~fPath: string~~)~~: string {\~~n return ~~path.basename(fPath);\~~n}\n\nexport function ~~pathToBlobInfo~~(fPath: string): ~~ResourceId~~ ~~\| undefined~~ {\n ~~const~~ ~~base =~~ path.basename(fPath);\n ~~return bigintToResourceId(BigInt(base));\n~~}\n"],"mappings":";;;;;AAGA,SAAgB,QAAQ,~~KAAyB~~;AAC/C,QAAO,GAAG~~,OAAO,IAAI~~;;~~AAGvB~~,SAAgB,UAAU,OAAuB;AAC/C,QAAOA,UAAK,SAAS,MAAM"}
1	+ {"version":3,"file":"blob_key.cjs","names":["path"],"sources":["../../../src/drivers/download_blob/blob_key.ts"],"sourcesContent":["import { parseSignedResourceId, SignedResourceId } from \"@milaboratories/pl-client\";\nimport * as path from \"node:path\";\n\nexport function blobKey(rId: SignedResourceId): string {\n const { globalId } = parseSignedResourceId(rId);\n return `${globalId}`;\n}\n\nexport function pathToKey(fPath: string): string {\n return path.basename(fPath);\n}\n"],"mappings":";;;;;AAGA,SAAgB,QAAQ,KAA+B;CACrD,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI;AAC/C,QAAO,GAAG;;AAGZ,SAAgB,UAAU,OAAuB;AAC/C,QAAOA,UAAK,SAAS,MAAM"}

package/dist/drivers/download_blob/blob_key.js CHANGED Viewed

@@ -1,8 +1,9 @@
-import "@milaboratories/pl-client";
+import { parseSignedResourceId } from "@milaboratories/pl-client";
 import * as path$1 from "node:path";
 //#region src/drivers/download_blob/blob_key.ts
 function blobKey(rId) {
-	return `${BigInt(rId)}`;
+	const { globalId } = parseSignedResourceId(rId);
+	return `${globalId}`;
 }
 function pathToKey(fPath) {
 	return path$1.basename(fPath);

package/dist/drivers/download_blob/blob_key.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"blob_key.js","names":["path"],"sources":["../../../src/drivers/download_blob/blob_key.ts"],"sourcesContent":["import { ~~bigintToResourceId~~, ~~ResourceId~~ } from \"@milaboratories/pl-client\";\nimport * as path from \"node:path\";\n\nexport function blobKey(rId: ~~ResourceId~~): string {\n ~~return~~ `${~~BigInt(rId)~~}~~`;\n}\n\nexport~~ ~~function~~ ~~pathToKey~~(~~fPath: string~~)~~: string {\~~n return ~~path.basename(fPath);\~~n}\n\nexport function ~~pathToBlobInfo~~(fPath: string): ~~ResourceId~~ ~~\| undefined~~ {\n ~~const~~ ~~base =~~ path.basename(fPath);\n ~~return bigintToResourceId(BigInt(base));\n~~}\n"],"mappings":";;;AAGA,SAAgB,QAAQ,~~KAAyB~~;AAC/C,QAAO,GAAG~~,OAAO,IAAI~~;;~~AAGvB~~,SAAgB,UAAU,OAAuB;AAC/C,QAAOA,OAAK,SAAS,MAAM"}
1	+ {"version":3,"file":"blob_key.js","names":["path"],"sources":["../../../src/drivers/download_blob/blob_key.ts"],"sourcesContent":["import { parseSignedResourceId, SignedResourceId } from \"@milaboratories/pl-client\";\nimport * as path from \"node:path\";\n\nexport function blobKey(rId: SignedResourceId): string {\n const { globalId } = parseSignedResourceId(rId);\n return `${globalId}`;\n}\n\nexport function pathToKey(fPath: string): string {\n return path.basename(fPath);\n}\n"],"mappings":";;;AAGA,SAAgB,QAAQ,KAA+B;CACrD,MAAM,EAAE,aAAa,sBAAsB,IAAI;AAC/C,QAAO,GAAG;;AAGZ,SAAgB,UAAU,OAAuB;AAC/C,QAAOA,OAAK,SAAS,MAAM"}

package/dist/drivers/download_blob/download_blob.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"download_blob.cjs","names":["FilesCache","SparseCacheFsRanges","SparseCacheFsFile","SparseCache","TaskProcessor","path","Computable","blobKey","DownloadBlobTask","newLocalHandle","nonRecoverableError","OnDemandBlobResourceSnapshot","getSize","newRemoteHandle","parseLocalHandle","isOffByOneError","isLocalBlobHandle","withFileContent","isRemoteBlobHandle","parseRemoteHandle","newLogHandle","getResourceInfoFromLogHandle","fsp","pathToKey","CallersCounter","ChangeSource","Updater","readline","Denque","os","WrongResourceTypeError"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n AnyLogHandle,\n BlobDriver,\n ContentHandler,\n GetContentOptions,\n LocalBlobHandle,\n LocalBlobHandleAndSize,\n ReadyLogHandle,\n RemoteBlobHandle,\n RemoteBlobHandleAndSize,\n StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n makeResourceSnapshot,\n treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n isLocalBlobHandle,\n newLocalHandle,\n parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n isRemoteBlobHandle,\n newRemoteHandle,\n parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n /*\n A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /\n A hard limit of the amount of sparse cache, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one.\n \n The sparse cache is used to store ranges of blobs.\n * /\n rangesCacheMaxSizeBytes: number;\n\n /\n Max number of concurrent downloads while calculating computable states\n * derived from this driver\n * /\n nConcurrentDownloads: number;\n};\n\n/* DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. /\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n /* Represents a unique key to the path of a blob as a map. /\n private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadBlobTask>;\n private rangesCache: SparseCache;\n\n /* Downloads files and writes them to the local dir. /\n private downloadQueue: TaskProcessor;\n\n private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n private idToLastLines: Map<string, LastLinesGetter> = new Map();\n private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n private readonly saveDir: string;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientDownload: ClientDownload,\n private readonly clientLogs: ClientLogs,\n saveDir: string,\n private readonly rangesCacheDir: string,\n private readonly signer: Signer,\n private readonly ops: DownloadDriverOps,\n ) {\n this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n this.rangesCache = new SparseCache(\n this.logger,\n this.ops.rangesCacheMaxSizeBytes,\n fsRanges,\n fsStorage,\n );\n\n this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n this.saveDir = path.resolve(saveDir);\n }\n\n static async init(\n logger: MiLogger,\n clientDownload: ClientDownload,\n clientLogs: ClientLogs,\n saveDir: string,\n rangesCacheDir: string,\n signer: Signer,\n ops: DownloadDriverOps,\n ): Promise<DownloadDriver> {\n const driver = new DownloadDriver(\n logger,\n clientDownload,\n clientLogs,\n saveDir,\n rangesCacheDir,\n signer,\n ops,\n );\n await driver.rangesCache.reset();\n\n return driver;\n }\n\n /* Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. /\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx: ComputableCtx,\n ): LocalBlobHandleAndSize \| undefined;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ): ComputableStableDefined<LocalBlobHandleAndSize>;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<LocalBlobHandleAndSize \| undefined> \| LocalBlobHandleAndSize \| undefined {\n if (ctx === undefined) {\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n }\n\n const rInfo = treeEntryToResourceInfo(res, ctx);\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n if (result == undefined) {\n ctx.markUnstable(\"download blob is still undefined\");\n }\n\n return result;\n }\n\n private getDownloadedBlobNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n callerId: string,\n ): LocalBlobHandleAndSize \| undefined {\n validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n // We don't need to request files with wider limits,\n // PFrame's engine does it disk-optimally by itself.\n\n const task = this.getOrSetNewTask(rInfo, callerId);\n task.attach(w, callerId);\n\n const result = task.getBlob();\n if (!result.done) {\n return undefined;\n }\n if (result.result.ok) {\n return result.result.value;\n }\n throw result.result.error;\n }\n\n private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n const key = blobKey(rInfo.id);\n\n const inMemoryTask = this.keyToDownload.get(key);\n if (inMemoryTask) {\n return inMemoryTask;\n }\n\n // schedule the blob downloading, then it'll be added to the cache.\n const fPath = path.resolve(this.saveDir, key);\n\n const newTask = new DownloadBlobTask(\n this.logger,\n this.clientDownload,\n rInfo,\n newLocalHandle(fPath, this.signer),\n fPath,\n );\n this.keyToDownload.set(key, newTask);\n\n this.downloadQueue.push({\n fn: () => this.downloadBlob(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask;\n }\n\n private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n await task.download();\n const blob = task.getBlob();\n if (blob.done && blob.result.ok) {\n this.cache.addCache(task, callerId);\n }\n }\n\n /* Gets on demand blob. /\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: undefined,\n fromBytes?: number,\n toBytes?: number,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n fromBytes?: number,\n toBytes?: number,\n ): RemoteBlobHandleAndSize;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): ComputableStableDefined<RemoteBlobHandleAndSize> \| RemoteBlobHandleAndSize \| undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n : res;\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n // note that the watcher is not needed,\n // the handler never changes.\n const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n return result;\n }\n\n private getOnDemandBlobNoCtx(\n info: OnDemandBlobResourceSnapshot,\n callerId: string,\n ): RemoteBlobHandleAndSize {\n validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n if (blob === undefined) {\n blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n this.keyToOnDemand.set(blobKey(info.id), blob);\n }\n\n blob.attach(callerId);\n\n return blob.getHandle();\n }\n\n /* Gets a path from a handle. /\n public getLocalPath(handle: LocalBlobHandle): string {\n const { path } = parseLocalHandle(handle, this.signer);\n return path;\n }\n\n /* Gets a content of a blob by a handle. /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array>;\n /* @deprecated Use {@link getContent} with {@link GetContentOptions} instead /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n range?: RangeBytes,\n ): Promise<Uint8Array>;\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n optionsOrRange?: GetContentOptions \| RangeBytes,\n ): Promise<Uint8Array> {\n let options: GetContentOptions = {};\n if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n if (\"range\" in optionsOrRange) {\n options = optionsOrRange;\n } else {\n const range = optionsOrRange as RangeBytes;\n validateRangeBytes(range, `getContent`);\n options = { range };\n }\n }\n\n const request = () =>\n this.withContent(handle, {\n ...options,\n handler: async (content) => {\n const chunks: Uint8Array[] = [];\n for await (const chunk of content) {\n options.signal?.throwIfAborted();\n chunks.push(chunk);\n }\n return Buffer.concat(chunks);\n },\n });\n\n try {\n return await request();\n } catch (error) {\n if (isOffByOneError(error)) {\n return await request();\n }\n throw error;\n }\n }\n\n /* Gets a content stream of a blob by a handle and calls handler with it. /\n public async withContent<T>(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options: GetContentOptions & {\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n const { range, signal, handler } = options;\n\n if (isLocalBlobHandle(handle)) {\n return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n }\n\n if (isRemoteBlobHandle(handle)) {\n const result = parseRemoteHandle(handle, this.signer);\n\n const key = blobKey(result.info.id);\n const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n signal?.throwIfAborted();\n\n if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n return await this.clientDownload.withBlobContent(\n result.info,\n { signal },\n options,\n async (content, size) => {\n const [handlerStream, cacheStream] = content.tee();\n\n const handlerPromise = handler(handlerStream, size);\n const _cachePromise = buffer(cacheStream)\n .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n .catch(() => {\n // Ignore cache errors - they shouldn't affect the main handler result\n // This prevents unhandled promise rejections when the stream fails\n });\n\n return await handlerPromise;\n },\n );\n }\n\n throw new Error(\"Malformed remote handle\");\n }\n\n /\n Creates computable that will return blob content once it is downloaded.\n * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n /\n public getComputableContent(\n res: ResourceInfo \| PlTreeEntry,\n range?: RangeBytes,\n ): ComputableStableDefined<Uint8Array> {\n if (range) {\n validateRangeBytes(range, `getComputableContent`);\n }\n\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n }).withStableType();\n }\n\n /* Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. /\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n if (result == undefined)\n ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, lines);\n this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ): Computable<string \| undefined>;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string \| undefined;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getProgressLogNoCtx(\n ctx.watcher,\n r as ResourceSnapshot,\n patternToSearch,\n callerId,\n );\n if (result === undefined)\n ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns an Id of a smart object, that can read logs directly from\n * the platform. /\n public getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<AnyLogHandle>;\n public getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n public getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<AnyLogHandle> \| AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n return this.getLogHandleNoCtx(r as ResourceSnapshot);\n }\n\n private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n return newLogHandle(false, rInfo);\n }\n\n public async lastLines(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number, // if 0n, then start from the end.\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n public async readText(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n const task = this.keyToDownload.get(blobKey(rInfo.id));\n if (task == undefined) {\n return;\n }\n\n if (this.cache.existsFile(blobKey(rInfo.id))) {\n const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n await Promise.all(\n toDelete.map(async (cachedFile) => {\n await fsp.rm(cachedFile.path);\n\n this.cache.removeCache(cachedFile);\n\n this.removeTask(\n mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted) {\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n }\n\n private removeTask(task: DownloadBlobTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n this.keyToDownload.delete(pathToKey(task.path));\n this.idToLastLines.delete(blobKey(task.rInfo.id));\n this.idToProgressLog.delete(blobKey(task.rInfo.id));\n }\n\n private async releaseOnDemandBlob(blobId: ResourceId, callerId: string) {\n const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n this.keyToDownload.forEach((task, key) => {\n this.keyToDownload.delete(key);\n task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n });\n }\n\n async dispose(): Promise<void> {\n await this.rangesCache.dispose();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.dispose();\n }\n}\n\n/* Keeps a counter to the on demand handle. /\nclass OnDemandBlobHolder {\n private readonly counter = new CallersCounter();\n\n constructor(\n private readonly size: number,\n private readonly handle: RemoteBlobHandle,\n ) {}\n\n public getHandle(): RemoteBlobHandleAndSize {\n return { handle: this.handle, size: this.size };\n }\n\n public attach(callerId: string) {\n this.counter.inc(callerId);\n }\n\n public release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n}\n\nclass LastLinesGetter {\n private updater: Updater;\n private log: string \| undefined;\n private readonly change: ChangeSource = new ChangeSource();\n private error: any \| undefined = undefined;\n\n constructor(\n private readonly path: string,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {\n this.updater = new Updater(async () => this.update());\n }\n\n getOrSchedule(w: Watcher): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n this.change.attachWatcher(w);\n\n this.updater.schedule();\n\n return {\n log: this.log,\n error: this.error,\n };\n }\n\n async update(): Promise<void> {\n try {\n const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n this.log = newLogs;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.log = \"\";\n this.error = e;\n this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n return;\n }\n\n throw e;\n }\n }\n}\n\n/* Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n fPath: string,\n nLines: number,\n patternToSearch?: string,\n): Promise<string> {\n let inStream: fs.ReadStream \| undefined;\n let rl: readline.Interface \| undefined;\n\n try {\n inStream = fs.createReadStream(fPath);\n rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n const lines = new Denque();\n\n for await (const line of rl) {\n if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n lines.push(line);\n if (lines.length > nLines) {\n lines.shift();\n }\n }\n\n // last EOL is for keeping backward compat with platforma implementation.\n return lines.toArray().join(os.EOL) + os.EOL;\n } finally {\n // Cleanup resources in finally block to ensure they're always cleaned up\n try {\n if (rl) {\n rl.close();\n }\n } catch (cleanupError) {\n console.error(\"Error closing readline interface:\", cleanupError);\n }\n\n try {\n if (inStream && !inStream.destroyed) {\n inStream.destroy();\n }\n } catch (cleanupError) {\n console.error(\"Error destroying read stream:\", cleanupError);\n }\n }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"Blob/\")) {\n let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n if (rType.name == \"Blob\")\n message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n throw new WrongResourceTypeError(message);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,gCAAuD,IAAI,KAAK;;;CAIhE;CACA;;CAGA;CAEA,gCAAyD,IAAI,KAAK;CAElE,gCAAsD,IAAI,KAAK;CAC/D,kCAAwD,IAAI,KAAK;CAEjE;CAEA,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;AAPiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,aAAA;AAEA,OAAA,iBAAA;AACA,OAAA,SAAA;AACA,OAAA,MAAA;AAEjB,OAAK,QAAQ,IAAIA,oBAAAA,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAIC,cAAAA,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAIC,cAAAA,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAIC,cAAAA,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAIC,2BAAAA,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUC,UAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,kBACE,KACA,KACqF;AACrF,MAAI,QAAQ,KAAA,EACV,QAAOC,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,SAAA,GAAA,wBAAA,yBAAgC,KAAK,IAAI;EAE/C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,uBACE,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAMC,iBAAAA,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQF,UAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAIG,2BAAAA,iBAClB,KAAK,QACL,KAAK,gBACL,OACAC,8BAAAA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,gBACE,KACA,KACwF;AACxF,MAAI,QAAQ,KAAA,EAAW,QAAOJ,2BAAAA,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,SAAA,GAAA,wBAAA,eAAoD,IAAI,IAAA,GAAA,wBAAA,sBACrC,KAAKK,cAAAA,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,qBACE,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAIJ,iBAAAA,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,KAAA,GAAW;AACtB,UAAO,IAAI,mBAAmBK,cAAAA,QAAQ,KAAK,EAAEC,+BAAAA,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAIN,iBAAAA,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAASO,8BAAAA,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,IAAA,GAAA,gCAAA,oBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;EAIvB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAIC,wBAAAA,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAGY;EACZ,MAAM,EAAE,OAAO,QAAQ,YAAY;AAEnC,MAAIC,8BAAAA,kBAAkB,OAAO,CAC3B,QAAO,MAAMC,kBAAAA,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAIC,+BAAAA,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAASC,+BAAAA,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAMZ,iBAAAA,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AACvF,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAMU,kBAAAA,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;IACvB,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,KAAA,GAAA,sBAAA,QAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,qBACE,KACA,OACqC;AACrC,MAAI,MACF,EAAA,GAAA,gCAAA,oBAAmB,OAAO,uBAAuB;AAGnD,SAAOX,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAA,GACtE,CAAC,CAAC,gBAAgB;;CAcrB,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAE9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAIP,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAIA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,KAAA,EACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAC9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAIP,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAIA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,aACE,KACA,KACyC;AACzC,MAAI,OAAO,KAAA,EAAW,QAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,kBAA0B,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAOc,oBAAAA,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjCC,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjCA,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAId,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,KAAA,EACV;AAGF,MAAI,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAMe,iBAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,YAAA,GAAA,2BAAA,QACI,KAAK,eAAeC,iBAAAA,UAAU,WAAW,KAAK,CAAC,EACtD,aAAA,GAAA,0BAAA,yBAAoC,WAAW,CAAC,qCAAA,GAAA,0BAAA,yBACG,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,WAAmB,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAOA,iBAAAA,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAOhB,iBAAAA,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAOA,iBAAAA,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAAoB,UAAkB;AAEtE,MADgB,KAAK,cAAc,IAAIA,iBAAAA,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAOA,iBAAAA,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,SAAA,GAAA,0BAAA,oBAA2B,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,UAA2B,IAAIiB,2BAAAA,gBAAgB;CAE/C,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,YAA4C;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,OAAc,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAe,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB;CACA;CACA,SAAwC,IAAIC,2BAAAA,cAAc;CAC1D,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;AAHiB,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;AAEjB,OAAK,UAAU,IAAIC,gBAAAA,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,QAAA,GAAA,0BAAA,iBAAoB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAW,GAAG,iBAAiB,MAAM;AACrC,OAAKC,uBAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAIC,OAAAA,SAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAKC,QAAG,IAAI,GAAGA,QAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAIC,gBAAAA,uBAAuB,QAAQ"}
1	+ {"version":3,"file":"download_blob.cjs","names":["FilesCache","SparseCacheFsRanges","SparseCacheFsFile","SparseCache","TaskProcessor","path","Computable","blobKey","DownloadBlobTask","newLocalHandle","nonRecoverableError","OnDemandBlobResourceSnapshot","getSize","newRemoteHandle","parseLocalHandle","isOffByOneError","isLocalBlobHandle","withFileContent","isRemoteBlobHandle","parseRemoteHandle","newLogHandle","getResourceInfoFromLogHandle","fsp","pathToKey","CallersCounter","ChangeSource","Updater","readline","Denque","os","WrongResourceTypeError"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n AnyLogHandle,\n BlobDriver,\n ContentHandler,\n GetContentOptions,\n LocalBlobHandle,\n LocalBlobHandleAndSize,\n ReadyLogHandle,\n RemoteBlobHandle,\n RemoteBlobHandleAndSize,\n StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n makeResourceSnapshot,\n treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n isLocalBlobHandle,\n newLocalHandle,\n parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n isRemoteBlobHandle,\n newRemoteHandle,\n parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n /*\n A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /\n A hard limit of the amount of sparse cache, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one.\n \n The sparse cache is used to store ranges of blobs.\n * /\n rangesCacheMaxSizeBytes: number;\n\n /\n Max number of concurrent downloads while calculating computable states\n * derived from this driver\n * /\n nConcurrentDownloads: number;\n};\n\n/* DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. /\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n /* Represents a unique key to the path of a blob as a map. /\n private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadBlobTask>;\n private rangesCache: SparseCache;\n\n /* Downloads files and writes them to the local dir. /\n private downloadQueue: TaskProcessor;\n\n private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n private idToLastLines: Map<string, LastLinesGetter> = new Map();\n private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n private readonly saveDir: string;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientDownload: ClientDownload,\n private readonly clientLogs: ClientLogs,\n saveDir: string,\n private readonly rangesCacheDir: string,\n private readonly signer: Signer,\n private readonly ops: DownloadDriverOps,\n ) {\n this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n this.rangesCache = new SparseCache(\n this.logger,\n this.ops.rangesCacheMaxSizeBytes,\n fsRanges,\n fsStorage,\n );\n\n this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n this.saveDir = path.resolve(saveDir);\n }\n\n static async init(\n logger: MiLogger,\n clientDownload: ClientDownload,\n clientLogs: ClientLogs,\n saveDir: string,\n rangesCacheDir: string,\n signer: Signer,\n ops: DownloadDriverOps,\n ): Promise<DownloadDriver> {\n const driver = new DownloadDriver(\n logger,\n clientDownload,\n clientLogs,\n saveDir,\n rangesCacheDir,\n signer,\n ops,\n );\n await driver.rangesCache.reset();\n\n return driver;\n }\n\n /* Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. /\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx: ComputableCtx,\n ): LocalBlobHandleAndSize \| undefined;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ): ComputableStableDefined<LocalBlobHandleAndSize>;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<LocalBlobHandleAndSize \| undefined> \| LocalBlobHandleAndSize \| undefined {\n if (ctx === undefined) {\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n }\n\n const rInfo = treeEntryToResourceInfo(res, ctx);\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n if (result == undefined) {\n ctx.markUnstable(\"download blob is still undefined\");\n }\n\n return result;\n }\n\n private getDownloadedBlobNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n callerId: string,\n ): LocalBlobHandleAndSize \| undefined {\n validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n // We don't need to request files with wider limits,\n // PFrame's engine does it disk-optimally by itself.\n\n const task = this.getOrSetNewTask(rInfo, callerId);\n task.attach(w, callerId);\n\n const result = task.getBlob();\n if (!result.done) {\n return undefined;\n }\n if (result.result.ok) {\n return result.result.value;\n }\n throw result.result.error;\n }\n\n private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n const key = blobKey(rInfo.id);\n\n const inMemoryTask = this.keyToDownload.get(key);\n if (inMemoryTask) {\n return inMemoryTask;\n }\n\n // schedule the blob downloading, then it'll be added to the cache.\n const fPath = path.resolve(this.saveDir, key);\n\n const newTask = new DownloadBlobTask(\n this.logger,\n this.clientDownload,\n rInfo,\n newLocalHandle(fPath, this.signer),\n fPath,\n );\n this.keyToDownload.set(key, newTask);\n\n this.downloadQueue.push({\n fn: () => this.downloadBlob(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask;\n }\n\n private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n await task.download();\n const blob = task.getBlob();\n if (blob.done && blob.result.ok) {\n this.cache.addCache(task, callerId);\n }\n }\n\n /* Gets on demand blob. /\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: undefined,\n fromBytes?: number,\n toBytes?: number,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n fromBytes?: number,\n toBytes?: number,\n ): RemoteBlobHandleAndSize;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): ComputableStableDefined<RemoteBlobHandleAndSize> \| RemoteBlobHandleAndSize \| undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n : res;\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n // note that the watcher is not needed,\n // the handler never changes.\n const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n return result;\n }\n\n private getOnDemandBlobNoCtx(\n info: OnDemandBlobResourceSnapshot,\n callerId: string,\n ): RemoteBlobHandleAndSize {\n validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n if (blob === undefined) {\n blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n this.keyToOnDemand.set(blobKey(info.id), blob);\n }\n\n blob.attach(callerId);\n\n return blob.getHandle();\n }\n\n /* Gets a path from a handle. /\n public getLocalPath(handle: LocalBlobHandle): string {\n const { path } = parseLocalHandle(handle, this.signer);\n return path;\n }\n\n /* Gets a content of a blob by a handle. /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array>;\n /* @deprecated Use {@link getContent} with {@link GetContentOptions} instead /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n range?: RangeBytes,\n ): Promise<Uint8Array>;\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n optionsOrRange?: GetContentOptions \| RangeBytes,\n ): Promise<Uint8Array> {\n let options: GetContentOptions = {};\n if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n if (\"range\" in optionsOrRange) {\n options = optionsOrRange;\n } else {\n const range = optionsOrRange as RangeBytes;\n validateRangeBytes(range, `getContent`);\n options = { range };\n }\n }\n\n const request = () =>\n this.withContent(handle, {\n ...options,\n handler: async (content) => {\n const chunks: Uint8Array[] = [];\n for await (const chunk of content) {\n options.signal?.throwIfAborted();\n chunks.push(chunk);\n }\n return Buffer.concat(chunks);\n },\n });\n\n try {\n return await request();\n } catch (error) {\n if (isOffByOneError(error)) {\n return await request();\n }\n throw error;\n }\n }\n\n /* Gets a content stream of a blob by a handle and calls handler with it. /\n public async withContent<T>(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options: GetContentOptions & {\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n const { range, signal, handler } = options;\n\n if (isLocalBlobHandle(handle)) {\n return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n }\n\n if (isRemoteBlobHandle(handle)) {\n const result = parseRemoteHandle(handle, this.signer);\n\n const key = blobKey(result.info.id);\n const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n signal?.throwIfAborted();\n\n if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n return await this.clientDownload.withBlobContent(\n result.info,\n { signal },\n options,\n async (content, size) => {\n const [handlerStream, cacheStream] = content.tee();\n\n const handlerPromise = handler(handlerStream, size);\n const _cachePromise = buffer(cacheStream)\n .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n .catch(() => {\n // Ignore cache errors - they shouldn't affect the main handler result\n // This prevents unhandled promise rejections when the stream fails\n });\n\n return await handlerPromise;\n },\n );\n }\n\n throw new Error(\"Malformed remote handle\");\n }\n\n /\n Creates computable that will return blob content once it is downloaded.\n * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n /\n public getComputableContent(\n res: ResourceInfo \| PlTreeEntry,\n range?: RangeBytes,\n ): ComputableStableDefined<Uint8Array> {\n if (range) {\n validateRangeBytes(range, `getComputableContent`);\n }\n\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n }).withStableType();\n }\n\n /* Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. /\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n if (result == undefined)\n ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, lines);\n this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ): Computable<string \| undefined>;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string \| undefined;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getProgressLogNoCtx(\n ctx.watcher,\n r as ResourceSnapshot,\n patternToSearch,\n callerId,\n );\n if (result === undefined)\n ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns an Id of a smart object, that can read logs directly from\n * the platform. /\n public getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<AnyLogHandle>;\n public getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n public getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<AnyLogHandle> \| AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n return this.getLogHandleNoCtx(r as ResourceSnapshot);\n }\n\n private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n return newLogHandle(false, rInfo);\n }\n\n public async lastLines(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number, // if 0n, then start from the end.\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n public async readText(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n const task = this.keyToDownload.get(blobKey(rInfo.id));\n if (task == undefined) {\n return;\n }\n\n if (this.cache.existsFile(blobKey(rInfo.id))) {\n const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n await Promise.all(\n toDelete.map(async (cachedFile) => {\n await fsp.rm(cachedFile.path);\n\n this.cache.removeCache(cachedFile);\n\n this.removeTask(\n mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted) {\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n }\n\n private removeTask(task: DownloadBlobTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n this.keyToDownload.delete(pathToKey(task.path));\n this.idToLastLines.delete(blobKey(task.rInfo.id));\n this.idToProgressLog.delete(blobKey(task.rInfo.id));\n }\n\n private async releaseOnDemandBlob(blobId: SignedResourceId, callerId: string) {\n const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n this.keyToDownload.forEach((task, key) => {\n this.keyToDownload.delete(key);\n task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n });\n }\n\n async dispose(): Promise<void> {\n await this.rangesCache.dispose();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.dispose();\n }\n}\n\n/* Keeps a counter to the on demand handle. /\nclass OnDemandBlobHolder {\n private readonly counter = new CallersCounter();\n\n constructor(\n private readonly size: number,\n private readonly handle: RemoteBlobHandle,\n ) {}\n\n public getHandle(): RemoteBlobHandleAndSize {\n return { handle: this.handle, size: this.size };\n }\n\n public attach(callerId: string) {\n this.counter.inc(callerId);\n }\n\n public release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n}\n\nclass LastLinesGetter {\n private updater: Updater;\n private log: string \| undefined;\n private readonly change: ChangeSource = new ChangeSource();\n private error: any \| undefined = undefined;\n\n constructor(\n private readonly path: string,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {\n this.updater = new Updater(async () => this.update());\n }\n\n getOrSchedule(w: Watcher): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n this.change.attachWatcher(w);\n\n this.updater.schedule();\n\n return {\n log: this.log,\n error: this.error,\n };\n }\n\n async update(): Promise<void> {\n try {\n const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n this.log = newLogs;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.log = \"\";\n this.error = e;\n this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n return;\n }\n\n throw e;\n }\n }\n}\n\n/* Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n fPath: string,\n nLines: number,\n patternToSearch?: string,\n): Promise<string> {\n let inStream: fs.ReadStream \| undefined;\n let rl: readline.Interface \| undefined;\n\n try {\n inStream = fs.createReadStream(fPath);\n rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n const lines = new Denque();\n\n for await (const line of rl) {\n if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n lines.push(line);\n if (lines.length > nLines) {\n lines.shift();\n }\n }\n\n // last EOL is for keeping backward compat with platforma implementation.\n return lines.toArray().join(os.EOL) + os.EOL;\n } finally {\n // Cleanup resources in finally block to ensure they're always cleaned up\n try {\n if (rl) {\n rl.close();\n }\n } catch (cleanupError) {\n console.error(\"Error closing readline interface:\", cleanupError);\n }\n\n try {\n if (inStream && !inStream.destroyed) {\n inStream.destroy();\n }\n } catch (cleanupError) {\n console.error(\"Error destroying read stream:\", cleanupError);\n }\n }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"Blob/\")) {\n let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n if (rType.name == \"Blob\")\n message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n throw new WrongResourceTypeError(message);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,gCAAuD,IAAI,KAAK;;;CAIhE;CACA;;CAGA;CAEA,gCAAyD,IAAI,KAAK;CAElE,gCAAsD,IAAI,KAAK;CAC/D,kCAAwD,IAAI,KAAK;CAEjE;CAEA,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;AAPiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,aAAA;AAEA,OAAA,iBAAA;AACA,OAAA,SAAA;AACA,OAAA,MAAA;AAEjB,OAAK,QAAQ,IAAIA,oBAAAA,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAIC,cAAAA,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAIC,cAAAA,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAIC,cAAAA,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAIC,2BAAAA,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUC,UAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,kBACE,KACA,KACqF;AACrF,MAAI,QAAQ,KAAA,EACV,QAAOC,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,SAAA,GAAA,wBAAA,yBAAgC,KAAK,IAAI;EAE/C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,uBACE,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAMC,iBAAAA,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQF,UAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAIG,2BAAAA,iBAClB,KAAK,QACL,KAAK,gBACL,OACAC,8BAAAA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,gBACE,KACA,KACwF;AACxF,MAAI,QAAQ,KAAA,EAAW,QAAOJ,2BAAAA,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,SAAA,GAAA,wBAAA,eAAoD,IAAI,IAAA,GAAA,wBAAA,sBACrC,KAAKK,cAAAA,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,qBACE,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAIJ,iBAAAA,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,KAAA,GAAW;AACtB,UAAO,IAAI,mBAAmBK,cAAAA,QAAQ,KAAK,EAAEC,+BAAAA,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAIN,iBAAAA,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAASO,8BAAAA,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,IAAA,GAAA,gCAAA,oBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;EAIvB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAIC,wBAAAA,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAGY;EACZ,MAAM,EAAE,OAAO,QAAQ,YAAY;AAEnC,MAAIC,8BAAAA,kBAAkB,OAAO,CAC3B,QAAO,MAAMC,kBAAAA,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAIC,+BAAAA,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAASC,+BAAAA,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAMZ,iBAAAA,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AACvF,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAMU,kBAAAA,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;IACvB,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,KAAA,GAAA,sBAAA,QAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,qBACE,KACA,OACqC;AACrC,MAAI,MACF,EAAA,GAAA,gCAAA,oBAAmB,OAAO,uBAAuB;AAGnD,SAAOX,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAA,GACtE,CAAC,CAAC,gBAAgB;;CAcrB,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAE9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAIP,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAIA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,KAAA,EACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAC9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAIP,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAIA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,aACE,KACA,KACyC;AACzC,MAAI,OAAO,KAAA,EAAW,QAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,kBAA0B,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAOc,oBAAAA,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjCC,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjCA,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAId,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,KAAA,EACV;AAGF,MAAI,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAMe,iBAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,YAAA,GAAA,2BAAA,QACI,KAAK,eAAeC,iBAAAA,UAAU,WAAW,KAAK,CAAC,EACtD,aAAA,GAAA,0BAAA,yBAAoC,WAAW,CAAC,qCAAA,GAAA,0BAAA,yBACG,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,WAAmB,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAOA,iBAAAA,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAOhB,iBAAAA,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAOA,iBAAAA,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAA0B,UAAkB;AAE5E,MADgB,KAAK,cAAc,IAAIA,iBAAAA,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAOA,iBAAAA,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,SAAA,GAAA,0BAAA,oBAA2B,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,UAA2B,IAAIiB,2BAAAA,gBAAgB;CAE/C,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,YAA4C;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,OAAc,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAe,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB;CACA;CACA,SAAwC,IAAIC,2BAAAA,cAAc;CAC1D,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;AAHiB,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;AAEjB,OAAK,UAAU,IAAIC,gBAAAA,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,QAAA,GAAA,0BAAA,iBAAoB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAW,GAAG,iBAAiB,MAAM;AACrC,OAAKC,uBAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAIC,OAAAA,SAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAKC,QAAG,IAAI,GAAGA,QAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAIC,gBAAAA,uBAAuB,QAAQ"}