@milaboratories/pl-drivers 1.11.5 → 1.11.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/dist/drivers/upload_task.cjs +1 -1
  2. package/dist/drivers/upload_task.cjs.map +1 -1
  3. package/dist/drivers/upload_task.js +1 -1
  4. package/dist/drivers/upload_task.js.map +1 -1
  5. package/package.json +4 -4
  6. package/src/drivers/upload_task.ts +1 -1
package/dist/drivers/upload_task.cjs CHANGED
@@ -99,7 +99,7 @@ class UploadTask {
99
99
  catch (e) {
100
100
  this.setRetriableError(e);
101
101
  if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {
102
- this.logger.warn(`deadline exceeded while getting a status of BlobImport`);
102
+ this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);
103
103
  return;
104
104
  }
105
105
  if (isResourceWasDeletedError(e)) {
package/dist/drivers/upload_task.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"upload_task.cjs","sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientProgress, ProgressStatus } from '../clients/progress';\nimport type { ClientUpload } from '../clients/upload';\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';\nimport type { ImportResourceSnapshot } from './types';\nimport { ImportFileHandleUploadData } from './types';\nimport assert from 'node:assert';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n private nMaxUploads: number;\n private nPartsWithThisUploadSpeed = 0;\n private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n /** If this is upload progress this field will be defined */\n private uploadData?: ImportFileHandleUploadData;\n public progress: sdk.ImportProgress;\n\n /** If failed, then getting a progress is terminally failed. */\n public failed?: boolean;\n\n /** True if the blob was existed.\n * At this case, the task will show progress == 1.0. */\n private alreadyExisted = false;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly maxNConcurrentPartsUpload: number,\n signer: Signer,\n public readonly res: ImportResourceSnapshot,\n ) {\n this.nMaxUploads = this.maxNConcurrentPartsUpload;\n const { uploadData, progress } = newProgress(res, signer);\n this.uploadData = uploadData;\n this.progress = progress;\n }\n\n public getProgress(w: Watcher, callerId: string) {\n this.incCounter(w, callerId);\n\n if (this.failed) {\n this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n throw new Error(this.progress.lastError);\n }\n\n return cloneProgress(this.progress);\n }\n\n public shouldScheduleUpload(): boolean {\n return isMyUpload(this.progress);\n }\n\n /** Uploads a blob if it's not BlobIndex. */\n public async uploadBlobTask() {\n try {\n await uploadBlob(\n this.logger,\n this.clientBlob,\n this.res,\n this.uploadData!,\n this.isComputableDone.bind(this),\n {\n nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n currentSpeed: this.nMaxUploads,\n maxSpeed: this.maxNConcurrentPartsUpload,\n },\n );\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n } catch (e: any) {\n\n if (isResourceWasDeletedError(e)) {\n this.setRetriableError(e);\n this.logger.warn(`resource was deleted while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);\n this.setDone(true);\n\n return;\n }\n\n this.logger.error(`error while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n if (nonRecoverableError(e)) {\n this.setTerminalError(e);\n return;\n }\n\n this.setRetriableError(e);\n\n if (isHeadersTimeoutError(e)) {\n // we probably have a slow internet, we need to slow things a bit.\n this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n }\n\n throw e;\n }\n }\n\n public async updateStatus() {\n try {\n // we do it with timeout in case we have slow internet.\n const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n const oldStatus = this.progress.status;\n const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n this.progress.status = newStatus;\n this.setDone(status.done);\n\n if (status.done || this.progress.status.progress != oldStatus?.progress) {\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n }\n } catch (e: any) {\n this.setRetriableError(e);\n\n if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {\n this.logger.warn(`deadline exceeded while getting a status of BlobImport`);\n return;\n }\n\n if (isResourceWasDeletedError(e)) {\n this.logger.warn(\n `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,\n );\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed, resource not found`);\n this.setDone(true);\n return;\n }\n\n this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n // It was a terminal error, but when a connection drops,\n // this will stop the whole task, so we make it retryable.\n // It was like that:\n // this.change.markChanged();\n // this.setTerminalError(e);\n }\n }\n\n /** Set non-terminal error, that task can be retried. */\n private setRetriableError(e: unknown) {\n this.progress.lastError = String(e);\n }\n\n /** Set a terminal error, the task will throw a error instead of a progress. */\n private setTerminalError(e: unknown) {\n this.progress.lastError = String(e);\n this.progress.done = false;\n this.failed = true;\n }\n\n public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n if (isImportResourceOutputSet(res)) {\n this.setDone(true);\n this.alreadyExisted = true;\n }\n }\n\n private setDone(done: boolean) {\n this.progress.done = done;\n if (done) this.progress.lastError = undefined;\n }\n\n public incCounter(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n public decCounter(callerId: string) {\n return this.counter.dec(callerId);\n }\n\n private isComputableDone() {\n return this.counter.isZero();\n }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n logger: MiLogger,\n clientBlob: ClientUpload,\n res: ResourceInfo,\n uploadData: ImportFileHandleUploadData,\n isDoneFn: () => boolean,\n speed: {\n nPartsWithThisUploadSpeed: number,\n nPartsToIncreaseUpload: number,\n currentSpeed: number,\n maxSpeed: number,\n },\n) {\n assert(isUpload(res), 'the upload operation can be done only for BlobUploads');\n const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n if (isDoneFn()) return;\n const parts = await clientBlob.initUpload(res, { timeout });\n logger.info(\n `started to upload blob ${res.id},`\n + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`\n + ` number of concurrent uploads: ${speed.currentSpeed}`,\n );\n\n const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n if (isDoneFn()) return;\n await clientBlob.partUpload(\n res,\n uploadData.localPath,\n BigInt(uploadData.modificationTime),\n part,\n parts.checksumAlgorithm,\n parts.checksumHeader,\n { timeout }\n );\n logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n // if we had a network freeze, it will be increased slowly.\n speed.nPartsWithThisUploadSpeed++;\n if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n speed.nPartsWithThisUploadSpeed = 0;\n speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n controller.setConcurrency(speed.currentSpeed);\n }\n };\n\n await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n if (isDoneFn()) return;\n await clientBlob.finalize(res, { timeout });\n\n logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n let isUploadSignMatch: boolean | undefined;\n let uploadData: ImportFileHandleUploadData | undefined;\n if (isUpload(res)) {\n uploadData = ImportFileHandleUploadData.parse(res.data);\n isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n }\n\n return {\n uploadData: uploadData,\n progress: {\n done: false,\n status: undefined,\n isUpload: isUpload(res),\n isUploadSignMatch: isUploadSignMatch,\n lastError: undefined,\n } satisfies sdk.ImportProgress,\n };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n const cloned: sdk.ImportProgress = {\n done: progress.done,\n isUpload: progress.isUpload,\n isUploadSignMatch: progress.isUploadSignMatch,\n lastError: progress.lastError,\n };\n\n if (progress.status)\n cloned.status = {\n progress: progress.status.progress,\n bytesProcessed: progress.status.bytesProcessed,\n bytesTotal: progress.status.bytesTotal,\n };\n\n return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n return 'blob' in res.fields\n ? res.fields.blob !== undefined\n : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n return res.type.name.startsWith('BlobUpload');\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n try {\n signer.verify(path, signature);\n return true;\n } catch (e) {\n return false;\n }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n return {\n progress: proto.progress ?? 0,\n bytesProcessed: Number(proto.bytesProcessed),\n bytesTotal: Number(proto.bytesTotal),\n };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n return {\n progress: 1.0,\n bytesProcessed: Number(status.bytesTotal),\n bytesTotal: Number(status.bytesTotal),\n };\n }\n\n return status;\n}\n\nexport function isResourceWasDeletedError(e: any) {\n return (\n e.name == 'RpcError'\n && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')\n );\n}\n\nexport function nonRecoverableError(e: any) {\n return e instanceof MTimeError ||\n e instanceof UnexpectedEOF ||\n e instanceof NoFileForUploading ||\n e instanceof BadRequestError;\n}\n\nfunction isHeadersTimeoutError(e: any) {\n return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n const newConcurrency = Math.min(current + 2, max);\n if (newConcurrency != current)\n logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n const newConcurrency = Math.max(Math.round(current / 2), min);\n if (newConcurrency != current)\n logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","asyncPool","ImportFileHandleUploadData","MTimeError","UnexpectedEOF","NoFileForUploading","BadRequestError"],"mappings":";;;;;;;;;AAcA;;AAEgC;MACnB,UAAU,CAAA;AAmBF,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,cAAA;AACA,IAAA,yBAAA;AAED,IAAA,GAAA;AAvBD,IAAA,MAAM,GAAiB,IAAIA,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;AACvD,IAAA,WAAW;IACX,yBAAyB,GAAG,CAAC;AAC7B,IAAA,sBAAsB,GAAG,EAAE,CAAC;;AAG5B,IAAA,UAAU;AACX,IAAA,QAAQ;;AAGR,IAAA,MAAM;AAEb;AACuD;IAC/C,cAAc,GAAG,KAAK;IAE9B,WAAA,CACmB,MAAgB,EAChB,UAAwB,EACxB,cAA8B,EAC9B,yBAAiC,EAClD,MAAc,EACE,GAA2B,EAAA;QAL1B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,yBAAyB,GAAzB,yBAAyB;QAE1B,IAAA,CAAA,GAAG,GAAH,GAAG;AAEnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,yBAAyB;AACjD,QAAA,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;AACzD,QAAA,IAAI,CAAC,UAAU,GAAG,UAAU;AAC5B,QAAA,IAAI,CAAC,QAAQ,GAAG,QAAQ;IAC1B;IAEO,WAAW,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC7C,QAAA,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC;AAE5B,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,6BAAA,EAAgC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAA,CAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC1C;AAEA,QAAA,OAAO,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;IACrC;IAEO,oBAAoB,GAAA;AACzB,QAAA,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClC;;AAGO,IAAA,MAAM,cAAc,GAAA;AACzB,QAAA,IAAI;YACF,MAAM,UAAU,CACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,UAAW,EAChB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAChC;gBACE,yBAAyB,EAAE,IAAI,CAAC,yBAAyB;gBACzD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;gBACnD,YAAY,EAAE,IAAI,CAAC,WAAW;gBAC9B,QAAQ,EAAE,IAAI,CAAC,yBAAyB;AACzC,aAAA,CACF;AACD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBC,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,6CAAA,EAAgD,CAAC,CAAA,CAAE,CAAC;AACrE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,8BAAA,CAAgC,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAElB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,8BAAA,EAAiC,CAAC,CAAA,CAAE,CAAC;AACvD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;AAEpF,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACxB;YACF;AAEA,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;AAEzB,YAAA,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE;;AAE5B,gBAAA,IAAI,CAAC,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1E;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEO,IAAA,MAAM,YAAY,GAAA;AACvB,QAAA,IAAI;;AAEF,YAAA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAEhF,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;AACtC,YAAA,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;AACnF,YAAA,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS;AAChC,YAAA,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;AAEzB,YAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,QAAQ,EAAE;AACvE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;YACzF;QACF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEzB,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,mBAAmB,KAAK,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,mBAAmB,CAAC,EAAE;AACxG,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,sDAAA,CAAwD,CAAC;gBAC1E;YACF;AAEA,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iEAAiE,CAAC,CAAA,EAAA,EAAKC,gCAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA,CAAE,CAC3G;AACD,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBD,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,4BAAA,CAA8B,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,uDAAA,EAA0D,CAAC,CAAA,CAAE,CAAC;;;;;;QAMlF;IACF;;AAGQ,IAAA,iBAAiB,CAAC,CAAU,EAAA;QAClC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IACrC;;AAGQ,IAAA,gBAAgB,CAAC,CAAU,EAAA;QACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;AACnC,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK;AAC1B,QAAA,IAAI,CAAC,MAAM,GAAG,IAAI;IACpB;AAEO,IAAA,kBAAkB,CAAC,GAA2B,EAAA;AACnD,QAAA,IAAI,yBAAyB,CAAC,GAAG,CAAC,EAAE;AAClC,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,cAAc,GAAG,IAAI;QAC5B;IACF;AAEQ,IAAA,OAAO,CAAC,IAAa,EAAA;AAC3B,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI;AACzB,QAAA,IAAI,IAAI;AAAE,YAAA,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,SAAS;IAC/C;IAEO,UAAU,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEO,IAAA,UAAU,CAAC,QAAgB,EAAA;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;IAEQ,gBAAgB,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;IAC9B;AACD;AAED;AACO,eAAe,UAAU,CAC9B,MAAgB,EAChB,UAAwB,EACxB,GAAiB,EACjB,UAAsC,EACtC,QAAuB,EACvB,KAKC,EAAA;IAED,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,uDAAuD,CAAC;AAC9E,IAAA,MAAM,OAAO,GAAG,KAAK,CAAC;AAEtB,IAAA,IAAI,QAAQ,EAAE;QAAE;AAChB,IAAA,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;AAC3D,IAAA,MAAM,CAAC,IAAI,CACT,0BAA0B,GAAG,CAAC,EAAE,CAAA,CAAA;UAC9B,CAAA,gBAAA,EAAmB,KAAK,CAAC,OAAO,CAAA,kBAAA,EAAqB,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAA,CAAA;AAC1E,UAAA,CAAA,+BAAA,EAAkC,KAAK,CAAC,YAAY,CAAA,CAAE,CACzD;IAED,MAAM,YAAY,GAAG,CAAC,IAAY,KAAK,OAAO,UAA+B,KAAI;AAC/E,QAAA,IAAI,QAAQ,EAAE;YAAE;AAChB,QAAA,MAAM,UAAU,CAAC,UAAU,CACzB,GAAG,EACH,UAAU,CAAC,SAAS,EACpB,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,EACnC,IAAI,EACJ,KAAK,CAAC,iBAAiB,EACvB,KAAK,CAAC,cAAc,EACpB,EAAE,OAAO,EAAE,CACZ;AACD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,OAAO,iBAAiB,GAAG,CAAC,EAAE,CAAA,CAAE,CAAC;;QAG7E,KAAK,CAAC,yBAAyB,EAAE;QACjC,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC,sBAAsB,EAAE;AACnE,YAAA,KAAK,CAAC,yBAAyB,GAAG,CAAC;AACnC,YAAA,KAAK,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC;AACpF,YAAA,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC;QAC/C;AACF,IAAA,CAAC;AAED,IAAA,MAAME,mBAAS,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAErE,IAAA,IAAI,QAAQ,EAAE;QAAE;IAChB,MAAM,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,CAAA,sBAAA,EAAyB,GAAG,CAAC,EAAE,CAAA,UAAA,CAAY,CAAC;AAC1D;AAEA,SAAS,WAAW,CAAC,GAA2B,EAAE,MAAc,EAAA;AAC9D,IAAA,IAAI,iBAAsC;AAC1C,IAAA,IAAI,UAAkD;AACtD,IAAA,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE;QACjB,UAAU,GAAGC,gCAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACvD,QAAA,iBAAiB,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,aAAa,CAAC;IACzF;IAEA,OAAO;AACL,QAAA,UAAU,EAAE,UAAU;AACtB,QAAA,QAAQ,EAAE;AACR,YAAA,IAAI,EAAE,KAAK;AACX,YAAA,MAAM,EAAE,SAAS;AACjB,YAAA,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvB,YAAA,iBAAiB,EAAE,iBAAiB;AACpC,YAAA,SAAS,EAAE,SAAS;AACQ,SAAA;KAC/B;AACH;AAEA;AACM,SAAU,UAAU,CAAC,CAAqB,EAAA;IAC9C,OAAO,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC;AACrD;AAEA;;;;AAIqD;AACrD,SAAS,aAAa,CAAC,QAA4B,EAAA;AACjD,KAAmC;QACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,SAAS,EAAE,QAAQ,CAAC,SAAS;;IAG/B,IAAI,QAAQ,CAAC,MAAM;SACD;AACd,YAAA,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;AAClC,YAAA,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc;AAC9C,YAAA,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;UACvC;AAEH,IAAA,OAAO,QAAQ;AACjB;AAEA,SAAS,yBAAyB,CAAC,GAA2B,EAAA;AAC5D,IAAA,OAAO,MAAM,IAAI,GAAG,CAAC;AACnB,UAAE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK;UACpB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;AAC1C;AAEM,SAAU,QAAQ,CAAC,GAAiB,EAAA;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;AAC/C;SAEgB,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAA;AACzE,IAAA,IAAI;AACF,QAAA,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC;AAC9B,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,CAAC,EAAE;AACV,QAAA,OAAO,KAAK;IACd;AACF;AAEA,SAAS,aAAa,CAAC,KAAqB,EAAA;IAC1C,OAAO;AACL,QAAA,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;AAC7B,QAAA,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;AAC5C,QAAA,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;KACrC;AACH;AAEA;;;;AAI+E;AAC/E,SAAS,qBAAqB,CAAC,cAAuB,EAAE,MAAwB,EAAA;AAC9E,IAAA,IAAI,cAAc,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,EAAE;QAC1E,OAAO;AACL,YAAA,QAAQ,EAAE,GAAG;AACb,YAAA,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;AACzC,YAAA,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;SACtC;IACH;AAEA,IAAA,OAAO,MAAM;AACf;AAEM,SAAU,yBAAyB,CAAC,CAAM,EAAA;AAC9C,IAAA,QACE,CAAC,CAAC,IAAI,IAAI;AACP,YAAC,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,gBAAgB,CAAC;AAEnF;AAEM,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,OAAO,CAAC,YAAYC,iBAAU;AAC5B,QAAA,CAAC,YAAYC,oBAAa;AAC1B,QAAA,CAAC,YAAYC,yBAAkB;QAC/B,CAAC,YAAYC,sBAAe;AAChC;AAEA,SAAS,qBAAqB,CAAC,CAAM,EAAA;IACnC,OAAQ,CAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA,uBAAA,CAAyB,CAAC;AAClE;AAEA;AACA,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC;IACjD,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;AAEA;AACuC;AACvC,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC;IAC7D,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;;;;;;;;;;"}
1
+ {"version":3,"file":"upload_task.cjs","sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientProgress, ProgressStatus } from '../clients/progress';\nimport type { ClientUpload } from '../clients/upload';\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';\nimport type { ImportResourceSnapshot } from './types';\nimport { ImportFileHandleUploadData } from './types';\nimport assert from 'node:assert';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n private nMaxUploads: number;\n private nPartsWithThisUploadSpeed = 0;\n private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n /** If this is upload progress this field will be defined */\n private uploadData?: ImportFileHandleUploadData;\n public progress: sdk.ImportProgress;\n\n /** If failed, then getting a progress is terminally failed. */\n public failed?: boolean;\n\n /** True if the blob was existed.\n * At this case, the task will show progress == 1.0. */\n private alreadyExisted = false;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly maxNConcurrentPartsUpload: number,\n signer: Signer,\n public readonly res: ImportResourceSnapshot,\n ) {\n this.nMaxUploads = this.maxNConcurrentPartsUpload;\n const { uploadData, progress } = newProgress(res, signer);\n this.uploadData = uploadData;\n this.progress = progress;\n }\n\n public getProgress(w: Watcher, callerId: string) {\n this.incCounter(w, callerId);\n\n if (this.failed) {\n this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n throw new Error(this.progress.lastError);\n }\n\n return cloneProgress(this.progress);\n }\n\n public shouldScheduleUpload(): boolean {\n return isMyUpload(this.progress);\n }\n\n /** Uploads a blob if it's not BlobIndex. */\n public async uploadBlobTask() {\n try {\n await uploadBlob(\n this.logger,\n this.clientBlob,\n this.res,\n this.uploadData!,\n this.isComputableDone.bind(this),\n {\n nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n currentSpeed: this.nMaxUploads,\n maxSpeed: this.maxNConcurrentPartsUpload,\n },\n );\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n } catch (e: any) {\n\n if (isResourceWasDeletedError(e)) {\n this.setRetriableError(e);\n this.logger.warn(`resource was deleted while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);\n this.setDone(true);\n\n return;\n }\n\n this.logger.error(`error while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n if (nonRecoverableError(e)) {\n this.setTerminalError(e);\n return;\n }\n\n this.setRetriableError(e);\n\n if (isHeadersTimeoutError(e)) {\n // we probably have a slow internet, we need to slow things a bit.\n this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n }\n\n throw e;\n }\n }\n\n public async updateStatus() {\n try {\n // we do it with timeout in case we have slow internet.\n const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n const oldStatus = this.progress.status;\n const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n this.progress.status = newStatus;\n this.setDone(status.done);\n\n if (status.done || this.progress.status.progress != oldStatus?.progress) {\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n }\n } catch (e: any) {\n this.setRetriableError(e);\n\n if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {\n this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);\n return;\n }\n\n if (isResourceWasDeletedError(e)) {\n this.logger.warn(\n `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,\n );\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed, resource not found`);\n this.setDone(true);\n return;\n }\n\n this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n // It was a terminal error, but when a connection drops,\n // this will stop the whole task, so we make it retryable.\n // It was like that:\n // this.change.markChanged();\n // this.setTerminalError(e);\n }\n }\n\n /** Set non-terminal error, that task can be retried. */\n private setRetriableError(e: unknown) {\n this.progress.lastError = String(e);\n }\n\n /** Set a terminal error, the task will throw a error instead of a progress. */\n private setTerminalError(e: unknown) {\n this.progress.lastError = String(e);\n this.progress.done = false;\n this.failed = true;\n }\n\n public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n if (isImportResourceOutputSet(res)) {\n this.setDone(true);\n this.alreadyExisted = true;\n }\n }\n\n private setDone(done: boolean) {\n this.progress.done = done;\n if (done) this.progress.lastError = undefined;\n }\n\n public incCounter(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n public decCounter(callerId: string) {\n return this.counter.dec(callerId);\n }\n\n private isComputableDone() {\n return this.counter.isZero();\n }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n logger: MiLogger,\n clientBlob: ClientUpload,\n res: ResourceInfo,\n uploadData: ImportFileHandleUploadData,\n isDoneFn: () => boolean,\n speed: {\n nPartsWithThisUploadSpeed: number,\n nPartsToIncreaseUpload: number,\n currentSpeed: number,\n maxSpeed: number,\n },\n) {\n assert(isUpload(res), 'the upload operation can be done only for BlobUploads');\n const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n if (isDoneFn()) return;\n const parts = await clientBlob.initUpload(res, { timeout });\n logger.info(\n `started to upload blob ${res.id},`\n + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`\n + ` number of concurrent uploads: ${speed.currentSpeed}`,\n );\n\n const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n if (isDoneFn()) return;\n await clientBlob.partUpload(\n res,\n uploadData.localPath,\n BigInt(uploadData.modificationTime),\n part,\n parts.checksumAlgorithm,\n parts.checksumHeader,\n { timeout }\n );\n logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n // if we had a network freeze, it will be increased slowly.\n speed.nPartsWithThisUploadSpeed++;\n if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n speed.nPartsWithThisUploadSpeed = 0;\n speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n controller.setConcurrency(speed.currentSpeed);\n }\n };\n\n await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n if (isDoneFn()) return;\n await clientBlob.finalize(res, { timeout });\n\n logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n let isUploadSignMatch: boolean | undefined;\n let uploadData: ImportFileHandleUploadData | undefined;\n if (isUpload(res)) {\n uploadData = ImportFileHandleUploadData.parse(res.data);\n isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n }\n\n return {\n uploadData: uploadData,\n progress: {\n done: false,\n status: undefined,\n isUpload: isUpload(res),\n isUploadSignMatch: isUploadSignMatch,\n lastError: undefined,\n } satisfies sdk.ImportProgress,\n };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n const cloned: sdk.ImportProgress = {\n done: progress.done,\n isUpload: progress.isUpload,\n isUploadSignMatch: progress.isUploadSignMatch,\n lastError: progress.lastError,\n };\n\n if (progress.status)\n cloned.status = {\n progress: progress.status.progress,\n bytesProcessed: progress.status.bytesProcessed,\n bytesTotal: progress.status.bytesTotal,\n };\n\n return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n return 'blob' in res.fields\n ? res.fields.blob !== undefined\n : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n return res.type.name.startsWith('BlobUpload');\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n try {\n signer.verify(path, signature);\n return true;\n } catch (e) {\n return false;\n }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n return {\n progress: proto.progress ?? 0,\n bytesProcessed: Number(proto.bytesProcessed),\n bytesTotal: Number(proto.bytesTotal),\n };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n return {\n progress: 1.0,\n bytesProcessed: Number(status.bytesTotal),\n bytesTotal: Number(status.bytesTotal),\n };\n }\n\n return status;\n}\n\nexport function isResourceWasDeletedError(e: any) {\n return (\n e.name == 'RpcError'\n && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')\n );\n}\n\nexport function nonRecoverableError(e: any) {\n return e instanceof MTimeError ||\n e instanceof UnexpectedEOF ||\n e instanceof NoFileForUploading ||\n e instanceof BadRequestError;\n}\n\nfunction isHeadersTimeoutError(e: any) {\n return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n const newConcurrency = Math.min(current + 2, max);\n if (newConcurrency != current)\n logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n const newConcurrency = Math.max(Math.round(current / 2), min);\n if (newConcurrency != current)\n logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","asyncPool","ImportFileHandleUploadData","MTimeError","UnexpectedEOF","NoFileForUploading","BadRequestError"],"mappings":";;;;;;;;;AAcA;;AAEgC;MACnB,UAAU,CAAA;AAmBF,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,cAAA;AACA,IAAA,yBAAA;AAED,IAAA,GAAA;AAvBD,IAAA,MAAM,GAAiB,IAAIA,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;AACvD,IAAA,WAAW;IACX,yBAAyB,GAAG,CAAC;AAC7B,IAAA,sBAAsB,GAAG,EAAE,CAAC;;AAG5B,IAAA,UAAU;AACX,IAAA,QAAQ;;AAGR,IAAA,MAAM;AAEb;AACuD;IAC/C,cAAc,GAAG,KAAK;IAE9B,WAAA,CACmB,MAAgB,EAChB,UAAwB,EACxB,cAA8B,EAC9B,yBAAiC,EAClD,MAAc,EACE,GAA2B,EAAA;QAL1B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,yBAAyB,GAAzB,yBAAyB;QAE1B,IAAA,CAAA,GAAG,GAAH,GAAG;AAEnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,yBAAyB;AACjD,QAAA,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;AACzD,QAAA,IAAI,CAAC,UAAU,GAAG,UAAU;AAC5B,QAAA,IAAI,CAAC,QAAQ,GAAG,QAAQ;IAC1B;IAEO,WAAW,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC7C,QAAA,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC;AAE5B,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,6BAAA,EAAgC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAA,CAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC1C;AAEA,QAAA,OAAO,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;IACrC;IAEO,oBAAoB,GAAA;AACzB,QAAA,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClC;;AAGO,IAAA,MAAM,cAAc,GAAA;AACzB,QAAA,IAAI;YACF,MAAM,UAAU,CACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,UAAW,EAChB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAChC;gBACE,yBAAyB,EAAE,IAAI,CAAC,yBAAyB;gBACzD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;gBACnD,YAAY,EAAE,IAAI,CAAC,WAAW;gBAC9B,QAAQ,EAAE,IAAI,CAAC,yBAAyB;AACzC,aAAA,CACF;AACD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBC,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,6CAAA,EAAgD,CAAC,CAAA,CAAE,CAAC;AACrE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,8BAAA,CAAgC,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAElB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,8BAAA,EAAiC,CAAC,CAAA,CAAE,CAAC;AACvD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;AAEpF,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACxB;YACF;AAEA,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;AAEzB,YAAA,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE;;AAE5B,gBAAA,IAAI,CAAC,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1E;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEO,IAAA,MAAM,YAAY,GAAA;AACvB,QAAA,IAAI;;AAEF,YAAA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAEhF,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;AACtC,YAAA,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;AACnF,YAAA,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS;AAChC,YAAA,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;AAEzB,YAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,QAAQ,EAAE;AACvE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBA,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;YACzF;QACF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEzB,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,mBAAmB,KAAK,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,mBAAmB,CAAC,EAAE;gBACxG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,wDAAA,EAA2D,CAAC,CAAC,OAAO,CAAA,CAAE,CAAC;gBACxF;YACF;AAEA,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iEAAiE,CAAC,CAAA,EAAA,EAAKC,gCAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA,CAAE,CAC3G;AACD,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBD,2BAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,4BAAA,CAA8B,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,uDAAA,EAA0D,CAAC,CAAA,CAAE,CAAC;;;;;;QAMlF;IACF;;AAGQ,IAAA,iBAAiB,CAAC,CAAU,EAAA;QAClC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IACrC;;AAGQ,IAAA,gBAAgB,CAAC,CAAU,EAAA;QACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;AACnC,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK;AAC1B,QAAA,IAAI,CAAC,MAAM,GAAG,IAAI;IACpB;AAEO,IAAA,kBAAkB,CAAC,GAA2B,EAAA;AACnD,QAAA,IAAI,yBAAyB,CAAC,GAAG,CAAC,EAAE;AAClC,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,cAAc,GAAG,IAAI;QAC5B;IACF;AAEQ,IAAA,OAAO,CAAC,IAAa,EAAA;AAC3B,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI;AACzB,QAAA,IAAI,IAAI;AAAE,YAAA,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,SAAS;IAC/C;IAEO,UAAU,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEO,IAAA,UAAU,CAAC,QAAgB,EAAA;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;IAEQ,gBAAgB,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;IAC9B;AACD;AAED;AACO,eAAe,UAAU,CAC9B,MAAgB,EAChB,UAAwB,EACxB,GAAiB,EACjB,UAAsC,EACtC,QAAuB,EACvB,KAKC,EAAA;IAED,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,uDAAuD,CAAC;AAC9E,IAAA,MAAM,OAAO,GAAG,KAAK,CAAC;AAEtB,IAAA,IAAI,QAAQ,EAAE;QAAE;AAChB,IAAA,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;AAC3D,IAAA,MAAM,CAAC,IAAI,CACT,0BAA0B,GAAG,CAAC,EAAE,CAAA,CAAA;UAC9B,CAAA,gBAAA,EAAmB,KAAK,CAAC,OAAO,CAAA,kBAAA,EAAqB,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAA,CAAA;AAC1E,UAAA,CAAA,+BAAA,EAAkC,KAAK,CAAC,YAAY,CAAA,CAAE,CACzD;IAED,MAAM,YAAY,GAAG,CAAC,IAAY,KAAK,OAAO,UAA+B,KAAI;AAC/E,QAAA,IAAI,QAAQ,EAAE;YAAE;AAChB,QAAA,MAAM,UAAU,CAAC,UAAU,CACzB,GAAG,EACH,UAAU,CAAC,SAAS,EACpB,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,EACnC,IAAI,EACJ,KAAK,CAAC,iBAAiB,EACvB,KAAK,CAAC,cAAc,EACpB,EAAE,OAAO,EAAE,CACZ;AACD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,OAAO,iBAAiB,GAAG,CAAC,EAAE,CAAA,CAAE,CAAC;;QAG7E,KAAK,CAAC,yBAAyB,EAAE;QACjC,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC,sBAAsB,EAAE;AACnE,YAAA,KAAK,CAAC,yBAAyB,GAAG,CAAC;AACnC,YAAA,KAAK,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC;AACpF,YAAA,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC;QAC/C;AACF,IAAA,CAAC;AAED,IAAA,MAAME,mBAAS,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAErE,IAAA,IAAI,QAAQ,EAAE;QAAE;IAChB,MAAM,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,CAAA,sBAAA,EAAyB,GAAG,CAAC,EAAE,CAAA,UAAA,CAAY,CAAC;AAC1D;AAEA,SAAS,WAAW,CAAC,GAA2B,EAAE,MAAc,EAAA;AAC9D,IAAA,IAAI,iBAAsC;AAC1C,IAAA,IAAI,UAAkD;AACtD,IAAA,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE;QACjB,UAAU,GAAGC,gCAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACvD,QAAA,iBAAiB,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,aAAa,CAAC;IACzF;IAEA,OAAO;AACL,QAAA,UAAU,EAAE,UAAU;AACtB,QAAA,QAAQ,EAAE;AACR,YAAA,IAAI,EAAE,KAAK;AACX,YAAA,MAAM,EAAE,SAAS;AACjB,YAAA,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvB,YAAA,iBAAiB,EAAE,iBAAiB;AACpC,YAAA,SAAS,EAAE,SAAS;AACQ,SAAA;KAC/B;AACH;AAEA;AACM,SAAU,UAAU,CAAC,CAAqB,EAAA;IAC9C,OAAO,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC;AACrD;AAEA;;;;AAIqD;AACrD,SAAS,aAAa,CAAC,QAA4B,EAAA;AACjD,KAAmC;QACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,SAAS,EAAE,QAAQ,CAAC,SAAS;;IAG/B,IAAI,QAAQ,CAAC,MAAM;SACD;AACd,YAAA,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;AAClC,YAAA,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc;AAC9C,YAAA,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;UACvC;AAEH,IAAA,OAAO,QAAQ;AACjB;AAEA,SAAS,yBAAyB,CAAC,GAA2B,EAAA;AAC5D,IAAA,OAAO,MAAM,IAAI,GAAG,CAAC;AACnB,UAAE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK;UACpB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;AAC1C;AAEM,SAAU,QAAQ,CAAC,GAAiB,EAAA;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;AAC/C;SAEgB,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAA;AACzE,IAAA,IAAI;AACF,QAAA,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC;AAC9B,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,CAAC,EAAE;AACV,QAAA,OAAO,KAAK;IACd;AACF;AAEA,SAAS,aAAa,CAAC,KAAqB,EAAA;IAC1C,OAAO;AACL,QAAA,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;AAC7B,QAAA,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;AAC5C,QAAA,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;KACrC;AACH;AAEA;;;;AAI+E;AAC/E,SAAS,qBAAqB,CAAC,cAAuB,EAAE,MAAwB,EAAA;AAC9E,IAAA,IAAI,cAAc,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,EAAE;QAC1E,OAAO;AACL,YAAA,QAAQ,EAAE,GAAG;AACb,YAAA,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;AACzC,YAAA,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;SACtC;IACH;AAEA,IAAA,OAAO,MAAM;AACf;AAEM,SAAU,yBAAyB,CAAC,CAAM,EAAA;AAC9C,IAAA,QACE,CAAC,CAAC,IAAI,IAAI;AACP,YAAC,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,gBAAgB,CAAC;AAEnF;AAEM,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,OAAO,CAAC,YAAYC,iBAAU;AAC5B,QAAA,CAAC,YAAYC,oBAAa;AAC1B,QAAA,CAAC,YAAYC,yBAAkB;QAC/B,CAAC,YAAYC,sBAAe;AAChC;AAEA,SAAS,qBAAqB,CAAC,CAAM,EAAA;IACnC,OAAQ,CAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA,uBAAA,CAAyB,CAAC;AAClE;AAEA;AACA,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC;IACjD,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;AAEA;AACuC;AACvC,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC;IAC7D,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;;;;;;;;;;"}
package/dist/drivers/upload_task.js CHANGED
@@ -97,7 +97,7 @@ class UploadTask {
97
97
  catch (e) {
98
98
  this.setRetriableError(e);
99
99
  if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {
100
- this.logger.warn(`deadline exceeded while getting a status of BlobImport`);
100
+ this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);
101
101
  return;
102
102
  }
103
103
  if (isResourceWasDeletedError(e)) {
package/dist/drivers/upload_task.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"upload_task.js","sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientProgress, ProgressStatus } from '../clients/progress';\nimport type { ClientUpload } from '../clients/upload';\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';\nimport type { ImportResourceSnapshot } from './types';\nimport { ImportFileHandleUploadData } from './types';\nimport assert from 'node:assert';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n private nMaxUploads: number;\n private nPartsWithThisUploadSpeed = 0;\n private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n /** If this is upload progress this field will be defined */\n private uploadData?: ImportFileHandleUploadData;\n public progress: sdk.ImportProgress;\n\n /** If failed, then getting a progress is terminally failed. */\n public failed?: boolean;\n\n /** True if the blob was existed.\n * At this case, the task will show progress == 1.0. */\n private alreadyExisted = false;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly maxNConcurrentPartsUpload: number,\n signer: Signer,\n public readonly res: ImportResourceSnapshot,\n ) {\n this.nMaxUploads = this.maxNConcurrentPartsUpload;\n const { uploadData, progress } = newProgress(res, signer);\n this.uploadData = uploadData;\n this.progress = progress;\n }\n\n public getProgress(w: Watcher, callerId: string) {\n this.incCounter(w, callerId);\n\n if (this.failed) {\n this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n throw new Error(this.progress.lastError);\n }\n\n return cloneProgress(this.progress);\n }\n\n public shouldScheduleUpload(): boolean {\n return isMyUpload(this.progress);\n }\n\n /** Uploads a blob if it's not BlobIndex. */\n public async uploadBlobTask() {\n try {\n await uploadBlob(\n this.logger,\n this.clientBlob,\n this.res,\n this.uploadData!,\n this.isComputableDone.bind(this),\n {\n nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n currentSpeed: this.nMaxUploads,\n maxSpeed: this.maxNConcurrentPartsUpload,\n },\n );\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n } catch (e: any) {\n\n if (isResourceWasDeletedError(e)) {\n this.setRetriableError(e);\n this.logger.warn(`resource was deleted while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);\n this.setDone(true);\n\n return;\n }\n\n this.logger.error(`error while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n if (nonRecoverableError(e)) {\n this.setTerminalError(e);\n return;\n }\n\n this.setRetriableError(e);\n\n if (isHeadersTimeoutError(e)) {\n // we probably have a slow internet, we need to slow things a bit.\n this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n }\n\n throw e;\n }\n }\n\n public async updateStatus() {\n try {\n // we do it with timeout in case we have slow internet.\n const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n const oldStatus = this.progress.status;\n const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n this.progress.status = newStatus;\n this.setDone(status.done);\n\n if (status.done || this.progress.status.progress != oldStatus?.progress) {\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n }\n } catch (e: any) {\n this.setRetriableError(e);\n\n if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {\n this.logger.warn(`deadline exceeded while getting a status of BlobImport`);\n return;\n }\n\n if (isResourceWasDeletedError(e)) {\n this.logger.warn(\n `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,\n );\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed, resource not found`);\n this.setDone(true);\n return;\n }\n\n this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n // It was a terminal error, but when a connection drops,\n // this will stop the whole task, so we make it retryable.\n // It was like that:\n // this.change.markChanged();\n // this.setTerminalError(e);\n }\n }\n\n /** Set non-terminal error, that task can be retried. */\n private setRetriableError(e: unknown) {\n this.progress.lastError = String(e);\n }\n\n /** Set a terminal error, the task will throw a error instead of a progress. */\n private setTerminalError(e: unknown) {\n this.progress.lastError = String(e);\n this.progress.done = false;\n this.failed = true;\n }\n\n public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n if (isImportResourceOutputSet(res)) {\n this.setDone(true);\n this.alreadyExisted = true;\n }\n }\n\n private setDone(done: boolean) {\n this.progress.done = done;\n if (done) this.progress.lastError = undefined;\n }\n\n public incCounter(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n public decCounter(callerId: string) {\n return this.counter.dec(callerId);\n }\n\n private isComputableDone() {\n return this.counter.isZero();\n }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n logger: MiLogger,\n clientBlob: ClientUpload,\n res: ResourceInfo,\n uploadData: ImportFileHandleUploadData,\n isDoneFn: () => boolean,\n speed: {\n nPartsWithThisUploadSpeed: number,\n nPartsToIncreaseUpload: number,\n currentSpeed: number,\n maxSpeed: number,\n },\n) {\n assert(isUpload(res), 'the upload operation can be done only for BlobUploads');\n const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n if (isDoneFn()) return;\n const parts = await clientBlob.initUpload(res, { timeout });\n logger.info(\n `started to upload blob ${res.id},`\n + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`\n + ` number of concurrent uploads: ${speed.currentSpeed}`,\n );\n\n const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n if (isDoneFn()) return;\n await clientBlob.partUpload(\n res,\n uploadData.localPath,\n BigInt(uploadData.modificationTime),\n part,\n parts.checksumAlgorithm,\n parts.checksumHeader,\n { timeout }\n );\n logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n // if we had a network freeze, it will be increased slowly.\n speed.nPartsWithThisUploadSpeed++;\n if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n speed.nPartsWithThisUploadSpeed = 0;\n speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n controller.setConcurrency(speed.currentSpeed);\n }\n };\n\n await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n if (isDoneFn()) return;\n await clientBlob.finalize(res, { timeout });\n\n logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n let isUploadSignMatch: boolean | undefined;\n let uploadData: ImportFileHandleUploadData | undefined;\n if (isUpload(res)) {\n uploadData = ImportFileHandleUploadData.parse(res.data);\n isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n }\n\n return {\n uploadData: uploadData,\n progress: {\n done: false,\n status: undefined,\n isUpload: isUpload(res),\n isUploadSignMatch: isUploadSignMatch,\n lastError: undefined,\n } satisfies sdk.ImportProgress,\n };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n const cloned: sdk.ImportProgress = {\n done: progress.done,\n isUpload: progress.isUpload,\n isUploadSignMatch: progress.isUploadSignMatch,\n lastError: progress.lastError,\n };\n\n if (progress.status)\n cloned.status = {\n progress: progress.status.progress,\n bytesProcessed: progress.status.bytesProcessed,\n bytesTotal: progress.status.bytesTotal,\n };\n\n return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n return 'blob' in res.fields\n ? res.fields.blob !== undefined\n : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n return res.type.name.startsWith('BlobUpload');\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n try {\n signer.verify(path, signature);\n return true;\n } catch (e) {\n return false;\n }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n return {\n progress: proto.progress ?? 0,\n bytesProcessed: Number(proto.bytesProcessed),\n bytesTotal: Number(proto.bytesTotal),\n };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n return {\n progress: 1.0,\n bytesProcessed: Number(status.bytesTotal),\n bytesTotal: Number(status.bytesTotal),\n };\n }\n\n return status;\n}\n\nexport function isResourceWasDeletedError(e: any) {\n return (\n e.name == 'RpcError'\n && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')\n );\n}\n\nexport function nonRecoverableError(e: any) {\n return e instanceof MTimeError ||\n e instanceof UnexpectedEOF ||\n e instanceof NoFileForUploading ||\n e instanceof BadRequestError;\n}\n\nfunction isHeadersTimeoutError(e: any) {\n return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n const newConcurrency = Math.min(current + 2, max);\n if (newConcurrency != current)\n logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n const newConcurrency = Math.max(Math.round(current / 2), min);\n if (newConcurrency != current)\n logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n"],"names":[],"mappings":";;;;;;;AAcA;;AAEgC;MACnB,UAAU,CAAA;AAmBF,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,cAAA;AACA,IAAA,yBAAA;AAED,IAAA,GAAA;AAvBD,IAAA,MAAM,GAAiB,IAAI,YAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAI,cAAc,EAAE;AACvD,IAAA,WAAW;IACX,yBAAyB,GAAG,CAAC;AAC7B,IAAA,sBAAsB,GAAG,EAAE,CAAC;;AAG5B,IAAA,UAAU;AACX,IAAA,QAAQ;;AAGR,IAAA,MAAM;AAEb;AACuD;IAC/C,cAAc,GAAG,KAAK;IAE9B,WAAA,CACmB,MAAgB,EAChB,UAAwB,EACxB,cAA8B,EAC9B,yBAAiC,EAClD,MAAc,EACE,GAA2B,EAAA;QAL1B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,yBAAyB,GAAzB,yBAAyB;QAE1B,IAAA,CAAA,GAAG,GAAH,GAAG;AAEnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,yBAAyB;AACjD,QAAA,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;AACzD,QAAA,IAAI,CAAC,UAAU,GAAG,UAAU;AAC5B,QAAA,IAAI,CAAC,QAAQ,GAAG,QAAQ;IAC1B;IAEO,WAAW,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC7C,QAAA,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC;AAE5B,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,6BAAA,EAAgC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAA,CAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC1C;AAEA,QAAA,OAAO,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;IACrC;IAEO,oBAAoB,GAAA;AACzB,QAAA,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClC;;AAGO,IAAA,MAAM,cAAc,GAAA;AACzB,QAAA,IAAI;YACF,MAAM,UAAU,CACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,UAAW,EAChB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAChC;gBACE,yBAAyB,EAAE,IAAI,CAAC,yBAAyB;gBACzD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;gBACnD,YAAY,EAAE,IAAI,CAAC,WAAW;gBAC9B,QAAQ,EAAE,IAAI,CAAC,yBAAyB;AACzC,aAAA,CACF;AACD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,6CAAA,EAAgD,CAAC,CAAA,CAAE,CAAC;AACrE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,8BAAA,CAAgC,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAElB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,8BAAA,EAAiC,CAAC,CAAA,CAAE,CAAC;AACvD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;AAEpF,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACxB;YACF;AAEA,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;AAEzB,YAAA,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE;;AAE5B,gBAAA,IAAI,CAAC,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1E;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEO,IAAA,MAAM,YAAY,GAAA;AACvB,QAAA,IAAI;;AAEF,YAAA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAEhF,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;AACtC,YAAA,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;AACnF,YAAA,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS;AAChC,YAAA,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;AAEzB,YAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,QAAQ,EAAE;AACvE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;YACzF;QACF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEzB,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,mBAAmB,KAAK,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,mBAAmB,CAAC,EAAE;AACxG,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,sDAAA,CAAwD,CAAC;gBAC1E;YACF;AAEA,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iEAAiE,CAAC,CAAA,EAAA,EAAK,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA,CAAE,CAC3G;AACD,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,4BAAA,CAA8B,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,uDAAA,EAA0D,CAAC,CAAA,CAAE,CAAC;;;;;;QAMlF;IACF;;AAGQ,IAAA,iBAAiB,CAAC,CAAU,EAAA;QAClC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IACrC;;AAGQ,IAAA,gBAAgB,CAAC,CAAU,EAAA;QACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;AACnC,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK;AAC1B,QAAA,IAAI,CAAC,MAAM,GAAG,IAAI;IACpB;AAEO,IAAA,kBAAkB,CAAC,GAA2B,EAAA;AACnD,QAAA,IAAI,yBAAyB,CAAC,GAAG,CAAC,EAAE;AAClC,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,cAAc,GAAG,IAAI;QAC5B;IACF;AAEQ,IAAA,OAAO,CAAC,IAAa,EAAA;AAC3B,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI;AACzB,QAAA,IAAI,IAAI;AAAE,YAAA,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,SAAS;IAC/C;IAEO,UAAU,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEO,IAAA,UAAU,CAAC,QAAgB,EAAA;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;IAEQ,gBAAgB,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;IAC9B;AACD;AAED;AACO,eAAe,UAAU,CAC9B,MAAgB,EAChB,UAAwB,EACxB,GAAiB,EACjB,UAAsC,EACtC,QAAuB,EACvB,KAKC,EAAA;IAED,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,uDAAuD,CAAC;AAC9E,IAAA,MAAM,OAAO,GAAG,KAAK,CAAC;AAEtB,IAAA,IAAI,QAAQ,EAAE;QAAE;AAChB,IAAA,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;AAC3D,IAAA,MAAM,CAAC,IAAI,CACT,0BAA0B,GAAG,CAAC,EAAE,CAAA,CAAA;UAC9B,CAAA,gBAAA,EAAmB,KAAK,CAAC,OAAO,CAAA,kBAAA,EAAqB,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAA,CAAA;AAC1E,UAAA,CAAA,+BAAA,EAAkC,KAAK,CAAC,YAAY,CAAA,CAAE,CACzD;IAED,MAAM,YAAY,GAAG,CAAC,IAAY,KAAK,OAAO,UAA+B,KAAI;AAC/E,QAAA,IAAI,QAAQ,EAAE;YAAE;AAChB,QAAA,MAAM,UAAU,CAAC,UAAU,CACzB,GAAG,EACH,UAAU,CAAC,SAAS,EACpB,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,EACnC,IAAI,EACJ,KAAK,CAAC,iBAAiB,EACvB,KAAK,CAAC,cAAc,EACpB,EAAE,OAAO,EAAE,CACZ;AACD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,OAAO,iBAAiB,GAAG,CAAC,EAAE,CAAA,CAAE,CAAC;;QAG7E,KAAK,CAAC,yBAAyB,EAAE;QACjC,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC,sBAAsB,EAAE;AACnE,YAAA,KAAK,CAAC,yBAAyB,GAAG,CAAC;AACnC,YAAA,KAAK,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC;AACpF,YAAA,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC;QAC/C;AACF,IAAA,CAAC;AAED,IAAA,MAAM,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAErE,IAAA,IAAI,QAAQ,EAAE;QAAE;IAChB,MAAM,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,CAAA,sBAAA,EAAyB,GAAG,CAAC,EAAE,CAAA,UAAA,CAAY,CAAC;AAC1D;AAEA,SAAS,WAAW,CAAC,GAA2B,EAAE,MAAc,EAAA;AAC9D,IAAA,IAAI,iBAAsC;AAC1C,IAAA,IAAI,UAAkD;AACtD,IAAA,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE;QACjB,UAAU,GAAG,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACvD,QAAA,iBAAiB,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,aAAa,CAAC;IACzF;IAEA,OAAO;AACL,QAAA,UAAU,EAAE,UAAU;AACtB,QAAA,QAAQ,EAAE;AACR,YAAA,IAAI,EAAE,KAAK;AACX,YAAA,MAAM,EAAE,SAAS;AACjB,YAAA,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvB,YAAA,iBAAiB,EAAE,iBAAiB;AACpC,YAAA,SAAS,EAAE,SAAS;AACQ,SAAA;KAC/B;AACH;AAEA;AACM,SAAU,UAAU,CAAC,CAAqB,EAAA;IAC9C,OAAO,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC;AACrD;AAEA;;;;AAIqD;AACrD,SAAS,aAAa,CAAC,QAA4B,EAAA;AACjD,KAAmC;QACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,SAAS,EAAE,QAAQ,CAAC,SAAS;;IAG/B,IAAI,QAAQ,CAAC,MAAM;SACD;AACd,YAAA,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;AAClC,YAAA,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc;AAC9C,YAAA,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;UACvC;AAEH,IAAA,OAAO,QAAQ;AACjB;AAEA,SAAS,yBAAyB,CAAC,GAA2B,EAAA;AAC5D,IAAA,OAAO,MAAM,IAAI,GAAG,CAAC;AACnB,UAAE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK;UACpB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;AAC1C;AAEM,SAAU,QAAQ,CAAC,GAAiB,EAAA;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;AAC/C;SAEgB,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAA;AACzE,IAAA,IAAI;AACF,QAAA,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC;AAC9B,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,CAAC,EAAE;AACV,QAAA,OAAO,KAAK;IACd;AACF;AAEA,SAAS,aAAa,CAAC,KAAqB,EAAA;IAC1C,OAAO;AACL,QAAA,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;AAC7B,QAAA,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;AAC5C,QAAA,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;KACrC;AACH;AAEA;;;;AAI+E;AAC/E,SAAS,qBAAqB,CAAC,cAAuB,EAAE,MAAwB,EAAA;AAC9E,IAAA,IAAI,cAAc,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,EAAE;QAC1E,OAAO;AACL,YAAA,QAAQ,EAAE,GAAG;AACb,YAAA,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;AACzC,YAAA,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;SACtC;IACH;AAEA,IAAA,OAAO,MAAM;AACf;AAEM,SAAU,yBAAyB,CAAC,CAAM,EAAA;AAC9C,IAAA,QACE,CAAC,CAAC,IAAI,IAAI;AACP,YAAC,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,gBAAgB,CAAC;AAEnF;AAEM,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,OAAO,CAAC,YAAY,UAAU;AAC5B,QAAA,CAAC,YAAY,aAAa;AAC1B,QAAA,CAAC,YAAY,kBAAkB;QAC/B,CAAC,YAAY,eAAe;AAChC;AAEA,SAAS,qBAAqB,CAAC,CAAM,EAAA;IACnC,OAAQ,CAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA,uBAAA,CAAyB,CAAC;AAClE;AAEA;AACA,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC;IACjD,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;AAEA;AACuC;AACvC,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC;IAC7D,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;;;;"}
1
+ {"version":3,"file":"upload_task.js","sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientProgress, ProgressStatus } from '../clients/progress';\nimport type { ClientUpload } from '../clients/upload';\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';\nimport type { ImportResourceSnapshot } from './types';\nimport { ImportFileHandleUploadData } from './types';\nimport assert from 'node:assert';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n private nMaxUploads: number;\n private nPartsWithThisUploadSpeed = 0;\n private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n /** If this is upload progress this field will be defined */\n private uploadData?: ImportFileHandleUploadData;\n public progress: sdk.ImportProgress;\n\n /** If failed, then getting a progress is terminally failed. */\n public failed?: boolean;\n\n /** True if the blob was existed.\n * At this case, the task will show progress == 1.0. */\n private alreadyExisted = false;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly maxNConcurrentPartsUpload: number,\n signer: Signer,\n public readonly res: ImportResourceSnapshot,\n ) {\n this.nMaxUploads = this.maxNConcurrentPartsUpload;\n const { uploadData, progress } = newProgress(res, signer);\n this.uploadData = uploadData;\n this.progress = progress;\n }\n\n public getProgress(w: Watcher, callerId: string) {\n this.incCounter(w, callerId);\n\n if (this.failed) {\n this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n throw new Error(this.progress.lastError);\n }\n\n return cloneProgress(this.progress);\n }\n\n public shouldScheduleUpload(): boolean {\n return isMyUpload(this.progress);\n }\n\n /** Uploads a blob if it's not BlobIndex. */\n public async uploadBlobTask() {\n try {\n await uploadBlob(\n this.logger,\n this.clientBlob,\n this.res,\n this.uploadData!,\n this.isComputableDone.bind(this),\n {\n nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n currentSpeed: this.nMaxUploads,\n maxSpeed: this.maxNConcurrentPartsUpload,\n },\n );\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n } catch (e: any) {\n\n if (isResourceWasDeletedError(e)) {\n this.setRetriableError(e);\n this.logger.warn(`resource was deleted while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);\n this.setDone(true);\n\n return;\n }\n\n this.logger.error(`error while uploading a blob: ${e}`);\n this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n if (nonRecoverableError(e)) {\n this.setTerminalError(e);\n return;\n }\n\n this.setRetriableError(e);\n\n if (isHeadersTimeoutError(e)) {\n // we probably have a slow internet, we need to slow things a bit.\n this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n }\n\n throw e;\n }\n }\n\n public async updateStatus() {\n try {\n // we do it with timeout in case we have slow internet.\n const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n const oldStatus = this.progress.status;\n const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n this.progress.status = newStatus;\n this.setDone(status.done);\n\n if (status.done || this.progress.status.progress != oldStatus?.progress) {\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n }\n } catch (e: any) {\n this.setRetriableError(e);\n\n if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {\n this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);\n return;\n }\n\n if (isResourceWasDeletedError(e)) {\n this.logger.warn(\n `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,\n );\n this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed, resource not found`);\n this.setDone(true);\n return;\n }\n\n this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n // It was a terminal error, but when a connection drops,\n // this will stop the whole task, so we make it retryable.\n // It was like that:\n // this.change.markChanged();\n // this.setTerminalError(e);\n }\n }\n\n /** Set non-terminal error, that task can be retried. */\n private setRetriableError(e: unknown) {\n this.progress.lastError = String(e);\n }\n\n /** Set a terminal error, the task will throw a error instead of a progress. */\n private setTerminalError(e: unknown) {\n this.progress.lastError = String(e);\n this.progress.done = false;\n this.failed = true;\n }\n\n public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n if (isImportResourceOutputSet(res)) {\n this.setDone(true);\n this.alreadyExisted = true;\n }\n }\n\n private setDone(done: boolean) {\n this.progress.done = done;\n if (done) this.progress.lastError = undefined;\n }\n\n public incCounter(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n public decCounter(callerId: string) {\n return this.counter.dec(callerId);\n }\n\n private isComputableDone() {\n return this.counter.isZero();\n }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n logger: MiLogger,\n clientBlob: ClientUpload,\n res: ResourceInfo,\n uploadData: ImportFileHandleUploadData,\n isDoneFn: () => boolean,\n speed: {\n nPartsWithThisUploadSpeed: number,\n nPartsToIncreaseUpload: number,\n currentSpeed: number,\n maxSpeed: number,\n },\n) {\n assert(isUpload(res), 'the upload operation can be done only for BlobUploads');\n const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n if (isDoneFn()) return;\n const parts = await clientBlob.initUpload(res, { timeout });\n logger.info(\n `started to upload blob ${res.id},`\n + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`\n + ` number of concurrent uploads: ${speed.currentSpeed}`,\n );\n\n const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n if (isDoneFn()) return;\n await clientBlob.partUpload(\n res,\n uploadData.localPath,\n BigInt(uploadData.modificationTime),\n part,\n parts.checksumAlgorithm,\n parts.checksumHeader,\n { timeout }\n );\n logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n // if we had a network freeze, it will be increased slowly.\n speed.nPartsWithThisUploadSpeed++;\n if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n speed.nPartsWithThisUploadSpeed = 0;\n speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n controller.setConcurrency(speed.currentSpeed);\n }\n };\n\n await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n if (isDoneFn()) return;\n await clientBlob.finalize(res, { timeout });\n\n logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n let isUploadSignMatch: boolean | undefined;\n let uploadData: ImportFileHandleUploadData | undefined;\n if (isUpload(res)) {\n uploadData = ImportFileHandleUploadData.parse(res.data);\n isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n }\n\n return {\n uploadData: uploadData,\n progress: {\n done: false,\n status: undefined,\n isUpload: isUpload(res),\n isUploadSignMatch: isUploadSignMatch,\n lastError: undefined,\n } satisfies sdk.ImportProgress,\n };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n const cloned: sdk.ImportProgress = {\n done: progress.done,\n isUpload: progress.isUpload,\n isUploadSignMatch: progress.isUploadSignMatch,\n lastError: progress.lastError,\n };\n\n if (progress.status)\n cloned.status = {\n progress: progress.status.progress,\n bytesProcessed: progress.status.bytesProcessed,\n bytesTotal: progress.status.bytesTotal,\n };\n\n return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n return 'blob' in res.fields\n ? res.fields.blob !== undefined\n : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n return res.type.name.startsWith('BlobUpload');\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n try {\n signer.verify(path, signature);\n return true;\n } catch (e) {\n return false;\n }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n return {\n progress: proto.progress ?? 0,\n bytesProcessed: Number(proto.bytesProcessed),\n bytesTotal: Number(proto.bytesTotal),\n };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n return {\n progress: 1.0,\n bytesProcessed: Number(status.bytesTotal),\n bytesTotal: Number(status.bytesTotal),\n };\n }\n\n return status;\n}\n\nexport function isResourceWasDeletedError(e: any) {\n return (\n e.name == 'RpcError'\n && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')\n );\n}\n\nexport function nonRecoverableError(e: any) {\n return e instanceof MTimeError ||\n e instanceof UnexpectedEOF ||\n e instanceof NoFileForUploading ||\n e instanceof BadRequestError;\n}\n\nfunction isHeadersTimeoutError(e: any) {\n return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n const newConcurrency = Math.min(current + 2, max);\n if (newConcurrency != current)\n logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n const newConcurrency = Math.max(Math.round(current / 2), min);\n if (newConcurrency != current)\n logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`)\n\n return newConcurrency;\n}\n"],"names":[],"mappings":";;;;;;;AAcA;;AAEgC;MACnB,UAAU,CAAA;AAmBF,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,cAAA;AACA,IAAA,yBAAA;AAED,IAAA,GAAA;AAvBD,IAAA,MAAM,GAAiB,IAAI,YAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAI,cAAc,EAAE;AACvD,IAAA,WAAW;IACX,yBAAyB,GAAG,CAAC;AAC7B,IAAA,sBAAsB,GAAG,EAAE,CAAC;;AAG5B,IAAA,UAAU;AACX,IAAA,QAAQ;;AAGR,IAAA,MAAM;AAEb;AACuD;IAC/C,cAAc,GAAG,KAAK;IAE9B,WAAA,CACmB,MAAgB,EAChB,UAAwB,EACxB,cAA8B,EAC9B,yBAAiC,EAClD,MAAc,EACE,GAA2B,EAAA;QAL1B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,yBAAyB,GAAzB,yBAAyB;QAE1B,IAAA,CAAA,GAAG,GAAH,GAAG;AAEnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,yBAAyB;AACjD,QAAA,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;AACzD,QAAA,IAAI,CAAC,UAAU,GAAG,UAAU;AAC5B,QAAA,IAAI,CAAC,QAAQ,GAAG,QAAQ;IAC1B;IAEO,WAAW,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC7C,QAAA,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC;AAE5B,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,6BAAA,EAAgC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAA,CAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC1C;AAEA,QAAA,OAAO,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;IACrC;IAEO,oBAAoB,GAAA;AACzB,QAAA,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClC;;AAGO,IAAA,MAAM,cAAc,GAAA;AACzB,QAAA,IAAI;YACF,MAAM,UAAU,CACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,UAAW,EAChB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAChC;gBACE,yBAAyB,EAAE,IAAI,CAAC,yBAAyB;gBACzD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;gBACnD,YAAY,EAAE,IAAI,CAAC,WAAW;gBAC9B,QAAQ,EAAE,IAAI,CAAC,yBAAyB;AACzC,aAAA,CACF;AACD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,6CAAA,EAAgD,CAAC,CAAA,CAAE,CAAC;AACrE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,8BAAA,CAAgC,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAElB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,8BAAA,EAAiC,CAAC,CAAA,CAAE,CAAC;AACvD,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;AAEpF,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACxB;YACF;AAEA,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;AAEzB,YAAA,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE;;AAE5B,gBAAA,IAAI,CAAC,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1E;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEO,IAAA,MAAM,YAAY,GAAA;AACvB,QAAA,IAAI;;AAEF,YAAA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAEhF,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;AACtC,YAAA,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;AACnF,YAAA,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS;AAChC,YAAA,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;AAEzB,YAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,QAAQ,EAAE;AACvE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;YACzF;QACF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEzB,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,mBAAmB,KAAK,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,mBAAmB,CAAC,EAAE;gBACxG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,wDAAA,EAA2D,CAAC,CAAC,OAAO,CAAA,CAAE,CAAC;gBACxF;YACF;AAEA,YAAA,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAChC,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iEAAiE,CAAC,CAAA,EAAA,EAAK,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA,CAAE,CAC3G;AACD,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA,4BAAA,CAA8B,CAAC;AAC3G,gBAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,uDAAA,EAA0D,CAAC,CAAA,CAAE,CAAC;;;;;;QAMlF;IACF;;AAGQ,IAAA,iBAAiB,CAAC,CAAU,EAAA;QAClC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IACrC;;AAGQ,IAAA,gBAAgB,CAAC,CAAU,EAAA;QACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;AACnC,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK;AAC1B,QAAA,IAAI,CAAC,MAAM,GAAG,IAAI;IACpB;AAEO,IAAA,kBAAkB,CAAC,GAA2B,EAAA;AACnD,QAAA,IAAI,yBAAyB,CAAC,GAAG,CAAC,EAAE;AAClC,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,cAAc,GAAG,IAAI;QAC5B;IACF;AAEQ,IAAA,OAAO,CAAC,IAAa,EAAA;AAC3B,QAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI;AACzB,QAAA,IAAI,IAAI;AAAE,YAAA,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,SAAS;IAC/C;IAEO,UAAU,CAAC,CAAU,EAAE,QAAgB,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEO,IAAA,UAAU,CAAC,QAAgB,EAAA;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;IAEQ,gBAAgB,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;IAC9B;AACD;AAED;AACO,eAAe,UAAU,CAC9B,MAAgB,EAChB,UAAwB,EACxB,GAAiB,EACjB,UAAsC,EACtC,QAAuB,EACvB,KAKC,EAAA;IAED,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,uDAAuD,CAAC;AAC9E,IAAA,MAAM,OAAO,GAAG,KAAK,CAAC;AAEtB,IAAA,IAAI,QAAQ,EAAE;QAAE;AAChB,IAAA,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;AAC3D,IAAA,MAAM,CAAC,IAAI,CACT,0BAA0B,GAAG,CAAC,EAAE,CAAA,CAAA;UAC9B,CAAA,gBAAA,EAAmB,KAAK,CAAC,OAAO,CAAA,kBAAA,EAAqB,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAA,CAAA;AAC1E,UAAA,CAAA,+BAAA,EAAkC,KAAK,CAAC,YAAY,CAAA,CAAE,CACzD;IAED,MAAM,YAAY,GAAG,CAAC,IAAY,KAAK,OAAO,UAA+B,KAAI;AAC/E,QAAA,IAAI,QAAQ,EAAE;YAAE;AAChB,QAAA,MAAM,UAAU,CAAC,UAAU,CACzB,GAAG,EACH,UAAU,CAAC,SAAS,EACpB,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,EACnC,IAAI,EACJ,KAAK,CAAC,iBAAiB,EACvB,KAAK,CAAC,cAAc,EACpB,EAAE,OAAO,EAAE,CACZ;AACD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,OAAO,iBAAiB,GAAG,CAAC,EAAE,CAAA,CAAE,CAAC;;QAG7E,KAAK,CAAC,yBAAyB,EAAE;QACjC,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC,sBAAsB,EAAE;AACnE,YAAA,KAAK,CAAC,yBAAyB,GAAG,CAAC;AACnC,YAAA,KAAK,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC;AACpF,YAAA,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC;QAC/C;AACF,IAAA,CAAC;AAED,IAAA,MAAM,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAErE,IAAA,IAAI,QAAQ,EAAE;QAAE;IAChB,MAAM,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,CAAA,sBAAA,EAAyB,GAAG,CAAC,EAAE,CAAA,UAAA,CAAY,CAAC;AAC1D;AAEA,SAAS,WAAW,CAAC,GAA2B,EAAE,MAAc,EAAA;AAC9D,IAAA,IAAI,iBAAsC;AAC1C,IAAA,IAAI,UAAkD;AACtD,IAAA,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE;QACjB,UAAU,GAAG,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACvD,QAAA,iBAAiB,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,aAAa,CAAC;IACzF;IAEA,OAAO;AACL,QAAA,UAAU,EAAE,UAAU;AACtB,QAAA,QAAQ,EAAE;AACR,YAAA,IAAI,EAAE,KAAK;AACX,YAAA,MAAM,EAAE,SAAS;AACjB,YAAA,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvB,YAAA,iBAAiB,EAAE,iBAAiB;AACpC,YAAA,SAAS,EAAE,SAAS;AACQ,SAAA;KAC/B;AACH;AAEA;AACM,SAAU,UAAU,CAAC,CAAqB,EAAA;IAC9C,OAAO,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC;AACrD;AAEA;;;;AAIqD;AACrD,SAAS,aAAa,CAAC,QAA4B,EAAA;AACjD,KAAmC;QACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,SAAS,EAAE,QAAQ,CAAC,SAAS;;IAG/B,IAAI,QAAQ,CAAC,MAAM;SACD;AACd,YAAA,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;AAClC,YAAA,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc;AAC9C,YAAA,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;UACvC;AAEH,IAAA,OAAO,QAAQ;AACjB;AAEA,SAAS,yBAAyB,CAAC,GAA2B,EAAA;AAC5D,IAAA,OAAO,MAAM,IAAI,GAAG,CAAC;AACnB,UAAE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK;UACpB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;AAC1C;AAEM,SAAU,QAAQ,CAAC,GAAiB,EAAA;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;AAC/C;SAEgB,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAA;AACzE,IAAA,IAAI;AACF,QAAA,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC;AAC9B,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,CAAC,EAAE;AACV,QAAA,OAAO,KAAK;IACd;AACF;AAEA,SAAS,aAAa,CAAC,KAAqB,EAAA;IAC1C,OAAO;AACL,QAAA,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;AAC7B,QAAA,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;AAC5C,QAAA,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;KACrC;AACH;AAEA;;;;AAI+E;AAC/E,SAAS,qBAAqB,CAAC,cAAuB,EAAE,MAAwB,EAAA;AAC9E,IAAA,IAAI,cAAc,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,EAAE;QAC1E,OAAO;AACL,YAAA,QAAQ,EAAE,GAAG;AACb,YAAA,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;AACzC,YAAA,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;SACtC;IACH;AAEA,IAAA,OAAO,MAAM;AACf;AAEM,SAAU,yBAAyB,CAAC,CAAM,EAAA;AAC9C,IAAA,QACE,CAAC,CAAC,IAAI,IAAI;AACP,YAAC,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,gBAAgB,CAAC;AAEnF;AAEM,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,OAAO,CAAC,YAAY,UAAU;AAC5B,QAAA,CAAC,YAAY,aAAa;AAC1B,QAAA,CAAC,YAAY,kBAAkB;QAC/B,CAAC,YAAY,eAAe;AAChC;AAEA,SAAS,qBAAqB,CAAC,CAAM,EAAA;IACnC,OAAQ,CAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA,uBAAA,CAAyB,CAAC;AAClE;AAEA;AACA,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC;IACjD,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;AAEA;AACuC;AACvC,SAAS,mBAAmB,CAAC,MAAgB,EAAE,OAAe,EAAE,GAAW,EAAA;AACzE,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC;IAC7D,IAAI,cAAc,IAAI,OAAO;QAC3B,MAAM,CAAC,IAAI,CAAC,CAAA,+CAAA,EAAkD,OAAO,CAAA,IAAA,EAAO,cAAc,CAAA,CAAE,CAAC;AAE/F,IAAA,OAAO,cAAc;AACvB;;;;"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@milaboratories/pl-drivers",
3
- "version": "1.11.5",
3
+ "version": "1.11.7",
4
4
  "engines": {
5
5
  "node": ">=22"
6
6
  },
@@ -32,10 +32,10 @@
32
32
  "upath": "^2.0.1",
33
33
  "zod": "~3.23.8",
34
34
  "@milaboratories/ts-helpers": "1.5.1",
35
- "@milaboratories/pl-client": "2.14.0",
35
+ "@milaboratories/helpers": "1.11.0",
36
+ "@milaboratories/pl-model-common": "1.21.1",
36
37
  "@milaboratories/computable": "2.7.1",
37
- "@milaboratories/pl-model-common": "1.20.1",
38
- "@milaboratories/helpers": "1.10.0",
38
+ "@milaboratories/pl-client": "2.14.0",
39
39
  "@milaboratories/pl-tree": "1.8.4"
40
40
  },
41
41
  "devDependencies": {
package/src/drivers/upload_task.ts CHANGED
@@ -126,7 +126,7 @@ export class UploadTask {
126
126
  this.setRetriableError(e);
127
127
 
128
128
  if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {
129
- this.logger.warn(`deadline exceeded while getting a status of BlobImport`);
129
+ this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);
130
130
  return;
131
131
  }
132
132