@milaboratories/pl-drivers 1.5.16 → 1.5.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.5.16",
+  "version": "1.5.17",
   "engines": {
     "node": ">=20"
   },
@@ -30,11 +30,11 @@
     "tar-fs": "^3.0.8",
     "undici": "^7.2.3",
     "zod": "~3.23.8",
-    "@milaboratories/ts-helpers": "^1.1.3",
-    "@milaboratories/pl-tree": "^1.4.23",
-    "@milaboratories/computable": "^2.3.4",
+    "@milaboratories/ts-helpers": "^1.1.4",
+    "@milaboratories/pl-client": "^2.7.5",
+    "@milaboratories/computable": "^2.3.5",
     "@milaboratories/pl-model-common": "^1.10.5",
-    "@milaboratories/pl-client": "^2.7.4"
+    "@milaboratories/pl-tree": "^1.4.24"
   },
   "devDependencies": {
     "eslint": "^9.16.0",
@@ -47,8 +47,8 @@
     "jest": "^29.7.0",
     "@jest/globals": "^29.7.0",
     "ts-jest": "^29.2.5",
-    "@milaboratories/eslint-config": "^1.0.1",
-    "@milaboratories/platforma-build-configs": "1.0.2"
+    "@milaboratories/platforma-build-configs": "1.0.2",
+    "@milaboratories/eslint-config": "^1.0.1"
   },
   "scripts": {
     "type-check": "tsc --noEmit --composite false",

package/src/clients/upload.ts

@@ -77,6 +77,12 @@ export class ClientUpload {
       } = await request(info.uploadUrl, {
         dispatcher: this.httpClient,
         body: chunk,
+        // We got headers only after we send
+        // the whole body (in case of S3 PUT requests it's 5 MB).
+        // It might be slow with a slow connection (or with SSH),
+        // that's why we got big timeout here.
+        headersTimeout: 60000,
+        bodyTimeout: 60000,
         headers: toHeadersMap(info.headers),
         method: info.method.toUpperCase() as Dispatcher.HttpMethod,
       });
@@ -85,10 +91,13 @@ export class ClientUpload {
       const body = await rawBody.text();
       checkStatusCodeOk(statusCode, body, headers, info);
     } catch (e: unknown) {
+      if (e instanceof NetworkError)
+        throw e;
+
       throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);
     }
 
-    await this.grpcUpdateProgress({ id, type }, info.chunkEnd - info.chunkStart, options);
+    await this.grpcUpdateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options);
   }
 
   public async finalize(info: ResourceInfo, options?: RpcOptions) {

package/src/drivers/download_blob_url/task.ts

@@ -179,7 +179,7 @@ export class DownloadAndUnarchiveTask {
 async function dirSize(dir: string): Promise<number> {
   const files = await fsp.readdir(dir, { withFileTypes: true });
   const sizes = await Promise.all(
-    files.map(async (file) => {
+    files.map(async (file: any) => {
       const fPath = path.join(dir, file.name);
 
       if (file.isDirectory()) return await dirSize(fPath);
@@ -189,7 +189,7 @@ async function dirSize(dir: string): Promise<number> {
     }),
   );
 
-  return sizes.reduce((sum, size) => sum + size, 0);
+  return sizes.reduce((sum: any, size: any) => sum + size, 0);
 }
 
 /** Do rm -rf on dir. */

package/src/drivers/download_url.ts

@@ -295,7 +295,7 @@ class URLAborted extends Error {}
 async function dirSize(dir: string): Promise<number> {
   const files = await fsp.readdir(dir, { withFileTypes: true });
   const sizes = await Promise.all(
-    files.map(async (file) => {
+    files.map(async (file: any) => {
       const fPath = path.join(dir, file.name);
 
       if (file.isDirectory()) return await dirSize(fPath);
@@ -305,7 +305,7 @@ async function dirSize(dir: string): Promise<number> {
     }),
   );
 
-  return sizes.reduce((sum, size) => sum + size, 0);
+  return sizes.reduce((sum: any, size: any) => sum + size, 0);
 }
 
 /** Do rm -rf on dir. */

package/src/drivers/upload_task.ts

@@ -2,7 +2,7 @@ import type { Watcher } from '@milaboratories/computable';
 import { ChangeSource } from '@milaboratories/computable';
 import { stringifyWithResourceId } from '@milaboratories/pl-client';
 import type * as sdk from '@milaboratories/pl-model-common';
-import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
+import type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-helpers';
 import { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';
 import type { ClientProgress, ProgressStatus } from '../clients/progress';
 import type { ClientUpload } from '../clients/upload';
@@ -17,6 +17,9 @@ import assert from 'node:assert';
 export class UploadTask {
   private readonly change: ChangeSource = new ChangeSource();
   private readonly counter: CallersCounter = new CallersCounter();
+  private nMaxUploads: number;
+  private nPartsWithThisUploadSpeed = 0;
+  private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.
 
   /** If this is upload progress this field will be defined */
   private uploadData?: ImportFileHandleUploadData;
@@ -33,10 +36,11 @@ export class UploadTask {
     private readonly logger: MiLogger,
     private readonly clientBlob: ClientUpload,
     private readonly clientProgress: ClientProgress,
-    private readonly nConcurrentPartsUpload: number,
+    private readonly maxNConcurrentPartsUpload: number,
     signer: Signer,
     public readonly res: ImportResourceSnapshot,
   ) {
+    this.nMaxUploads = this.maxNConcurrentPartsUpload;
     const { uploadData, progress } = newProgress(res, signer);
     this.uploadData = uploadData;
     this.progress = progress;
@@ -60,30 +64,41 @@ export class UploadTask {
   /** Uploads a blob if it's not BlobIndex. */
   public async uploadBlobTask() {
     assert(isUpload(this.res), 'the upload operation can be done only for BlobUploads');
+    const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.
 
     try {
       if (this.isComputableDone()) return;
-      const parts = await this.clientBlob.initUpload(this.res);
+      const parts = await this.clientBlob.initUpload(this.res, { timeout });
       this.logger.info(
         `started to upload blob ${this.res.id},`
-        + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length}`,
+          + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`
+          + ` number of concurrent uploads: ${this.nMaxUploads}`,
       );
 
-      const partUploadFn = (part: bigint) => async () => {
+      const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {
         if (this.isComputableDone()) return;
         await this.clientBlob.partUpload(
           this.res,
           this.uploadData!.localPath,
           BigInt(this.uploadData!.modificationTime),
           part,
+          { timeout }
         );
         this.logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${this.res.id}`);
+
+        // if we had a network freeze, it will be increased slowly.
+        this.nPartsWithThisUploadSpeed++;
+        if (this.nPartsWithThisUploadSpeed >= this.nPartsToIncreaseUpload) {
+          this.nPartsWithThisUploadSpeed = 0;
+          this.nMaxUploads = increaseConcurrency(this.logger, this.nMaxUploads, this.maxNConcurrentPartsUpload);
+          controller.setConcurrency(this.nMaxUploads);
+        }
       };
 
-      await asyncPool(this.nConcurrentPartsUpload, parts.toUpload.map(partUploadFn));
+      await asyncPool(this.nMaxUploads, parts.toUpload.map(partUploadFn));
 
       if (this.isComputableDone()) return;
-      await this.clientBlob.finalize(this.res);
+      await this.clientBlob.finalize(this.res, { timeout });
 
       this.logger.info(`uploading of resource ${this.res.id} finished.`);
       this.change.markChanged();
@@ -106,13 +121,19 @@ export class UploadTask {
         return;
       }
 
+      if (isHeadersTimeoutError(e)) {
+        // we probably have a slow internet, we need to slow things a bit.
+        this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);
+      }
+
       throw e;
     }
   }
 
   public async updateStatus() {
     try {
-      const status = await this.clientProgress.getStatus(this.res);
+      // we do it with timeout in case we have slow internet.
+      const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });
 
       const oldStatus = this.progress.status;
       const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));
@@ -125,7 +146,7 @@ export class UploadTask {
     } catch (e: any) {
       this.setRetriableError(e);
 
-      if (e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') {
+      if ((e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') || e?.message?.includes('DEADLINE_EXCEEDED')) {
         this.logger.warn(`deadline exceeded while getting a status of BlobImport`);
         return;
       }
@@ -139,9 +160,12 @@ export class UploadTask {
         return;
       }
 
-      this.logger.error(`error while updating a status of BlobImport: ${e}`);
-      this.change.markChanged();
-      this.setTerminalError(e);
+      this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);
+      // It was a terminal error, but when a connection drops,
+      // this will stop the whole task, so we make it retryable.
+      // It was like that:
+      // this.change.markChanged();
+      // this.setTerminalError(e);
     }
   }
 
@@ -280,3 +304,26 @@ export function isResourceWasDeletedError(e: any) {
 export function nonRecoverableError(e: any) {
   return e instanceof MTimeError || e instanceof UnexpectedEOF || e instanceof NoFileForUploading;
 }
+
+function isHeadersTimeoutError(e: any) {
+  return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);
+}
+
+/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */
+function increaseConcurrency(logger: MiLogger, current: number, max: number): number {
+  const newConcurrency = Math.min(current + 2, max);
+  if (newConcurrency != current)
+    logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`)
+
+  return newConcurrency;
+}
+
+/** When a error happens, this will half the concurrency level, so the next time
+ * we'll try to upload blobs slower. */
+function decreaseConcurrency(logger: MiLogger, current: number, min: number): number {
+  const newConcurrency = Math.max(Math.round(current / 2), min);
+  if (newConcurrency != current)
+    logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`)
+
+  return newConcurrency;
+}