@milaboratories/pl-drivers 1.5.50 → 1.5.51

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.5.50",
+  "version": "1.5.51",
   "engines": {
     "node": ">=20"
   },
@@ -30,14 +30,14 @@
     "tar-fs": "^3.0.8",
     "undici": "~7.5.0",
     "zod": "~3.23.8",
+    "@milaboratories/pl-client": "^2.9.0",
     "@milaboratories/computable": "^2.4.7",
+    "@milaboratories/pl-model-common": "^1.14.0",
     "@milaboratories/ts-helpers": "^1.2.0",
-    "@milaboratories/pl-tree": "^1.6.0",
-    "@milaboratories/pl-client": "^2.9.0",
-    "@milaboratories/pl-model-common": "^1.14.0"
+    "@milaboratories/pl-tree": "^1.6.0"
   },
   "devDependencies": {
-    "eslint": "^9.23.0",
+    "eslint": "^9.25.1",
     "@types/decompress": "^4.2.7",
     "typescript": "~5.5.4",
     "vite": "^5.4.11",

package/src/drivers/download_blob.ts

@@ -252,7 +252,7 @@ export class DownloadDriver implements BlobDriver {
   public getComputableContent(
     res: ResourceInfo | PlTreeEntry
   ): ComputableStableDefined<Uint8Array>{
-    return Computable.make((ctx) => 
+    return Computable.make((ctx) =>
       this.getDownloadedBlob(res, ctx), {
         postprocessValue: (v) => v ? this.getContent(v.handle) : undefined
       }

package/src/drivers/upload.ts

@@ -26,7 +26,7 @@ import { scheduler } from 'node:timers/promises';
 import type { PollingOps } from './helpers/polling_ops';
 import type { ImportResourceSnapshot } from './types';
 import { IndexResourceSnapshot, UploadResourceSnapshot } from './types';
-import { nonRecoverableError, UploadTask } from './upload_task';
+import { isMyUpload, nonRecoverableError, UploadTask } from './upload_task';
 import { WrongResourceTypeError } from './helpers/helpers';
 
 export function makeBlobImportSnapshot(
@@ -145,11 +145,12 @@ export class UploadDriver {
 
     this.idToProgress.set(res.id, newTask);
 
-    if (newTask.shouldScheduleUpload())
+    if (newTask.shouldScheduleUpload()) {
       this.uploadQueue.push({
         fn: () => newTask.uploadBlobTask(),
         recoverableErrorPredicate: (e) => !nonRecoverableError(e),
       });
+    }
 
     newTask.setDoneIfOutputSet(res);
     return newTask.getProgress(w, callerId);

package/src/drivers/upload_task.ts

@@ -10,6 +10,7 @@ import { MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload
 import type { ImportResourceSnapshot } from './types';
 import { ImportFileHandleUploadData } from './types';
 import assert from 'node:assert';
+import { ResourceInfo } from '@milaboratories/pl-tree';
 
 /** Holds all info needed to upload a file and a status of uploading
  * and indexing. Also, has a method to update a status of the progress.
@@ -58,49 +59,25 @@ export class UploadTask {
   }
 
   public shouldScheduleUpload(): boolean {
-    return this.progress.isUpload && this.progress.isUploadSignMatch!;
+    return isMyUpload(this.progress);
   }
 
   /** Uploads a blob if it's not BlobIndex. */
   public async uploadBlobTask() {
-    assert(isUpload(this.res), 'the upload operation can be done only for BlobUploads');
-    const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.
-
     try {
-      if (this.isComputableDone()) return;
-      const parts = await this.clientBlob.initUpload(this.res, { timeout });
-      this.logger.info(
-        `started to upload blob ${this.res.id},`
-          + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`
-          + ` number of concurrent uploads: ${this.nMaxUploads}`,
+      await uploadBlob(
+        this.logger,
+        this.clientBlob,
+        this.res,
+        this.uploadData!,
+        this.isComputableDone.bind(this),
+        {
+          nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,
+          nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,
+          currentSpeed: this.nMaxUploads,
+          maxSpeed: this.maxNConcurrentPartsUpload,
+        },
       );
-
-      const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {
-        if (this.isComputableDone()) return;
-        await this.clientBlob.partUpload(
-          this.res,
-          this.uploadData!.localPath,
-          BigInt(this.uploadData!.modificationTime),
-          part,
-          { timeout }
-        );
-        this.logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${this.res.id}`);
-
-        // if we had a network freeze, it will be increased slowly.
-        this.nPartsWithThisUploadSpeed++;
-        if (this.nPartsWithThisUploadSpeed >= this.nPartsToIncreaseUpload) {
-          this.nPartsWithThisUploadSpeed = 0;
-          this.nMaxUploads = increaseConcurrency(this.logger, this.nMaxUploads, this.maxNConcurrentPartsUpload);
-          controller.setConcurrency(this.nMaxUploads);
-        }
-      };
-
-      await asyncPool(this.nMaxUploads, parts.toUpload.map(partUploadFn));
-
-      if (this.isComputableDone()) return;
-      await this.clientBlob.finalize(this.res, { timeout });
-
-      this.logger.info(`uploading of resource ${this.res.id} finished.`);
       this.change.markChanged();
     } catch (e: any) {
       this.setRetriableError(e);
@@ -207,6 +184,59 @@ export class UploadTask {
   }
 }
 
+/** Uploads a blob if it's not BlobIndex. */
+export async function uploadBlob(
+  logger: MiLogger,
+  clientBlob: ClientUpload,
+  res: ResourceInfo,
+  uploadData: ImportFileHandleUploadData,
+  isDoneFn: () => boolean,
+  speed: {
+    nPartsWithThisUploadSpeed: number,
+    nPartsToIncreaseUpload: number,
+    currentSpeed: number,
+    maxSpeed: number,
+  },
+) {
+  assert(isUpload(res), 'the upload operation can be done only for BlobUploads');
+  const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.
+
+  if (isDoneFn()) return;
+  const parts = await clientBlob.initUpload(res, { timeout });
+  logger.info(
+    `started to upload blob ${res.id},`
+    + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},`
+    + ` number of concurrent uploads: ${speed.currentSpeed}`,
+  );
+
+  const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {
+    if (isDoneFn()) return;
+    await clientBlob.partUpload(
+      res,
+      uploadData.localPath,
+      BigInt(uploadData.modificationTime),
+      part,
+      { timeout }
+    );
+    logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);
+
+    // if we had a network freeze, it will be increased slowly.
+    speed.nPartsWithThisUploadSpeed++;
+    if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {
+      speed.nPartsWithThisUploadSpeed = 0;
+      speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);
+      controller.setConcurrency(speed.currentSpeed);
+    }
+  };
+
+  await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));
+
+  if (isDoneFn()) return;
+  await clientBlob.finalize(res, { timeout });
+
+  logger.info(`uploading of resource ${res.id} finished.`);
+}
+
 function newProgress(res: ImportResourceSnapshot, signer: Signer) {
   let isUploadSignMatch: boolean | undefined;
   let uploadData: ImportFileHandleUploadData | undefined;
@@ -223,10 +253,15 @@ function newProgress(res: ImportResourceSnapshot, signer: Signer) {
       isUpload: isUpload(res),
       isUploadSignMatch: isUploadSignMatch,
       lastError: undefined,
-    },
+    } satisfies sdk.ImportProgress,
   };
 }
 
+/** Returns true if we need to upload the blob that got from this progress. */
+export function isMyUpload(p: sdk.ImportProgress): boolean {
+  return p.isUpload && (p.isUploadSignMatch ?? false);
+}
+
 /** Creates a deep copy of progress,
  * since we do not want to pass a mutable object
  * to API, it led to bugs before.
@@ -256,7 +291,7 @@ function isImportResourceOutputSet(res: ImportResourceSnapshot) {
     : res.fields.incarnation !== undefined;
 }
 
-function isUpload(res: ImportResourceSnapshot) {
+function isUpload(res: ResourceInfo) {
   return res.type.name.startsWith('BlobUpload');
 }
 

package/src/index.ts

@@ -9,6 +9,7 @@ export * from './drivers/download_blob';
 export * from './drivers/download_blob_url/driver';
 export * from './drivers/download_blob_url/snapshot';
 export * from './drivers/upload';
+export * from './drivers/upload_task';
 export * from './drivers/logs_stream';
 export * from './drivers/logs';
 export * from './drivers/download_url';