@cloudbase/manager-node 3.11.0 → 3.11.1-beta

package/lib/storage/index.js

@@ -382,27 +382,32 @@ class StorageService {
      * @returns {Promise<(NodeJS.ReadableStream | string)[]>}
      */
     async downloadDirectory(options) {
-        const { cloudPath, localPath } = options;
+        const { cloudPath, localPath, parallel = 20 } = options;
         const resolveLocalPath = path_1.default.resolve(localPath);
         utils_1.checkFullAccess(resolveLocalPath, true);
         const cloudDirectoryKey = this.getCloudKey(cloudPath);
         const files = await this.walkCloudDir(cloudDirectoryKey);
-        const promises = files.map(async (file) => {
-            const fileRelativePath = file.Key.replace(cloudDirectoryKey, '');
-            // 空路径和文件夹跳过
-            if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
-                return;
-            }
-            const localFilePath = path_1.default.join(resolveLocalPath, fileRelativePath);
-            // 创建文件的父文件夹
-            const fileDir = path_1.default.dirname(localFilePath);
-            await make_dir_1.default(fileDir);
-            return this.downloadFile({
-                cloudPath: file.Key,
-                localPath: localFilePath
-            });
+        const promises = files.map(file => async () => {
+            return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath });
         });
-        return Promise.all(promises);
+        const asyncTaskController = new parallel_1.AsyncTaskParallelController(parallel, 50);
+        asyncTaskController.loadTasks(promises);
+        let res = await asyncTaskController.run();
+        const errorIndexArr = [];
+        res.map((item, index) => /Error/gi.test(Object.prototype.toString.call(item)) && errorIndexArr.push(index));
+        // 重试逻辑
+        if (errorIndexArr.length) {
+            const errorFiles = errorIndexArr.map(errorIndex => files[errorIndex]);
+            asyncTaskController.loadTasks(errorFiles.map(file => async () => {
+                return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath });
+            }));
+            res = await asyncTaskController.run();
+        }
+        const errorResultArr = this.determineDownLoadResultIsError(res);
+        if (errorResultArr.length) {
+            throw errorResultArr[0];
+        }
+        return res;
     }
     /**
      * 列出文件夹下的文件
@@ -911,6 +916,38 @@ class StorageService {
             }), interval);
         }
     }
+    /**
+     * 拼接路径下载单文件
+     * @param file
+     * @param cloudDirectoryKey
+     * @param resolveLocalPath
+     * @returns
+     */
+    async downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath }) {
+        const fileRelativePath = file.Key.replace(cloudDirectoryKey, '');
+        // 空路径和文件夹跳过
+        if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
+            return;
+        }
+        const localFilePath = path_1.default.join(resolveLocalPath, fileRelativePath);
+        // 创建文件的父文件夹
+        const fileDir = path_1.default.dirname(localFilePath);
+        await make_dir_1.default(fileDir);
+        return this.downloadFile({
+            cloudPath: file.Key,
+            localPath: localFilePath
+        });
+    }
+    /**
+     * 根据下载结果返回错误列表
+     * @param res
+     * @returns
+     */
+    determineDownLoadResultIsError(res) {
+        const resultErrorArr = [];
+        res.map(item => /Error/gi.test(Object.prototype.toString.call(item)) && resultErrorArr.push(item));
+        return resultErrorArr;
+    }
 }
 __decorate([
     utils_1.preLazy()

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cloudbase/manager-node",
-    "version": "3.11.0",
+    "version": "3.11.1-beta",
     "description": "The node manage service api for cloudbase.",
     "main": "lib/index.js",
     "scripts": {

package/src/storage/index.ts

@@ -62,7 +62,6 @@ export interface IFilesOptions extends IOptions {
     ignore?: string | string[]
     // 文件列表
     files: { localPath: string; cloudPath?: string }[]
-
     // 重试次数
     retryCount?: number
     // 重试时间间隔(毫秒)
@@ -146,8 +145,8 @@ export class StorageService {
      */
     @preLazy()
     public async uploadFiles(options: IFilesOptions): Promise<void> {
-        const { files, onProgress, parallel, onFileFinish, ignore, retryCount,
-            retryInterval } = options
+        const { files, onProgress, parallel, onFileFinish, ignore, retryCount, retryInterval } =
+            options
         const { bucket, region } = this.getStorageConfig()
 
         return this.uploadFilesCustom({
@@ -455,7 +454,6 @@ export class StorageService {
 
         const cos = this.getCos(parallel)
         const uploadFiles = Util.promisify(cos.uploadFiles).bind(cos)
-
         const params = {
             files: fileList,
             SliceSize: BIG_FILE_SIZE,
@@ -567,32 +565,42 @@ export class StorageService {
     public async downloadDirectory(options: {
         cloudPath: string
         localPath?: string
+        parallel?: number
     }): Promise<(NodeJS.ReadableStream | string)[]> {
-        const { cloudPath, localPath } = options
+        const { cloudPath, localPath, parallel = 20 } = options
         const resolveLocalPath = path.resolve(localPath)
 
         checkFullAccess(resolveLocalPath, true)
 
         const cloudDirectoryKey = this.getCloudKey(cloudPath)
         const files = await this.walkCloudDir(cloudDirectoryKey)
-
-        const promises = files.map(async file => {
-            const fileRelativePath = file.Key.replace(cloudDirectoryKey, '')
-            // 空路径和文件夹跳过
-            if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
-                return
-            }
-            const localFilePath = path.join(resolveLocalPath, fileRelativePath)
-            // 创建文件的父文件夹
-            const fileDir = path.dirname(localFilePath)
-            await makeDir(fileDir)
-            return this.downloadFile({
-                cloudPath: file.Key,
-                localPath: localFilePath
-            })
+        const promises = files.map(file => async () => {
+            return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath })
         })
+        const asyncTaskController = new AsyncTaskParallelController(parallel, 50)
+        asyncTaskController.loadTasks(promises)
+        let res = await asyncTaskController.run()
+        const errorIndexArr = []
 
-        return Promise.all(promises)
+        res.map(
+            (item, index) =>
+                /Error/gi.test(Object.prototype.toString.call(item)) && errorIndexArr.push(index)
+        )
+        // 重试逻辑
+        if (errorIndexArr.length) {
+            const errorFiles = errorIndexArr.map(errorIndex => files[errorIndex])
+            asyncTaskController.loadTasks(
+                errorFiles.map(file => async () => {
+                    return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath })
+                })
+            )
+            res = await asyncTaskController.run()
+        }
+        const errorResultArr = this.determineDownLoadResultIsError(res)
+        if (errorResultArr.length) {
+            throw errorResultArr[0]
+        }
+        return res
     }
 
     /**
@@ -1219,4 +1227,41 @@ export class StorageService {
             )
         }
     }
+
+    /**
+     * 拼接路径下载单文件
+     * @param file
+     * @param cloudDirectoryKey
+     * @param resolveLocalPath
+     * @returns
+     */
+    private async downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath }) {
+        const fileRelativePath = file.Key.replace(cloudDirectoryKey, '')
+        // 空路径和文件夹跳过
+        if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
+            return
+        }
+        const localFilePath = path.join(resolveLocalPath, fileRelativePath)
+        // 创建文件的父文件夹
+        const fileDir = path.dirname(localFilePath)
+        await makeDir(fileDir)
+        return this.downloadFile({
+            cloudPath: file.Key,
+            localPath: localFilePath
+        })
+    }
+
+    /**
+     * 根据下载结果返回错误列表
+     * @param res
+     * @returns
+     */
+    private determineDownLoadResultIsError(res) {
+        const resultErrorArr = []
+        res.map(
+            item =>
+                /Error/gi.test(Object.prototype.toString.call(item)) && resultErrorArr.push(item)
+        )
+        return resultErrorArr
+    }
 }

package/types/storage/index.d.ts

@@ -151,6 +151,7 @@ export declare class StorageService {
     downloadDirectory(options: {
         cloudPath: string;
         localPath?: string;
+        parallel?: number;
     }): Promise<(NodeJS.ReadableStream | string)[]>;
     /**
      * 列出文件夹下的文件
@@ -305,5 +306,19 @@ export declare class StorageService {
      * @returns
      */
     private uploadFilesWithRetry;
+    /**
+     * 拼接路径下载单文件
+     * @param file
+     * @param cloudDirectoryKey
+     * @param resolveLocalPath
+     * @returns
+     */
+    private downloadWithFilePath;
+    /**
+     * 根据下载结果返回错误列表
+     * @param res
+     * @returns
+     */
+    private determineDownLoadResultIsError;
 }
 export {};