@cloudbase/manager-node 3.9.5-beta → 3.11.1-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/hosting/index.js +7 -3
- package/lib/storage/index.js +128 -27
- package/package.json +1 -1
- package/src/hosting/index.ts +17 -3
- package/src/storage/index.ts +180 -43
- package/types/hosting/index.d.ts +4 -0
- package/types/storage/index.d.ts +36 -0
package/lib/hosting/index.js
CHANGED
|
@@ -131,7 +131,7 @@ class HostingService {
|
|
|
131
131
|
* @param options
|
|
132
132
|
*/
|
|
133
133
|
async uploadFiles(options) {
|
|
134
|
-
const { localPath, cloudPath, files = [], onProgress, onFileFinish, parallel = 20, ignore } = options;
|
|
134
|
+
const { localPath, cloudPath, files = [], onProgress, onFileFinish, parallel = 20, ignore, retryCount, retryInterval } = options;
|
|
135
135
|
const hosting = await this.checkStatus();
|
|
136
136
|
const { Bucket, Regoin } = hosting;
|
|
137
137
|
const storageService = await this.environment.getStorageService();
|
|
@@ -150,7 +150,9 @@ class HostingService {
|
|
|
150
150
|
onProgress,
|
|
151
151
|
onFileFinish,
|
|
152
152
|
fileId: false,
|
|
153
|
-
ignore
|
|
153
|
+
ignore,
|
|
154
|
+
retryCount,
|
|
155
|
+
retryInterval,
|
|
154
156
|
});
|
|
155
157
|
}
|
|
156
158
|
else {
|
|
@@ -171,7 +173,9 @@ class HostingService {
|
|
|
171
173
|
bucket: Bucket,
|
|
172
174
|
region: Regoin,
|
|
173
175
|
files: uploadFiles,
|
|
174
|
-
fileId: false
|
|
176
|
+
fileId: false,
|
|
177
|
+
retryCount,
|
|
178
|
+
retryInterval,
|
|
175
179
|
});
|
|
176
180
|
}
|
|
177
181
|
/**
|
package/lib/storage/index.js
CHANGED
|
@@ -49,7 +49,7 @@ class StorageService {
|
|
|
49
49
|
* @param options
|
|
50
50
|
*/
|
|
51
51
|
async uploadFiles(options) {
|
|
52
|
-
const { files, onProgress, parallel, onFileFinish, ignore } = options;
|
|
52
|
+
const { files, onProgress, parallel, onFileFinish, ignore, retryCount, retryInterval } = options;
|
|
53
53
|
const { bucket, region } = this.getStorageConfig();
|
|
54
54
|
return this.uploadFilesCustom({
|
|
55
55
|
files,
|
|
@@ -58,7 +58,9 @@ class StorageService {
|
|
|
58
58
|
ignore,
|
|
59
59
|
parallel,
|
|
60
60
|
onProgress,
|
|
61
|
-
onFileFinish
|
|
61
|
+
onFileFinish,
|
|
62
|
+
retryCount,
|
|
63
|
+
retryInterval
|
|
62
64
|
});
|
|
63
65
|
}
|
|
64
66
|
/**
|
|
@@ -134,18 +136,24 @@ class StorageService {
|
|
|
134
136
|
* 上传文件夹
|
|
135
137
|
* @param {string} localPath 本地文件夹路径
|
|
136
138
|
* @param {string} cloudPath 云端文件夹
|
|
139
|
+
* @param {number} parallel 并发量
|
|
140
|
+
* @param {number} retryCount 重试次数
|
|
141
|
+
* @param {number} retryInterval 重试时间间隔(毫秒)
|
|
137
142
|
* @param {(string | string[])} ignore
|
|
138
143
|
* @param {(string | string[])} ignore
|
|
139
144
|
* @returns {Promise<void>}
|
|
140
145
|
*/
|
|
141
146
|
async uploadDirectory(options) {
|
|
142
|
-
const { localPath, cloudPath = '', ignore, onProgress, onFileFinish } = options;
|
|
147
|
+
const { localPath, cloudPath = '', ignore, onProgress, onFileFinish, parallel, retryCount, retryInterval } = options;
|
|
143
148
|
// 此处不检查路径是否存在
|
|
144
149
|
// 绝对路径 /var/blog/xxxx
|
|
145
150
|
const { bucket, region } = this.getStorageConfig();
|
|
146
151
|
return this.uploadDirectoryCustom({
|
|
147
152
|
localPath,
|
|
148
153
|
cloudPath,
|
|
154
|
+
parallel,
|
|
155
|
+
retryCount,
|
|
156
|
+
retryInterval,
|
|
149
157
|
bucket,
|
|
150
158
|
region,
|
|
151
159
|
ignore,
|
|
@@ -157,13 +165,16 @@ class StorageService {
|
|
|
157
165
|
* 上传文件夹,支持自定义 Region 和 Bucket
|
|
158
166
|
* @param {string} localPath
|
|
159
167
|
* @param {string} cloudPath
|
|
168
|
+
* @param {number} parallel
|
|
169
|
+
* @param {number} retryCount
|
|
170
|
+
* @param {number} retryInterval
|
|
160
171
|
* @param {string} bucket
|
|
161
172
|
* @param {string} region
|
|
162
173
|
* @param {IOptions} options
|
|
163
174
|
* @returns {Promise<void>}
|
|
164
175
|
*/
|
|
165
176
|
async uploadDirectoryCustom(options) {
|
|
166
|
-
const { localPath, cloudPath, bucket, region, onProgress, onFileFinish, ignore, fileId = true, parallel = 20 } = options;
|
|
177
|
+
const { localPath, cloudPath, bucket, region, onProgress, onFileFinish, ignore, fileId = true, parallel = 20, retryCount = 0, retryInterval = 500 } = options;
|
|
167
178
|
// 此处不检查路径是否存在
|
|
168
179
|
// 绝对路径 /var/blog/xxxx
|
|
169
180
|
const resolvePath = path_1.default.resolve(localPath);
|
|
@@ -229,11 +240,18 @@ class StorageService {
|
|
|
229
240
|
// 对文件上传进行处理
|
|
230
241
|
const cos = this.getCos(parallel);
|
|
231
242
|
const uploadFiles = util_1.default.promisify(cos.uploadFiles).bind(cos);
|
|
232
|
-
|
|
243
|
+
const params = {
|
|
233
244
|
files,
|
|
234
245
|
SliceSize: BIG_FILE_SIZE,
|
|
235
246
|
onProgress,
|
|
236
247
|
onFileFinish
|
|
248
|
+
};
|
|
249
|
+
return this.uploadFilesWithRetry({
|
|
250
|
+
uploadFiles,
|
|
251
|
+
options: params,
|
|
252
|
+
times: retryCount,
|
|
253
|
+
interval: retryInterval,
|
|
254
|
+
failedFiles: []
|
|
237
255
|
});
|
|
238
256
|
}
|
|
239
257
|
/**
|
|
@@ -241,7 +259,7 @@ class StorageService {
|
|
|
241
259
|
* @param options
|
|
242
260
|
*/
|
|
243
261
|
async uploadFilesCustom(options) {
|
|
244
|
-
const { files, bucket, region, ignore, onProgress, onFileFinish, fileId = true, parallel = 20 } = options;
|
|
262
|
+
const { files, bucket, region, ignore, onProgress, onFileFinish, fileId = true, parallel = 20, retryCount = 0, retryInterval = 500 } = options;
|
|
245
263
|
if (!files || !files.length) {
|
|
246
264
|
return;
|
|
247
265
|
}
|
|
@@ -275,11 +293,24 @@ class StorageService {
|
|
|
275
293
|
fileList = await asyncTaskController.run();
|
|
276
294
|
const cos = this.getCos(parallel);
|
|
277
295
|
const uploadFiles = util_1.default.promisify(cos.uploadFiles).bind(cos);
|
|
278
|
-
|
|
279
|
-
onProgress,
|
|
280
|
-
onFileFinish,
|
|
296
|
+
const params = {
|
|
281
297
|
files: fileList,
|
|
282
|
-
SliceSize: BIG_FILE_SIZE
|
|
298
|
+
SliceSize: BIG_FILE_SIZE,
|
|
299
|
+
onProgress,
|
|
300
|
+
onFileFinish
|
|
301
|
+
};
|
|
302
|
+
// return uploadFiles({
|
|
303
|
+
// onProgress,
|
|
304
|
+
// onFileFinish,
|
|
305
|
+
// files: fileList,
|
|
306
|
+
// SliceSize: BIG_FILE_SIZE
|
|
307
|
+
// })
|
|
308
|
+
return this.uploadFilesWithRetry({
|
|
309
|
+
uploadFiles,
|
|
310
|
+
options: params,
|
|
311
|
+
times: retryCount,
|
|
312
|
+
interval: retryInterval,
|
|
313
|
+
failedFiles: []
|
|
283
314
|
});
|
|
284
315
|
}
|
|
285
316
|
/**
|
|
@@ -351,27 +382,32 @@ class StorageService {
|
|
|
351
382
|
* @returns {Promise<(NodeJS.ReadableStream | string)[]>}
|
|
352
383
|
*/
|
|
353
384
|
async downloadDirectory(options) {
|
|
354
|
-
const { cloudPath, localPath } = options;
|
|
385
|
+
const { cloudPath, localPath, parallel = 20 } = options;
|
|
355
386
|
const resolveLocalPath = path_1.default.resolve(localPath);
|
|
356
387
|
utils_1.checkFullAccess(resolveLocalPath, true);
|
|
357
388
|
const cloudDirectoryKey = this.getCloudKey(cloudPath);
|
|
358
389
|
const files = await this.walkCloudDir(cloudDirectoryKey);
|
|
359
|
-
const promises = files.map(async (
|
|
360
|
-
|
|
361
|
-
// 空路径和文件夹跳过
|
|
362
|
-
if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
|
|
363
|
-
return;
|
|
364
|
-
}
|
|
365
|
-
const localFilePath = path_1.default.join(resolveLocalPath, fileRelativePath);
|
|
366
|
-
// 创建文件的父文件夹
|
|
367
|
-
const fileDir = path_1.default.dirname(localFilePath);
|
|
368
|
-
await make_dir_1.default(fileDir);
|
|
369
|
-
return this.downloadFile({
|
|
370
|
-
cloudPath: file.Key,
|
|
371
|
-
localPath: localFilePath
|
|
372
|
-
});
|
|
390
|
+
const promises = files.map(file => async () => {
|
|
391
|
+
return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath });
|
|
373
392
|
});
|
|
374
|
-
|
|
393
|
+
const asyncTaskController = new parallel_1.AsyncTaskParallelController(parallel, 50);
|
|
394
|
+
asyncTaskController.loadTasks(promises);
|
|
395
|
+
let res = await asyncTaskController.run();
|
|
396
|
+
const errorIndexArr = [];
|
|
397
|
+
res.map((item, index) => /Error/gi.test(Object.prototype.toString.call(item)) && errorIndexArr.push(index));
|
|
398
|
+
// 重试逻辑
|
|
399
|
+
if (errorIndexArr.length) {
|
|
400
|
+
const errorFiles = errorIndexArr.map(errorIndex => files[errorIndex]);
|
|
401
|
+
asyncTaskController.loadTasks(errorFiles.map(file => async () => {
|
|
402
|
+
return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath });
|
|
403
|
+
}));
|
|
404
|
+
res = await asyncTaskController.run();
|
|
405
|
+
}
|
|
406
|
+
const errorResultArr = this.determineDownLoadResultIsError(res);
|
|
407
|
+
if (errorResultArr.length) {
|
|
408
|
+
throw errorResultArr[0];
|
|
409
|
+
}
|
|
410
|
+
return res;
|
|
375
411
|
}
|
|
376
412
|
/**
|
|
377
413
|
* 列出文件夹下的文件
|
|
@@ -774,7 +810,6 @@ class StorageService {
|
|
|
774
810
|
params.WebsiteConfiguration.RoutingRules.push(routeItem);
|
|
775
811
|
}
|
|
776
812
|
}
|
|
777
|
-
console.log('params:', JSON.stringify(params));
|
|
778
813
|
const res = await putBucketWebsite(params);
|
|
779
814
|
return res;
|
|
780
815
|
}
|
|
@@ -847,6 +882,72 @@ class StorageService {
|
|
|
847
882
|
env: envConfig.EnvId
|
|
848
883
|
};
|
|
849
884
|
}
|
|
885
|
+
/**
|
|
886
|
+
* 带重试功能的上传多文件函数
|
|
887
|
+
* @param uploadFiles sdk上传函数
|
|
888
|
+
* @param options sdk上传函数参数
|
|
889
|
+
* @param times 重试次数
|
|
890
|
+
* @param interval 重试时间间隔(毫秒)
|
|
891
|
+
* @param failedFiles 失败文件列表
|
|
892
|
+
* @returns
|
|
893
|
+
*/
|
|
894
|
+
async uploadFilesWithRetry({ uploadFiles, options, times, interval, failedFiles }) {
|
|
895
|
+
const { files, onFileFinish } = options;
|
|
896
|
+
const tempFailedFiles = [];
|
|
897
|
+
const res = await uploadFiles(Object.assign(Object.assign({}, options), { files: failedFiles.length
|
|
898
|
+
? files.filter(file => failedFiles.includes(file.Key))
|
|
899
|
+
: files, onFileFinish: (...args) => {
|
|
900
|
+
const error = args[0];
|
|
901
|
+
const fileInfo = args[2];
|
|
902
|
+
if (error) {
|
|
903
|
+
tempFailedFiles.push(fileInfo.Key);
|
|
904
|
+
}
|
|
905
|
+
onFileFinish === null || onFileFinish === void 0 ? void 0 : onFileFinish.apply(null, args);
|
|
906
|
+
} }));
|
|
907
|
+
if (!(tempFailedFiles === null || tempFailedFiles === void 0 ? void 0 : tempFailedFiles.length) || times <= 0)
|
|
908
|
+
return res;
|
|
909
|
+
if (times > 0) {
|
|
910
|
+
setTimeout(() => this.uploadFilesWithRetry({
|
|
911
|
+
uploadFiles,
|
|
912
|
+
options,
|
|
913
|
+
times: times - 1,
|
|
914
|
+
interval,
|
|
915
|
+
failedFiles: tempFailedFiles
|
|
916
|
+
}), interval);
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
/**
|
|
920
|
+
* 拼接路径下载单文件
|
|
921
|
+
* @param file
|
|
922
|
+
* @param cloudDirectoryKey
|
|
923
|
+
* @param resolveLocalPath
|
|
924
|
+
* @returns
|
|
925
|
+
*/
|
|
926
|
+
async downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath }) {
|
|
927
|
+
const fileRelativePath = file.Key.replace(cloudDirectoryKey, '');
|
|
928
|
+
// 空路径和文件夹跳过
|
|
929
|
+
if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
const localFilePath = path_1.default.join(resolveLocalPath, fileRelativePath);
|
|
933
|
+
// 创建文件的父文件夹
|
|
934
|
+
const fileDir = path_1.default.dirname(localFilePath);
|
|
935
|
+
await make_dir_1.default(fileDir);
|
|
936
|
+
return this.downloadFile({
|
|
937
|
+
cloudPath: file.Key,
|
|
938
|
+
localPath: localFilePath
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
/**
|
|
942
|
+
* 根据下载结果返回错误列表
|
|
943
|
+
* @param res
|
|
944
|
+
* @returns
|
|
945
|
+
*/
|
|
946
|
+
determineDownLoadResultIsError(res) {
|
|
947
|
+
const resultErrorArr = [];
|
|
948
|
+
res.map(item => /Error/gi.test(Object.prototype.toString.call(item)) && resultErrorArr.push(item));
|
|
949
|
+
return resultErrorArr;
|
|
950
|
+
}
|
|
850
951
|
}
|
|
851
952
|
__decorate([
|
|
852
953
|
utils_1.preLazy()
|
package/package.json
CHANGED
package/src/hosting/index.ts
CHANGED
|
@@ -38,6 +38,10 @@ export interface IHostingFileOptions {
|
|
|
38
38
|
onProgress?: OnProgress
|
|
39
39
|
onFileFinish?: OnFileFinish
|
|
40
40
|
ignore?: string | string[]
|
|
41
|
+
// 重试次数
|
|
42
|
+
retryCount?: number
|
|
43
|
+
// 重试时间间隔(毫秒)
|
|
44
|
+
retryInterval?: number
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
export interface IHostingFilesOptions {
|
|
@@ -52,6 +56,10 @@ export interface IHostingFilesOptions {
|
|
|
52
56
|
onProgress?: OnProgress
|
|
53
57
|
onFileFinish?: OnFileFinish
|
|
54
58
|
ignore?: string | string[]
|
|
59
|
+
// 重试次数
|
|
60
|
+
retryCount?: number
|
|
61
|
+
// 重试时间间隔(毫秒)
|
|
62
|
+
retryInterval?: number
|
|
55
63
|
}
|
|
56
64
|
|
|
57
65
|
export type IHostingOptions = IHostingFileOptions | IHostingFilesOptions
|
|
@@ -331,7 +339,9 @@ export class HostingService {
|
|
|
331
339
|
onProgress,
|
|
332
340
|
onFileFinish,
|
|
333
341
|
parallel = 20,
|
|
334
|
-
ignore
|
|
342
|
+
ignore,
|
|
343
|
+
retryCount,
|
|
344
|
+
retryInterval
|
|
335
345
|
} = options
|
|
336
346
|
|
|
337
347
|
const hosting = await this.checkStatus()
|
|
@@ -355,7 +365,9 @@ export class HostingService {
|
|
|
355
365
|
onProgress,
|
|
356
366
|
onFileFinish,
|
|
357
367
|
fileId: false,
|
|
358
|
-
ignore
|
|
368
|
+
ignore,
|
|
369
|
+
retryCount,
|
|
370
|
+
retryInterval,
|
|
359
371
|
})
|
|
360
372
|
} else {
|
|
361
373
|
// 文件上传统一通过批量上传接口
|
|
@@ -376,7 +388,9 @@ export class HostingService {
|
|
|
376
388
|
bucket: Bucket,
|
|
377
389
|
region: Regoin,
|
|
378
390
|
files: uploadFiles,
|
|
379
|
-
fileId: false
|
|
391
|
+
fileId: false,
|
|
392
|
+
retryCount,
|
|
393
|
+
retryInterval,
|
|
380
394
|
})
|
|
381
395
|
}
|
|
382
396
|
|
package/src/storage/index.ts
CHANGED
|
@@ -49,6 +49,12 @@ export interface IFileOptions extends IOptions {
|
|
|
49
49
|
localPath: string
|
|
50
50
|
// cloudPath 可以为空
|
|
51
51
|
cloudPath?: string
|
|
52
|
+
// 并发数量
|
|
53
|
+
parallel?: number
|
|
54
|
+
// 重试次数
|
|
55
|
+
retryCount?: number
|
|
56
|
+
// 重试时间间隔(毫秒)
|
|
57
|
+
retryInterval?: number
|
|
52
58
|
}
|
|
53
59
|
|
|
54
60
|
export interface IFilesOptions extends IOptions {
|
|
@@ -56,6 +62,10 @@ export interface IFilesOptions extends IOptions {
|
|
|
56
62
|
ignore?: string | string[]
|
|
57
63
|
// 文件列表
|
|
58
64
|
files: { localPath: string; cloudPath?: string }[]
|
|
65
|
+
// 重试次数
|
|
66
|
+
retryCount?: number
|
|
67
|
+
// 重试时间间隔(毫秒)
|
|
68
|
+
retryInterval?: number
|
|
59
69
|
}
|
|
60
70
|
|
|
61
71
|
export interface ICustomOptions {
|
|
@@ -135,7 +145,8 @@ export class StorageService {
|
|
|
135
145
|
*/
|
|
136
146
|
@preLazy()
|
|
137
147
|
public async uploadFiles(options: IFilesOptions): Promise<void> {
|
|
138
|
-
const { files, onProgress, parallel, onFileFinish, ignore } =
|
|
148
|
+
const { files, onProgress, parallel, onFileFinish, ignore, retryCount, retryInterval } =
|
|
149
|
+
options
|
|
139
150
|
const { bucket, region } = this.getStorageConfig()
|
|
140
151
|
|
|
141
152
|
return this.uploadFilesCustom({
|
|
@@ -145,7 +156,9 @@ export class StorageService {
|
|
|
145
156
|
ignore,
|
|
146
157
|
parallel,
|
|
147
158
|
onProgress,
|
|
148
|
-
onFileFinish
|
|
159
|
+
onFileFinish,
|
|
160
|
+
retryCount,
|
|
161
|
+
retryInterval
|
|
149
162
|
})
|
|
150
163
|
}
|
|
151
164
|
|
|
@@ -230,19 +243,34 @@ export class StorageService {
|
|
|
230
243
|
* 上传文件夹
|
|
231
244
|
* @param {string} localPath 本地文件夹路径
|
|
232
245
|
* @param {string} cloudPath 云端文件夹
|
|
246
|
+
* @param {number} parallel 并发量
|
|
247
|
+
* @param {number} retryCount 重试次数
|
|
248
|
+
* @param {number} retryInterval 重试时间间隔(毫秒)
|
|
233
249
|
* @param {(string | string[])} ignore
|
|
234
250
|
* @param {(string | string[])} ignore
|
|
235
251
|
* @returns {Promise<void>}
|
|
236
252
|
*/
|
|
237
253
|
@preLazy()
|
|
238
254
|
public async uploadDirectory(options: IFileOptions): Promise<void> {
|
|
239
|
-
const {
|
|
255
|
+
const {
|
|
256
|
+
localPath,
|
|
257
|
+
cloudPath = '',
|
|
258
|
+
ignore,
|
|
259
|
+
onProgress,
|
|
260
|
+
onFileFinish,
|
|
261
|
+
parallel,
|
|
262
|
+
retryCount,
|
|
263
|
+
retryInterval
|
|
264
|
+
} = options
|
|
240
265
|
// 此处不检查路径是否存在
|
|
241
266
|
// 绝对路径 /var/blog/xxxx
|
|
242
267
|
const { bucket, region } = this.getStorageConfig()
|
|
243
268
|
return this.uploadDirectoryCustom({
|
|
244
269
|
localPath,
|
|
245
270
|
cloudPath,
|
|
271
|
+
parallel,
|
|
272
|
+
retryCount,
|
|
273
|
+
retryInterval,
|
|
246
274
|
bucket,
|
|
247
275
|
region,
|
|
248
276
|
ignore,
|
|
@@ -255,6 +283,9 @@ export class StorageService {
|
|
|
255
283
|
* 上传文件夹,支持自定义 Region 和 Bucket
|
|
256
284
|
* @param {string} localPath
|
|
257
285
|
* @param {string} cloudPath
|
|
286
|
+
* @param {number} parallel
|
|
287
|
+
* @param {number} retryCount
|
|
288
|
+
* @param {number} retryInterval
|
|
258
289
|
* @param {string} bucket
|
|
259
290
|
* @param {string} region
|
|
260
291
|
* @param {IOptions} options
|
|
@@ -271,7 +302,9 @@ export class StorageService {
|
|
|
271
302
|
onFileFinish,
|
|
272
303
|
ignore,
|
|
273
304
|
fileId = true,
|
|
274
|
-
parallel = 20
|
|
305
|
+
parallel = 20,
|
|
306
|
+
retryCount = 0,
|
|
307
|
+
retryInterval = 500
|
|
275
308
|
} = options
|
|
276
309
|
// 此处不检查路径是否存在
|
|
277
310
|
// 绝对路径 /var/blog/xxxx
|
|
@@ -310,12 +343,13 @@ export class StorageService {
|
|
|
310
343
|
const creatingDirController = new AsyncTaskParallelController(parallel, 50)
|
|
311
344
|
const creatingDirTasks = fileStatsList
|
|
312
345
|
.filter(info => info.isDir)
|
|
313
|
-
.map(
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
346
|
+
.map(
|
|
347
|
+
info => () =>
|
|
348
|
+
this.createCloudDirectroyCustom({
|
|
349
|
+
cloudPath: info.cloudFileKey,
|
|
350
|
+
bucket,
|
|
351
|
+
region
|
|
352
|
+
})
|
|
319
353
|
)
|
|
320
354
|
|
|
321
355
|
creatingDirController.loadTasks(creatingDirTasks)
|
|
@@ -348,15 +382,20 @@ export class StorageService {
|
|
|
348
382
|
// 对文件上传进行处理
|
|
349
383
|
const cos = this.getCos(parallel)
|
|
350
384
|
const uploadFiles = Util.promisify(cos.uploadFiles).bind(cos)
|
|
351
|
-
|
|
352
|
-
return uploadFiles({
|
|
385
|
+
const params = {
|
|
353
386
|
files,
|
|
354
387
|
SliceSize: BIG_FILE_SIZE,
|
|
355
388
|
onProgress,
|
|
356
389
|
onFileFinish
|
|
390
|
+
}
|
|
391
|
+
return this.uploadFilesWithRetry({
|
|
392
|
+
uploadFiles,
|
|
393
|
+
options: params,
|
|
394
|
+
times: retryCount,
|
|
395
|
+
interval: retryInterval,
|
|
396
|
+
failedFiles: []
|
|
357
397
|
})
|
|
358
398
|
}
|
|
359
|
-
|
|
360
399
|
/**
|
|
361
400
|
* 批量上传文件
|
|
362
401
|
* @param options
|
|
@@ -371,7 +410,9 @@ export class StorageService {
|
|
|
371
410
|
onProgress,
|
|
372
411
|
onFileFinish,
|
|
373
412
|
fileId = true,
|
|
374
|
-
parallel = 20
|
|
413
|
+
parallel = 20,
|
|
414
|
+
retryCount = 0,
|
|
415
|
+
retryInterval = 500
|
|
375
416
|
} = options
|
|
376
417
|
|
|
377
418
|
if (!files || !files.length) {
|
|
@@ -413,12 +454,25 @@ export class StorageService {
|
|
|
413
454
|
|
|
414
455
|
const cos = this.getCos(parallel)
|
|
415
456
|
const uploadFiles = Util.promisify(cos.uploadFiles).bind(cos)
|
|
416
|
-
|
|
417
|
-
return uploadFiles({
|
|
418
|
-
onProgress,
|
|
419
|
-
onFileFinish,
|
|
457
|
+
const params = {
|
|
420
458
|
files: fileList,
|
|
421
|
-
SliceSize: BIG_FILE_SIZE
|
|
459
|
+
SliceSize: BIG_FILE_SIZE,
|
|
460
|
+
onProgress,
|
|
461
|
+
onFileFinish
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// return uploadFiles({
|
|
465
|
+
// onProgress,
|
|
466
|
+
// onFileFinish,
|
|
467
|
+
// files: fileList,
|
|
468
|
+
// SliceSize: BIG_FILE_SIZE
|
|
469
|
+
// })
|
|
470
|
+
return this.uploadFilesWithRetry({
|
|
471
|
+
uploadFiles,
|
|
472
|
+
options: params,
|
|
473
|
+
times: retryCount,
|
|
474
|
+
interval: retryInterval,
|
|
475
|
+
failedFiles: []
|
|
422
476
|
})
|
|
423
477
|
}
|
|
424
478
|
|
|
@@ -511,32 +565,42 @@ export class StorageService {
|
|
|
511
565
|
public async downloadDirectory(options: {
|
|
512
566
|
cloudPath: string
|
|
513
567
|
localPath?: string
|
|
568
|
+
parallel?: number
|
|
514
569
|
}): Promise<(NodeJS.ReadableStream | string)[]> {
|
|
515
|
-
const { cloudPath, localPath } = options
|
|
570
|
+
const { cloudPath, localPath, parallel = 20 } = options
|
|
516
571
|
const resolveLocalPath = path.resolve(localPath)
|
|
517
572
|
|
|
518
573
|
checkFullAccess(resolveLocalPath, true)
|
|
519
574
|
|
|
520
575
|
const cloudDirectoryKey = this.getCloudKey(cloudPath)
|
|
521
576
|
const files = await this.walkCloudDir(cloudDirectoryKey)
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
const fileRelativePath = file.Key.replace(cloudDirectoryKey, '')
|
|
525
|
-
// 空路径和文件夹跳过
|
|
526
|
-
if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
|
|
527
|
-
return
|
|
528
|
-
}
|
|
529
|
-
const localFilePath = path.join(resolveLocalPath, fileRelativePath)
|
|
530
|
-
// 创建文件的父文件夹
|
|
531
|
-
const fileDir = path.dirname(localFilePath)
|
|
532
|
-
await makeDir(fileDir)
|
|
533
|
-
return this.downloadFile({
|
|
534
|
-
cloudPath: file.Key,
|
|
535
|
-
localPath: localFilePath
|
|
536
|
-
})
|
|
577
|
+
const promises = files.map(file => async () => {
|
|
578
|
+
return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath })
|
|
537
579
|
})
|
|
580
|
+
const asyncTaskController = new AsyncTaskParallelController(parallel, 50)
|
|
581
|
+
asyncTaskController.loadTasks(promises)
|
|
582
|
+
let res = await asyncTaskController.run()
|
|
583
|
+
const errorIndexArr = []
|
|
538
584
|
|
|
539
|
-
|
|
585
|
+
res.map(
|
|
586
|
+
(item, index) =>
|
|
587
|
+
/Error/gi.test(Object.prototype.toString.call(item)) && errorIndexArr.push(index)
|
|
588
|
+
)
|
|
589
|
+
// 重试逻辑
|
|
590
|
+
if (errorIndexArr.length) {
|
|
591
|
+
const errorFiles = errorIndexArr.map(errorIndex => files[errorIndex])
|
|
592
|
+
asyncTaskController.loadTasks(
|
|
593
|
+
errorFiles.map(file => async () => {
|
|
594
|
+
return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath })
|
|
595
|
+
})
|
|
596
|
+
)
|
|
597
|
+
res = await asyncTaskController.run()
|
|
598
|
+
}
|
|
599
|
+
const errorResultArr = this.determineDownLoadResultIsError(res)
|
|
600
|
+
if (errorResultArr.length) {
|
|
601
|
+
throw errorResultArr[0]
|
|
602
|
+
}
|
|
603
|
+
return res
|
|
540
604
|
}
|
|
541
605
|
|
|
542
606
|
/**
|
|
@@ -735,9 +799,7 @@ export class StorageService {
|
|
|
735
799
|
* @returns {Promise<void>}
|
|
736
800
|
*/
|
|
737
801
|
@preLazy()
|
|
738
|
-
public async deleteDirectory(
|
|
739
|
-
cloudPath: string
|
|
740
|
-
): Promise<{
|
|
802
|
+
public async deleteDirectory(cloudPath: string): Promise<{
|
|
741
803
|
Deleted: { Key: string }[]
|
|
742
804
|
Error: Object[]
|
|
743
805
|
}> {
|
|
@@ -758,9 +820,7 @@ export class StorageService {
|
|
|
758
820
|
* @returns {Promise<void>}
|
|
759
821
|
*/
|
|
760
822
|
@preLazy()
|
|
761
|
-
public async deleteDirectoryCustom(
|
|
762
|
-
options: { cloudPath: string } & ICustomOptions
|
|
763
|
-
): Promise<{
|
|
823
|
+
public async deleteDirectoryCustom(options: { cloudPath: string } & ICustomOptions): Promise<{
|
|
764
824
|
Deleted: { Key: string }[]
|
|
765
825
|
Error: Object[]
|
|
766
826
|
}> {
|
|
@@ -1040,7 +1100,6 @@ export class StorageService {
|
|
|
1040
1100
|
}
|
|
1041
1101
|
}
|
|
1042
1102
|
|
|
1043
|
-
console.log('params:', JSON.stringify(params))
|
|
1044
1103
|
const res = await putBucketWebsite(params)
|
|
1045
1104
|
|
|
1046
1105
|
return res
|
|
@@ -1127,4 +1186,82 @@ export class StorageService {
|
|
|
1127
1186
|
env: envConfig.EnvId
|
|
1128
1187
|
}
|
|
1129
1188
|
}
|
|
1189
|
+
/**
|
|
1190
|
+
* 带重试功能的上传多文件函数
|
|
1191
|
+
* @param uploadFiles sdk上传函数
|
|
1192
|
+
* @param options sdk上传函数参数
|
|
1193
|
+
* @param times 重试次数
|
|
1194
|
+
* @param interval 重试时间间隔(毫秒)
|
|
1195
|
+
* @param failedFiles 失败文件列表
|
|
1196
|
+
* @returns
|
|
1197
|
+
*/
|
|
1198
|
+
private async uploadFilesWithRetry({ uploadFiles, options, times, interval, failedFiles }) {
|
|
1199
|
+
const { files, onFileFinish } = options
|
|
1200
|
+
const tempFailedFiles = []
|
|
1201
|
+
const res = await uploadFiles({
|
|
1202
|
+
...options,
|
|
1203
|
+
files: failedFiles.length
|
|
1204
|
+
? files.filter(file => failedFiles.includes(file.Key))
|
|
1205
|
+
: files,
|
|
1206
|
+
onFileFinish: (...args) => {
|
|
1207
|
+
const error = args[0]
|
|
1208
|
+
const fileInfo = (args as any)[2]
|
|
1209
|
+
if (error) {
|
|
1210
|
+
tempFailedFiles.push(fileInfo.Key)
|
|
1211
|
+
}
|
|
1212
|
+
onFileFinish?.apply(null, args)
|
|
1213
|
+
}
|
|
1214
|
+
})
|
|
1215
|
+
if (!tempFailedFiles?.length || times <= 0) return res
|
|
1216
|
+
if (times > 0) {
|
|
1217
|
+
setTimeout(
|
|
1218
|
+
() =>
|
|
1219
|
+
this.uploadFilesWithRetry({
|
|
1220
|
+
uploadFiles,
|
|
1221
|
+
options,
|
|
1222
|
+
times: times - 1,
|
|
1223
|
+
interval,
|
|
1224
|
+
failedFiles: tempFailedFiles
|
|
1225
|
+
}),
|
|
1226
|
+
interval
|
|
1227
|
+
)
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
/**
|
|
1232
|
+
* 拼接路径下载单文件
|
|
1233
|
+
* @param file
|
|
1234
|
+
* @param cloudDirectoryKey
|
|
1235
|
+
* @param resolveLocalPath
|
|
1236
|
+
* @returns
|
|
1237
|
+
*/
|
|
1238
|
+
private async downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath }) {
|
|
1239
|
+
const fileRelativePath = file.Key.replace(cloudDirectoryKey, '')
|
|
1240
|
+
// 空路径和文件夹跳过
|
|
1241
|
+
if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
|
|
1242
|
+
return
|
|
1243
|
+
}
|
|
1244
|
+
const localFilePath = path.join(resolveLocalPath, fileRelativePath)
|
|
1245
|
+
// 创建文件的父文件夹
|
|
1246
|
+
const fileDir = path.dirname(localFilePath)
|
|
1247
|
+
await makeDir(fileDir)
|
|
1248
|
+
return this.downloadFile({
|
|
1249
|
+
cloudPath: file.Key,
|
|
1250
|
+
localPath: localFilePath
|
|
1251
|
+
})
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
/**
|
|
1255
|
+
* 根据下载结果返回错误列表
|
|
1256
|
+
* @param res
|
|
1257
|
+
* @returns
|
|
1258
|
+
*/
|
|
1259
|
+
private determineDownLoadResultIsError(res) {
|
|
1260
|
+
const resultErrorArr = []
|
|
1261
|
+
res.map(
|
|
1262
|
+
item =>
|
|
1263
|
+
/Error/gi.test(Object.prototype.toString.call(item)) && resultErrorArr.push(item)
|
|
1264
|
+
)
|
|
1265
|
+
return resultErrorArr
|
|
1266
|
+
}
|
|
1130
1267
|
}
|
package/types/hosting/index.d.ts
CHANGED
|
@@ -20,6 +20,8 @@ export interface IHostingFileOptions {
|
|
|
20
20
|
onProgress?: OnProgress;
|
|
21
21
|
onFileFinish?: OnFileFinish;
|
|
22
22
|
ignore?: string | string[];
|
|
23
|
+
retryCount?: number;
|
|
24
|
+
retryInterval?: number;
|
|
23
25
|
}
|
|
24
26
|
export interface IHostingFilesOptions {
|
|
25
27
|
localPath?: string;
|
|
@@ -32,6 +34,8 @@ export interface IHostingFilesOptions {
|
|
|
32
34
|
onProgress?: OnProgress;
|
|
33
35
|
onFileFinish?: OnFileFinish;
|
|
34
36
|
ignore?: string | string[];
|
|
37
|
+
retryCount?: number;
|
|
38
|
+
retryInterval?: number;
|
|
35
39
|
}
|
|
36
40
|
export declare type IHostingOptions = IHostingFileOptions | IHostingFilesOptions;
|
|
37
41
|
export interface IHostingCloudOptions {
|
package/types/storage/index.d.ts
CHANGED
|
@@ -17,6 +17,9 @@ export interface IOptions {
|
|
|
17
17
|
export interface IFileOptions extends IOptions {
|
|
18
18
|
localPath: string;
|
|
19
19
|
cloudPath?: string;
|
|
20
|
+
parallel?: number;
|
|
21
|
+
retryCount?: number;
|
|
22
|
+
retryInterval?: number;
|
|
20
23
|
}
|
|
21
24
|
export interface IFilesOptions extends IOptions {
|
|
22
25
|
ignore?: string | string[];
|
|
@@ -24,6 +27,8 @@ export interface IFilesOptions extends IOptions {
|
|
|
24
27
|
localPath: string;
|
|
25
28
|
cloudPath?: string;
|
|
26
29
|
}[];
|
|
30
|
+
retryCount?: number;
|
|
31
|
+
retryInterval?: number;
|
|
27
32
|
}
|
|
28
33
|
export interface ICustomOptions {
|
|
29
34
|
bucket: string;
|
|
@@ -87,6 +92,9 @@ export declare class StorageService {
|
|
|
87
92
|
* 上传文件夹
|
|
88
93
|
* @param {string} localPath 本地文件夹路径
|
|
89
94
|
* @param {string} cloudPath 云端文件夹
|
|
95
|
+
* @param {number} parallel 并发量
|
|
96
|
+
* @param {number} retryCount 重试次数
|
|
97
|
+
* @param {number} retryInterval 重试时间间隔(毫秒)
|
|
90
98
|
* @param {(string | string[])} ignore
|
|
91
99
|
* @param {(string | string[])} ignore
|
|
92
100
|
* @returns {Promise<void>}
|
|
@@ -96,6 +104,9 @@ export declare class StorageService {
|
|
|
96
104
|
* 上传文件夹,支持自定义 Region 和 Bucket
|
|
97
105
|
* @param {string} localPath
|
|
98
106
|
* @param {string} cloudPath
|
|
107
|
+
* @param {number} parallel
|
|
108
|
+
* @param {number} retryCount
|
|
109
|
+
* @param {number} retryInterval
|
|
99
110
|
* @param {string} bucket
|
|
100
111
|
* @param {string} region
|
|
101
112
|
* @param {IOptions} options
|
|
@@ -140,6 +151,7 @@ export declare class StorageService {
|
|
|
140
151
|
downloadDirectory(options: {
|
|
141
152
|
cloudPath: string;
|
|
142
153
|
localPath?: string;
|
|
154
|
+
parallel?: number;
|
|
143
155
|
}): Promise<(NodeJS.ReadableStream | string)[]>;
|
|
144
156
|
/**
|
|
145
157
|
* 列出文件夹下的文件
|
|
@@ -284,5 +296,29 @@ export declare class StorageService {
|
|
|
284
296
|
* 获取存储桶配置
|
|
285
297
|
*/
|
|
286
298
|
private getStorageConfig;
|
|
299
|
+
/**
|
|
300
|
+
* 带重试功能的上传多文件函数
|
|
301
|
+
* @param uploadFiles sdk上传函数
|
|
302
|
+
* @param options sdk上传函数参数
|
|
303
|
+
* @param times 重试次数
|
|
304
|
+
* @param interval 重试时间间隔(毫秒)
|
|
305
|
+
* @param failedFiles 失败文件列表
|
|
306
|
+
* @returns
|
|
307
|
+
*/
|
|
308
|
+
private uploadFilesWithRetry;
|
|
309
|
+
/**
|
|
310
|
+
* 拼接路径下载单文件
|
|
311
|
+
* @param file
|
|
312
|
+
* @param cloudDirectoryKey
|
|
313
|
+
* @param resolveLocalPath
|
|
314
|
+
* @returns
|
|
315
|
+
*/
|
|
316
|
+
private downloadWithFilePath;
|
|
317
|
+
/**
|
|
318
|
+
* 根据下载结果返回错误列表
|
|
319
|
+
* @param res
|
|
320
|
+
* @returns
|
|
321
|
+
*/
|
|
322
|
+
private determineDownLoadResultIsError;
|
|
287
323
|
}
|
|
288
324
|
export {};
|