@cloudbase/manager-node 4.2.3 → 4.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/lib/constant.js +5 -1
- package/lib/env/index.js +10 -209
- package/lib/function/index.js +0 -3
- package/lib/storage/index.js +24 -10
- package/lib/utils/cloud-api-request.js +7 -0
- package/lib/utils/http-request.js +3 -3
- package/lib/utils/runenv.js +8 -0
- package/package.json +4 -3
- package/types/constant.d.ts +7 -0
- package/types/env/index.d.ts +0 -17
- package/types/function/types.d.ts +0 -2
- package/types/utils/runenv.d.ts +1 -0
- package/src/access/index.ts +0 -168
- package/src/access/types.ts +0 -55
- package/src/billing/index.ts +0 -43
- package/src/cam/index.ts +0 -106
- package/src/cloudBaseRun/index.ts +0 -40
- package/src/cloudBaseRun/types.ts +0 -24
- package/src/common/index.ts +0 -54
- package/src/constant.ts +0 -56
- package/src/context.ts +0 -18
- package/src/database/index.ts +0 -369
- package/src/debug.ts +0 -34
- package/src/env/index.ts +0 -614
- package/src/environment.ts +0 -156
- package/src/environmentManager.ts +0 -50
- package/src/error.ts +0 -27
- package/src/function/index.ts +0 -1378
- package/src/function/packer.ts +0 -164
- package/src/function/types.ts +0 -165
- package/src/hosting/index.ts +0 -698
- package/src/index.ts +0 -127
- package/src/interfaces/base.interface.ts +0 -8
- package/src/interfaces/billing.interface.ts +0 -21
- package/src/interfaces/cam.interface.ts +0 -28
- package/src/interfaces/flexdb.interface.ts +0 -104
- package/src/interfaces/function.interface.ts +0 -75
- package/src/interfaces/index.ts +0 -7
- package/src/interfaces/storage.interface.ts +0 -29
- package/src/interfaces/tcb.interface.ts +0 -642
- package/src/storage/index.ts +0 -1290
- package/src/third/index.ts +0 -24
- package/src/user/index.ts +0 -174
- package/src/user/types.ts +0 -21
- package/src/utils/auth.ts +0 -112
- package/src/utils/cloud-api-request.ts +0 -252
- package/src/utils/cloudbase-request.ts +0 -109
- package/src/utils/envLazy.ts +0 -15
- package/src/utils/fs.ts +0 -57
- package/src/utils/http-request.ts +0 -37
- package/src/utils/index.ts +0 -103
- package/src/utils/parallel.ts +0 -82
- package/src/utils/uuid.ts +0 -14
package/src/storage/index.ts
DELETED
|
@@ -1,1290 +0,0 @@
|
|
|
1
|
-
import fs from 'fs'
|
|
2
|
-
import Util from 'util'
|
|
3
|
-
import path from 'path'
|
|
4
|
-
import makeDir from 'make-dir'
|
|
5
|
-
import walkdir from 'walkdir'
|
|
6
|
-
import micromatch from 'micromatch'
|
|
7
|
-
import COS from 'cos-nodejs-sdk-v5'
|
|
8
|
-
import {
|
|
9
|
-
cloudBaseRequest,
|
|
10
|
-
CloudService,
|
|
11
|
-
fetchStream,
|
|
12
|
-
preLazy,
|
|
13
|
-
isDirectory,
|
|
14
|
-
checkFullAccess
|
|
15
|
-
} from '../utils'
|
|
16
|
-
import { CloudBaseError } from '../error'
|
|
17
|
-
import { Environment } from '../environment'
|
|
18
|
-
|
|
19
|
-
import {
|
|
20
|
-
IUploadMetadata,
|
|
21
|
-
IListFileInfo,
|
|
22
|
-
IFileInfo,
|
|
23
|
-
ITempUrlInfo,
|
|
24
|
-
IResponseInfo
|
|
25
|
-
} from '../interfaces'
|
|
26
|
-
import { AsyncTaskParallelController } from '../utils/parallel'
|
|
27
|
-
|
|
28
|
-
export interface IProgressData {
|
|
29
|
-
loaded: number // 已经上传的部分 字节
|
|
30
|
-
total: number // 整个文件的大小 字节
|
|
31
|
-
speed: number // 文件上传速度 字节/秒
|
|
32
|
-
percent: number // 百分比 小数 0 - 1
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export interface IOptions {
|
|
36
|
-
// 进度
|
|
37
|
-
onProgress?: OnProgress
|
|
38
|
-
// 文件上传完成的回调
|
|
39
|
-
onFileFinish?: OnFileFinish
|
|
40
|
-
// 忽略文件匹配规则
|
|
41
|
-
ignore?: string | string[]
|
|
42
|
-
// 是否获取文件 fileId
|
|
43
|
-
fileId?: boolean
|
|
44
|
-
// 并发数量
|
|
45
|
-
parallel?: number
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export interface IFileOptions extends IOptions {
|
|
49
|
-
localPath: string
|
|
50
|
-
// cloudPath 可以为空
|
|
51
|
-
cloudPath?: string
|
|
52
|
-
// 并发数量
|
|
53
|
-
parallel?: number
|
|
54
|
-
// 重试次数
|
|
55
|
-
retryCount?: number
|
|
56
|
-
// 重试时间间隔(毫秒)
|
|
57
|
-
retryInterval?: number
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export interface IFilesOptions extends IOptions {
|
|
61
|
-
// 忽略文件
|
|
62
|
-
ignore?: string | string[]
|
|
63
|
-
// 文件列表
|
|
64
|
-
files: { localPath: string; cloudPath?: string }[]
|
|
65
|
-
// 重试次数
|
|
66
|
-
retryCount?: number
|
|
67
|
-
// 重试时间间隔(毫秒)
|
|
68
|
-
retryInterval?: number
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export interface ICustomOptions {
|
|
72
|
-
bucket: string
|
|
73
|
-
region: string
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export interface IWalkCloudDirOptions {
|
|
77
|
-
prefix: string
|
|
78
|
-
bucket: string
|
|
79
|
-
region: string
|
|
80
|
-
marker?: string
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export interface IRoutingRules {
|
|
84
|
-
keyPrefixEquals?: string // 前缀匹配
|
|
85
|
-
httpErrorCodeReturnedEquals?: string // 错误码
|
|
86
|
-
replaceKeyWith?: string // 替换内容
|
|
87
|
-
replaceKeyPrefixWith?: string // condition设置为KeyPrefixEquals 前缀匹配时可设置
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
export interface IBucketWebsiteOptions {
|
|
91
|
-
indexDocument: string
|
|
92
|
-
errorDocument?: string
|
|
93
|
-
routingRules?: Array<IRoutingRules>
|
|
94
|
-
region: string // 地域
|
|
95
|
-
bucket: string // 桶名
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
export interface IGetBucketOpions {
|
|
99
|
-
bucket?: string
|
|
100
|
-
region?: string
|
|
101
|
-
prefix?: string
|
|
102
|
-
marker?: string
|
|
103
|
-
maxKeys?: number
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
export type AclType = 'READONLY' | 'PRIVATE' | 'ADMINWRITE' | 'ADMINONLY'
|
|
107
|
-
type OnProgress = (progressData: IProgressData) => void
|
|
108
|
-
type OnFileFinish = (error: Error, res: any, fileData: any) => void
|
|
109
|
-
|
|
110
|
-
const BIG_FILE_SIZE = 5242880 // 5MB 1024*1024*5
|
|
111
|
-
|
|
112
|
-
export class StorageService {
|
|
113
|
-
private environment: Environment
|
|
114
|
-
private tcbService: CloudService
|
|
115
|
-
|
|
116
|
-
constructor(environment: Environment) {
|
|
117
|
-
this.environment = environment
|
|
118
|
-
this.tcbService = new CloudService(environment.cloudBaseContext, 'tcb', '2018-06-08')
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* 上传文件
|
|
123
|
-
* localPath 为文件夹时,会尝试在文件夹中寻找 cloudPath 中的文件名
|
|
124
|
-
* @param {string} localPath 本地文件的绝对路径
|
|
125
|
-
* @param {string} cloudPath 云端文件路径,如 img/test.png
|
|
126
|
-
* @returns {Promise<any>}
|
|
127
|
-
*/
|
|
128
|
-
@preLazy()
|
|
129
|
-
public async uploadFile(options: IFileOptions): Promise<any> {
|
|
130
|
-
const { localPath, cloudPath = '', onProgress } = options
|
|
131
|
-
const { bucket, region } = this.getStorageConfig()
|
|
132
|
-
|
|
133
|
-
return this.uploadFileCustom({
|
|
134
|
-
localPath,
|
|
135
|
-
cloudPath,
|
|
136
|
-
bucket,
|
|
137
|
-
region,
|
|
138
|
-
onProgress
|
|
139
|
-
})
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
/**
|
|
143
|
-
* 批量上传文件,默认并发 5
|
|
144
|
-
* @param options
|
|
145
|
-
*/
|
|
146
|
-
@preLazy()
|
|
147
|
-
public async uploadFiles(options: IFilesOptions): Promise<void> {
|
|
148
|
-
const { files, onProgress, parallel, onFileFinish, ignore, retryCount, retryInterval } =
|
|
149
|
-
options
|
|
150
|
-
const { bucket, region } = this.getStorageConfig()
|
|
151
|
-
|
|
152
|
-
return this.uploadFilesCustom({
|
|
153
|
-
files,
|
|
154
|
-
bucket,
|
|
155
|
-
region,
|
|
156
|
-
ignore,
|
|
157
|
-
parallel,
|
|
158
|
-
onProgress,
|
|
159
|
-
onFileFinish,
|
|
160
|
-
retryCount,
|
|
161
|
-
retryInterval
|
|
162
|
-
})
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
/**
|
|
166
|
-
* 上传文件,支持自定义 Bucket 和 Region
|
|
167
|
-
* @param {string} localPath
|
|
168
|
-
* @param {string} cloudPath
|
|
169
|
-
* @param {string} bucket
|
|
170
|
-
* @param {string} region
|
|
171
|
-
*/
|
|
172
|
-
@preLazy()
|
|
173
|
-
public async uploadFileCustom(options: IFileOptions & ICustomOptions): Promise<any> {
|
|
174
|
-
const { localPath, cloudPath, bucket, region, onProgress, fileId = true } = options
|
|
175
|
-
let localFilePath = ''
|
|
176
|
-
let resolveLocalPath = path.resolve(localPath)
|
|
177
|
-
checkFullAccess(resolveLocalPath, true)
|
|
178
|
-
|
|
179
|
-
// 如果 localPath 是一个文件夹,尝试在文件下寻找 cloudPath 中的文件
|
|
180
|
-
const fileStats = fs.statSync(resolveLocalPath)
|
|
181
|
-
if (fileStats.isDirectory()) {
|
|
182
|
-
const fileName = path.parse(cloudPath).base
|
|
183
|
-
const attemptFilePath = path.join(localPath, fileName)
|
|
184
|
-
if (checkFullAccess(attemptFilePath)) {
|
|
185
|
-
localFilePath = path.resolve(attemptFilePath)
|
|
186
|
-
}
|
|
187
|
-
} else {
|
|
188
|
-
localFilePath = resolveLocalPath
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
if (!localFilePath) {
|
|
192
|
-
throw new CloudBaseError('本地文件不存在!')
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
const cos = this.getCos()
|
|
196
|
-
const putObject = Util.promisify(cos.putObject).bind(cos)
|
|
197
|
-
const sliceUploadFile = Util.promisify(cos.sliceUploadFile).bind(cos)
|
|
198
|
-
let cosFileId
|
|
199
|
-
|
|
200
|
-
// 针对静态托管,fileId 不是必须的
|
|
201
|
-
if (fileId) {
|
|
202
|
-
// 针对文件存储,cosFileId 是必须的,区分上传人员,否则无法获取下载连接
|
|
203
|
-
const res = await this.getUploadMetadata(cloudPath)
|
|
204
|
-
cosFileId = res.cosFileId
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
let res
|
|
208
|
-
|
|
209
|
-
// 小文件,直接上传
|
|
210
|
-
if (fileStats.size < BIG_FILE_SIZE) {
|
|
211
|
-
res = await putObject({
|
|
212
|
-
onProgress,
|
|
213
|
-
Bucket: bucket,
|
|
214
|
-
Region: region,
|
|
215
|
-
Key: cloudPath,
|
|
216
|
-
StorageClass: 'STANDARD',
|
|
217
|
-
ContentLength: fileStats.size,
|
|
218
|
-
Body: fs.createReadStream(localFilePath),
|
|
219
|
-
'x-cos-meta-fileid': cosFileId
|
|
220
|
-
})
|
|
221
|
-
} else {
|
|
222
|
-
// 大文件,分块上传
|
|
223
|
-
res = await sliceUploadFile({
|
|
224
|
-
Bucket: bucket,
|
|
225
|
-
Region: region,
|
|
226
|
-
Key: cloudPath,
|
|
227
|
-
FilePath: localFilePath,
|
|
228
|
-
StorageClass: 'STANDARD',
|
|
229
|
-
AsyncLimit: 3,
|
|
230
|
-
onProgress,
|
|
231
|
-
'x-cos-meta-fileid': cosFileId
|
|
232
|
-
})
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
if (res.statusCode !== 200) {
|
|
236
|
-
throw new CloudBaseError(`上传文件错误:${JSON.stringify(res)}`)
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
return res
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/**
|
|
243
|
-
* 上传文件夹
|
|
244
|
-
* @param {string} localPath 本地文件夹路径
|
|
245
|
-
* @param {string} cloudPath 云端文件夹
|
|
246
|
-
* @param {number} parallel 并发量
|
|
247
|
-
* @param {number} retryCount 重试次数
|
|
248
|
-
* @param {number} retryInterval 重试时间间隔(毫秒)
|
|
249
|
-
* @param {(string | string[])} ignore
|
|
250
|
-
* @param {(string | string[])} ignore
|
|
251
|
-
* @returns {Promise<void>}
|
|
252
|
-
*/
|
|
253
|
-
@preLazy()
|
|
254
|
-
public async uploadDirectory(options: IFileOptions): Promise<void> {
|
|
255
|
-
const {
|
|
256
|
-
localPath,
|
|
257
|
-
cloudPath = '',
|
|
258
|
-
ignore,
|
|
259
|
-
onProgress,
|
|
260
|
-
onFileFinish,
|
|
261
|
-
parallel,
|
|
262
|
-
retryCount,
|
|
263
|
-
retryInterval
|
|
264
|
-
} = options
|
|
265
|
-
// 此处不检查路径是否存在
|
|
266
|
-
// 绝对路径 /var/blog/xxxx
|
|
267
|
-
const { bucket, region } = this.getStorageConfig()
|
|
268
|
-
return this.uploadDirectoryCustom({
|
|
269
|
-
localPath,
|
|
270
|
-
cloudPath,
|
|
271
|
-
parallel,
|
|
272
|
-
retryCount,
|
|
273
|
-
retryInterval,
|
|
274
|
-
bucket,
|
|
275
|
-
region,
|
|
276
|
-
ignore,
|
|
277
|
-
onProgress,
|
|
278
|
-
onFileFinish
|
|
279
|
-
})
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
/**
|
|
283
|
-
* 上传文件夹,支持自定义 Region 和 Bucket
|
|
284
|
-
* @param {string} localPath
|
|
285
|
-
* @param {string} cloudPath
|
|
286
|
-
* @param {number} parallel
|
|
287
|
-
* @param {number} retryCount
|
|
288
|
-
* @param {number} retryInterval
|
|
289
|
-
* @param {string} bucket
|
|
290
|
-
* @param {string} region
|
|
291
|
-
* @param {IOptions} options
|
|
292
|
-
* @returns {Promise<void>}
|
|
293
|
-
*/
|
|
294
|
-
@preLazy()
|
|
295
|
-
public async uploadDirectoryCustom(options: IFileOptions & ICustomOptions): Promise<void> {
|
|
296
|
-
const {
|
|
297
|
-
localPath,
|
|
298
|
-
cloudPath,
|
|
299
|
-
bucket,
|
|
300
|
-
region,
|
|
301
|
-
onProgress,
|
|
302
|
-
onFileFinish,
|
|
303
|
-
ignore,
|
|
304
|
-
fileId = true,
|
|
305
|
-
parallel = 20,
|
|
306
|
-
retryCount = 0,
|
|
307
|
-
retryInterval = 500
|
|
308
|
-
} = options
|
|
309
|
-
// 此处不检查路径是否存在
|
|
310
|
-
// 绝对路径 /var/blog/xxxx
|
|
311
|
-
const resolvePath = path.resolve(localPath)
|
|
312
|
-
// 在路径结尾加上 '/'
|
|
313
|
-
const resolveLocalPath = path.join(resolvePath, path.sep)
|
|
314
|
-
const filePaths = await this.walkLocalDir(resolveLocalPath, ignore)
|
|
315
|
-
|
|
316
|
-
if (!filePaths || !filePaths.length) {
|
|
317
|
-
return
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
const fileStatsList = filePaths.map(filePath => {
|
|
321
|
-
// 处理 windows 路径
|
|
322
|
-
const fileKeyPath = filePath.replace(resolveLocalPath, '').replace(/\\/g, '/')
|
|
323
|
-
// 解析 cloudPath
|
|
324
|
-
let cloudFileKey = path.join(cloudPath, fileKeyPath).replace(/\\/g, '/')
|
|
325
|
-
|
|
326
|
-
if (isDirectory(filePath)) {
|
|
327
|
-
cloudFileKey = this.getCloudKey(cloudFileKey)
|
|
328
|
-
return {
|
|
329
|
-
filePath,
|
|
330
|
-
cloudFileKey,
|
|
331
|
-
isDir: true
|
|
332
|
-
}
|
|
333
|
-
} else {
|
|
334
|
-
return {
|
|
335
|
-
filePath,
|
|
336
|
-
cloudFileKey,
|
|
337
|
-
isDir: false
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
})
|
|
341
|
-
|
|
342
|
-
// 创建目录请求
|
|
343
|
-
const creatingDirController = new AsyncTaskParallelController(parallel, 50)
|
|
344
|
-
const creatingDirTasks = fileStatsList
|
|
345
|
-
.filter(info => info.isDir)
|
|
346
|
-
.map(
|
|
347
|
-
info => () =>
|
|
348
|
-
this.createCloudDirectroyCustom({
|
|
349
|
-
cloudPath: info.cloudFileKey,
|
|
350
|
-
bucket,
|
|
351
|
-
region
|
|
352
|
-
})
|
|
353
|
-
)
|
|
354
|
-
|
|
355
|
-
creatingDirController.loadTasks(creatingDirTasks)
|
|
356
|
-
await creatingDirController.run()
|
|
357
|
-
|
|
358
|
-
// 上传文件对象
|
|
359
|
-
const tasks = fileStatsList
|
|
360
|
-
.filter(stats => !stats.isDir)
|
|
361
|
-
.map(stats => async () => {
|
|
362
|
-
let cosFileId
|
|
363
|
-
if (fileId) {
|
|
364
|
-
const res = await this.getUploadMetadata(stats.cloudFileKey)
|
|
365
|
-
cosFileId = res.cosFileId
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
return {
|
|
369
|
-
Bucket: bucket,
|
|
370
|
-
Region: region,
|
|
371
|
-
Key: stats.cloudFileKey,
|
|
372
|
-
FilePath: stats.filePath,
|
|
373
|
-
'x-cos-meta-fileid': cosFileId
|
|
374
|
-
}
|
|
375
|
-
})
|
|
376
|
-
|
|
377
|
-
// 控制请求并发
|
|
378
|
-
const getMetadataController = new AsyncTaskParallelController(parallel, 50)
|
|
379
|
-
getMetadataController.loadTasks(tasks)
|
|
380
|
-
const files = await getMetadataController.run()
|
|
381
|
-
|
|
382
|
-
// 对文件上传进行处理
|
|
383
|
-
const cos = this.getCos(parallel)
|
|
384
|
-
const uploadFiles = Util.promisify(cos.uploadFiles).bind(cos)
|
|
385
|
-
const params = {
|
|
386
|
-
files,
|
|
387
|
-
SliceSize: BIG_FILE_SIZE,
|
|
388
|
-
onProgress,
|
|
389
|
-
onFileFinish
|
|
390
|
-
}
|
|
391
|
-
return this.uploadFilesWithRetry({
|
|
392
|
-
uploadFiles,
|
|
393
|
-
options: params,
|
|
394
|
-
times: retryCount,
|
|
395
|
-
interval: retryInterval,
|
|
396
|
-
failedFiles: []
|
|
397
|
-
})
|
|
398
|
-
}
|
|
399
|
-
/**
|
|
400
|
-
* 批量上传文件
|
|
401
|
-
* @param options
|
|
402
|
-
*/
|
|
403
|
-
@preLazy()
|
|
404
|
-
public async uploadFilesCustom(options: IFilesOptions & ICustomOptions): Promise<any> {
|
|
405
|
-
const {
|
|
406
|
-
files,
|
|
407
|
-
bucket,
|
|
408
|
-
region,
|
|
409
|
-
ignore,
|
|
410
|
-
onProgress,
|
|
411
|
-
onFileFinish,
|
|
412
|
-
fileId = true,
|
|
413
|
-
parallel = 20,
|
|
414
|
-
retryCount = 0,
|
|
415
|
-
retryInterval = 500
|
|
416
|
-
} = options
|
|
417
|
-
|
|
418
|
-
if (!files || !files.length) {
|
|
419
|
-
return
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
let fileList = files
|
|
423
|
-
.map(item => {
|
|
424
|
-
const { localPath, cloudPath } = item
|
|
425
|
-
|
|
426
|
-
return {
|
|
427
|
-
filePath: localPath,
|
|
428
|
-
cloudFileKey: cloudPath
|
|
429
|
-
}
|
|
430
|
-
})
|
|
431
|
-
.filter(item => (ignore?.length ? !micromatch.isMatch(item.filePath, ignore) : true))
|
|
432
|
-
|
|
433
|
-
// 生成上传文件属性
|
|
434
|
-
const tasks = fileList.map(stats => async () => {
|
|
435
|
-
let cosFileId
|
|
436
|
-
if (fileId) {
|
|
437
|
-
const res = await this.getUploadMetadata(stats.cloudFileKey)
|
|
438
|
-
cosFileId = res.cosFileId
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
return {
|
|
442
|
-
Bucket: bucket,
|
|
443
|
-
Region: region,
|
|
444
|
-
Key: stats.cloudFileKey,
|
|
445
|
-
FilePath: stats.filePath,
|
|
446
|
-
'x-cos-meta-fileid': cosFileId
|
|
447
|
-
}
|
|
448
|
-
})
|
|
449
|
-
|
|
450
|
-
// 控制请求并发
|
|
451
|
-
const asyncTaskController = new AsyncTaskParallelController(parallel, 50)
|
|
452
|
-
asyncTaskController.loadTasks(tasks)
|
|
453
|
-
fileList = await asyncTaskController.run()
|
|
454
|
-
|
|
455
|
-
const cos = this.getCos(parallel)
|
|
456
|
-
const uploadFiles = Util.promisify(cos.uploadFiles).bind(cos)
|
|
457
|
-
const params = {
|
|
458
|
-
files: fileList,
|
|
459
|
-
SliceSize: BIG_FILE_SIZE,
|
|
460
|
-
onProgress,
|
|
461
|
-
onFileFinish
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
// return uploadFiles({
|
|
465
|
-
// onProgress,
|
|
466
|
-
// onFileFinish,
|
|
467
|
-
// files: fileList,
|
|
468
|
-
// SliceSize: BIG_FILE_SIZE
|
|
469
|
-
// })
|
|
470
|
-
return this.uploadFilesWithRetry({
|
|
471
|
-
uploadFiles,
|
|
472
|
-
options: params,
|
|
473
|
-
times: retryCount,
|
|
474
|
-
interval: retryInterval,
|
|
475
|
-
failedFiles: []
|
|
476
|
-
})
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
/**
|
|
480
|
-
* 创建一个空的文件夹
|
|
481
|
-
* @param {string} cloudPath
|
|
482
|
-
*/
|
|
483
|
-
@preLazy()
|
|
484
|
-
public async createCloudDirectroy(cloudPath: string) {
|
|
485
|
-
const { bucket, region } = this.getStorageConfig()
|
|
486
|
-
await this.createCloudDirectroyCustom({
|
|
487
|
-
cloudPath,
|
|
488
|
-
bucket,
|
|
489
|
-
region
|
|
490
|
-
})
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
/**
|
|
494
|
-
* 创建一个空的文件夹,支持自定义 Region 和 Bucket
|
|
495
|
-
* @param {string} cloudPath
|
|
496
|
-
* @param {string} bucket
|
|
497
|
-
* @param {string} region
|
|
498
|
-
*/
|
|
499
|
-
@preLazy()
|
|
500
|
-
public async createCloudDirectroyCustom(options: ICustomOptions & { cloudPath: string }) {
|
|
501
|
-
const { cloudPath, bucket, region } = options
|
|
502
|
-
const cos = this.getCos()
|
|
503
|
-
const putObject = Util.promisify(cos.putObject).bind(cos)
|
|
504
|
-
|
|
505
|
-
const dirKey = this.getCloudKey(cloudPath)
|
|
506
|
-
|
|
507
|
-
const res = await putObject({
|
|
508
|
-
Bucket: bucket,
|
|
509
|
-
Region: region,
|
|
510
|
-
Key: dirKey,
|
|
511
|
-
Body: ''
|
|
512
|
-
})
|
|
513
|
-
|
|
514
|
-
if (res.statusCode !== 200) {
|
|
515
|
-
throw new CloudBaseError(`创建文件夹失败:${JSON.stringify(res)}`)
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
/**
|
|
520
|
-
* 下载文件
|
|
521
|
-
* @param {string} cloudPath 云端文件路径
|
|
522
|
-
* @param {string} localPath 文件本地存储路径,文件需指定文件名称
|
|
523
|
-
* @returns {Promise<NodeJS.ReadableStream>}
|
|
524
|
-
*/
|
|
525
|
-
@preLazy()
|
|
526
|
-
public async downloadFile(options: {
|
|
527
|
-
cloudPath: string
|
|
528
|
-
localPath?: string
|
|
529
|
-
}): Promise<NodeJS.ReadableStream | string> {
|
|
530
|
-
const { cloudPath, localPath } = options
|
|
531
|
-
const resolveLocalPath = path.resolve(localPath)
|
|
532
|
-
const fileDir = path.dirname(localPath)
|
|
533
|
-
|
|
534
|
-
checkFullAccess(fileDir, true)
|
|
535
|
-
|
|
536
|
-
const urlList = await this.getTemporaryUrl([cloudPath])
|
|
537
|
-
const { url } = urlList[0]
|
|
538
|
-
|
|
539
|
-
const { proxy } = await this.environment.getAuthConfig()
|
|
540
|
-
const res = await fetchStream(url, {}, proxy)
|
|
541
|
-
|
|
542
|
-
// localPath 不存在时,返回 ReadableStream
|
|
543
|
-
if (!localPath) {
|
|
544
|
-
return res.body
|
|
545
|
-
}
|
|
546
|
-
const dest = fs.createWriteStream(resolveLocalPath)
|
|
547
|
-
res.body.pipe(dest)
|
|
548
|
-
|
|
549
|
-
// 写完成后返回
|
|
550
|
-
return new Promise(resolve => {
|
|
551
|
-
dest.on('close', () => {
|
|
552
|
-
// 返回文件地址
|
|
553
|
-
resolve(resolveLocalPath)
|
|
554
|
-
})
|
|
555
|
-
})
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
/**
|
|
559
|
-
* 下载文件夹
|
|
560
|
-
* @param {string} cloudPath 云端文件路径
|
|
561
|
-
* @param {string} localPath 本地文件夹存储路径
|
|
562
|
-
* @returns {Promise<(NodeJS.ReadableStream | string)[]>}
|
|
563
|
-
*/
|
|
564
|
-
@preLazy()
|
|
565
|
-
public async downloadDirectory(options: {
|
|
566
|
-
cloudPath: string
|
|
567
|
-
localPath?: string
|
|
568
|
-
parallel?: number
|
|
569
|
-
}): Promise<(NodeJS.ReadableStream | string)[]> {
|
|
570
|
-
const { cloudPath, localPath, parallel = 20 } = options
|
|
571
|
-
const resolveLocalPath = path.resolve(localPath)
|
|
572
|
-
|
|
573
|
-
checkFullAccess(resolveLocalPath, true)
|
|
574
|
-
|
|
575
|
-
const cloudDirectoryKey = this.getCloudKey(cloudPath)
|
|
576
|
-
const files = await this.walkCloudDir(cloudDirectoryKey)
|
|
577
|
-
const promises = files.map(file => async () => {
|
|
578
|
-
return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath })
|
|
579
|
-
})
|
|
580
|
-
const asyncTaskController = new AsyncTaskParallelController(parallel, 50)
|
|
581
|
-
asyncTaskController.loadTasks(promises)
|
|
582
|
-
let res = await asyncTaskController.run()
|
|
583
|
-
const errorIndexArr = []
|
|
584
|
-
|
|
585
|
-
res.map(
|
|
586
|
-
(item, index) =>
|
|
587
|
-
/Error/gi.test(Object.prototype.toString.call(item)) && errorIndexArr.push(index)
|
|
588
|
-
)
|
|
589
|
-
// 重试逻辑
|
|
590
|
-
if (errorIndexArr.length) {
|
|
591
|
-
const errorFiles = errorIndexArr.map(errorIndex => files[errorIndex])
|
|
592
|
-
asyncTaskController.loadTasks(
|
|
593
|
-
errorFiles.map(file => async () => {
|
|
594
|
-
return this.downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath })
|
|
595
|
-
})
|
|
596
|
-
)
|
|
597
|
-
res = await asyncTaskController.run()
|
|
598
|
-
}
|
|
599
|
-
const errorResultArr = this.determineDownLoadResultIsError(res)
|
|
600
|
-
if (errorResultArr.length) {
|
|
601
|
-
throw errorResultArr[0]
|
|
602
|
-
}
|
|
603
|
-
return res
|
|
604
|
-
}
|
|
605
|
-
|
|
606
|
-
/**
|
|
607
|
-
* 列出文件夹下的文件
|
|
608
|
-
* @link https://cloud.tencent.com/document/product/436/7734
|
|
609
|
-
* @param {string} cloudPath 云端文件夹,如果为空字符串,则表示根目录
|
|
610
|
-
* @returns {Promise<ListFileInfo[]>}
|
|
611
|
-
*/
|
|
612
|
-
@preLazy()
|
|
613
|
-
public async listDirectoryFiles(cloudPath: string): Promise<IListFileInfo[]> {
|
|
614
|
-
return this.walkCloudDir(cloudPath)
|
|
615
|
-
}
|
|
616
|
-
|
|
617
|
-
/**
|
|
618
|
-
* 获取文件临时下载链接
|
|
619
|
-
* @param {((string | ITempUrlInfo)[])} fileList 文件路径或文件信息数组
|
|
620
|
-
* @returns {Promise<{ fileId: string; url: string }[]>}
|
|
621
|
-
*/
|
|
622
|
-
@preLazy()
|
|
623
|
-
public async getTemporaryUrl(
|
|
624
|
-
fileList: (string | ITempUrlInfo)[]
|
|
625
|
-
): Promise<{ fileId: string; url: string }[]> {
|
|
626
|
-
if (!fileList || !Array.isArray(fileList)) {
|
|
627
|
-
throw new CloudBaseError('fileList 必须是非空的数组')
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
const files: ITempUrlInfo[] = fileList.map(item => {
|
|
631
|
-
if (typeof item === 'string') {
|
|
632
|
-
return { cloudPath: item, maxAge: 3600 }
|
|
633
|
-
} else {
|
|
634
|
-
return item
|
|
635
|
-
}
|
|
636
|
-
})
|
|
637
|
-
|
|
638
|
-
const invalidData = files.find(
|
|
639
|
-
item => !item.cloudPath || !item.maxAge || typeof item.cloudPath !== 'string'
|
|
640
|
-
)
|
|
641
|
-
|
|
642
|
-
if (invalidData) {
|
|
643
|
-
throw new CloudBaseError(`非法参数:${JSON.stringify(invalidData)}`)
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
const notExistsFiles = []
|
|
647
|
-
|
|
648
|
-
const checkFileRequests = files.map(file =>
|
|
649
|
-
(async () => {
|
|
650
|
-
try {
|
|
651
|
-
await this.getFileInfo(file.cloudPath)
|
|
652
|
-
} catch (e) {
|
|
653
|
-
if (e.statusCode === 404) {
|
|
654
|
-
notExistsFiles.push(file.cloudPath)
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
})()
|
|
658
|
-
)
|
|
659
|
-
|
|
660
|
-
await Promise.all(checkFileRequests)
|
|
661
|
-
|
|
662
|
-
// 文件路径不存在
|
|
663
|
-
if (notExistsFiles.length) {
|
|
664
|
-
throw new CloudBaseError(`以下文件不存在:${notExistsFiles.join(', ')}`)
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
const data = files.map(item => ({
|
|
668
|
-
fileid: this.cloudPathToFileId(item.cloudPath),
|
|
669
|
-
max_age: item.maxAge
|
|
670
|
-
}))
|
|
671
|
-
|
|
672
|
-
const config = this.environment.getAuthConfig()
|
|
673
|
-
|
|
674
|
-
const res = await cloudBaseRequest({
|
|
675
|
-
config,
|
|
676
|
-
params: {
|
|
677
|
-
file_list: data,
|
|
678
|
-
action: 'storage.batchGetDownloadUrl'
|
|
679
|
-
},
|
|
680
|
-
method: 'POST'
|
|
681
|
-
})
|
|
682
|
-
|
|
683
|
-
const downloadList = res.data.download_list.map(item => ({
|
|
684
|
-
url: item.download_url,
|
|
685
|
-
fileId: item.fileid || item.fileID
|
|
686
|
-
}))
|
|
687
|
-
|
|
688
|
-
return downloadList
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
/**
|
|
692
|
-
* 删除文件
|
|
693
|
-
* @param {string[]} cloudPathList 云端文件路径数组
|
|
694
|
-
* @returns {Promise<void>}
|
|
695
|
-
*/
|
|
696
|
-
@preLazy()
|
|
697
|
-
public async deleteFile(cloudPathList: string[]): Promise<void> {
|
|
698
|
-
if (!cloudPathList || !Array.isArray(cloudPathList)) {
|
|
699
|
-
throw new CloudBaseError('fileList必须是非空的数组')
|
|
700
|
-
}
|
|
701
|
-
|
|
702
|
-
const hasInvalidFileId = cloudPathList.some(file => !file || typeof file !== 'string')
|
|
703
|
-
if (hasInvalidFileId) {
|
|
704
|
-
throw new CloudBaseError('fileList的元素必须是非空的字符串')
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
const { bucket, env } = this.getStorageConfig()
|
|
708
|
-
const fileIdList = cloudPathList.map(filePath => this.cloudPathToFileId(filePath))
|
|
709
|
-
|
|
710
|
-
const config = this.environment.getAuthConfig()
|
|
711
|
-
const res = await cloudBaseRequest({
|
|
712
|
-
config,
|
|
713
|
-
params: {
|
|
714
|
-
action: 'storage.batchDeleteFile',
|
|
715
|
-
fileid_list: fileIdList
|
|
716
|
-
},
|
|
717
|
-
method: 'POST'
|
|
718
|
-
})
|
|
719
|
-
|
|
720
|
-
const failedList = res.data.delete_list
|
|
721
|
-
.filter(item => item.code !== 'SUCCESS')
|
|
722
|
-
.map(item => `${item.fileID} : ${item.code}`)
|
|
723
|
-
if (failedList.length) {
|
|
724
|
-
throw new CloudBaseError(`部分删除文件失败:${JSON.stringify(failedList)}`)
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
/**
|
|
729
|
-
* 删除文件,可以指定 Bucket 和 Region
|
|
730
|
-
* @param {string[]} cloudPathList
|
|
731
|
-
* @param {string} bucket
|
|
732
|
-
* @param {string} region
|
|
733
|
-
* @returns {Promise<void>}
|
|
734
|
-
*/
|
|
735
|
-
@preLazy()
|
|
736
|
-
public async deleteFileCustom(
|
|
737
|
-
cloudPathList: string[],
|
|
738
|
-
bucket: string,
|
|
739
|
-
region: string
|
|
740
|
-
): Promise<void> {
|
|
741
|
-
if (!cloudPathList || !Array.isArray(cloudPathList)) {
|
|
742
|
-
throw new CloudBaseError('fileList必须是非空的数组')
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
const hasInvalidFileId = cloudPathList.some(file => !file || typeof file !== 'string')
|
|
746
|
-
if (hasInvalidFileId) {
|
|
747
|
-
throw new CloudBaseError('fileList的元素必须是非空的字符串')
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
const cos = this.getCos()
|
|
751
|
-
const deleteObject = Util.promisify(cos.deleteObject).bind(cos)
|
|
752
|
-
|
|
753
|
-
const promises = cloudPathList.map(async file =>
|
|
754
|
-
deleteObject({
|
|
755
|
-
Bucket: bucket,
|
|
756
|
-
Region: region,
|
|
757
|
-
Key: file
|
|
758
|
-
})
|
|
759
|
-
)
|
|
760
|
-
|
|
761
|
-
await Promise.all(promises)
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
/**
|
|
765
|
-
* 获取文件信息
|
|
766
|
-
* @param {string} cloudPath 云端文件路径
|
|
767
|
-
* @returns {Promise<FileInfo>}
|
|
768
|
-
*/
|
|
769
|
-
@preLazy()
|
|
770
|
-
public async getFileInfo(cloudPath: string): Promise<IFileInfo> {
|
|
771
|
-
const cos = this.getCos()
|
|
772
|
-
const headObject = Util.promisify(cos.headObject).bind(cos)
|
|
773
|
-
const { bucket, region } = this.getStorageConfig()
|
|
774
|
-
|
|
775
|
-
const { headers } = await headObject({
|
|
776
|
-
Bucket: bucket,
|
|
777
|
-
Region: region,
|
|
778
|
-
Key: cloudPath
|
|
779
|
-
})
|
|
780
|
-
|
|
781
|
-
if (!headers) {
|
|
782
|
-
throw new CloudBaseError(`[${cloudPath}] 获取文件信息失败`)
|
|
783
|
-
}
|
|
784
|
-
|
|
785
|
-
// 文件大小 KB
|
|
786
|
-
const size = Number(Number(headers['content-length']) / 1024).toFixed(2)
|
|
787
|
-
|
|
788
|
-
return {
|
|
789
|
-
Size: size,
|
|
790
|
-
Type: headers['content-type'],
|
|
791
|
-
Date: headers['date'],
|
|
792
|
-
ETag: headers['etag']
|
|
793
|
-
}
|
|
794
|
-
}
|
|
795
|
-
|
|
796
|
-
/**
|
|
797
|
-
* 删除文件夹
|
|
798
|
-
* @param {string} cloudPath 云端文件夹路径
|
|
799
|
-
* @returns {Promise<void>}
|
|
800
|
-
*/
|
|
801
|
-
@preLazy()
|
|
802
|
-
public async deleteDirectory(cloudPath: string): Promise<{
|
|
803
|
-
Deleted: { Key: string }[]
|
|
804
|
-
Error: Object[]
|
|
805
|
-
}> {
|
|
806
|
-
const { bucket, region } = this.getStorageConfig()
|
|
807
|
-
|
|
808
|
-
return this.deleteDirectoryCustom({
|
|
809
|
-
cloudPath,
|
|
810
|
-
bucket,
|
|
811
|
-
region
|
|
812
|
-
})
|
|
813
|
-
}
|
|
814
|
-
|
|
815
|
-
/**
|
|
816
|
-
* 删除文件,可以指定 bucket 和 region
|
|
817
|
-
* @param {string} cloudPath
|
|
818
|
-
* @param {string} bucket
|
|
819
|
-
* @param {string} region
|
|
820
|
-
* @returns {Promise<void>}
|
|
821
|
-
*/
|
|
822
|
-
@preLazy()
|
|
823
|
-
public async deleteDirectoryCustom(options: { cloudPath: string } & ICustomOptions): Promise<{
|
|
824
|
-
Deleted: { Key: string }[]
|
|
825
|
-
Error: Object[]
|
|
826
|
-
}> {
|
|
827
|
-
const { cloudPath, bucket, region } = options
|
|
828
|
-
const key = this.getCloudKey(cloudPath)
|
|
829
|
-
|
|
830
|
-
const cos = this.getCos()
|
|
831
|
-
const deleteMultipleObject = Util.promisify(cos.deleteMultipleObject).bind(cos)
|
|
832
|
-
|
|
833
|
-
// 遍历获取全部文件
|
|
834
|
-
const files = await this.walkCloudDirCustom({
|
|
835
|
-
bucket,
|
|
836
|
-
region,
|
|
837
|
-
prefix: key
|
|
838
|
-
})
|
|
839
|
-
|
|
840
|
-
// 文件为空时,不能调用删除接口
|
|
841
|
-
if (!files.length) {
|
|
842
|
-
return {
|
|
843
|
-
Deleted: [],
|
|
844
|
-
Error: []
|
|
845
|
-
}
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
// COS 接口最大一次删除 1000 个 Key
|
|
849
|
-
// 将数组切分为 500 个文件一组
|
|
850
|
-
const sliceGroup = []
|
|
851
|
-
const total = Math.ceil(files.length / 500)
|
|
852
|
-
for (let i = 0; i < total; i++) {
|
|
853
|
-
sliceGroup.push(files.splice(0, 500))
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
const tasks = sliceGroup.map(group =>
|
|
857
|
-
deleteMultipleObject({
|
|
858
|
-
Bucket: bucket,
|
|
859
|
-
Region: region,
|
|
860
|
-
Objects: group.map(file => ({ Key: file.Key }))
|
|
861
|
-
})
|
|
862
|
-
)
|
|
863
|
-
|
|
864
|
-
// 删除多个文件
|
|
865
|
-
const taskRes = await Promise.all(tasks)
|
|
866
|
-
|
|
867
|
-
// 合并响应结果
|
|
868
|
-
const Deleted = taskRes.map(_ => _.Deleted).reduce((prev, next) => [...prev, ...next], [])
|
|
869
|
-
const Error = taskRes.map(_ => _.Error).reduce((prev, next) => [...prev, ...next], [])
|
|
870
|
-
return {
|
|
871
|
-
Deleted,
|
|
872
|
-
Error
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
/**
|
|
877
|
-
* 获取文件存储权限
|
|
878
|
-
* READONLY:所有用户可读,仅创建者和管理员可写
|
|
879
|
-
* PRIVATE:仅创建者及管理员可读写
|
|
880
|
-
* ADMINWRITE:所有用户可读,仅管理员可写
|
|
881
|
-
* ADMINONLY:仅管理员可读写
|
|
882
|
-
* @returns
|
|
883
|
-
*/
|
|
884
|
-
@preLazy()
|
|
885
|
-
public async getStorageAcl(): Promise<AclType> {
|
|
886
|
-
const { bucket, env } = this.getStorageConfig()
|
|
887
|
-
|
|
888
|
-
const res = await this.tcbService.request<{
|
|
889
|
-
AclTag: AclType
|
|
890
|
-
}>('DescribeStorageACL', {
|
|
891
|
-
EnvId: env,
|
|
892
|
-
Bucket: bucket
|
|
893
|
-
})
|
|
894
|
-
|
|
895
|
-
return res.AclTag
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
/**
|
|
899
|
-
* 设置文件存储权限
|
|
900
|
-
* READONLY:所有用户可读,仅创建者和管理员可写
|
|
901
|
-
* PRIVATE:仅创建者及管理员可读写
|
|
902
|
-
* ADMINWRITE:所有用户可读,仅管理员可写
|
|
903
|
-
* ADMINONLY:仅管理员可读写
|
|
904
|
-
* @param {string} acl
|
|
905
|
-
* @returns
|
|
906
|
-
*/
|
|
907
|
-
@preLazy()
|
|
908
|
-
public async setStorageAcl(acl: AclType): Promise<IResponseInfo> {
|
|
909
|
-
const validAcl = ['READONLY', 'PRIVATE', 'ADMINWRITE', 'ADMINONLY']
|
|
910
|
-
if (!validAcl.includes(acl)) {
|
|
911
|
-
throw new CloudBaseError('非法的权限类型')
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
const { bucket, env } = this.getStorageConfig()
|
|
915
|
-
|
|
916
|
-
return this.tcbService.request<IResponseInfo>('ModifyStorageACL', {
|
|
917
|
-
EnvId: env,
|
|
918
|
-
Bucket: bucket,
|
|
919
|
-
AclTag: acl
|
|
920
|
-
})
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
/**
|
|
924
|
-
* 遍历云端文件夹
|
|
925
|
-
* @param {string} prefix
|
|
926
|
-
* @param {string} [marker] 路径开始标志
|
|
927
|
-
* @returns {Promise<IListFileInfo[]>}
|
|
928
|
-
*/
|
|
929
|
-
@preLazy()
|
|
930
|
-
public async walkCloudDir(prefix: string, marker?: string): Promise<IListFileInfo[]> {
|
|
931
|
-
const { bucket, region } = this.getStorageConfig()
|
|
932
|
-
return this.walkCloudDirCustom({
|
|
933
|
-
prefix,
|
|
934
|
-
bucket,
|
|
935
|
-
region,
|
|
936
|
-
marker
|
|
937
|
-
})
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
/**
|
|
941
|
-
* 遍历云端文件夹,支持自定义 Bucket 和 Region
|
|
942
|
-
* @param {string} prefix
|
|
943
|
-
* @param {string} [marker]
|
|
944
|
-
* @param {string} bucket
|
|
945
|
-
* @param {string} region
|
|
946
|
-
* @returns {Promise<IListFileInfo[]>}
|
|
947
|
-
*/
|
|
948
|
-
@preLazy()
|
|
949
|
-
public async walkCloudDirCustom(options: IWalkCloudDirOptions): Promise<IListFileInfo[]> {
|
|
950
|
-
const { prefix, bucket, region, marker = '/' } = options
|
|
951
|
-
let fileList = []
|
|
952
|
-
const cos = this.getCos()
|
|
953
|
-
const getBucket = Util.promisify(cos.getBucket).bind(cos)
|
|
954
|
-
|
|
955
|
-
const prefixKey = this.getCloudKey(prefix)
|
|
956
|
-
|
|
957
|
-
const res = await getBucket({
|
|
958
|
-
Bucket: bucket,
|
|
959
|
-
Region: region,
|
|
960
|
-
Prefix: prefixKey,
|
|
961
|
-
MaxKeys: 100,
|
|
962
|
-
Marker: marker
|
|
963
|
-
})
|
|
964
|
-
|
|
965
|
-
fileList.push(...res.Contents)
|
|
966
|
-
|
|
967
|
-
let moreFiles = []
|
|
968
|
-
if (res.IsTruncated === 'true' || res.IsTruncated === true) {
|
|
969
|
-
moreFiles = await this.walkCloudDirCustom({
|
|
970
|
-
bucket,
|
|
971
|
-
region,
|
|
972
|
-
prefix: prefixKey,
|
|
973
|
-
marker: res.NextMarker
|
|
974
|
-
})
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
fileList.push(...moreFiles)
|
|
978
|
-
return fileList
|
|
979
|
-
}
|
|
980
|
-
|
|
981
|
-
/**
|
|
982
|
-
* 遍历本地文件夹
|
|
983
|
-
* 忽略不包含 dir 路径,即如果 ignore 匹配 dir,dir 也不会被忽略
|
|
984
|
-
* @private
|
|
985
|
-
* @param {string} dir
|
|
986
|
-
* @param {(string | string[])} [ignore]
|
|
987
|
-
* @returns
|
|
988
|
-
*/
|
|
989
|
-
public async walkLocalDir(dir: string, ignore?: string | string[]) {
|
|
990
|
-
try {
|
|
991
|
-
return walkdir.async(dir, {
|
|
992
|
-
filter: (currDir: string, files: string[]) => {
|
|
993
|
-
// NOTE: ignore 为空数组时会忽略全部文件
|
|
994
|
-
if (!ignore || !ignore.length) return files
|
|
995
|
-
|
|
996
|
-
return files.filter(item => {
|
|
997
|
-
// 当前文件全路径
|
|
998
|
-
const fullPath = path.join(currDir, item)
|
|
999
|
-
// 文件相对于传入目录的路径
|
|
1000
|
-
const fileRelativePath = fullPath.replace(path.join(dir, path.sep), '')
|
|
1001
|
-
// 匹配
|
|
1002
|
-
return !micromatch.isMatch(fileRelativePath, ignore)
|
|
1003
|
-
})
|
|
1004
|
-
}
|
|
1005
|
-
})
|
|
1006
|
-
} catch (e) {
|
|
1007
|
-
throw new CloudBaseError(e.message)
|
|
1008
|
-
}
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
/**
|
|
1012
|
-
* 获取文件上传链接属性
|
|
1013
|
-
*/
|
|
1014
|
-
public async getUploadMetadata(path: string): Promise<IUploadMetadata> {
|
|
1015
|
-
const config = this.environment.getAuthConfig()
|
|
1016
|
-
|
|
1017
|
-
const res = await cloudBaseRequest({
|
|
1018
|
-
config,
|
|
1019
|
-
params: {
|
|
1020
|
-
path,
|
|
1021
|
-
action: 'storage.getUploadMetadata'
|
|
1022
|
-
},
|
|
1023
|
-
method: 'POST'
|
|
1024
|
-
})
|
|
1025
|
-
if (res.code) {
|
|
1026
|
-
throw new CloudBaseError(`${res.code}: ${res.message || ''}`, {
|
|
1027
|
-
requestId: res.requestId
|
|
1028
|
-
})
|
|
1029
|
-
}
|
|
1030
|
-
return res.data
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
|
-
/**
|
|
1034
|
-
* 获取静态网站配置
|
|
1035
|
-
*/
|
|
1036
|
-
async getWebsiteConfig(options: { bucket: string; region: string }) {
|
|
1037
|
-
const { bucket, region } = options
|
|
1038
|
-
const cos = this.getCos()
|
|
1039
|
-
const getBucketWebsite = Util.promisify(cos.getBucketWebsite).bind(cos)
|
|
1040
|
-
|
|
1041
|
-
const res = await getBucketWebsite({
|
|
1042
|
-
Bucket: bucket,
|
|
1043
|
-
Region: region
|
|
1044
|
-
})
|
|
1045
|
-
|
|
1046
|
-
return res
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
/**
|
|
1050
|
-
* 配置文档
|
|
1051
|
-
*/
|
|
1052
|
-
@preLazy()
|
|
1053
|
-
async putBucketWebsite(options: IBucketWebsiteOptions) {
|
|
1054
|
-
const { indexDocument, errorDocument, bucket, region, routingRules } = options
|
|
1055
|
-
|
|
1056
|
-
const cos = this.getCos()
|
|
1057
|
-
const putBucketWebsite = Util.promisify(cos.putBucketWebsite).bind(cos)
|
|
1058
|
-
|
|
1059
|
-
let params: any = {
|
|
1060
|
-
Bucket: bucket,
|
|
1061
|
-
Region: region,
|
|
1062
|
-
WebsiteConfiguration: {
|
|
1063
|
-
IndexDocument: {
|
|
1064
|
-
Suffix: indexDocument
|
|
1065
|
-
},
|
|
1066
|
-
ErrorDocument: {
|
|
1067
|
-
Key: errorDocument
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
}
|
|
1071
|
-
|
|
1072
|
-
if (routingRules) {
|
|
1073
|
-
params.WebsiteConfiguration.RoutingRules = []
|
|
1074
|
-
for (let value of routingRules) {
|
|
1075
|
-
const routeItem: any = {}
|
|
1076
|
-
if (value.keyPrefixEquals) {
|
|
1077
|
-
routeItem.Condition = {
|
|
1078
|
-
KeyPrefixEquals: value.keyPrefixEquals
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
|
|
1082
|
-
if (value.httpErrorCodeReturnedEquals) {
|
|
1083
|
-
routeItem.Condition = {
|
|
1084
|
-
HttpErrorCodeReturnedEquals: value.httpErrorCodeReturnedEquals
|
|
1085
|
-
}
|
|
1086
|
-
}
|
|
1087
|
-
|
|
1088
|
-
if (value.replaceKeyWith) {
|
|
1089
|
-
routeItem.Redirect = {
|
|
1090
|
-
ReplaceKeyWith: value.replaceKeyWith
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
|
-
if (value.replaceKeyPrefixWith) {
|
|
1095
|
-
routeItem.Redirect = {
|
|
1096
|
-
ReplaceKeyPrefixWith: value.replaceKeyPrefixWith
|
|
1097
|
-
}
|
|
1098
|
-
}
|
|
1099
|
-
params.WebsiteConfiguration.RoutingRules.push(routeItem)
|
|
1100
|
-
}
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
const res = await putBucketWebsite(params)
|
|
1104
|
-
|
|
1105
|
-
return res
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
/**
|
|
1109
|
-
* 查询object列表
|
|
1110
|
-
* @param {IGetBucketOpions} options
|
|
1111
|
-
* @memberof StorageService
|
|
1112
|
-
*/
|
|
1113
|
-
@preLazy()
|
|
1114
|
-
async getBucket(options: IGetBucketOpions) {
|
|
1115
|
-
// const { bucket } = this.getStorageConfig()
|
|
1116
|
-
const { prefix, maxKeys, marker, bucket, region } = options
|
|
1117
|
-
|
|
1118
|
-
const cos = this.getCos()
|
|
1119
|
-
const getBucket = Util.promisify(cos.getBucket).bind(cos)
|
|
1120
|
-
|
|
1121
|
-
const prefixKey = this.getCloudKey(prefix)
|
|
1122
|
-
|
|
1123
|
-
const res = await getBucket({
|
|
1124
|
-
Bucket: bucket,
|
|
1125
|
-
Region: region,
|
|
1126
|
-
Prefix: prefixKey,
|
|
1127
|
-
MaxKeys: maxKeys,
|
|
1128
|
-
Marker: marker
|
|
1129
|
-
})
|
|
1130
|
-
|
|
1131
|
-
return res
|
|
1132
|
-
}
|
|
1133
|
-
|
|
1134
|
-
/**
|
|
1135
|
-
* 获取 COS 配置
|
|
1136
|
-
*/
|
|
1137
|
-
private getCos(parallel = 20) {
|
|
1138
|
-
const { secretId, secretKey, token, proxy } = this.environment.getAuthConfig()
|
|
1139
|
-
const cosProxy = process.env.TCB_COS_PROXY
|
|
1140
|
-
const useCosInternalDomain = !!process.env.USE_COS_INTERNAL_DOMAIN
|
|
1141
|
-
|
|
1142
|
-
return new COS({
|
|
1143
|
-
FileParallelLimit: parallel,
|
|
1144
|
-
SecretId: secretId,
|
|
1145
|
-
SecretKey: secretKey,
|
|
1146
|
-
Proxy: cosProxy || proxy,
|
|
1147
|
-
SecurityToken: token,
|
|
1148
|
-
...(useCosInternalDomain
|
|
1149
|
-
? {
|
|
1150
|
-
Domain: '{Bucket}.cos-internal.{Region}.tencentcos.cn',
|
|
1151
|
-
ServiceDomain: 'service.cos.tencentcos.cn',
|
|
1152
|
-
Protocol: 'http:'
|
|
1153
|
-
}
|
|
1154
|
-
: {})
|
|
1155
|
-
})
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
/**
|
|
1159
|
-
* 将 cloudPath 转换成 cloudPath/ 形式
|
|
1160
|
-
*/
|
|
1161
|
-
private getCloudKey(cloudPath: string): string {
|
|
1162
|
-
if (!cloudPath) {
|
|
1163
|
-
return ''
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
// 单个 / 转换成根目录
|
|
1167
|
-
if (cloudPath === '/') {
|
|
1168
|
-
return ''
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
return cloudPath[cloudPath.length - 1] === '/' ? cloudPath : `${cloudPath}/`
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
/**
|
|
1175
|
-
* 将 cloudPath 转换成 fileId
|
|
1176
|
-
*/
|
|
1177
|
-
private cloudPathToFileId(cloudPath: string): string {
|
|
1178
|
-
const { env, bucket } = this.getStorageConfig()
|
|
1179
|
-
return `cloud://${env}.${bucket}/${cloudPath}`
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
/**
|
|
1183
|
-
* 获取存储桶配置
|
|
1184
|
-
*/
|
|
1185
|
-
private getStorageConfig() {
|
|
1186
|
-
const envConfig = this.environment.lazyEnvironmentConfig
|
|
1187
|
-
const storageConfig = envConfig?.Storages?.[0]
|
|
1188
|
-
const { Region, Bucket } = storageConfig
|
|
1189
|
-
const region = process.env.TCB_COS_REGION || Region
|
|
1190
|
-
|
|
1191
|
-
return {
|
|
1192
|
-
region,
|
|
1193
|
-
bucket: Bucket,
|
|
1194
|
-
env: envConfig.EnvId
|
|
1195
|
-
}
|
|
1196
|
-
}
|
|
1197
|
-
/**
|
|
1198
|
-
* 带重试功能的上传多文件函数
|
|
1199
|
-
* @param uploadFiles sdk上传函数
|
|
1200
|
-
* @param options sdk上传函数参数
|
|
1201
|
-
* @param times 重试次数
|
|
1202
|
-
* @param interval 重试时间间隔(毫秒)
|
|
1203
|
-
* @param failedFiles 失败文件列表
|
|
1204
|
-
* @returns
|
|
1205
|
-
*/
|
|
1206
|
-
private async uploadFilesWithRetry({ uploadFiles, options, times, interval, failedFiles }) {
|
|
1207
|
-
const { files, onFileFinish } = options
|
|
1208
|
-
const tempFailedFiles = []
|
|
1209
|
-
let curError: any = null
|
|
1210
|
-
const res = await uploadFiles({
|
|
1211
|
-
...options,
|
|
1212
|
-
files: failedFiles.length
|
|
1213
|
-
? files.filter(file => failedFiles.includes(file.Key))
|
|
1214
|
-
: files,
|
|
1215
|
-
onFileFinish: (...args) => {
|
|
1216
|
-
const error = args[0]
|
|
1217
|
-
const fileInfo = (args as any)[2]
|
|
1218
|
-
if (error) {
|
|
1219
|
-
curError = error
|
|
1220
|
-
tempFailedFiles.push(fileInfo.Key)
|
|
1221
|
-
}
|
|
1222
|
-
onFileFinish?.apply(null, args)
|
|
1223
|
-
}
|
|
1224
|
-
})
|
|
1225
|
-
// if (!tempFailedFiles?.length || times <= 0) return res
|
|
1226
|
-
|
|
1227
|
-
if (!tempFailedFiles?.length) {
|
|
1228
|
-
return res
|
|
1229
|
-
} else {
|
|
1230
|
-
if (times > 0) {
|
|
1231
|
-
return await new Promise((resolve, reject) => {
|
|
1232
|
-
setTimeout(
|
|
1233
|
-
() =>
|
|
1234
|
-
this.uploadFilesWithRetry({
|
|
1235
|
-
uploadFiles,
|
|
1236
|
-
options,
|
|
1237
|
-
times: times - 1,
|
|
1238
|
-
interval,
|
|
1239
|
-
failedFiles: tempFailedFiles
|
|
1240
|
-
})
|
|
1241
|
-
.then(res => resolve(res))
|
|
1242
|
-
.catch(err => reject(err)),
|
|
1243
|
-
interval
|
|
1244
|
-
)
|
|
1245
|
-
})
|
|
1246
|
-
} else {
|
|
1247
|
-
if (curError) {
|
|
1248
|
-
throw curError
|
|
1249
|
-
}
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
/**
|
|
1255
|
-
* 拼接路径下载单文件
|
|
1256
|
-
* @param file
|
|
1257
|
-
* @param cloudDirectoryKey
|
|
1258
|
-
* @param resolveLocalPath
|
|
1259
|
-
* @returns
|
|
1260
|
-
*/
|
|
1261
|
-
private async downloadWithFilePath({ file, cloudDirectoryKey, resolveLocalPath }) {
|
|
1262
|
-
const fileRelativePath = file.Key.replace(cloudDirectoryKey, '')
|
|
1263
|
-
// 空路径和文件夹跳过
|
|
1264
|
-
if (!fileRelativePath || /\/$/g.test(fileRelativePath)) {
|
|
1265
|
-
return
|
|
1266
|
-
}
|
|
1267
|
-
const localFilePath = path.join(resolveLocalPath, fileRelativePath)
|
|
1268
|
-
// 创建文件的父文件夹
|
|
1269
|
-
const fileDir = path.dirname(localFilePath)
|
|
1270
|
-
await makeDir(fileDir)
|
|
1271
|
-
return this.downloadFile({
|
|
1272
|
-
cloudPath: file.Key,
|
|
1273
|
-
localPath: localFilePath
|
|
1274
|
-
})
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
/**
|
|
1278
|
-
* 根据下载结果返回错误列表
|
|
1279
|
-
* @param res
|
|
1280
|
-
* @returns
|
|
1281
|
-
*/
|
|
1282
|
-
private determineDownLoadResultIsError(res) {
|
|
1283
|
-
const resultErrorArr = []
|
|
1284
|
-
res.map(
|
|
1285
|
-
item =>
|
|
1286
|
-
/Error/gi.test(Object.prototype.toString.call(item)) && resultErrorArr.push(item)
|
|
1287
|
-
)
|
|
1288
|
-
return resultErrorArr
|
|
1289
|
-
}
|
|
1290
|
-
}
|