@nocobase/plugin-file-manager 2.1.0-beta.30 → 2.1.0-beta.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +1 -1
- package/dist/client-v2/125.01d5562df948d974.js +10 -0
- package/dist/client-v2/229.bd72c2d7aa088310.js +10 -0
- package/dist/client-v2/336.1dd1b32466d0c778.js +10 -0
- package/dist/client-v2/43.eb45d53ba3e9828b.js +10 -0
- package/dist/client-v2/450.f590b4c220108742.js +10 -0
- package/dist/client-v2/929.d7e783304cc1f236.js +10 -0
- package/dist/client-v2/942.c10c97317af6dd02.js +10 -0
- package/dist/{client/StorageOptions.d.ts → client-v2/components/BaseUrlField.d.ts} +1 -1
- package/dist/client-v2/components/DefaultField.d.ts +18 -0
- package/dist/client-v2/{storageTypes/index.d.ts → components/FileSizeField.d.ts} +2 -2
- package/dist/client-v2/components/MimetypeField.d.ts +10 -0
- package/dist/client-v2/components/NameField.d.ts +10 -0
- package/dist/client-v2/components/ParanoidField.d.ts +10 -0
- package/dist/client-v2/components/PathField.d.ts +18 -0
- package/dist/client-v2/components/RenameModeField.d.ts +10 -0
- package/dist/client-v2/components/TitleField.d.ts +10 -0
- package/dist/client-v2/components/index.d.ts +17 -0
- package/dist/client-v2/index.d.ts +5 -0
- package/dist/client-v2/index.js +1 -1
- package/dist/client-v2/plugin.d.ts +47 -6
- package/dist/client-v2/storage-forms/AliOssStorageForm.d.ts +10 -0
- package/dist/client-v2/storage-forms/LocalStorageForm.d.ts +10 -0
- package/dist/client-v2/storage-forms/S3StorageForm.d.ts +10 -0
- package/dist/client-v2/storage-forms/TxCosStorageForm.d.ts +10 -0
- package/dist/externalVersion.js +10 -13
- package/dist/node_modules/@aws-sdk/client-s3/package.json +1 -1
- package/dist/node_modules/@aws-sdk/lib-storage/package.json +1 -1
- package/dist/node_modules/ali-oss/package.json +1 -1
- package/dist/node_modules/cos-nodejs-sdk-v5/.github/workflows/auto-changelog.yml +55 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/.prettierrc +10 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/.travis.yml +16 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/LICENSE +21 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/demo/crc64.js +9 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/demo/demo-sts-scope.js +75 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/demo/demo-sts.js +65 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/demo/demo.js +4542 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/demo/util.js +135 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/index.d.ts +2610 -0
- package/dist/node_modules/{multer-cos → cos-nodejs-sdk-v5}/index.js +2 -2
- package/dist/node_modules/cos-nodejs-sdk-v5/package.json +1 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/advance.js +1659 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/async.js +59 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/base.js +4404 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/cos.js +137 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/event.js +34 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/select-stream.js +181 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/session.js +126 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/task.js +255 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/sdk/util.js +776 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/test/csp.js +1302 -0
- package/dist/node_modules/cos-nodejs-sdk-v5/test/test.js +6119 -0
- package/dist/node_modules/mime-match/package.json +1 -1
- package/dist/node_modules/mime-types/package.json +1 -1
- package/dist/node_modules/mkdirp/package.json +1 -1
- package/dist/node_modules/url-join/package.json +1 -1
- package/dist/server/storages/tx-cos.d.ts +16 -1
- package/dist/server/storages/tx-cos.js +111 -10
- package/dist/shared/previewer/filePreviewTypes.d.ts +1 -0
- package/dist/shared/previewer/filePreviewTypes.js +21 -0
- package/package.json +2 -3
- package/dist/client-v2/855.e7d2e24a0b457a89.js +0 -10
- package/dist/client-v2/storageTypes/types.d.ts +0 -26
- package/dist/node_modules/multer-cos/LICENSE +0 -24
- package/dist/node_modules/multer-cos/demo/index.js +0 -39
- package/dist/node_modules/multer-cos/demo/myMulter.js +0 -88
- package/dist/node_modules/multer-cos/package.json +0 -1
|
@@ -0,0 +1,1659 @@
|
|
|
1
|
+
var session = require('./session');
|
|
2
|
+
var fs = require('fs');
|
|
3
|
+
var Async = require('./async');
|
|
4
|
+
var EventProxy = require('./event').EventProxy;
|
|
5
|
+
var util = require('./util');
|
|
6
|
+
|
|
7
|
+
// 文件分块上传全过程,暴露的分块上传接口
|
|
8
|
+
function sliceUploadFile(params, callback) {
|
|
9
|
+
var self = this;
|
|
10
|
+
var ep = new EventProxy();
|
|
11
|
+
var TaskId = params.TaskId;
|
|
12
|
+
var Bucket = params.Bucket;
|
|
13
|
+
var Region = params.Region;
|
|
14
|
+
var Key = params.Key;
|
|
15
|
+
var FilePath = params.FilePath;
|
|
16
|
+
var ChunkSize = params.ChunkSize || params.SliceSize || self.options.ChunkSize;
|
|
17
|
+
var AsyncLimit = params.AsyncLimit;
|
|
18
|
+
var StorageClass = params.StorageClass;
|
|
19
|
+
var ServerSideEncryption = params.ServerSideEncryption;
|
|
20
|
+
var FileSize;
|
|
21
|
+
|
|
22
|
+
var onProgress;
|
|
23
|
+
var onHashProgress = params.onHashProgress;
|
|
24
|
+
|
|
25
|
+
// 上传过程中出现错误,返回错误
|
|
26
|
+
ep.on('error', function (err) {
|
|
27
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
28
|
+
var _err = {
|
|
29
|
+
UploadId: params.UploadData.UploadId || '',
|
|
30
|
+
err: err,
|
|
31
|
+
};
|
|
32
|
+
return callback(_err);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// 上传分块完成,开始 uploadSliceComplete 操作
|
|
36
|
+
ep.on('upload_complete', function (UploadCompleteData) {
|
|
37
|
+
var _UploadCompleteData = util.extend(
|
|
38
|
+
{
|
|
39
|
+
UploadId: params.UploadData.UploadId || '',
|
|
40
|
+
},
|
|
41
|
+
UploadCompleteData
|
|
42
|
+
);
|
|
43
|
+
callback(null, _UploadCompleteData);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// 上传分块完成,开始 uploadSliceComplete 操作
|
|
47
|
+
ep.on('upload_slice_complete', function (UploadData) {
|
|
48
|
+
var metaHeaders = {};
|
|
49
|
+
util.each(params.Headers, function (val, k) {
|
|
50
|
+
var shortKey = k.toLowerCase();
|
|
51
|
+
if (shortKey.indexOf('x-cos-meta-') === 0 || shortKey === 'pic-operations') metaHeaders[k] = val;
|
|
52
|
+
});
|
|
53
|
+
uploadSliceComplete.call(
|
|
54
|
+
self,
|
|
55
|
+
{
|
|
56
|
+
Bucket: Bucket,
|
|
57
|
+
Region: Region,
|
|
58
|
+
Key: Key,
|
|
59
|
+
UploadId: UploadData.UploadId,
|
|
60
|
+
SliceList: UploadData.SliceList,
|
|
61
|
+
Headers: metaHeaders,
|
|
62
|
+
},
|
|
63
|
+
function (err, data) {
|
|
64
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
65
|
+
session.removeUsing(UploadData.UploadId);
|
|
66
|
+
if (err) {
|
|
67
|
+
onProgress(null, true);
|
|
68
|
+
return ep.emit('error', err);
|
|
69
|
+
}
|
|
70
|
+
session.removeUploadId.call(self, UploadData.UploadId);
|
|
71
|
+
onProgress({ loaded: FileSize, total: FileSize }, true);
|
|
72
|
+
ep.emit('upload_complete', data);
|
|
73
|
+
}
|
|
74
|
+
);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
// 获取 UploadId 完成,开始上传每个分片
|
|
78
|
+
ep.on('get_upload_data_finish', function (UploadData) {
|
|
79
|
+
// 处理 UploadId 缓存
|
|
80
|
+
var uuid = session.getFileId(params.FileStat, params.ChunkSize, Bucket, Key);
|
|
81
|
+
uuid && session.saveUploadId.call(self, uuid, UploadData.UploadId, self.options.UploadIdCacheLimit); // 缓存 UploadId
|
|
82
|
+
session.setUsing(UploadData.UploadId); // 标记 UploadId 为正在使用
|
|
83
|
+
|
|
84
|
+
// 获取 UploadId
|
|
85
|
+
onProgress(null, true); // 任务状态开始 uploading
|
|
86
|
+
uploadSliceList.call(
|
|
87
|
+
self,
|
|
88
|
+
{
|
|
89
|
+
TaskId: TaskId,
|
|
90
|
+
Bucket: Bucket,
|
|
91
|
+
Region: Region,
|
|
92
|
+
Key: Key,
|
|
93
|
+
FilePath: FilePath,
|
|
94
|
+
FileSize: FileSize,
|
|
95
|
+
SliceSize: ChunkSize,
|
|
96
|
+
AsyncLimit: AsyncLimit,
|
|
97
|
+
ServerSideEncryption: ServerSideEncryption,
|
|
98
|
+
UploadData: UploadData,
|
|
99
|
+
Headers: params.Headers,
|
|
100
|
+
onProgress: onProgress,
|
|
101
|
+
},
|
|
102
|
+
function (err, data) {
|
|
103
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
104
|
+
if (err) {
|
|
105
|
+
onProgress(null, true);
|
|
106
|
+
return ep.emit('error', err);
|
|
107
|
+
}
|
|
108
|
+
ep.emit('upload_slice_complete', data);
|
|
109
|
+
}
|
|
110
|
+
);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
// 开始获取文件 UploadId,里面会视情况计算 ETag,并比对,保证文件一致性,也优化上传
|
|
114
|
+
ep.on('get_file_size_finish', function () {
|
|
115
|
+
onProgress = util.throttleOnProgress.call(self, FileSize, params.onProgress);
|
|
116
|
+
|
|
117
|
+
if (params.UploadData.UploadId) {
|
|
118
|
+
ep.emit('get_upload_data_finish', params.UploadData);
|
|
119
|
+
} else {
|
|
120
|
+
var _params = util.extend(
|
|
121
|
+
{
|
|
122
|
+
TaskId: TaskId,
|
|
123
|
+
Bucket: Bucket,
|
|
124
|
+
Region: Region,
|
|
125
|
+
Key: Key,
|
|
126
|
+
Headers: params.Headers,
|
|
127
|
+
StorageClass: StorageClass,
|
|
128
|
+
FilePath: FilePath,
|
|
129
|
+
FileSize: FileSize,
|
|
130
|
+
SliceSize: ChunkSize,
|
|
131
|
+
onHashProgress: onHashProgress,
|
|
132
|
+
},
|
|
133
|
+
params
|
|
134
|
+
);
|
|
135
|
+
getUploadIdAndPartList.call(self, _params, function (err, UploadData) {
|
|
136
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
137
|
+
if (err) return ep.emit('error', err);
|
|
138
|
+
params.UploadData.UploadId = UploadData.UploadId;
|
|
139
|
+
params.UploadData.PartList = UploadData.PartList;
|
|
140
|
+
ep.emit('get_upload_data_finish', params.UploadData);
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
// 获取上传文件大小
|
|
146
|
+
FileSize = params.ContentLength;
|
|
147
|
+
delete params.ContentLength;
|
|
148
|
+
!params.Headers && (params.Headers = {});
|
|
149
|
+
util.each(params.Headers, function (item, key) {
|
|
150
|
+
if (key.toLowerCase() === 'content-length') {
|
|
151
|
+
delete params.Headers[key];
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
// 控制分片大小
|
|
156
|
+
(function () {
|
|
157
|
+
var SIZE = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1024 * 2, 1024 * 4, 1024 * 5];
|
|
158
|
+
var AutoChunkSize = 1024 * 1024;
|
|
159
|
+
for (var i = 0; i < SIZE.length; i++) {
|
|
160
|
+
AutoChunkSize = SIZE[i] * 1024 * 1024;
|
|
161
|
+
if (FileSize / AutoChunkSize <= self.options.MaxPartNumber) break;
|
|
162
|
+
}
|
|
163
|
+
params.ChunkSize = params.SliceSize = ChunkSize = Math.max(ChunkSize, AutoChunkSize);
|
|
164
|
+
})();
|
|
165
|
+
|
|
166
|
+
// 开始上传
|
|
167
|
+
if (FileSize === 0) {
|
|
168
|
+
params.Body = '';
|
|
169
|
+
params.ContentLength = 0;
|
|
170
|
+
params.SkipTask = true;
|
|
171
|
+
self.putObject(params, callback);
|
|
172
|
+
} else {
|
|
173
|
+
ep.emit('get_file_size_finish');
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// 获取上传任务的 UploadId
|
|
178
|
+
function getUploadIdAndPartList(params, callback) {
|
|
179
|
+
var TaskId = params.TaskId;
|
|
180
|
+
var Bucket = params.Bucket;
|
|
181
|
+
var Region = params.Region;
|
|
182
|
+
var Key = params.Key;
|
|
183
|
+
var StorageClass = params.StorageClass;
|
|
184
|
+
var self = this;
|
|
185
|
+
|
|
186
|
+
// 计算 ETag
|
|
187
|
+
var ETagMap = {};
|
|
188
|
+
var FileSize = params.FileSize;
|
|
189
|
+
var SliceSize = params.SliceSize;
|
|
190
|
+
var SliceCount = Math.ceil(FileSize / SliceSize);
|
|
191
|
+
var FinishSliceCount = 0;
|
|
192
|
+
var FinishSize = 0;
|
|
193
|
+
var onHashProgress = util.throttleOnProgress.call(self, FileSize, params.onHashProgress);
|
|
194
|
+
var getChunkETag = function (PartNumber, callback) {
|
|
195
|
+
var start = SliceSize * (PartNumber - 1);
|
|
196
|
+
var end = Math.min(start + SliceSize, FileSize);
|
|
197
|
+
var ChunkSize = end - start;
|
|
198
|
+
|
|
199
|
+
if (ETagMap[PartNumber]) {
|
|
200
|
+
callback(null, {
|
|
201
|
+
PartNumber: PartNumber,
|
|
202
|
+
ETag: ETagMap[PartNumber],
|
|
203
|
+
Size: ChunkSize,
|
|
204
|
+
});
|
|
205
|
+
} else {
|
|
206
|
+
util.fileSlice(params.FilePath, start, end, function (chunkItem) {
|
|
207
|
+
util.getFileMd5(chunkItem, function (err, md5) {
|
|
208
|
+
if (err) return callback(util.error(err));
|
|
209
|
+
var ETag = '"' + md5 + '"';
|
|
210
|
+
ETagMap[PartNumber] = ETag;
|
|
211
|
+
FinishSliceCount += 1;
|
|
212
|
+
FinishSize += ChunkSize;
|
|
213
|
+
onHashProgress({ loaded: FinishSize, total: FileSize });
|
|
214
|
+
callback(null, {
|
|
215
|
+
PartNumber: PartNumber,
|
|
216
|
+
ETag: ETag,
|
|
217
|
+
Size: ChunkSize,
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
// 通过和文件的 md5 对比,判断 UploadId 是否可用
|
|
225
|
+
var isAvailableUploadList = function (PartList, callback) {
|
|
226
|
+
var PartCount = PartList.length;
|
|
227
|
+
// 如果没有分片,通过
|
|
228
|
+
if (PartCount === 0) {
|
|
229
|
+
return callback(null, true);
|
|
230
|
+
}
|
|
231
|
+
// 检查分片数量
|
|
232
|
+
if (PartCount > SliceCount) {
|
|
233
|
+
return callback(null, false);
|
|
234
|
+
}
|
|
235
|
+
// 检查分片大小
|
|
236
|
+
if (PartCount > 1) {
|
|
237
|
+
var PartSliceSize = Math.max(PartList[0].Size, PartList[1].Size);
|
|
238
|
+
if (PartSliceSize !== SliceSize) {
|
|
239
|
+
return callback(null, false);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
// 逐个分片计算并检查 ETag 是否一致
|
|
243
|
+
var next = function (index) {
|
|
244
|
+
if (index < PartCount) {
|
|
245
|
+
var Part = PartList[index];
|
|
246
|
+
getChunkETag(Part.PartNumber, function (err, chunk) {
|
|
247
|
+
if (chunk && chunk.ETag === Part.ETag && chunk.Size === Part.Size) {
|
|
248
|
+
next(index + 1);
|
|
249
|
+
} else {
|
|
250
|
+
callback(null, false);
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
} else {
|
|
254
|
+
callback(null, true);
|
|
255
|
+
}
|
|
256
|
+
};
|
|
257
|
+
next(0);
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
var ep = new EventProxy();
|
|
261
|
+
ep.on('error', function (errData) {
|
|
262
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
263
|
+
return callback(errData);
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
// 存在 UploadId
|
|
267
|
+
ep.on('upload_id_available', function (UploadData) {
|
|
268
|
+
// 转换成 map
|
|
269
|
+
var map = {};
|
|
270
|
+
var list = [];
|
|
271
|
+
util.each(UploadData.PartList, function (item) {
|
|
272
|
+
map[item.PartNumber] = item;
|
|
273
|
+
});
|
|
274
|
+
for (var PartNumber = 1; PartNumber <= SliceCount; PartNumber++) {
|
|
275
|
+
var item = map[PartNumber];
|
|
276
|
+
if (item) {
|
|
277
|
+
item.PartNumber = PartNumber;
|
|
278
|
+
item.Uploaded = true;
|
|
279
|
+
} else {
|
|
280
|
+
item = {
|
|
281
|
+
PartNumber: PartNumber,
|
|
282
|
+
ETag: null,
|
|
283
|
+
Uploaded: false,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
list.push(item);
|
|
287
|
+
}
|
|
288
|
+
UploadData.PartList = list;
|
|
289
|
+
callback(null, UploadData);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
// 不存在 UploadId, 初始化生成 UploadId
|
|
293
|
+
ep.on('no_available_upload_id', function () {
|
|
294
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
295
|
+
var _params = util.extend(
|
|
296
|
+
{
|
|
297
|
+
Bucket: Bucket,
|
|
298
|
+
Region: Region,
|
|
299
|
+
Key: Key,
|
|
300
|
+
Headers: util.clone(params.Headers),
|
|
301
|
+
Query: util.clone(params.Query),
|
|
302
|
+
StorageClass: StorageClass,
|
|
303
|
+
},
|
|
304
|
+
params
|
|
305
|
+
);
|
|
306
|
+
self.multipartInit(_params, function (err, data) {
|
|
307
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
308
|
+
if (err) return ep.emit('error', err);
|
|
309
|
+
var UploadId = data.UploadId;
|
|
310
|
+
if (!UploadId) {
|
|
311
|
+
return callback(util.error(new Error('no such upload id')));
|
|
312
|
+
}
|
|
313
|
+
ep.emit('upload_id_available', { UploadId: UploadId, PartList: [] });
|
|
314
|
+
});
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
// 如果已存在 UploadId,找一个可以用的 UploadId
|
|
318
|
+
ep.on('has_and_check_upload_id', function (UploadIdList) {
|
|
319
|
+
// 串行地,找一个内容一致的 UploadId
|
|
320
|
+
UploadIdList = UploadIdList.reverse();
|
|
321
|
+
Async.eachLimit(
|
|
322
|
+
UploadIdList,
|
|
323
|
+
1,
|
|
324
|
+
function (UploadId, asyncCallback) {
|
|
325
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
326
|
+
// 如果正在上传,跳过
|
|
327
|
+
if (session.using[UploadId]) {
|
|
328
|
+
asyncCallback(); // 检查下一个 UploadId
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
// 判断 UploadId 是否可用
|
|
332
|
+
wholeMultipartListPart.call(
|
|
333
|
+
self,
|
|
334
|
+
{
|
|
335
|
+
Bucket: Bucket,
|
|
336
|
+
Region: Region,
|
|
337
|
+
Key: Key,
|
|
338
|
+
UploadId: UploadId,
|
|
339
|
+
},
|
|
340
|
+
function (err, PartListData) {
|
|
341
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
342
|
+
if (err) {
|
|
343
|
+
session.removeUsing(UploadId);
|
|
344
|
+
return ep.emit('error', err);
|
|
345
|
+
}
|
|
346
|
+
var PartList = PartListData.PartList;
|
|
347
|
+
PartList.forEach(function (item) {
|
|
348
|
+
item.PartNumber *= 1;
|
|
349
|
+
item.Size *= 1;
|
|
350
|
+
item.ETag = item.ETag || '';
|
|
351
|
+
});
|
|
352
|
+
isAvailableUploadList(PartList, function (err, isAvailable) {
|
|
353
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
354
|
+
if (err) return ep.emit('error', err);
|
|
355
|
+
if (isAvailable) {
|
|
356
|
+
asyncCallback({
|
|
357
|
+
UploadId: UploadId,
|
|
358
|
+
PartList: PartList,
|
|
359
|
+
}); // 马上结束
|
|
360
|
+
} else {
|
|
361
|
+
asyncCallback(); // 检查下一个 UploadId
|
|
362
|
+
}
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
);
|
|
366
|
+
},
|
|
367
|
+
function (AvailableUploadData) {
|
|
368
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
369
|
+
onHashProgress(null, true);
|
|
370
|
+
if (AvailableUploadData && AvailableUploadData.UploadId) {
|
|
371
|
+
ep.emit('upload_id_available', AvailableUploadData);
|
|
372
|
+
} else {
|
|
373
|
+
ep.emit('no_available_upload_id');
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
);
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
// 在本地缓存找可用的 UploadId
|
|
380
|
+
ep.on('seek_local_avail_upload_id', function (RemoteUploadIdList) {
|
|
381
|
+
// 在本地找可用的 UploadId
|
|
382
|
+
var uuid = session.getFileId(params.FileStat, params.ChunkSize, Bucket, Key);
|
|
383
|
+
var LocalUploadIdList = session.getUploadIdList.call(self, uuid);
|
|
384
|
+
if (!uuid || !LocalUploadIdList) {
|
|
385
|
+
ep.emit('has_and_check_upload_id', RemoteUploadIdList);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
var next = function (index) {
|
|
389
|
+
// 如果本地找不到可用 UploadId,再一个个遍历校验远端
|
|
390
|
+
if (index >= LocalUploadIdList.length) {
|
|
391
|
+
ep.emit('has_and_check_upload_id', RemoteUploadIdList);
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
var UploadId = LocalUploadIdList[index];
|
|
395
|
+
// 如果不在远端 UploadId 列表里,跳过并删除
|
|
396
|
+
if (!util.isInArray(RemoteUploadIdList, UploadId)) {
|
|
397
|
+
session.removeUploadId.call(self, UploadId);
|
|
398
|
+
next(index + 1);
|
|
399
|
+
return;
|
|
400
|
+
}
|
|
401
|
+
// 如果正在上传,跳过
|
|
402
|
+
if (session.using[UploadId]) {
|
|
403
|
+
next(index + 1);
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
// 判断 UploadId 是否存在线上
|
|
407
|
+
wholeMultipartListPart.call(
|
|
408
|
+
self,
|
|
409
|
+
{
|
|
410
|
+
Bucket: Bucket,
|
|
411
|
+
Region: Region,
|
|
412
|
+
Key: Key,
|
|
413
|
+
UploadId: UploadId,
|
|
414
|
+
},
|
|
415
|
+
function (err, PartListData) {
|
|
416
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
417
|
+
if (err) {
|
|
418
|
+
// 如果 UploadId 获取会出错,跳过并删除
|
|
419
|
+
session.removeUploadId.call(self, UploadId);
|
|
420
|
+
next(index + 1);
|
|
421
|
+
} else {
|
|
422
|
+
// 找到可用 UploadId
|
|
423
|
+
ep.emit('upload_id_available', {
|
|
424
|
+
UploadId: UploadId,
|
|
425
|
+
PartList: PartListData.PartList,
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
);
|
|
430
|
+
};
|
|
431
|
+
next(0);
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
// 获取线上 UploadId 列表
|
|
435
|
+
ep.on('get_remote_upload_id_list', function () {
|
|
436
|
+
// 获取符合条件的 UploadId 列表,因为同一个文件可以有多个上传任务。
|
|
437
|
+
wholeMultipartList.call(
|
|
438
|
+
self,
|
|
439
|
+
{
|
|
440
|
+
Bucket: Bucket,
|
|
441
|
+
Region: Region,
|
|
442
|
+
Key: Key,
|
|
443
|
+
},
|
|
444
|
+
function (err, data) {
|
|
445
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
446
|
+
if (err) return ep.emit('error', err);
|
|
447
|
+
// 整理远端 UploadId 列表
|
|
448
|
+
var RemoteUploadIdList = util
|
|
449
|
+
.filter(data.UploadList, function (item) {
|
|
450
|
+
return (
|
|
451
|
+
item.Key === Key && (!StorageClass || item.StorageClass.toUpperCase() === StorageClass.toUpperCase())
|
|
452
|
+
);
|
|
453
|
+
})
|
|
454
|
+
.reverse()
|
|
455
|
+
.map(function (item) {
|
|
456
|
+
return item.UploadId || item.UploadID;
|
|
457
|
+
});
|
|
458
|
+
if (RemoteUploadIdList.length) {
|
|
459
|
+
ep.emit('seek_local_avail_upload_id', RemoteUploadIdList);
|
|
460
|
+
} else {
|
|
461
|
+
// 远端没有 UploadId,清理缓存的 UploadId
|
|
462
|
+
var uuid = session.getFileId(params.FileStat, params.ChunkSize, Bucket, Key),
|
|
463
|
+
LocalUploadIdList;
|
|
464
|
+
if (uuid && (LocalUploadIdList = session.getUploadIdList.call(self, uuid))) {
|
|
465
|
+
util.each(LocalUploadIdList, function (UploadId) {
|
|
466
|
+
session.removeUploadId.call(self, UploadId);
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
ep.emit('no_available_upload_id');
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
);
|
|
473
|
+
});
|
|
474
|
+
|
|
475
|
+
// 开始找可用 UploadId
|
|
476
|
+
ep.emit('get_remote_upload_id_list');
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// 获取符合条件的全部上传任务 (条件包括 Bucket, Region, Prefix)
|
|
480
|
+
function wholeMultipartList(params, callback) {
|
|
481
|
+
var self = this;
|
|
482
|
+
var UploadList = [];
|
|
483
|
+
var sendParams = {
|
|
484
|
+
Bucket: params.Bucket,
|
|
485
|
+
Region: params.Region,
|
|
486
|
+
Prefix: params.Key,
|
|
487
|
+
};
|
|
488
|
+
var next = function () {
|
|
489
|
+
self.multipartList(sendParams, function (err, data) {
|
|
490
|
+
if (err) return callback(err);
|
|
491
|
+
UploadList.push.apply(UploadList, data.Upload || []);
|
|
492
|
+
if (data.IsTruncated === 'true') {
|
|
493
|
+
// 列表不完整
|
|
494
|
+
sendParams.KeyMarker = data.NextKeyMarker;
|
|
495
|
+
sendParams.UploadIdMarker = data.NextUploadIdMarker;
|
|
496
|
+
next();
|
|
497
|
+
} else {
|
|
498
|
+
callback(null, { UploadList: UploadList });
|
|
499
|
+
}
|
|
500
|
+
});
|
|
501
|
+
};
|
|
502
|
+
next();
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// 获取指定上传任务的分块列表
|
|
506
|
+
function wholeMultipartListPart(params, callback) {
|
|
507
|
+
var self = this;
|
|
508
|
+
var PartList = [];
|
|
509
|
+
var sendParams = {
|
|
510
|
+
Bucket: params.Bucket,
|
|
511
|
+
Region: params.Region,
|
|
512
|
+
Key: params.Key,
|
|
513
|
+
UploadId: params.UploadId,
|
|
514
|
+
};
|
|
515
|
+
var next = function () {
|
|
516
|
+
self.multipartListPart(sendParams, function (err, data) {
|
|
517
|
+
if (err) return callback(err);
|
|
518
|
+
PartList.push.apply(PartList, data.Part || []);
|
|
519
|
+
if (data.IsTruncated === 'true') {
|
|
520
|
+
// 列表不完整
|
|
521
|
+
sendParams.PartNumberMarker = data.NextPartNumberMarker;
|
|
522
|
+
next();
|
|
523
|
+
} else {
|
|
524
|
+
callback(null, { PartList: PartList });
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
};
|
|
528
|
+
next();
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// 上传文件分块,包括
|
|
532
|
+
/*
|
|
533
|
+
UploadId (上传任务编号)
|
|
534
|
+
AsyncLimit (并发量),
|
|
535
|
+
SliceList (上传的分块数组),
|
|
536
|
+
FilePath (本地文件的位置),
|
|
537
|
+
SliceSize (文件分块大小)
|
|
538
|
+
FileSize (文件大小)
|
|
539
|
+
onProgress (上传成功之后的回调函数)
|
|
540
|
+
*/
|
|
541
|
+
function uploadSliceList(params, cb) {
|
|
542
|
+
var self = this;
|
|
543
|
+
var TaskId = params.TaskId;
|
|
544
|
+
var Bucket = params.Bucket;
|
|
545
|
+
var Region = params.Region;
|
|
546
|
+
var Key = params.Key;
|
|
547
|
+
var UploadData = params.UploadData;
|
|
548
|
+
var FileSize = params.FileSize;
|
|
549
|
+
var SliceSize = params.SliceSize;
|
|
550
|
+
var ChunkParallel = Math.min(params.AsyncLimit || self.options.ChunkParallelLimit || 1, 256);
|
|
551
|
+
var FilePath = params.FilePath;
|
|
552
|
+
var SliceCount = Math.ceil(FileSize / SliceSize);
|
|
553
|
+
var FinishSize = 0;
|
|
554
|
+
var ServerSideEncryption = params.ServerSideEncryption;
|
|
555
|
+
var needUploadSlices = util.filter(UploadData.PartList, function (SliceItem) {
|
|
556
|
+
if (SliceItem['Uploaded']) {
|
|
557
|
+
FinishSize += SliceItem['PartNumber'] >= SliceCount ? FileSize % SliceSize || SliceSize : SliceSize;
|
|
558
|
+
}
|
|
559
|
+
return !SliceItem['Uploaded'];
|
|
560
|
+
});
|
|
561
|
+
var onProgress = params.onProgress;
|
|
562
|
+
|
|
563
|
+
Async.eachLimit(
|
|
564
|
+
needUploadSlices,
|
|
565
|
+
ChunkParallel,
|
|
566
|
+
function (SliceItem, asyncCallback) {
|
|
567
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
568
|
+
var PartNumber = SliceItem['PartNumber'];
|
|
569
|
+
var currentSize =
|
|
570
|
+
Math.min(FileSize, SliceItem['PartNumber'] * SliceSize) - (SliceItem['PartNumber'] - 1) * SliceSize;
|
|
571
|
+
var preAddSize = 0;
|
|
572
|
+
uploadSliceItem.call(
|
|
573
|
+
self,
|
|
574
|
+
{
|
|
575
|
+
TaskId: TaskId,
|
|
576
|
+
Bucket: Bucket,
|
|
577
|
+
Region: Region,
|
|
578
|
+
Key: Key,
|
|
579
|
+
SliceSize: SliceSize,
|
|
580
|
+
FileSize: FileSize,
|
|
581
|
+
PartNumber: PartNumber,
|
|
582
|
+
ServerSideEncryption: ServerSideEncryption,
|
|
583
|
+
FilePath: FilePath,
|
|
584
|
+
UploadData: UploadData,
|
|
585
|
+
Headers: params.Headers,
|
|
586
|
+
onProgress: function (data) {
|
|
587
|
+
FinishSize += data.loaded - preAddSize;
|
|
588
|
+
preAddSize = data.loaded;
|
|
589
|
+
onProgress({ loaded: FinishSize, total: FileSize });
|
|
590
|
+
},
|
|
591
|
+
},
|
|
592
|
+
function (err, data) {
|
|
593
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
594
|
+
if (err) {
|
|
595
|
+
FinishSize -= preAddSize;
|
|
596
|
+
} else {
|
|
597
|
+
FinishSize += currentSize - preAddSize;
|
|
598
|
+
SliceItem.ETag = data.ETag;
|
|
599
|
+
}
|
|
600
|
+
onProgress({ loaded: FinishSize, total: FileSize });
|
|
601
|
+
asyncCallback(err || null, data);
|
|
602
|
+
}
|
|
603
|
+
);
|
|
604
|
+
},
|
|
605
|
+
function (err) {
|
|
606
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
607
|
+
if (err) return cb(err);
|
|
608
|
+
cb(null, {
|
|
609
|
+
UploadId: UploadData.UploadId,
|
|
610
|
+
SliceList: UploadData.PartList,
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// 上传指定分片
|
|
617
|
+
function uploadSliceItem(params, callback) {
|
|
618
|
+
var self = this;
|
|
619
|
+
var TaskId = params.TaskId;
|
|
620
|
+
var Bucket = params.Bucket;
|
|
621
|
+
var Region = params.Region;
|
|
622
|
+
var Key = params.Key;
|
|
623
|
+
var FileSize = params.FileSize;
|
|
624
|
+
var FilePath = params.FilePath;
|
|
625
|
+
var PartNumber = params.PartNumber * 1;
|
|
626
|
+
var SliceSize = params.SliceSize;
|
|
627
|
+
var ServerSideEncryption = params.ServerSideEncryption;
|
|
628
|
+
var UploadData = params.UploadData;
|
|
629
|
+
var ChunkRetryTimes = self.options.ChunkRetryTimes + 1;
|
|
630
|
+
var Headers = params.Headers || {};
|
|
631
|
+
|
|
632
|
+
var start = SliceSize * (PartNumber - 1);
|
|
633
|
+
|
|
634
|
+
var ContentLength = SliceSize;
|
|
635
|
+
|
|
636
|
+
var end = start + SliceSize;
|
|
637
|
+
|
|
638
|
+
if (end > FileSize) {
|
|
639
|
+
end = FileSize;
|
|
640
|
+
ContentLength = end - start;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
var headersWhiteList = ['x-cos-traffic-limit', 'x-cos-mime-limit'];
|
|
644
|
+
var headers = {};
|
|
645
|
+
util.each(Headers, function (v, k) {
|
|
646
|
+
if (headersWhiteList.indexOf(k) > -1) {
|
|
647
|
+
headers[k] = v;
|
|
648
|
+
}
|
|
649
|
+
});
|
|
650
|
+
|
|
651
|
+
util.fileSlice(FilePath, start, end, function (md5Body) {
|
|
652
|
+
util.getFileMd5(md5Body, function (err, md5) {
|
|
653
|
+
var contentMd5 = md5 ? util.binaryBase64(md5) : '';
|
|
654
|
+
var PartItem = UploadData.PartList[PartNumber - 1];
|
|
655
|
+
Async.retry(
|
|
656
|
+
ChunkRetryTimes,
|
|
657
|
+
function (tryCallback) {
|
|
658
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
659
|
+
util.fileSlice(FilePath, start, end, function (Body) {
|
|
660
|
+
self.multipartUpload(
|
|
661
|
+
{
|
|
662
|
+
TaskId: TaskId,
|
|
663
|
+
Bucket: Bucket,
|
|
664
|
+
Region: Region,
|
|
665
|
+
Key: Key,
|
|
666
|
+
ContentLength: ContentLength,
|
|
667
|
+
PartNumber: PartNumber,
|
|
668
|
+
UploadId: UploadData.UploadId,
|
|
669
|
+
ServerSideEncryption: ServerSideEncryption,
|
|
670
|
+
Body: Body,
|
|
671
|
+
Headers: headers,
|
|
672
|
+
onProgress: params.onProgress,
|
|
673
|
+
ContentMD5: contentMd5,
|
|
674
|
+
},
|
|
675
|
+
function (err, data) {
|
|
676
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
677
|
+
if (err) return tryCallback(err);
|
|
678
|
+
PartItem.Uploaded = true;
|
|
679
|
+
return tryCallback(null, data);
|
|
680
|
+
}
|
|
681
|
+
);
|
|
682
|
+
});
|
|
683
|
+
},
|
|
684
|
+
function (err, data) {
|
|
685
|
+
if (!self._isRunningTask(TaskId)) return;
|
|
686
|
+
return callback(err, data);
|
|
687
|
+
}
|
|
688
|
+
);
|
|
689
|
+
});
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
// 完成分块上传
|
|
694
|
+
function uploadSliceComplete(params, callback) {
|
|
695
|
+
var Bucket = params.Bucket;
|
|
696
|
+
var Region = params.Region;
|
|
697
|
+
var Key = params.Key;
|
|
698
|
+
var UploadId = params.UploadId;
|
|
699
|
+
var SliceList = params.SliceList;
|
|
700
|
+
var self = this;
|
|
701
|
+
var ChunkRetryTimes = this.options.ChunkRetryTimes + 1;
|
|
702
|
+
var Headers = params.Headers;
|
|
703
|
+
var Parts = SliceList.map(function (item) {
|
|
704
|
+
return {
|
|
705
|
+
PartNumber: item.PartNumber,
|
|
706
|
+
ETag: item.ETag,
|
|
707
|
+
};
|
|
708
|
+
});
|
|
709
|
+
// 完成上传的请求也做重试
|
|
710
|
+
Async.retry(
|
|
711
|
+
ChunkRetryTimes,
|
|
712
|
+
function (tryCallback) {
|
|
713
|
+
self.multipartComplete(
|
|
714
|
+
{
|
|
715
|
+
Bucket: Bucket,
|
|
716
|
+
Region: Region,
|
|
717
|
+
Key: Key,
|
|
718
|
+
UploadId: UploadId,
|
|
719
|
+
Parts: Parts,
|
|
720
|
+
Headers: Headers,
|
|
721
|
+
},
|
|
722
|
+
tryCallback
|
|
723
|
+
);
|
|
724
|
+
},
|
|
725
|
+
function (err, data) {
|
|
726
|
+
callback(err, data);
|
|
727
|
+
}
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
// 抛弃分块上传任务
|
|
732
|
+
/*
|
|
733
|
+
AsyncLimit (抛弃上传任务的并发量),
|
|
734
|
+
UploadId (上传任务的编号,当 Level 为 task 时候需要)
|
|
735
|
+
Level (抛弃分块上传任务的级别,task : 抛弃指定的上传任务,file : 抛弃指定的文件对应的上传任务,其他值 :抛弃指定Bucket 的全部上传任务)
|
|
736
|
+
*/
|
|
737
|
+
function abortUploadTask(params, callback) {
|
|
738
|
+
var Bucket = params.Bucket;
|
|
739
|
+
var Region = params.Region;
|
|
740
|
+
var Key = params.Key;
|
|
741
|
+
var UploadId = params.UploadId;
|
|
742
|
+
var Level = params.Level || 'task';
|
|
743
|
+
var AsyncLimit = params.AsyncLimit;
|
|
744
|
+
var self = this;
|
|
745
|
+
|
|
746
|
+
var ep = new EventProxy();
|
|
747
|
+
|
|
748
|
+
ep.on('error', function (errData) {
|
|
749
|
+
return callback(errData);
|
|
750
|
+
});
|
|
751
|
+
|
|
752
|
+
// 已经获取到需要抛弃的任务列表
|
|
753
|
+
ep.on('get_abort_array', function (AbortArray) {
|
|
754
|
+
abortUploadTaskArray.call(
|
|
755
|
+
self,
|
|
756
|
+
{
|
|
757
|
+
Bucket: Bucket,
|
|
758
|
+
Region: Region,
|
|
759
|
+
Key: Key,
|
|
760
|
+
Headers: params.Headers,
|
|
761
|
+
AsyncLimit: AsyncLimit,
|
|
762
|
+
AbortArray: AbortArray,
|
|
763
|
+
},
|
|
764
|
+
callback
|
|
765
|
+
);
|
|
766
|
+
});
|
|
767
|
+
|
|
768
|
+
if (Level === 'bucket') {
|
|
769
|
+
// Bucket 级别的任务抛弃,抛弃该 Bucket 下的全部上传任务
|
|
770
|
+
wholeMultipartList.call(
|
|
771
|
+
self,
|
|
772
|
+
{
|
|
773
|
+
Bucket: Bucket,
|
|
774
|
+
Region: Region,
|
|
775
|
+
},
|
|
776
|
+
function (err, data) {
|
|
777
|
+
if (err) return callback(err);
|
|
778
|
+
ep.emit('get_abort_array', data.UploadList || []);
|
|
779
|
+
}
|
|
780
|
+
);
|
|
781
|
+
} else if (Level === 'file') {
|
|
782
|
+
// 文件级别的任务抛弃,抛弃该文件的全部上传任务
|
|
783
|
+
if (!Key) return callback(util.error(new Error('abort_upload_task_no_key')));
|
|
784
|
+
wholeMultipartList.call(
|
|
785
|
+
self,
|
|
786
|
+
{
|
|
787
|
+
Bucket: Bucket,
|
|
788
|
+
Region: Region,
|
|
789
|
+
Key: Key,
|
|
790
|
+
},
|
|
791
|
+
function (err, data) {
|
|
792
|
+
if (err) return callback(err);
|
|
793
|
+
ep.emit('get_abort_array', data.UploadList || []);
|
|
794
|
+
}
|
|
795
|
+
);
|
|
796
|
+
} else if (Level === 'task') {
|
|
797
|
+
// 单个任务级别的任务抛弃,抛弃指定 UploadId 的上传任务
|
|
798
|
+
if (!UploadId) return callback(util.error(new Error('abort_upload_task_no_id')));
|
|
799
|
+
if (!Key) return callback(util.error(new Error('abort_upload_task_no_key')));
|
|
800
|
+
ep.emit('get_abort_array', [
|
|
801
|
+
{
|
|
802
|
+
Key: Key,
|
|
803
|
+
UploadId: UploadId,
|
|
804
|
+
},
|
|
805
|
+
]);
|
|
806
|
+
} else {
|
|
807
|
+
return callback(util.error(new Error('abort_unknown_level')));
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// 批量抛弃分块上传任务
|
|
812
|
+
function abortUploadTaskArray(params, callback) {
|
|
813
|
+
var Bucket = params.Bucket;
|
|
814
|
+
var Region = params.Region;
|
|
815
|
+
var Key = params.Key;
|
|
816
|
+
var AbortArray = params.AbortArray;
|
|
817
|
+
var AsyncLimit = params.AsyncLimit || 1;
|
|
818
|
+
var self = this;
|
|
819
|
+
|
|
820
|
+
var index = 0;
|
|
821
|
+
var resultList = new Array(AbortArray.length);
|
|
822
|
+
Async.eachLimit(
|
|
823
|
+
AbortArray,
|
|
824
|
+
AsyncLimit,
|
|
825
|
+
function (AbortItem, nextItem) {
|
|
826
|
+
var eachIndex = index;
|
|
827
|
+
if (Key && Key !== AbortItem.Key) {
|
|
828
|
+
resultList[eachIndex] = { error: { KeyNotMatch: true } };
|
|
829
|
+
nextItem(null);
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
var UploadId = AbortItem.UploadId || AbortItem.UploadID;
|
|
833
|
+
|
|
834
|
+
self.multipartAbort(
|
|
835
|
+
{
|
|
836
|
+
Bucket: Bucket,
|
|
837
|
+
Region: Region,
|
|
838
|
+
Key: AbortItem.Key,
|
|
839
|
+
Headers: params.Headers,
|
|
840
|
+
UploadId: UploadId,
|
|
841
|
+
},
|
|
842
|
+
function (err) {
|
|
843
|
+
var task = {
|
|
844
|
+
Bucket: Bucket,
|
|
845
|
+
Region: Region,
|
|
846
|
+
Key: AbortItem.Key,
|
|
847
|
+
UploadId: UploadId,
|
|
848
|
+
};
|
|
849
|
+
resultList[eachIndex] = { error: err, task: task };
|
|
850
|
+
nextItem(null);
|
|
851
|
+
}
|
|
852
|
+
);
|
|
853
|
+
index++;
|
|
854
|
+
},
|
|
855
|
+
function (err) {
|
|
856
|
+
if (err) return callback(err);
|
|
857
|
+
|
|
858
|
+
var successList = [];
|
|
859
|
+
var errorList = [];
|
|
860
|
+
|
|
861
|
+
for (var i = 0, len = resultList.length; i < len; i++) {
|
|
862
|
+
var item = resultList[i];
|
|
863
|
+
if (item['task']) {
|
|
864
|
+
if (item['error']) {
|
|
865
|
+
errorList.push(item['task']);
|
|
866
|
+
} else {
|
|
867
|
+
successList.push(item['task']);
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
return callback(null, {
|
|
873
|
+
successList: successList,
|
|
874
|
+
errorList: errorList,
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// 高级上传
|
|
881
|
+
function uploadFile(params, callback) {
|
|
882
|
+
var self = this;
|
|
883
|
+
|
|
884
|
+
// 判断多大的文件使用分片上传
|
|
885
|
+
var SliceSize = params.SliceSize === undefined ? self.options.SliceSize : params.SliceSize;
|
|
886
|
+
|
|
887
|
+
// 开始处理每个文件
|
|
888
|
+
var taskList = [];
|
|
889
|
+
|
|
890
|
+
fs.stat(params.FilePath, function (err, stat) {
|
|
891
|
+
if (err) {
|
|
892
|
+
return callback(err);
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
var isDir = stat.isDirectory();
|
|
896
|
+
var FileSize = (params.ContentLength = stat.size || 0);
|
|
897
|
+
var fileInfo = { TaskId: '' };
|
|
898
|
+
|
|
899
|
+
// 整理 option,用于返回给回调
|
|
900
|
+
util.each(params, function (v, k) {
|
|
901
|
+
if (typeof v !== 'object' && typeof v !== 'function') {
|
|
902
|
+
fileInfo[k] = v;
|
|
903
|
+
}
|
|
904
|
+
});
|
|
905
|
+
|
|
906
|
+
// 处理文件 TaskReady
|
|
907
|
+
var _onTaskReady = params.onTaskReady;
|
|
908
|
+
var onTaskReady = function (tid) {
|
|
909
|
+
fileInfo.TaskId = tid;
|
|
910
|
+
_onTaskReady && _onTaskReady(tid);
|
|
911
|
+
};
|
|
912
|
+
params.onTaskReady = onTaskReady;
|
|
913
|
+
|
|
914
|
+
// 处理文件完成
|
|
915
|
+
var _onFileFinish = params.onFileFinish;
|
|
916
|
+
var onFileFinish = function (err, data) {
|
|
917
|
+
_onFileFinish && _onFileFinish(err, data, fileInfo);
|
|
918
|
+
callback && callback(err, data);
|
|
919
|
+
};
|
|
920
|
+
|
|
921
|
+
// 添加上传任务
|
|
922
|
+
var api = FileSize <= SliceSize || isDir ? 'putObject' : 'sliceUploadFile';
|
|
923
|
+
if (api === 'putObject') {
|
|
924
|
+
params.Body = isDir ? '' : fs.createReadStream(params.FilePath);
|
|
925
|
+
params.Body.isSdkCreated = true;
|
|
926
|
+
}
|
|
927
|
+
taskList.push({
|
|
928
|
+
api: api,
|
|
929
|
+
params: params,
|
|
930
|
+
callback: onFileFinish,
|
|
931
|
+
});
|
|
932
|
+
self._addTasks(taskList);
|
|
933
|
+
});
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
// 批量上传文件
|
|
937
|
+
function uploadFiles(params, callback) {
|
|
938
|
+
var self = this;
|
|
939
|
+
|
|
940
|
+
// 判断多大的文件使用分片上传
|
|
941
|
+
var SliceSize = params.SliceSize === undefined ? self.options.SliceSize : params.SliceSize;
|
|
942
|
+
|
|
943
|
+
// 汇总返回进度
|
|
944
|
+
var TotalSize = 0;
|
|
945
|
+
var TotalFinish = 0;
|
|
946
|
+
var onTotalProgress = util.throttleOnProgress.call(self, TotalFinish, params.onProgress);
|
|
947
|
+
|
|
948
|
+
// 汇总返回回调
|
|
949
|
+
var unFinishCount = params.files.length;
|
|
950
|
+
var _onTotalFileFinish = params.onFileFinish;
|
|
951
|
+
var resultList = Array(unFinishCount);
|
|
952
|
+
var onTotalFileFinish = function (err, data, options) {
|
|
953
|
+
onTotalProgress(null, true);
|
|
954
|
+
_onTotalFileFinish && _onTotalFileFinish(err, data, options);
|
|
955
|
+
resultList[options.Index] = {
|
|
956
|
+
options: options,
|
|
957
|
+
error: err,
|
|
958
|
+
data: data,
|
|
959
|
+
};
|
|
960
|
+
if (--unFinishCount <= 0 && callback) {
|
|
961
|
+
callback(null, { files: resultList });
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
|
|
965
|
+
// 开始处理每个文件
|
|
966
|
+
var taskList = [];
|
|
967
|
+
var count = params.files.length;
|
|
968
|
+
util.each(params.files, function (fileParams, index) {
|
|
969
|
+
fs.stat(fileParams.FilePath, function (err, stat) {
|
|
970
|
+
var isDir = stat ? stat.isDirectory() : false;
|
|
971
|
+
var FileSize = (fileParams.ContentLength = stat ? stat.size : 0);
|
|
972
|
+
var fileInfo = { Index: index, TaskId: '' };
|
|
973
|
+
|
|
974
|
+
// 更新文件总大小
|
|
975
|
+
TotalSize += FileSize;
|
|
976
|
+
|
|
977
|
+
// 整理 option,用于返回给回调
|
|
978
|
+
util.each(fileParams, function (v, k) {
|
|
979
|
+
if (typeof v !== 'object' && typeof v !== 'function') {
|
|
980
|
+
fileInfo[k] = v;
|
|
981
|
+
}
|
|
982
|
+
});
|
|
983
|
+
|
|
984
|
+
// 处理单个文件 TaskReady
|
|
985
|
+
var _onTaskReady = fileParams.onTaskReady;
|
|
986
|
+
var onTaskReady = function (tid) {
|
|
987
|
+
fileInfo.TaskId = tid;
|
|
988
|
+
_onTaskReady && _onTaskReady(tid);
|
|
989
|
+
};
|
|
990
|
+
fileParams.onTaskReady = onTaskReady;
|
|
991
|
+
|
|
992
|
+
// 处理单个文件进度
|
|
993
|
+
var PreAddSize = 0;
|
|
994
|
+
var _onProgress = fileParams.onProgress;
|
|
995
|
+
var onProgress = function (info) {
|
|
996
|
+
TotalFinish = TotalFinish - PreAddSize + info.loaded;
|
|
997
|
+
PreAddSize = info.loaded;
|
|
998
|
+
_onProgress && _onProgress(info);
|
|
999
|
+
onTotalProgress({ loaded: TotalFinish, total: TotalSize });
|
|
1000
|
+
};
|
|
1001
|
+
fileParams.onProgress = onProgress;
|
|
1002
|
+
|
|
1003
|
+
// 处理单个文件完成
|
|
1004
|
+
var _onFileFinish = fileParams.onFileFinish;
|
|
1005
|
+
var onFileFinish = function (err, data) {
|
|
1006
|
+
_onFileFinish && _onFileFinish(err, data);
|
|
1007
|
+
onTotalFileFinish && onTotalFileFinish(err, data, fileInfo);
|
|
1008
|
+
};
|
|
1009
|
+
|
|
1010
|
+
// 添加上传任务
|
|
1011
|
+
var api = FileSize <= SliceSize || isDir ? 'putObject' : 'sliceUploadFile';
|
|
1012
|
+
if (api === 'putObject') {
|
|
1013
|
+
fileParams.Body = isDir ? '' : fs.createReadStream(fileParams.FilePath);
|
|
1014
|
+
fileParams.Body.isSdkCreated = true;
|
|
1015
|
+
}
|
|
1016
|
+
taskList.push({
|
|
1017
|
+
api: api,
|
|
1018
|
+
params: fileParams,
|
|
1019
|
+
callback: onFileFinish,
|
|
1020
|
+
});
|
|
1021
|
+
--count === 0 && self._addTasks(taskList);
|
|
1022
|
+
});
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
// 分片复制文件
|
|
1027
|
+
function sliceCopyFile(params, callback) {
|
|
1028
|
+
var ep = new EventProxy();
|
|
1029
|
+
|
|
1030
|
+
var self = this;
|
|
1031
|
+
var Bucket = params.Bucket;
|
|
1032
|
+
var Region = params.Region;
|
|
1033
|
+
var Key = params.Key;
|
|
1034
|
+
var CopySource = params.CopySource;
|
|
1035
|
+
var m = util.getSourceParams.call(this, CopySource);
|
|
1036
|
+
if (!m) {
|
|
1037
|
+
callback(util.error(new Error('CopySource format error')));
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
var SourceBucket = m.Bucket;
|
|
1042
|
+
var SourceRegion = m.Region;
|
|
1043
|
+
var SourceKey = decodeURIComponent(m.Key);
|
|
1044
|
+
var CopySliceSize = params.CopySliceSize === undefined ? self.options.CopySliceSize : params.CopySliceSize;
|
|
1045
|
+
CopySliceSize = Math.max(0, CopySliceSize);
|
|
1046
|
+
|
|
1047
|
+
var ChunkSize = params.CopyChunkSize || this.options.CopyChunkSize;
|
|
1048
|
+
var ChunkParallel = this.options.CopyChunkParallelLimit;
|
|
1049
|
+
var ChunkRetryTimes = this.options.ChunkRetryTimes + 1;
|
|
1050
|
+
|
|
1051
|
+
var ChunkCount = 0;
|
|
1052
|
+
var FinishSize = 0;
|
|
1053
|
+
var FileSize;
|
|
1054
|
+
var onProgress;
|
|
1055
|
+
var SourceResHeaders = {};
|
|
1056
|
+
var SourceHeaders = {};
|
|
1057
|
+
var TargetHeader = {};
|
|
1058
|
+
|
|
1059
|
+
// 分片复制完成,开始 multipartComplete 操作
|
|
1060
|
+
ep.on('copy_slice_complete', function (UploadData) {
|
|
1061
|
+
var metaHeaders = {};
|
|
1062
|
+
util.each(params.Headers, function (val, k) {
|
|
1063
|
+
if (k.toLowerCase().indexOf('x-cos-meta-') === 0) metaHeaders[k] = val;
|
|
1064
|
+
});
|
|
1065
|
+
var Parts = util.map(UploadData.PartList, function (item) {
|
|
1066
|
+
return {
|
|
1067
|
+
PartNumber: item.PartNumber,
|
|
1068
|
+
ETag: item.ETag,
|
|
1069
|
+
};
|
|
1070
|
+
});
|
|
1071
|
+
// 完成上传的请求也做重试
|
|
1072
|
+
Async.retry(
|
|
1073
|
+
ChunkRetryTimes,
|
|
1074
|
+
function (tryCallback) {
|
|
1075
|
+
self.multipartComplete(
|
|
1076
|
+
{
|
|
1077
|
+
Bucket: Bucket,
|
|
1078
|
+
Region: Region,
|
|
1079
|
+
Key: Key,
|
|
1080
|
+
UploadId: UploadData.UploadId,
|
|
1081
|
+
Parts: Parts,
|
|
1082
|
+
},
|
|
1083
|
+
tryCallback
|
|
1084
|
+
);
|
|
1085
|
+
},
|
|
1086
|
+
function (err, data) {
|
|
1087
|
+
session.removeUsing(UploadData.UploadId); // 标记 UploadId 没被使用了,因为复制没提供重试,所以只要出错,就是 UploadId 停用了。
|
|
1088
|
+
if (err) {
|
|
1089
|
+
onProgress(null, true);
|
|
1090
|
+
return callback(err);
|
|
1091
|
+
}
|
|
1092
|
+
session.removeUploadId.call(self, UploadData.UploadId);
|
|
1093
|
+
onProgress({ loaded: FileSize, total: FileSize }, true);
|
|
1094
|
+
callback(null, data);
|
|
1095
|
+
}
|
|
1096
|
+
);
|
|
1097
|
+
});
|
|
1098
|
+
|
|
1099
|
+
ep.on('get_copy_data_finish', function (UploadData) {
|
|
1100
|
+
// 处理 UploadId 缓存
|
|
1101
|
+
var uuid = session.getCopyFileId(CopySource, SourceResHeaders, ChunkSize, Bucket, Key);
|
|
1102
|
+
uuid && session.saveUploadId.call(self, uuid, UploadData.UploadId, self.options.UploadIdCacheLimit); // 缓存 UploadId
|
|
1103
|
+
session.setUsing(UploadData.UploadId); // 标记 UploadId 为正在使用
|
|
1104
|
+
|
|
1105
|
+
var needCopySlices = util.filter(UploadData.PartList, function (SliceItem) {
|
|
1106
|
+
if (SliceItem['Uploaded']) {
|
|
1107
|
+
FinishSize += SliceItem['PartNumber'] >= ChunkCount ? FileSize % ChunkSize || ChunkSize : ChunkSize;
|
|
1108
|
+
}
|
|
1109
|
+
return !SliceItem['Uploaded'];
|
|
1110
|
+
});
|
|
1111
|
+
Async.eachLimit(
|
|
1112
|
+
needCopySlices,
|
|
1113
|
+
ChunkParallel,
|
|
1114
|
+
function (SliceItem, asyncCallback) {
|
|
1115
|
+
var PartNumber = SliceItem.PartNumber;
|
|
1116
|
+
var CopySourceRange = SliceItem.CopySourceRange;
|
|
1117
|
+
var currentSize = SliceItem.end - SliceItem.start;
|
|
1118
|
+
Async.retry(
|
|
1119
|
+
ChunkRetryTimes,
|
|
1120
|
+
function (tryCallback) {
|
|
1121
|
+
copySliceItem.call(
|
|
1122
|
+
self,
|
|
1123
|
+
{
|
|
1124
|
+
Bucket: Bucket,
|
|
1125
|
+
Region: Region,
|
|
1126
|
+
Key: Key,
|
|
1127
|
+
CopySource: CopySource,
|
|
1128
|
+
UploadId: UploadData.UploadId,
|
|
1129
|
+
PartNumber: PartNumber,
|
|
1130
|
+
CopySourceRange: CopySourceRange,
|
|
1131
|
+
},
|
|
1132
|
+
tryCallback
|
|
1133
|
+
);
|
|
1134
|
+
},
|
|
1135
|
+
function (err, data) {
|
|
1136
|
+
if (err) return asyncCallback(err);
|
|
1137
|
+
FinishSize += currentSize;
|
|
1138
|
+
onProgress({ loaded: FinishSize, total: FileSize });
|
|
1139
|
+
SliceItem.ETag = data.ETag;
|
|
1140
|
+
asyncCallback(err || null, data);
|
|
1141
|
+
}
|
|
1142
|
+
);
|
|
1143
|
+
},
|
|
1144
|
+
function (err) {
|
|
1145
|
+
if (err) {
|
|
1146
|
+
session.removeUsing(UploadData.UploadId); // 标记 UploadId 没被使用了,因为复制没提供重试,所以只要出错,就是 UploadId 停用了。
|
|
1147
|
+
onProgress(null, true);
|
|
1148
|
+
return callback(err);
|
|
1149
|
+
}
|
|
1150
|
+
ep.emit('copy_slice_complete', UploadData);
|
|
1151
|
+
}
|
|
1152
|
+
);
|
|
1153
|
+
});
|
|
1154
|
+
|
|
1155
|
+
ep.on('get_chunk_size_finish', function () {
|
|
1156
|
+
var createNewUploadId = function () {
|
|
1157
|
+
self.multipartInit(
|
|
1158
|
+
{
|
|
1159
|
+
Bucket: Bucket,
|
|
1160
|
+
Region: Region,
|
|
1161
|
+
Key: Key,
|
|
1162
|
+
Headers: TargetHeader,
|
|
1163
|
+
},
|
|
1164
|
+
function (err, data) {
|
|
1165
|
+
if (err) return callback(err);
|
|
1166
|
+
params.UploadId = data.UploadId;
|
|
1167
|
+
ep.emit('get_copy_data_finish', { UploadId: params.UploadId, PartList: params.PartList });
|
|
1168
|
+
}
|
|
1169
|
+
);
|
|
1170
|
+
};
|
|
1171
|
+
|
|
1172
|
+
// 在本地找可用的 UploadId
|
|
1173
|
+
var uuid = session.getCopyFileId(CopySource, SourceResHeaders, ChunkSize, Bucket, Key);
|
|
1174
|
+
var LocalUploadIdList = session.getUploadIdList.call(self, uuid);
|
|
1175
|
+
if (!uuid || !LocalUploadIdList) return createNewUploadId();
|
|
1176
|
+
|
|
1177
|
+
var next = function (index) {
|
|
1178
|
+
// 如果本地找不到可用 UploadId,再一个个遍历校验远端
|
|
1179
|
+
if (index >= LocalUploadIdList.length) return createNewUploadId();
|
|
1180
|
+
var UploadId = LocalUploadIdList[index];
|
|
1181
|
+
// 如果正在被使用,跳过
|
|
1182
|
+
if (session.using[UploadId]) return next(index + 1);
|
|
1183
|
+
// 判断 UploadId 是否存在线上
|
|
1184
|
+
wholeMultipartListPart.call(
|
|
1185
|
+
self,
|
|
1186
|
+
{
|
|
1187
|
+
Bucket: Bucket,
|
|
1188
|
+
Region: Region,
|
|
1189
|
+
Key: Key,
|
|
1190
|
+
UploadId: UploadId,
|
|
1191
|
+
},
|
|
1192
|
+
function (err, PartListData) {
|
|
1193
|
+
if (err) {
|
|
1194
|
+
// 如果 UploadId 获取会出错,跳过并删除
|
|
1195
|
+
session.removeUploadId.call(self, UploadId);
|
|
1196
|
+
next(index + 1);
|
|
1197
|
+
} else {
|
|
1198
|
+
// 如果异步回来 UploadId 已经被用了,也跳过
|
|
1199
|
+
if (session.using[UploadId]) return next(index + 1);
|
|
1200
|
+
// 找到可用 UploadId
|
|
1201
|
+
var finishETagMap = {};
|
|
1202
|
+
var offset = 0;
|
|
1203
|
+
util.each(PartListData.PartList, function (PartItem) {
|
|
1204
|
+
var size = parseInt(PartItem.Size);
|
|
1205
|
+
var end = offset + size - 1;
|
|
1206
|
+
finishETagMap[PartItem.PartNumber + '|' + offset + '|' + end] = PartItem.ETag;
|
|
1207
|
+
offset += size;
|
|
1208
|
+
});
|
|
1209
|
+
util.each(params.PartList, function (PartItem) {
|
|
1210
|
+
var ETag = finishETagMap[PartItem.PartNumber + '|' + PartItem.start + '|' + PartItem.end];
|
|
1211
|
+
if (ETag) {
|
|
1212
|
+
PartItem.ETag = ETag;
|
|
1213
|
+
PartItem.Uploaded = true;
|
|
1214
|
+
}
|
|
1215
|
+
});
|
|
1216
|
+
ep.emit('get_copy_data_finish', { UploadId: UploadId, PartList: params.PartList });
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
);
|
|
1220
|
+
};
|
|
1221
|
+
next(0);
|
|
1222
|
+
});
|
|
1223
|
+
|
|
1224
|
+
ep.on('get_file_size_finish', function () {
|
|
1225
|
+
// 控制分片大小
|
|
1226
|
+
(function () {
|
|
1227
|
+
var SIZE = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1024 * 2, 1024 * 4, 1024 * 5];
|
|
1228
|
+
var AutoChunkSize = 1024 * 1024;
|
|
1229
|
+
for (var i = 0; i < SIZE.length; i++) {
|
|
1230
|
+
AutoChunkSize = SIZE[i] * 1024 * 1024;
|
|
1231
|
+
if (FileSize / AutoChunkSize <= self.options.MaxPartNumber) break;
|
|
1232
|
+
}
|
|
1233
|
+
params.ChunkSize = ChunkSize = Math.max(ChunkSize, AutoChunkSize);
|
|
1234
|
+
ChunkCount = Math.ceil(FileSize / ChunkSize);
|
|
1235
|
+
|
|
1236
|
+
var list = [];
|
|
1237
|
+
for (var partNumber = 1; partNumber <= ChunkCount; partNumber++) {
|
|
1238
|
+
var start = (partNumber - 1) * ChunkSize;
|
|
1239
|
+
var end = partNumber * ChunkSize < FileSize ? partNumber * ChunkSize - 1 : FileSize - 1;
|
|
1240
|
+
var item = {
|
|
1241
|
+
PartNumber: partNumber,
|
|
1242
|
+
start: start,
|
|
1243
|
+
end: end,
|
|
1244
|
+
CopySourceRange: 'bytes=' + start + '-' + end,
|
|
1245
|
+
};
|
|
1246
|
+
list.push(item);
|
|
1247
|
+
}
|
|
1248
|
+
params.PartList = list;
|
|
1249
|
+
})();
|
|
1250
|
+
|
|
1251
|
+
if (params.Headers['x-cos-metadata-directive'] === 'Replaced') {
|
|
1252
|
+
TargetHeader = params.Headers;
|
|
1253
|
+
} else {
|
|
1254
|
+
TargetHeader = SourceHeaders;
|
|
1255
|
+
}
|
|
1256
|
+
TargetHeader['x-cos-storage-class'] = params.Headers['x-cos-storage-class'] || SourceHeaders['x-cos-storage-class'];
|
|
1257
|
+
TargetHeader = util.clearKey(TargetHeader);
|
|
1258
|
+
/**
|
|
1259
|
+
* 对于归档存储的对象,如果未恢复副本,则不允许 Copy
|
|
1260
|
+
*/
|
|
1261
|
+
if (SourceHeaders['x-cos-storage-class'] === 'ARCHIVE' || SourceHeaders['x-cos-storage-class'] === 'DEEP_ARCHIVE') {
|
|
1262
|
+
var restoreHeader = SourceHeaders['x-cos-restore'];
|
|
1263
|
+
if (!restoreHeader || restoreHeader === 'ongoing-request="true"') {
|
|
1264
|
+
callback(util.error(new Error('Unrestored archive object is not allowed to be copied')));
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
/**
|
|
1269
|
+
* 去除一些无用的头部,规避 multipartInit 出错
|
|
1270
|
+
* 这些头部通常是在 putObjectCopy 时才使用
|
|
1271
|
+
*/
|
|
1272
|
+
delete TargetHeader['x-cos-copy-source'];
|
|
1273
|
+
delete TargetHeader['x-cos-metadata-directive'];
|
|
1274
|
+
delete TargetHeader['x-cos-copy-source-If-Modified-Since'];
|
|
1275
|
+
delete TargetHeader['x-cos-copy-source-If-Unmodified-Since'];
|
|
1276
|
+
delete TargetHeader['x-cos-copy-source-If-Match'];
|
|
1277
|
+
delete TargetHeader['x-cos-copy-source-If-None-Match'];
|
|
1278
|
+
ep.emit('get_chunk_size_finish');
|
|
1279
|
+
});
|
|
1280
|
+
|
|
1281
|
+
// 获取远端复制源文件的大小
|
|
1282
|
+
self.headObject(
|
|
1283
|
+
{
|
|
1284
|
+
Bucket: SourceBucket,
|
|
1285
|
+
Region: SourceRegion,
|
|
1286
|
+
Key: SourceKey,
|
|
1287
|
+
},
|
|
1288
|
+
function (err, data) {
|
|
1289
|
+
if (err) {
|
|
1290
|
+
if (err.statusCode && err.statusCode === 404) {
|
|
1291
|
+
callback(util.error(err, { ErrorStatus: SourceKey + ' Not Exist' }));
|
|
1292
|
+
} else {
|
|
1293
|
+
callback(err);
|
|
1294
|
+
}
|
|
1295
|
+
return;
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1298
|
+
FileSize = params.FileSize = data.headers['content-length'];
|
|
1299
|
+
if (FileSize === undefined || !FileSize) {
|
|
1300
|
+
callback(
|
|
1301
|
+
util.error(
|
|
1302
|
+
new Error(
|
|
1303
|
+
'get Content-Length error, please add "Content-Length" to CORS ExposeHeader setting.( 获取Content-Length失败,请在CORS ExposeHeader设置中添加Content-Length,请参考文档:https://cloud.tencent.com/document/product/436/13318 )'
|
|
1304
|
+
)
|
|
1305
|
+
)
|
|
1306
|
+
);
|
|
1307
|
+
return;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
onProgress = util.throttleOnProgress.call(self, FileSize, params.onProgress);
|
|
1311
|
+
|
|
1312
|
+
// 开始上传
|
|
1313
|
+
if (FileSize <= CopySliceSize) {
|
|
1314
|
+
if (!params.Headers['x-cos-metadata-directive']) {
|
|
1315
|
+
params.Headers['x-cos-metadata-directive'] = 'Copy';
|
|
1316
|
+
}
|
|
1317
|
+
self.putObjectCopy(params, function (err, data) {
|
|
1318
|
+
if (err) {
|
|
1319
|
+
onProgress(null, true);
|
|
1320
|
+
return callback(err);
|
|
1321
|
+
}
|
|
1322
|
+
onProgress({ loaded: FileSize, total: FileSize }, true);
|
|
1323
|
+
callback(err, data);
|
|
1324
|
+
});
|
|
1325
|
+
} else {
|
|
1326
|
+
var resHeaders = data.headers;
|
|
1327
|
+
SourceResHeaders = resHeaders;
|
|
1328
|
+
SourceHeaders = {
|
|
1329
|
+
'Cache-Control': resHeaders['cache-control'],
|
|
1330
|
+
'Content-Disposition': resHeaders['content-disposition'],
|
|
1331
|
+
'Content-Encoding': resHeaders['content-encoding'],
|
|
1332
|
+
'Content-Type': resHeaders['content-type'],
|
|
1333
|
+
Expires: resHeaders['expires'],
|
|
1334
|
+
'x-cos-storage-class': resHeaders['x-cos-storage-class'],
|
|
1335
|
+
};
|
|
1336
|
+
util.each(resHeaders, function (v, k) {
|
|
1337
|
+
var metaPrefix = 'x-cos-meta-';
|
|
1338
|
+
if (k.indexOf(metaPrefix) === 0 && k.length > metaPrefix.length) {
|
|
1339
|
+
SourceHeaders[k] = v;
|
|
1340
|
+
}
|
|
1341
|
+
});
|
|
1342
|
+
ep.emit('get_file_size_finish');
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
);
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
// 复制指定分片
|
|
1349
|
+
function copySliceItem(params, callback) {
|
|
1350
|
+
var TaskId = params.TaskId;
|
|
1351
|
+
var Bucket = params.Bucket;
|
|
1352
|
+
var Region = params.Region;
|
|
1353
|
+
var Key = params.Key;
|
|
1354
|
+
var CopySource = params.CopySource;
|
|
1355
|
+
var UploadId = params.UploadId;
|
|
1356
|
+
var PartNumber = params.PartNumber * 1;
|
|
1357
|
+
var CopySourceRange = params.CopySourceRange;
|
|
1358
|
+
|
|
1359
|
+
var ChunkRetryTimes = this.options.ChunkRetryTimes + 1;
|
|
1360
|
+
var self = this;
|
|
1361
|
+
|
|
1362
|
+
Async.retry(
|
|
1363
|
+
ChunkRetryTimes,
|
|
1364
|
+
function (tryCallback) {
|
|
1365
|
+
self.uploadPartCopy(
|
|
1366
|
+
{
|
|
1367
|
+
TaskId: TaskId,
|
|
1368
|
+
Bucket: Bucket,
|
|
1369
|
+
Region: Region,
|
|
1370
|
+
Key: Key,
|
|
1371
|
+
CopySource: CopySource,
|
|
1372
|
+
UploadId: UploadId,
|
|
1373
|
+
PartNumber: PartNumber,
|
|
1374
|
+
CopySourceRange: CopySourceRange,
|
|
1375
|
+
},
|
|
1376
|
+
function (err, data) {
|
|
1377
|
+
tryCallback(err || null, data);
|
|
1378
|
+
}
|
|
1379
|
+
);
|
|
1380
|
+
},
|
|
1381
|
+
function (err, data) {
|
|
1382
|
+
return callback(err, data);
|
|
1383
|
+
}
|
|
1384
|
+
);
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
// 分片下载文件
|
|
1388
|
+
function downloadFile(params, callback) {
|
|
1389
|
+
var self = this;
|
|
1390
|
+
var TaskId = params.TaskId || util.uuid();
|
|
1391
|
+
var Bucket = params.Bucket;
|
|
1392
|
+
var Region = params.Region;
|
|
1393
|
+
var Key = params.Key;
|
|
1394
|
+
var FilePath = params.FilePath;
|
|
1395
|
+
var FileSize;
|
|
1396
|
+
var FinishSize = 0;
|
|
1397
|
+
var onProgress;
|
|
1398
|
+
var ChunkSize = params.ChunkSize || 1024 * 1024;
|
|
1399
|
+
var ParallelLimit = params.ParallelLimit || 5;
|
|
1400
|
+
var RetryTimes = params.RetryTimes || 3;
|
|
1401
|
+
var ep = new EventProxy();
|
|
1402
|
+
var PartList;
|
|
1403
|
+
var aborted = false;
|
|
1404
|
+
var head = {};
|
|
1405
|
+
|
|
1406
|
+
ep.on('error', function (err) {
|
|
1407
|
+
callback(err);
|
|
1408
|
+
});
|
|
1409
|
+
|
|
1410
|
+
ep.on('get_file_info', function () {
|
|
1411
|
+
// 获取远端复制源文件的大小
|
|
1412
|
+
self.headObject(
|
|
1413
|
+
{
|
|
1414
|
+
Bucket: Bucket,
|
|
1415
|
+
Region: Region,
|
|
1416
|
+
Key: Key,
|
|
1417
|
+
},
|
|
1418
|
+
function (err, data) {
|
|
1419
|
+
if (err) return ep.emit('error', err);
|
|
1420
|
+
|
|
1421
|
+
// 获取文件大小
|
|
1422
|
+
FileSize = params.FileSize = parseInt(data.headers['content-length']);
|
|
1423
|
+
if (FileSize === undefined || !FileSize) {
|
|
1424
|
+
callback(
|
|
1425
|
+
util.error(
|
|
1426
|
+
new Error(
|
|
1427
|
+
'get Content-Length error, please add "Content-Length" to CORS ExposeHeader setting.( 获取Content-Length失败,请在CORS ExposeHeader设置中添加Content-Length,请参考文档:https://cloud.tencent.com/document/product/436/13318 )'
|
|
1428
|
+
)
|
|
1429
|
+
)
|
|
1430
|
+
);
|
|
1431
|
+
return;
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
// 归档文件不支持下载
|
|
1435
|
+
const resHeaders = data.headers;
|
|
1436
|
+
const storageClass = resHeaders['x-cos-storage-class'] || '';
|
|
1437
|
+
const restoreStatus = resHeaders['x-cos-restore'] || '';
|
|
1438
|
+
if (
|
|
1439
|
+
['DEEP_ARCHIVE', 'ARCHIVE'].includes(storageClass) &&
|
|
1440
|
+
(!restoreStatus || restoreStatus === 'ongoing-request="true"')
|
|
1441
|
+
) {
|
|
1442
|
+
// 自定义返回的错误码 与cos api无关
|
|
1443
|
+
return callback({
|
|
1444
|
+
statusCode: 403,
|
|
1445
|
+
header: resHeaders,
|
|
1446
|
+
code: 'CannotDownload',
|
|
1447
|
+
message: 'Archive object can not download, please restore to Standard storage class.',
|
|
1448
|
+
});
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
// 整理文件信息
|
|
1452
|
+
head = {
|
|
1453
|
+
ETag: data.ETag,
|
|
1454
|
+
size: FileSize,
|
|
1455
|
+
mtime: resHeaders['last-modified'],
|
|
1456
|
+
crc64ecma: resHeaders['x-cos-hash-crc64ecma'],
|
|
1457
|
+
};
|
|
1458
|
+
|
|
1459
|
+
// 处理进度反馈
|
|
1460
|
+
onProgress = util.throttleOnProgress.call(self, FileSize, function (info) {
|
|
1461
|
+
if (aborted) return;
|
|
1462
|
+
params.onProgress(info);
|
|
1463
|
+
});
|
|
1464
|
+
|
|
1465
|
+
if (FileSize <= ChunkSize) {
|
|
1466
|
+
// 小文件直接单请求下载
|
|
1467
|
+
self.getObject(
|
|
1468
|
+
{
|
|
1469
|
+
TaskId: TaskId,
|
|
1470
|
+
Bucket: Bucket,
|
|
1471
|
+
Region: Region,
|
|
1472
|
+
Key: Key,
|
|
1473
|
+
onProgress: onProgress,
|
|
1474
|
+
Output: fs.createWriteStream(FilePath),
|
|
1475
|
+
},
|
|
1476
|
+
function (err, data) {
|
|
1477
|
+
if (err) {
|
|
1478
|
+
onProgress(null, true);
|
|
1479
|
+
return callback(err);
|
|
1480
|
+
}
|
|
1481
|
+
onProgress({ loaded: FileSize, total: FileSize }, true);
|
|
1482
|
+
callback(err, data);
|
|
1483
|
+
}
|
|
1484
|
+
);
|
|
1485
|
+
} else {
|
|
1486
|
+
// 大文件分片下载
|
|
1487
|
+
ep.emit('calc_suitable_chunk_size');
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
);
|
|
1491
|
+
});
|
|
1492
|
+
|
|
1493
|
+
// 计算合适的分片大小
|
|
1494
|
+
ep.on('calc_suitable_chunk_size', function (SourceHeaders) {
|
|
1495
|
+
// 控制分片大小
|
|
1496
|
+
var SIZE = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1024 * 2, 1024 * 4, 1024 * 5];
|
|
1497
|
+
var AutoChunkSize = 1024 * 1024;
|
|
1498
|
+
for (var i = 0; i < SIZE.length; i++) {
|
|
1499
|
+
AutoChunkSize = SIZE[i] * 1024 * 1024;
|
|
1500
|
+
if (FileSize / AutoChunkSize <= self.options.MaxPartNumber) break;
|
|
1501
|
+
}
|
|
1502
|
+
params.ChunkSize = ChunkSize = Math.max(ChunkSize, AutoChunkSize);
|
|
1503
|
+
|
|
1504
|
+
var ChunkCount = Math.ceil(FileSize / ChunkSize);
|
|
1505
|
+
|
|
1506
|
+
var list = [];
|
|
1507
|
+
for (var partNumber = 1; partNumber <= ChunkCount; partNumber++) {
|
|
1508
|
+
var start = (partNumber - 1) * ChunkSize;
|
|
1509
|
+
var end = partNumber * ChunkSize < FileSize ? partNumber * ChunkSize - 1 : FileSize - 1;
|
|
1510
|
+
var item = {
|
|
1511
|
+
PartNumber: partNumber,
|
|
1512
|
+
start: start,
|
|
1513
|
+
end: end,
|
|
1514
|
+
};
|
|
1515
|
+
list.push(item);
|
|
1516
|
+
}
|
|
1517
|
+
PartList = list;
|
|
1518
|
+
|
|
1519
|
+
ep.emit('prepare_file');
|
|
1520
|
+
});
|
|
1521
|
+
|
|
1522
|
+
// 准备要下载的空文件
|
|
1523
|
+
ep.on('prepare_file', function (SourceHeaders) {
|
|
1524
|
+
fs.writeFile(FilePath, '', (err) => {
|
|
1525
|
+
if (err) {
|
|
1526
|
+
ep.emit('error', err.code === 'EISDIR' ? { code: 'exist_same_dir', message: FilePath } : err);
|
|
1527
|
+
} else {
|
|
1528
|
+
ep.emit('start_download_chunks');
|
|
1529
|
+
}
|
|
1530
|
+
});
|
|
1531
|
+
});
|
|
1532
|
+
|
|
1533
|
+
// 计算合适的分片大小
|
|
1534
|
+
var result;
|
|
1535
|
+
ep.on('start_download_chunks', function (SourceHeaders) {
|
|
1536
|
+
onProgress({ loaded: 0, total: FileSize }, true);
|
|
1537
|
+
var maxPartNumber = PartList.length;
|
|
1538
|
+
Async.eachLimit(
|
|
1539
|
+
PartList,
|
|
1540
|
+
ParallelLimit,
|
|
1541
|
+
function (part, nextChunk) {
|
|
1542
|
+
if (aborted) return;
|
|
1543
|
+
Async.retry(
|
|
1544
|
+
RetryTimes,
|
|
1545
|
+
function (tryCallback) {
|
|
1546
|
+
if (aborted) return;
|
|
1547
|
+
// FinishSize
|
|
1548
|
+
var Headers = util.clone(params.Headers);
|
|
1549
|
+
Headers.Range = 'bytes=' + part.start + '-' + part.end;
|
|
1550
|
+
const writeStream = fs.createWriteStream(FilePath, {
|
|
1551
|
+
start: part.start,
|
|
1552
|
+
flags: 'r+',
|
|
1553
|
+
});
|
|
1554
|
+
var preAddSize = 0;
|
|
1555
|
+
var chunkReadSize = part.end - part.start;
|
|
1556
|
+
self.getObject(
|
|
1557
|
+
{
|
|
1558
|
+
TaskId: TaskId,
|
|
1559
|
+
Bucket: params.Bucket,
|
|
1560
|
+
Region: params.Region,
|
|
1561
|
+
Key: params.Key,
|
|
1562
|
+
Query: params.Query,
|
|
1563
|
+
Headers: Headers,
|
|
1564
|
+
onProgress: function (data) {
|
|
1565
|
+
if (aborted) return;
|
|
1566
|
+
FinishSize += data.loaded - preAddSize;
|
|
1567
|
+
preAddSize = data.loaded;
|
|
1568
|
+
onProgress({ loaded: FinishSize, total: FileSize });
|
|
1569
|
+
},
|
|
1570
|
+
Output: writeStream,
|
|
1571
|
+
},
|
|
1572
|
+
function (err, data) {
|
|
1573
|
+
if (aborted) return;
|
|
1574
|
+
|
|
1575
|
+
// 处理错误和进度
|
|
1576
|
+
if (err) {
|
|
1577
|
+
FinishSize -= preAddSize;
|
|
1578
|
+
return tryCallback(err);
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
// 处理返回值
|
|
1582
|
+
if (part.PartNumber === maxPartNumber) result = data;
|
|
1583
|
+
var chunkHeaders = data.headers || {};
|
|
1584
|
+
|
|
1585
|
+
var contentRanges = chunkHeaders['content-range'] || ''; // content-range 格式:"bytes 3145728-4194303/68577051"
|
|
1586
|
+
var totalSize = parseInt(contentRanges.split('/')[1] || 0);
|
|
1587
|
+
|
|
1588
|
+
// 只校验文件大小和 crc64 是否有变更
|
|
1589
|
+
var changed;
|
|
1590
|
+
if (chunkHeaders['x-cos-hash-crc64ecma'] !== head.crc64ecma)
|
|
1591
|
+
changed = 'download error, x-cos-hash-crc64ecma has changed.';
|
|
1592
|
+
else if (totalSize !== head.size) changed = 'download error, Last-Modified has changed.';
|
|
1593
|
+
// else if (data.ETag !== head.ETag) error = 'download error, ETag has changed.';
|
|
1594
|
+
// else if (chunkHeaders['last-modified'] !== head.mtime) error = 'download error, Last-Modified has changed.';
|
|
1595
|
+
|
|
1596
|
+
// 如果
|
|
1597
|
+
if (changed) {
|
|
1598
|
+
FinishSize -= preAddSize;
|
|
1599
|
+
onProgress({ loaded: FinishSize, total: FileSize });
|
|
1600
|
+
ep.emit('error', {
|
|
1601
|
+
code: 'ObjectHasChanged',
|
|
1602
|
+
message: changed,
|
|
1603
|
+
statusCode: data.statusCode,
|
|
1604
|
+
header: chunkHeaders,
|
|
1605
|
+
});
|
|
1606
|
+
self.emit('inner-kill-task', { TaskId: TaskId });
|
|
1607
|
+
} else {
|
|
1608
|
+
FinishSize += chunkReadSize - preAddSize;
|
|
1609
|
+
part.loaded = true;
|
|
1610
|
+
onProgress({ loaded: FinishSize, total: FileSize });
|
|
1611
|
+
tryCallback(err, data);
|
|
1612
|
+
}
|
|
1613
|
+
}
|
|
1614
|
+
);
|
|
1615
|
+
},
|
|
1616
|
+
function (err, data) {
|
|
1617
|
+
if (aborted) return;
|
|
1618
|
+
nextChunk(err, data);
|
|
1619
|
+
}
|
|
1620
|
+
);
|
|
1621
|
+
},
|
|
1622
|
+
function (err, data) {
|
|
1623
|
+
if (aborted) return;
|
|
1624
|
+
onProgress({ loaded: FileSize, total: FileSize }, true);
|
|
1625
|
+
if (err) return ep.emit('error', err);
|
|
1626
|
+
ep.emit('download_chunks_complete');
|
|
1627
|
+
}
|
|
1628
|
+
);
|
|
1629
|
+
});
|
|
1630
|
+
|
|
1631
|
+
// 下载已完成
|
|
1632
|
+
ep.on('download_chunks_complete', function () {
|
|
1633
|
+
callback(null, result);
|
|
1634
|
+
});
|
|
1635
|
+
|
|
1636
|
+
// 监听 取消任务
|
|
1637
|
+
var killTask = function () {
|
|
1638
|
+
aborted = true;
|
|
1639
|
+
};
|
|
1640
|
+
TaskId && self.on('inner-kill-task', killTask);
|
|
1641
|
+
|
|
1642
|
+
ep.emit('get_file_info');
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
var API_MAP = {
|
|
1646
|
+
sliceUploadFile: sliceUploadFile,
|
|
1647
|
+
abortUploadTask: abortUploadTask,
|
|
1648
|
+
uploadFile: uploadFile,
|
|
1649
|
+
uploadFiles: uploadFiles,
|
|
1650
|
+
sliceCopyFile: sliceCopyFile,
|
|
1651
|
+
downloadFile: downloadFile,
|
|
1652
|
+
};
|
|
1653
|
+
|
|
1654
|
+
module.exports.init = function (COS, task) {
|
|
1655
|
+
task.transferToTaskMethod(API_MAP, 'sliceUploadFile');
|
|
1656
|
+
util.each(API_MAP, function (fn, apiName) {
|
|
1657
|
+
COS.prototype[apiName] = util.apiWrapper(apiName, fn);
|
|
1658
|
+
});
|
|
1659
|
+
};
|