@webiny/api-sync-system 6.3.0 → 6.4.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/constants.js +2 -1
- package/constants.js.map +1 -1
- package/index.js +0 -2
- package/package.json +14 -14
- package/resolver/app/RecordHandler.js +83 -105
- package/resolver/app/RecordHandler.js.map +1 -1
- package/resolver/app/RecordsValidation.js +11 -14
- package/resolver/app/RecordsValidation.js.map +1 -1
- package/resolver/app/ResolverApplication.js +24 -31
- package/resolver/app/ResolverApplication.js.map +1 -1
- package/resolver/app/abstractions/RecordHandler.js +0 -3
- package/resolver/app/abstractions/RecordsValidation.js +0 -3
- package/resolver/app/abstractions/ResolverApplication.js +0 -3
- package/resolver/app/abstractions/ResolverRecord.js +0 -3
- package/resolver/app/bundler/BaseBundle.js +17 -16
- package/resolver/app/bundler/BaseBundle.js.map +1 -1
- package/resolver/app/bundler/Bundler.js +13 -18
- package/resolver/app/bundler/Bundler.js.map +1 -1
- package/resolver/app/bundler/Bundles.js +21 -26
- package/resolver/app/bundler/Bundles.js.map +1 -1
- package/resolver/app/bundler/CommandBundle.js +7 -16
- package/resolver/app/bundler/CommandBundle.js.map +1 -1
- package/resolver/app/bundler/TableBundle.js +6 -13
- package/resolver/app/bundler/TableBundle.js.map +1 -1
- package/resolver/app/bundler/types.js +0 -3
- package/resolver/app/commandHandler/DeleteCommandHandler.js +22 -29
- package/resolver/app/commandHandler/DeleteCommandHandler.js.map +1 -1
- package/resolver/app/commandHandler/PutCommandHandler.js +16 -22
- package/resolver/app/commandHandler/PutCommandHandler.js.map +1 -1
- package/resolver/app/commandHandler/delete.js +11 -12
- package/resolver/app/commandHandler/delete.js.map +1 -1
- package/resolver/app/commandHandler/put.js +11 -12
- package/resolver/app/commandHandler/put.js.map +1 -1
- package/resolver/app/data/SourceDataContainer.js +28 -30
- package/resolver/app/data/SourceDataContainer.js.map +1 -1
- package/resolver/app/data/types.js +0 -3
- package/resolver/app/fetcher/Fetcher.js +84 -109
- package/resolver/app/fetcher/Fetcher.js.map +1 -1
- package/resolver/app/fetcher/types.js +0 -3
- package/resolver/app/ingestor/Ingestor.js +27 -32
- package/resolver/app/ingestor/Ingestor.js.map +1 -1
- package/resolver/app/ingestor/IngestorResult.js +28 -32
- package/resolver/app/ingestor/IngestorResult.js.map +1 -1
- package/resolver/app/ingestor/types.js +0 -3
- package/resolver/app/storer/Storer.js +71 -85
- package/resolver/app/storer/Storer.js.map +1 -1
- package/resolver/app/storer/types.js +0 -3
- package/resolver/app/transform/TransformHandler.js +45 -56
- package/resolver/app/transform/TransformHandler.js.map +1 -1
- package/resolver/app/transform/middleware.js +13 -20
- package/resolver/app/transform/middleware.js.map +1 -1
- package/resolver/app/utils/Retry.js +21 -24
- package/resolver/app/utils/Retry.js.map +1 -1
- package/resolver/app/utils/sleep.js +4 -5
- package/resolver/app/utils/sleep.js.map +1 -1
- package/resolver/app/validation/body.js +26 -31
- package/resolver/app/validation/body.js.map +1 -1
- package/resolver/app/validation/createJsonTransform.js +12 -13
- package/resolver/app/validation/createJsonTransform.js.map +1 -1
- package/resolver/app/validation/detail.js +30 -35
- package/resolver/app/validation/detail.js.map +1 -1
- package/resolver/app/validation/event.js +17 -18
- package/resolver/app/validation/event.js.map +1 -1
- package/resolver/app/validation/numericString.js +8 -11
- package/resolver/app/validation/numericString.js.map +1 -1
- package/resolver/app/validation/system.js +8 -9
- package/resolver/app/validation/system.js.map +1 -1
- package/resolver/createEventHandlerPlugin.js +70 -99
- package/resolver/createEventHandlerPlugin.js.map +1 -1
- package/resolver/createResolverHandler.js +49 -57
- package/resolver/createResolverHandler.js.map +1 -1
- package/resolver/deployment/Deployment.js +28 -31
- package/resolver/deployment/Deployment.js.map +1 -1
- package/resolver/deployment/Deployments.js +20 -23
- package/resolver/deployment/Deployments.js.map +1 -1
- package/resolver/deployment/DeploymentsFetcher.js +75 -84
- package/resolver/deployment/DeploymentsFetcher.js.map +1 -1
- package/resolver/deployment/types.js +0 -3
- package/resolver/lambda/LambdaTrigger.js +24 -26
- package/resolver/lambda/LambdaTrigger.js.map +1 -1
- package/resolver/plugins/CommandHandlerPlugin.js +16 -15
- package/resolver/plugins/CommandHandlerPlugin.js.map +1 -1
- package/resolver/plugins/StorerAfterEachPlugin.js +20 -19
- package/resolver/plugins/StorerAfterEachPlugin.js.map +1 -1
- package/resolver/plugins/TransformRecordPlugin.js +19 -18
- package/resolver/plugins/TransformRecordPlugin.js.map +1 -1
- package/resolver/recordTypes/fileManager/CopyFile.js +44 -67
- package/resolver/recordTypes/fileManager/CopyFile.js.map +1 -1
- package/resolver/recordTypes/fileManager/DeleteFile.js +40 -58
- package/resolver/recordTypes/fileManager/DeleteFile.js.map +1 -1
- package/resolver/recordTypes/fileManager/fileManager.js +5 -3
- package/resolver/recordTypes/fileManager/fileManager.js.map +1 -1
- package/resolver/recordTypes/fileManager/fileManagerOnDelete.js +24 -35
- package/resolver/recordTypes/fileManager/fileManagerOnDelete.js.map +1 -1
- package/resolver/recordTypes/fileManager/fileManagerOnPut.js +24 -35
- package/resolver/recordTypes/fileManager/fileManagerOnPut.js.map +1 -1
- package/resolver/recordTypes/fileManager/shouldBeHandled.js +10 -27
- package/resolver/recordTypes/fileManager/shouldBeHandled.js.map +1 -1
- package/resolver/recordTypes/fileManager/types.js +0 -3
- package/resolver/recordTypes/users/CopyUser.js +44 -54
- package/resolver/recordTypes/users/CopyUser.js.map +1 -1
- package/resolver/recordTypes/users/DeleteUser.js +40 -51
- package/resolver/recordTypes/users/DeleteUser.js.map +1 -1
- package/resolver/recordTypes/users/shouldBeHandled.js +10 -26
- package/resolver/recordTypes/users/shouldBeHandled.js.map +1 -1
- package/resolver/recordTypes/users/types.js +0 -3
- package/resolver/recordTypes/users/users.js +5 -3
- package/resolver/recordTypes/users/users.js.map +1 -1
- package/resolver/recordTypes/users/usersOnDelete.js +23 -34
- package/resolver/recordTypes/users/usersOnDelete.js.map +1 -1
- package/resolver/recordTypes/users/usersOnPut.js +24 -35
- package/resolver/recordTypes/users/usersOnPut.js.map +1 -1
- package/sync/FilterOutRecord.js +9 -17
- package/sync/FilterOutRecord.js.map +1 -1
- package/sync/attachToDynamoDbDocument.js +46 -65
- package/sync/attachToDynamoDbDocument.js.map +1 -1
- package/sync/createHandler.js +31 -40
- package/sync/createHandler.js.map +1 -1
- package/sync/createSendDataToEventBridgeOnRequestEnd.js +32 -37
- package/sync/createSendDataToEventBridgeOnRequestEnd.js.map +1 -1
- package/sync/createSyncSystem.js +24 -26
- package/sync/createSyncSystem.js.map +1 -1
- package/sync/filter/createDefaultFilterOutRecordPlugins.js +37 -76
- package/sync/filter/createDefaultFilterOutRecordPlugins.js.map +1 -1
- package/sync/handler/Handler.js +64 -76
- package/sync/handler/Handler.js.map +1 -1
- package/sync/handler/HandlerConverter.js +21 -29
- package/sync/handler/HandlerConverter.js.map +1 -1
- package/sync/handler/converter/BatchGetCommandConverter.js +12 -11
- package/sync/handler/converter/BatchGetCommandConverter.js.map +1 -1
- package/sync/handler/converter/BatchWriteCommandConverter.js +12 -11
- package/sync/handler/converter/BatchWriteCommandConverter.js.map +1 -1
- package/sync/handler/converter/DeleteCommandConverter.js +12 -11
- package/sync/handler/converter/DeleteCommandConverter.js.map +1 -1
- package/sync/handler/converter/GetCommandConverter.js +12 -11
- package/sync/handler/converter/GetCommandConverter.js.map +1 -1
- package/sync/handler/converter/PutCommandConverter.js +12 -11
- package/sync/handler/converter/PutCommandConverter.js.map +1 -1
- package/sync/handler/converter/QueryCommandConverter.js +12 -11
- package/sync/handler/converter/QueryCommandConverter.js.map +1 -1
- package/sync/handler/converter/ScanCommandConverter.js +12 -11
- package/sync/handler/converter/ScanCommandConverter.js.map +1 -1
- package/sync/handler/converter/UpdateCommandConverter.js +12 -11
- package/sync/handler/converter/UpdateCommandConverter.js.map +1 -1
- package/sync/handler/converter/commands/BatchWriteCommandValue.js +34 -37
- package/sync/handler/converter/commands/BatchWriteCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/DeleteCommandValue.js +19 -16
- package/sync/handler/converter/commands/DeleteCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/NullCommandValue.js +8 -8
- package/sync/handler/converter/commands/NullCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/PutCommandValue.js +19 -16
- package/sync/handler/converter/commands/PutCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/UpdateCommandValue.js +19 -16
- package/sync/handler/converter/commands/UpdateCommandValue.js.map +1 -1
- package/sync/handler/types.js +0 -3
- package/sync/plugins/FilterOutRecordPlugin.js +17 -19
- package/sync/plugins/FilterOutRecordPlugin.js.map +1 -1
- package/sync/requestPlugin.js +33 -48
- package/sync/requestPlugin.js.map +1 -1
- package/sync/types.js +0 -3
- package/sync/utils/getTableType.js +10 -9
- package/sync/utils/getTableType.js.map +1 -1
- package/sync/utils/manifest.js +28 -35
- package/sync/utils/manifest.js.map +1 -1
- package/sync/utils/validateSystemInput.js +21 -29
- package/sync/utils/validateSystemInput.js.map +1 -1
- package/types.js +6 -5
- package/types.js.map +1 -1
- package/utils/createSystemName.js +5 -3
- package/utils/createSystemName.js.map +1 -1
- package/worker/actions/copyFile/CopyFile.js +183 -258
- package/worker/actions/copyFile/CopyFile.js.map +1 -1
- package/worker/actions/copyFile/copyFileAction.js +26 -34
- package/worker/actions/copyFile/copyFileAction.js.map +1 -1
- package/worker/actions/copyFile/copyFileSchema.js +13 -14
- package/worker/actions/copyFile/copyFileSchema.js.map +1 -1
- package/worker/actions/copyFile/types.js +0 -3
- package/worker/actions/createUser/CreateUser.js +51 -74
- package/worker/actions/createUser/CreateUser.js.map +1 -1
- package/worker/actions/createUser/createUserAction.js +24 -28
- package/worker/actions/createUser/createUserAction.js.map +1 -1
- package/worker/actions/createUser/createUserSchema.js +13 -14
- package/worker/actions/createUser/createUserSchema.js.map +1 -1
- package/worker/actions/createUser/types.js +0 -3
- package/worker/actions/deleteFile/DeleteFile.js +52 -67
- package/worker/actions/deleteFile/DeleteFile.js.map +1 -1
- package/worker/actions/deleteFile/deleteFileAction.js +21 -29
- package/worker/actions/deleteFile/deleteFileAction.js.map +1 -1
- package/worker/actions/deleteFile/deleteFileSchema.js +13 -14
- package/worker/actions/deleteFile/deleteFileSchema.js.map +1 -1
- package/worker/actions/deleteFile/types.js +0 -3
- package/worker/actions/deleteUser/DeleteUser.js +38 -52
- package/worker/actions/deleteUser/DeleteUser.js.map +1 -1
- package/worker/actions/deleteUser/deleteUserAction.js +22 -26
- package/worker/actions/deleteUser/deleteUserAction.js.map +1 -1
- package/worker/actions/deleteUser/deleteUserSchema.js +12 -13
- package/worker/actions/deleteUser/deleteUserSchema.js.map +1 -1
- package/worker/actions/deleteUser/types.js +0 -3
- package/worker/actions/logValidationError.js +5 -6
- package/worker/actions/logValidationError.js.map +1 -1
- package/worker/actions/removeCognitoUserAttributes.js +9 -15
- package/worker/actions/removeCognitoUserAttributes.js.map +1 -1
- package/worker/actions/updateUser/UpdateUser.js +48 -65
- package/worker/actions/updateUser/UpdateUser.js.map +1 -1
- package/worker/actions/updateUser/types.js +0 -3
- package/worker/actions/updateUser/updateUserAction.js +24 -28
- package/worker/actions/updateUser/updateUserAction.js.map +1 -1
- package/worker/actions/updateUser/updateUserSchema.js +13 -14
- package/worker/actions/updateUser/updateUserSchema.js.map +1 -1
- package/worker/createWorkerHandler.js +24 -30
- package/worker/createWorkerHandler.js.map +1 -1
- package/worker/handler/WorkerActionHandler.js +19 -21
- package/worker/handler/WorkerActionHandler.js.map +1 -1
- package/worker/handler/eventHandler.js +14 -16
- package/worker/handler/eventHandler.js.map +1 -1
- package/worker/plugins/WorkerActionPlugin.js +17 -16
- package/worker/plugins/WorkerActionPlugin.js.map +1 -1
- package/worker/types.js +0 -3
- package/index.js.map +0 -1
- package/resolver/app/abstractions/RecordHandler.js.map +0 -1
- package/resolver/app/abstractions/RecordsValidation.js.map +0 -1
- package/resolver/app/abstractions/ResolverApplication.js.map +0 -1
- package/resolver/app/abstractions/ResolverRecord.js.map +0 -1
- package/resolver/app/bundler/types.js.map +0 -1
- package/resolver/app/data/types.js.map +0 -1
- package/resolver/app/fetcher/types.js.map +0 -1
- package/resolver/app/ingestor/types.js.map +0 -1
- package/resolver/app/storer/types.js.map +0 -1
- package/resolver/deployment/types.js.map +0 -1
- package/resolver/recordTypes/fileManager/types.js.map +0 -1
- package/resolver/recordTypes/users/types.js.map +0 -1
- package/sync/handler/types.js.map +0 -1
- package/sync/types.js.map +0 -1
- package/worker/actions/copyFile/types.js.map +0 -1
- package/worker/actions/createUser/types.js.map +0 -1
- package/worker/actions/deleteFile/types.js.map +0 -1
- package/worker/actions/deleteUser/types.js.map +0 -1
- package/worker/actions/updateUser/types.js.map +0 -1
- package/worker/types.js.map +0 -1
|
@@ -1,275 +1,200 @@
|
|
|
1
1
|
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CopyObjectCommand, CreateMultipartUploadCommand, HeadObjectCommand, UploadPartCopyCommand } from "@webiny/aws-sdk/client-s3/index.js";
|
|
2
2
|
import bytes from "bytes";
|
|
3
3
|
import { convertException } from "@webiny/utils";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
this.targetClient = createS3Client({
|
|
18
|
-
region: targetRegion
|
|
19
|
-
});
|
|
20
|
-
this.maxConcurrency = maxConcurrency;
|
|
21
|
-
this.minPartSizeBytes = this.parseByteSize(minPartSize);
|
|
22
|
-
this.maxPartSizeBytes = this.parseByteSize(maxPartSize);
|
|
23
|
-
if (this.minPartSizeBytes < 5 * 1024 * 1024) {
|
|
24
|
-
throw new Error("minPartSize must be at least 5MB.");
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
async copy(params) {
|
|
28
|
-
const {
|
|
29
|
-
key,
|
|
30
|
-
sourceBucket,
|
|
31
|
-
targetBucket
|
|
32
|
-
} = params;
|
|
33
|
-
const sourceHead = await this.headObject({
|
|
34
|
-
client: this.sourceClient,
|
|
35
|
-
bucket: sourceBucket,
|
|
36
|
-
key
|
|
37
|
-
});
|
|
38
|
-
if (!sourceHead?.ContentLength || !sourceHead.ETag) {
|
|
39
|
-
throw new Error("Source object metadata is invalid or missing.");
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* If the target object already exists, we can skip the copy operation.
|
|
43
|
-
*/
|
|
44
|
-
const targetHead = await this.headObject({
|
|
45
|
-
client: this.targetClient,
|
|
46
|
-
bucket: targetBucket,
|
|
47
|
-
key
|
|
48
|
-
});
|
|
49
|
-
if (targetHead) {
|
|
50
|
-
return;
|
|
4
|
+
class CopyFile {
|
|
5
|
+
constructor(params){
|
|
6
|
+
const { createS3Client, sourceRegion, targetRegion, maxConcurrency = 100, minPartSize = "5MB", maxPartSize = "1GB" } = params;
|
|
7
|
+
this.sourceClient = createS3Client({
|
|
8
|
+
region: sourceRegion
|
|
9
|
+
});
|
|
10
|
+
this.targetClient = createS3Client({
|
|
11
|
+
region: targetRegion
|
|
12
|
+
});
|
|
13
|
+
this.maxConcurrency = maxConcurrency;
|
|
14
|
+
this.minPartSizeBytes = this.parseByteSize(minPartSize);
|
|
15
|
+
this.maxPartSizeBytes = this.parseByteSize(maxPartSize);
|
|
16
|
+
if (this.minPartSizeBytes < 5242880) throw new Error("minPartSize must be at least 5MB.");
|
|
51
17
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
});
|
|
59
|
-
const completedParts = [];
|
|
60
|
-
try {
|
|
61
|
-
let partNumber = 1;
|
|
62
|
-
let start = 0;
|
|
63
|
-
const promises = [];
|
|
64
|
-
let active = 0;
|
|
65
|
-
while (start < totalSize) {
|
|
66
|
-
const end = Math.min(start + partSize - 1, totalSize - 1);
|
|
67
|
-
const copyPromise = this.uploadPartCopy({
|
|
68
|
-
sourceBucket,
|
|
69
|
-
sourceKey: key,
|
|
70
|
-
targetBucket,
|
|
71
|
-
targetKey: key,
|
|
72
|
-
uploadId,
|
|
73
|
-
partNumber,
|
|
74
|
-
start,
|
|
75
|
-
end,
|
|
76
|
-
completedParts
|
|
18
|
+
async copy(params) {
|
|
19
|
+
const { key, sourceBucket, targetBucket } = params;
|
|
20
|
+
const sourceHead = await this.headObject({
|
|
21
|
+
client: this.sourceClient,
|
|
22
|
+
bucket: sourceBucket,
|
|
23
|
+
key
|
|
77
24
|
});
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
25
|
+
if (!sourceHead?.ContentLength || !sourceHead.ETag) throw new Error("Source object metadata is invalid or missing.");
|
|
26
|
+
const targetHead = await this.headObject({
|
|
27
|
+
client: this.targetClient,
|
|
28
|
+
bucket: targetBucket,
|
|
29
|
+
key
|
|
30
|
+
});
|
|
31
|
+
if (targetHead) return;
|
|
32
|
+
const totalSize = sourceHead.ContentLength;
|
|
33
|
+
let partSize = Math.max(this.minPartSizeBytes, Math.min(this.maxPartSizeBytes, Math.ceil(totalSize / this.maxConcurrency)));
|
|
34
|
+
partSize = Math.min(partSize, totalSize);
|
|
35
|
+
const uploadId = await this.createMultipartUpload({
|
|
36
|
+
targetBucket,
|
|
37
|
+
targetKey: key
|
|
38
|
+
});
|
|
39
|
+
const completedParts = [];
|
|
40
|
+
try {
|
|
41
|
+
let partNumber = 1;
|
|
42
|
+
let start = 0;
|
|
43
|
+
const promises = [];
|
|
44
|
+
let active = 0;
|
|
45
|
+
while(start < totalSize){
|
|
46
|
+
const end = Math.min(start + partSize - 1, totalSize - 1);
|
|
47
|
+
const copyPromise = this.uploadPartCopy({
|
|
48
|
+
sourceBucket,
|
|
49
|
+
sourceKey: key,
|
|
50
|
+
targetBucket,
|
|
51
|
+
targetKey: key,
|
|
52
|
+
uploadId,
|
|
53
|
+
partNumber,
|
|
54
|
+
start,
|
|
55
|
+
end,
|
|
56
|
+
completedParts
|
|
57
|
+
});
|
|
58
|
+
promises.push(copyPromise);
|
|
59
|
+
active++;
|
|
60
|
+
if (active >= this.maxConcurrency) {
|
|
61
|
+
await Promise.race(promises);
|
|
62
|
+
active--;
|
|
63
|
+
}
|
|
64
|
+
partNumber++;
|
|
65
|
+
start += partSize;
|
|
66
|
+
}
|
|
67
|
+
await Promise.all(promises);
|
|
68
|
+
await this.completeMultipartUpload({
|
|
69
|
+
targetBucket,
|
|
70
|
+
targetKey: key,
|
|
71
|
+
uploadId,
|
|
72
|
+
completedParts
|
|
73
|
+
});
|
|
74
|
+
} catch (ex) {
|
|
75
|
+
await this.abortMultipartUpload({
|
|
76
|
+
targetBucket,
|
|
77
|
+
targetKey: key,
|
|
78
|
+
uploadId
|
|
79
|
+
});
|
|
80
|
+
throw ex;
|
|
81
|
+
}
|
|
82
|
+
try {
|
|
83
|
+
await this.copyMetadata({
|
|
84
|
+
sourceBucket,
|
|
85
|
+
targetBucket,
|
|
86
|
+
key
|
|
87
|
+
});
|
|
88
|
+
} catch (ex) {
|
|
89
|
+
console.error(`Failed to copy metadata for ${key} from ${sourceBucket} to ${targetBucket}.`);
|
|
90
|
+
console.log(convertException(ex));
|
|
83
91
|
}
|
|
84
|
-
partNumber++;
|
|
85
|
-
start += partSize;
|
|
86
|
-
}
|
|
87
|
-
await Promise.all(promises);
|
|
88
|
-
/**
|
|
89
|
-
* Complete the multipart upload with all the completed parts.
|
|
90
|
-
*/
|
|
91
|
-
await this.completeMultipartUpload({
|
|
92
|
-
targetBucket,
|
|
93
|
-
targetKey: key,
|
|
94
|
-
uploadId,
|
|
95
|
-
completedParts
|
|
96
|
-
});
|
|
97
|
-
} catch (ex) {
|
|
98
|
-
await this.abortMultipartUpload({
|
|
99
|
-
targetBucket,
|
|
100
|
-
targetKey: key,
|
|
101
|
-
uploadId
|
|
102
|
-
});
|
|
103
|
-
throw ex;
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* And then copy the metadata file.
|
|
107
|
-
*/
|
|
108
|
-
|
|
109
|
-
try {
|
|
110
|
-
await this.copyMetadata({
|
|
111
|
-
sourceBucket,
|
|
112
|
-
targetBucket,
|
|
113
|
-
key
|
|
114
|
-
});
|
|
115
|
-
} catch (ex) {
|
|
116
|
-
console.error(`Failed to copy metadata for ${key} from ${sourceBucket} to ${targetBucket}.`);
|
|
117
|
-
console.log(convertException(ex));
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
parseByteSize(value) {
|
|
121
|
-
if (typeof value === "number") {
|
|
122
|
-
return value;
|
|
123
92
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
93
|
+
parseByteSize(value) {
|
|
94
|
+
if ("number" == typeof value) return value;
|
|
95
|
+
const parsed = bytes.parse(value);
|
|
96
|
+
if ("number" == typeof parsed) return parsed;
|
|
97
|
+
const typeOfValue = typeof value;
|
|
98
|
+
throw new Error(`Invalid byte size value type "${typeOfValue}".`);
|
|
127
99
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
const result = await client.send(command);
|
|
144
|
-
if (result.$metadata?.httpStatusCode === 200) {
|
|
145
|
-
return result;
|
|
146
|
-
}
|
|
147
|
-
} catch (ex) {
|
|
148
|
-
if (ex.name === "NotFound" || ex.$metadata?.httpStatusCode === 404) {
|
|
100
|
+
async headObject(params) {
|
|
101
|
+
const { client, bucket, key } = params;
|
|
102
|
+
const input = {
|
|
103
|
+
Bucket: bucket,
|
|
104
|
+
Key: key
|
|
105
|
+
};
|
|
106
|
+
const command = new HeadObjectCommand(input);
|
|
107
|
+
try {
|
|
108
|
+
const result = await client.send(command);
|
|
109
|
+
if (result.$metadata?.httpStatusCode === 200) return result;
|
|
110
|
+
} catch (ex) {
|
|
111
|
+
if ("NotFound" === ex.name || ex.$metadata?.httpStatusCode === 404) return null;
|
|
112
|
+
console.error(`Failed to head object ${key} in bucket ${bucket}.`);
|
|
113
|
+
console.log(convertException(ex));
|
|
114
|
+
}
|
|
149
115
|
return null;
|
|
150
|
-
}
|
|
151
|
-
console.error(`Failed to head object ${key} in bucket ${bucket}.`);
|
|
152
|
-
console.log(convertException(ex));
|
|
153
116
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
117
|
+
async createMultipartUpload(params) {
|
|
118
|
+
const { targetBucket, targetKey } = params;
|
|
119
|
+
const input = {
|
|
120
|
+
Bucket: targetBucket,
|
|
121
|
+
Key: targetKey
|
|
122
|
+
};
|
|
123
|
+
const command = new CreateMultipartUploadCommand(input);
|
|
124
|
+
const response = await this.targetClient.send(command);
|
|
125
|
+
if (!response.UploadId) throw new Error("Failed to create multipart upload.");
|
|
126
|
+
return response.UploadId;
|
|
127
|
+
}
|
|
128
|
+
async uploadPartCopy(params) {
|
|
129
|
+
const { sourceBucket, sourceKey, targetBucket, targetKey, uploadId, partNumber, start, end, completedParts } = params;
|
|
130
|
+
const input = {
|
|
131
|
+
Bucket: targetBucket,
|
|
132
|
+
Key: targetKey,
|
|
133
|
+
UploadId: uploadId,
|
|
134
|
+
PartNumber: partNumber,
|
|
135
|
+
CopySource: encodeURIComponent(`${sourceBucket}/${sourceKey}`),
|
|
136
|
+
CopySourceRange: `bytes=${start}-${end}`
|
|
137
|
+
};
|
|
138
|
+
const command = new UploadPartCopyCommand(input);
|
|
139
|
+
const response = await this.targetClient.send(command);
|
|
140
|
+
if (!response.CopyPartResult?.ETag) throw new Error(`UploadPartCopy failed for part ${partNumber}`);
|
|
141
|
+
completedParts.push({
|
|
142
|
+
ETag: response.CopyPartResult.ETag,
|
|
143
|
+
PartNumber: partNumber
|
|
144
|
+
});
|
|
169
145
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
Bucket: targetBucket,
|
|
186
|
-
Key: targetKey,
|
|
187
|
-
UploadId: uploadId,
|
|
188
|
-
PartNumber: partNumber,
|
|
189
|
-
CopySource: encodeURIComponent(`${sourceBucket}/${sourceKey}`),
|
|
190
|
-
CopySourceRange: `bytes=${start}-${end}`
|
|
191
|
-
};
|
|
192
|
-
const command = new UploadPartCopyCommand(input);
|
|
193
|
-
const response = await this.targetClient.send(command);
|
|
194
|
-
if (!response.CopyPartResult?.ETag) {
|
|
195
|
-
throw new Error(`UploadPartCopy failed for part ${partNumber}`);
|
|
146
|
+
async completeMultipartUpload(params) {
|
|
147
|
+
const { targetBucket, targetKey, uploadId, completedParts } = params;
|
|
148
|
+
const sortedParts = [
|
|
149
|
+
...completedParts
|
|
150
|
+
].sort((a, b)=>a.PartNumber - b.PartNumber);
|
|
151
|
+
const input = {
|
|
152
|
+
Bucket: targetBucket,
|
|
153
|
+
Key: targetKey,
|
|
154
|
+
UploadId: uploadId,
|
|
155
|
+
MultipartUpload: {
|
|
156
|
+
Parts: sortedParts
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
const command = new CompleteMultipartUploadCommand(input);
|
|
160
|
+
return await this.targetClient.send(command);
|
|
196
161
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
uploadId,
|
|
207
|
-
completedParts
|
|
208
|
-
} = params;
|
|
209
|
-
const sortedParts = [...completedParts].sort((a, b) => {
|
|
210
|
-
return a.PartNumber - b.PartNumber;
|
|
211
|
-
});
|
|
212
|
-
const input = {
|
|
213
|
-
Bucket: targetBucket,
|
|
214
|
-
Key: targetKey,
|
|
215
|
-
UploadId: uploadId,
|
|
216
|
-
MultipartUpload: {
|
|
217
|
-
Parts: sortedParts
|
|
218
|
-
}
|
|
219
|
-
};
|
|
220
|
-
const command = new CompleteMultipartUploadCommand(input);
|
|
221
|
-
return await this.targetClient.send(command);
|
|
222
|
-
}
|
|
223
|
-
async abortMultipartUpload(params) {
|
|
224
|
-
const {
|
|
225
|
-
targetBucket,
|
|
226
|
-
targetKey,
|
|
227
|
-
uploadId
|
|
228
|
-
} = params;
|
|
229
|
-
const input = {
|
|
230
|
-
Bucket: targetBucket,
|
|
231
|
-
Key: targetKey,
|
|
232
|
-
UploadId: uploadId
|
|
233
|
-
};
|
|
234
|
-
const command = new AbortMultipartUploadCommand(input);
|
|
235
|
-
await this.targetClient.send(command);
|
|
236
|
-
}
|
|
237
|
-
async copyMetadata(params) {
|
|
238
|
-
const {
|
|
239
|
-
sourceBucket,
|
|
240
|
-
targetBucket,
|
|
241
|
-
key
|
|
242
|
-
} = params;
|
|
243
|
-
const metadataKey = `${key}.metadata`;
|
|
244
|
-
const [sourceMetadata, targetMetadata] = await Promise.all([this.headObject({
|
|
245
|
-
client: this.sourceClient,
|
|
246
|
-
bucket: sourceBucket,
|
|
247
|
-
key: metadataKey
|
|
248
|
-
}).catch(() => null), this.headObject({
|
|
249
|
-
client: this.targetClient,
|
|
250
|
-
bucket: targetBucket,
|
|
251
|
-
key: metadataKey
|
|
252
|
-
}).catch(() => null)]);
|
|
253
|
-
/**
|
|
254
|
-
* If the source metadata does not exist, we cannot copy it.
|
|
255
|
-
*/
|
|
256
|
-
if (!sourceMetadata?.ETag) {
|
|
257
|
-
return null;
|
|
162
|
+
async abortMultipartUpload(params) {
|
|
163
|
+
const { targetBucket, targetKey, uploadId } = params;
|
|
164
|
+
const input = {
|
|
165
|
+
Bucket: targetBucket,
|
|
166
|
+
Key: targetKey,
|
|
167
|
+
UploadId: uploadId
|
|
168
|
+
};
|
|
169
|
+
const command = new AbortMultipartUploadCommand(input);
|
|
170
|
+
await this.targetClient.send(command);
|
|
258
171
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
172
|
+
async copyMetadata(params) {
|
|
173
|
+
const { sourceBucket, targetBucket, key } = params;
|
|
174
|
+
const metadataKey = `${key}.metadata`;
|
|
175
|
+
const [sourceMetadata, targetMetadata] = await Promise.all([
|
|
176
|
+
this.headObject({
|
|
177
|
+
client: this.sourceClient,
|
|
178
|
+
bucket: sourceBucket,
|
|
179
|
+
key: metadataKey
|
|
180
|
+
}).catch(()=>null),
|
|
181
|
+
this.headObject({
|
|
182
|
+
client: this.targetClient,
|
|
183
|
+
bucket: targetBucket,
|
|
184
|
+
key: metadataKey
|
|
185
|
+
}).catch(()=>null)
|
|
186
|
+
]);
|
|
187
|
+
if (!sourceMetadata?.ETag) return null;
|
|
188
|
+
if (targetMetadata) return null;
|
|
189
|
+
const input = {
|
|
190
|
+
Bucket: targetBucket,
|
|
191
|
+
Key: metadataKey,
|
|
192
|
+
CopySource: encodeURIComponent(`${sourceBucket}/${metadataKey}`)
|
|
193
|
+
};
|
|
194
|
+
const command = new CopyObjectCommand(input);
|
|
195
|
+
return await this.targetClient.send(command);
|
|
264
196
|
}
|
|
265
|
-
const input = {
|
|
266
|
-
Bucket: targetBucket,
|
|
267
|
-
Key: metadataKey,
|
|
268
|
-
CopySource: encodeURIComponent(`${sourceBucket}/${metadataKey}`)
|
|
269
|
-
};
|
|
270
|
-
const command = new CopyObjectCommand(input);
|
|
271
|
-
return await this.targetClient.send(command);
|
|
272
|
-
}
|
|
273
197
|
}
|
|
198
|
+
export { CopyFile };
|
|
274
199
|
|
|
275
200
|
//# sourceMappingURL=CopyFile.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["AbortMultipartUploadCommand","CompleteMultipartUploadCommand","CopyObjectCommand","CreateMultipartUploadCommand","HeadObjectCommand","UploadPartCopyCommand","bytes","convertException","CopyFile","constructor","params","createS3Client","sourceRegion","targetRegion","maxConcurrency","minPartSize","maxPartSize","sourceClient","region","targetClient","minPartSizeBytes","parseByteSize","maxPartSizeBytes","Error","copy","key","sourceBucket","targetBucket","sourceHead","headObject","client","bucket","ContentLength","ETag","targetHead","totalSize","partSize","Math","max","min","ceil","uploadId","createMultipartUpload","targetKey","completedParts","partNumber","start","promises","active","end","copyPromise","uploadPartCopy","sourceKey","push","Promise","race","all","completeMultipartUpload","ex","abortMultipartUpload","copyMetadata","console","error","log","value","parsed","parse","typeOfValue","input","Bucket","Key","command","result","send","$metadata","httpStatusCode","name","response","UploadId","PartNumber","CopySource","encodeURIComponent","CopySourceRange","CopyPartResult","sortedParts","sort","a","b","MultipartUpload","Parts","metadataKey","sourceMetadata","targetMetadata","catch"],"sources":["CopyFile.ts"],"sourcesContent":["import type {\n AbortMultipartUploadCommandInput,\n CompletedPart,\n CompleteMultipartUploadCommandInput,\n CopyObjectCommandInput,\n CopyObjectCommandOutput,\n CreateMultipartUploadCommandInput,\n HeadObjectCommandInput,\n S3Client,\n S3ClientConfig,\n UploadPartCopyCommandInput\n} from \"@webiny/aws-sdk/client-s3/index.js\";\nimport {\n AbortMultipartUploadCommand,\n CompleteMultipartUploadCommand,\n CopyObjectCommand,\n CreateMultipartUploadCommand,\n HeadObjectCommand,\n UploadPartCopyCommand\n} from \"@webiny/aws-sdk/client-s3/index.js\";\nimport bytes from \"bytes\";\nimport { convertException } from \"@webiny/utils\";\n\ntype ByteSize = number | `${number}B` | `${number}KB` | `${number}MB` | `${number}GB`;\n\nexport interface ICopyFileParams {\n createS3Client: (config: Partial<S3ClientConfig>) => Pick<S3Client, \"send\">;\n sourceRegion: string;\n targetRegion: string;\n maxConcurrency?: number;\n minPartSize?: ByteSize;\n maxPartSize?: ByteSize;\n}\n\nexport interface ICopyParams {\n key: string;\n sourceBucket: string;\n targetBucket: string;\n}\n\nexport interface IHeadObjectParams {\n client: Pick<S3Client, \"send\">;\n bucket: string;\n key: string;\n}\n\nexport interface ICreateMultipartUploadParams {\n targetBucket: string;\n targetKey: string;\n}\n\nexport interface IUploadPartCopyParams {\n sourceBucket: string;\n sourceKey: string;\n targetBucket: string;\n targetKey: string;\n uploadId: string;\n partNumber: number;\n start: number;\n end: number;\n completedParts: CompletedPart[];\n}\n\nexport interface ICompleteMultipartUploadParams {\n targetBucket: string;\n targetKey: string;\n uploadId: string;\n completedParts: CompletedPart[];\n}\n\nexport interface IAbortMultipartUploadParams {\n targetBucket: string;\n targetKey: string;\n uploadId: string;\n}\n\ninterface ICopyMetadataParams {\n sourceBucket: string;\n targetBucket: string;\n key: string;\n}\n\nexport class CopyFile {\n private readonly sourceClient: Pick<S3Client, \"send\">;\n private readonly targetClient: Pick<S3Client, \"send\">;\n private readonly maxConcurrency: number;\n private readonly minPartSizeBytes: number;\n private readonly maxPartSizeBytes: number;\n\n public constructor(params: ICopyFileParams) {\n const {\n createS3Client,\n sourceRegion,\n targetRegion,\n maxConcurrency = 100,\n minPartSize = \"5MB\",\n maxPartSize = \"1GB\"\n } = params;\n\n this.sourceClient = createS3Client({\n region: sourceRegion\n });\n this.targetClient = createS3Client({\n region: targetRegion\n });\n this.maxConcurrency = maxConcurrency;\n\n this.minPartSizeBytes = this.parseByteSize(minPartSize);\n this.maxPartSizeBytes = this.parseByteSize(maxPartSize);\n\n if (this.minPartSizeBytes < 5 * 1024 * 1024) {\n throw new Error(\"minPartSize must be at least 5MB.\");\n }\n }\n\n public async copy(params: ICopyParams): Promise<void> {\n const { key, sourceBucket, targetBucket } = params;\n\n const sourceHead = await this.headObject({\n client: this.sourceClient,\n bucket: sourceBucket,\n key\n });\n\n if (!sourceHead?.ContentLength || !sourceHead.ETag) {\n throw new Error(\"Source object metadata is invalid or missing.\");\n }\n /**\n * If the target object already exists, we can skip the copy operation.\n */\n const targetHead = await this.headObject({\n client: this.targetClient,\n bucket: targetBucket,\n key\n });\n\n if (targetHead) {\n return;\n }\n\n const totalSize = sourceHead.ContentLength;\n let partSize = Math.max(\n this.minPartSizeBytes,\n Math.min(this.maxPartSizeBytes, Math.ceil(totalSize / this.maxConcurrency))\n );\n partSize = Math.min(partSize, totalSize);\n\n const uploadId = await this.createMultipartUpload({\n targetBucket,\n targetKey: key\n });\n const completedParts: CompletedPart[] = [];\n\n try {\n let partNumber = 1;\n let start = 0;\n const promises: Promise<void>[] = [];\n let active = 0;\n\n while (start < totalSize) {\n const end = Math.min(start + partSize - 1, totalSize - 1);\n\n const copyPromise = this.uploadPartCopy({\n sourceBucket,\n sourceKey: key,\n targetBucket,\n targetKey: key,\n uploadId,\n partNumber,\n start,\n end,\n completedParts\n });\n\n promises.push(copyPromise);\n active++;\n\n if (active >= this.maxConcurrency) {\n await Promise.race(promises);\n active--;\n }\n\n partNumber++;\n start += partSize;\n }\n\n await Promise.all(promises);\n /**\n * Complete the multipart upload with all the completed parts.\n */\n await this.completeMultipartUpload({\n targetBucket,\n targetKey: key,\n uploadId,\n completedParts\n });\n } catch (ex) {\n await this.abortMultipartUpload({ targetBucket, targetKey: key, uploadId });\n throw ex;\n }\n /**\n * And then copy the metadata file.\n */\n\n try {\n await this.copyMetadata({\n sourceBucket,\n targetBucket,\n key\n });\n } catch (ex) {\n console.error(\n `Failed to copy metadata for ${key} from ${sourceBucket} to ${targetBucket}.`\n );\n console.log(convertException(ex));\n }\n }\n\n private parseByteSize(value: ByteSize): number {\n if (typeof value === \"number\") {\n return value;\n }\n const parsed = bytes.parse(value);\n if (typeof parsed === \"number\") {\n return parsed;\n }\n const typeOfValue = typeof value;\n throw new Error(`Invalid byte size value type \"${typeOfValue}\".`);\n }\n\n private async headObject(params: IHeadObjectParams) {\n const { client, bucket, key } = params;\n const input: HeadObjectCommandInput = {\n Bucket: bucket,\n Key: key\n };\n const command = new HeadObjectCommand(input);\n try {\n const result = await client.send(command);\n if (result.$metadata?.httpStatusCode === 200) {\n return result;\n }\n } catch (ex) {\n if (ex.name === \"NotFound\" || ex.$metadata?.httpStatusCode === 404) {\n return null;\n }\n console.error(`Failed to head object ${key} in bucket ${bucket}.`);\n console.log(convertException(ex));\n }\n return null;\n }\n\n private async createMultipartUpload(params: ICreateMultipartUploadParams): Promise<string> {\n const { targetBucket, targetKey } = params;\n const input: CreateMultipartUploadCommandInput = {\n Bucket: targetBucket,\n Key: targetKey\n };\n const command = new CreateMultipartUploadCommand(input);\n const response = await this.targetClient.send(command);\n if (!response.UploadId) {\n throw new Error(\"Failed to create multipart upload.\");\n }\n return response.UploadId;\n }\n\n private async uploadPartCopy(params: IUploadPartCopyParams): Promise<void> {\n const {\n sourceBucket,\n sourceKey,\n targetBucket,\n targetKey,\n uploadId,\n partNumber,\n start,\n end,\n completedParts\n } = params;\n\n const input: UploadPartCopyCommandInput = {\n Bucket: targetBucket,\n Key: targetKey,\n UploadId: uploadId,\n PartNumber: partNumber,\n CopySource: encodeURIComponent(`${sourceBucket}/${sourceKey}`),\n CopySourceRange: `bytes=${start}-${end}`\n };\n\n const command = new UploadPartCopyCommand(input);\n const response = await this.targetClient.send(command);\n\n if (!response.CopyPartResult?.ETag) {\n throw new Error(`UploadPartCopy failed for part ${partNumber}`);\n }\n\n completedParts.push({\n ETag: response.CopyPartResult.ETag,\n PartNumber: partNumber\n });\n }\n\n private async completeMultipartUpload(params: ICompleteMultipartUploadParams) {\n const { targetBucket, targetKey, uploadId, completedParts } = params;\n\n const sortedParts = [...completedParts].sort((a, b) => {\n return a.PartNumber! - b.PartNumber!;\n });\n\n const input: CompleteMultipartUploadCommandInput = {\n Bucket: targetBucket,\n Key: targetKey,\n UploadId: uploadId,\n MultipartUpload: { Parts: sortedParts }\n };\n\n const command = new CompleteMultipartUploadCommand(input);\n return await this.targetClient.send(command);\n }\n\n private async abortMultipartUpload(params: IAbortMultipartUploadParams) {\n const { targetBucket, targetKey, uploadId } = params;\n\n const input: AbortMultipartUploadCommandInput = {\n Bucket: targetBucket,\n Key: targetKey,\n UploadId: uploadId\n };\n const command = new AbortMultipartUploadCommand(input);\n await this.targetClient.send(command);\n }\n\n private async copyMetadata(\n params: ICopyMetadataParams\n ): Promise<CopyObjectCommandOutput | null> {\n const { sourceBucket, targetBucket, key } = params;\n\n const metadataKey = `${key}.metadata`;\n\n const [sourceMetadata, targetMetadata] = await Promise.all([\n this.headObject({\n client: this.sourceClient,\n bucket: sourceBucket,\n key: metadataKey\n }).catch(() => null),\n this.headObject({\n client: this.targetClient,\n bucket: targetBucket,\n key: metadataKey\n }).catch(() => null)\n ]);\n /**\n * If the source metadata does not exist, we cannot copy it.\n */\n if (!sourceMetadata?.ETag) {\n return null;\n }\n /**\n * If the target metadata already exists, we can skip copying it.\n */\n if (targetMetadata) {\n return null;\n }\n\n const input: CopyObjectCommandInput = {\n Bucket: targetBucket,\n Key: metadataKey,\n CopySource: encodeURIComponent(`${sourceBucket}/${metadataKey}`)\n };\n\n const command = new CopyObjectCommand(input);\n\n return await this.targetClient.send(command);\n }\n}\n"],"mappings":"AAYA,SACIA,2BAA2B,EAC3BC,8BAA8B,EAC9BC,iBAAiB,EACjBC,4BAA4B,EAC5BC,iBAAiB,EACjBC,qBAAqB,QAClB,oCAAoC;AAC3C,OAAOC,KAAK,MAAM,OAAO;AACzB,SAASC,gBAAgB,QAAQ,eAAe;AA6DhD,OAAO,MAAMC,QAAQ,CAAC;EAOXC,WAAWA,CAACC,MAAuB,EAAE;IACxC,MAAM;MACFC,cAAc;MACdC,YAAY;MACZC,YAAY;MACZC,cAAc,GAAG,GAAG;MACpBC,WAAW,GAAG,KAAK;MACnBC,WAAW,GAAG;IAClB,CAAC,GAAGN,MAAM;IAEV,IAAI,CAACO,YAAY,GAAGN,cAAc,CAAC;MAC/BO,MAAM,EAAEN;IACZ,CAAC,CAAC;IACF,IAAI,CAACO,YAAY,GAAGR,cAAc,CAAC;MAC/BO,MAAM,EAAEL;IACZ,CAAC,CAAC;IACF,IAAI,CAACC,cAAc,GAAGA,cAAc;IAEpC,IAAI,CAACM,gBAAgB,GAAG,IAAI,CAACC,aAAa,CAACN,WAAW,CAAC;IACvD,IAAI,CAACO,gBAAgB,GAAG,IAAI,CAACD,aAAa,CAACL,WAAW,CAAC;IAEvD,IAAI,IAAI,CAACI,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE;MACzC,MAAM,IAAIG,KAAK,CAAC,mCAAmC,CAAC;IACxD;EACJ;EAEA,MAAaC,IAAIA,CAACd,MAAmB,EAAiB;IAClD,MAAM;MAAEe,GAAG;MAAEC,YAAY;MAAEC;IAAa,CAAC,GAAGjB,MAAM;IAElD,MAAMkB,UAAU,GAAG,MAAM,IAAI,CAACC,UAAU,CAAC;MACrCC,MAAM,EAAE,IAAI,CAACb,YAAY;MACzBc,MAAM,EAAEL,YAAY;MACpBD;IACJ,CAAC,CAAC;IAEF,IAAI,CAACG,UAAU,EAAEI,aAAa,IAAI,CAACJ,UAAU,CAACK,IAAI,EAAE;MAChD,MAAM,IAAIV,KAAK,CAAC,+CAA+C,CAAC;IACpE;IACA;AACR;AACA;IACQ,MAAMW,UAAU,GAAG,MAAM,IAAI,CAACL,UAAU,CAAC;MACrCC,MAAM,EAAE,IAAI,CAACX,YAAY;MACzBY,MAAM,EAAEJ,YAAY;MACpBF;IACJ,CAAC,CAAC;IAEF,IAAIS,UAAU,EAAE;MACZ;IACJ;IAEA,MAAMC,SAAS,GAAGP,UAAU,CAACI,aAAa;IAC1C,IAAII,QAAQ,GAAGC,IAAI,CAACC,GAAG,CACnB,IAAI,CAAClB,gBAAgB,EACrBiB,IAAI,CAACE,GAAG,CAAC,IAAI,CAACjB,gBAAgB,EAAEe,IAAI,CAACG,IAAI,CAACL,SAAS,GAAG,IAAI,CAACrB,cAAc,CAAC,CAC9E,CAAC;IACDsB,QAAQ,GAAGC,IAAI,CAACE,GAAG,CAACH,QAAQ,EAAED,SAAS,CAAC;IAExC,MAAMM,QAAQ,GAAG,MAAM,IAAI,CAACC,qBAAqB,CAAC;MAC9Cf,YAAY;MACZgB,SAAS,EAAElB;IACf,CAAC,CAAC;IACF,MAAMmB,cAA+B,GAAG,EAAE;IAE1C,IAAI;MACA,IAAIC,UAAU,GAAG,CAAC;MAClB,IAAIC,KAAK,GAAG,CAAC;MACb,MAAMC,QAAyB,GAAG,EAAE;MACpC,IAAIC,MAAM,GAAG,CAAC;MAEd,OAAOF,KAAK,GAAGX,SAAS,EAAE;QACtB,MAAMc,GAAG,GAAGZ,IAAI,CAACE,GAAG,CAACO,KAAK,GAAGV,QAAQ,GAAG,CAAC,EAAED,SAAS,GAAG,CAAC,CAAC;QAEzD,MAAMe,WAAW,GAAG,IAAI,CAACC,cAAc,CAAC;UACpCzB,YAAY;UACZ0B,SAAS,EAAE3B,GAAG;UACdE,YAAY;UACZgB,SAAS,EAAElB,GAAG;UACdgB,QAAQ;UACRI,UAAU;UACVC,KAAK;UACLG,GAAG;UACHL;QACJ,CAAC,CAAC;QAEFG,QAAQ,CAACM,IAAI,CAACH,WAAW,CAAC;QAC1BF,MAAM,EAAE;QAER,IAAIA,MAAM,IAAI,IAAI,CAAClC,cAAc,EAAE;UAC/B,MAAMwC,OAAO,CAACC,IAAI,CAACR,QAAQ,CAAC;UAC5BC,MAAM,EAAE;QACZ;QAEAH,UAAU,EAAE;QACZC,KAAK,IAAIV,QAAQ;MACrB;MAEA,MAAMkB,OAAO,CAACE,GAAG,CAACT,QAAQ,CAAC;MAC3B;AACZ;AACA;MACY,MAAM,IAAI,CAACU,uBAAuB,CAAC;QAC/B9B,YAAY;QACZgB,SAAS,EAAElB,GAAG;QACdgB,QAAQ;QACRG;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOc,EAAE,EAAE;MACT,MAAM,IAAI,CAACC,oBAAoB,CAAC;QAAEhC,YAAY;QAAEgB,SAAS,EAAElB,GAAG;QAAEgB;MAAS,CAAC,CAAC;MAC3E,MAAMiB,EAAE;IACZ;IACA;AACR;AACA;;IAEQ,IAAI;MACA,MAAM,IAAI,CAACE,YAAY,CAAC;QACpBlC,YAAY;QACZC,YAAY;QACZF;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOiC,EAAE,EAAE;MACTG,OAAO,CAACC,KAAK,CACT,+BAA+BrC,GAAG,SAASC,YAAY,OAAOC,YAAY,GAC9E,CAAC;MACDkC,OAAO,CAACE,GAAG,CAACxD,gBAAgB,CAACmD,EAAE,CAAC,CAAC;IACrC;EACJ;EAEQrC,aAAaA,CAAC2C,KAAe,EAAU;IAC3C,IAAI,OAAOA,KAAK,KAAK,QAAQ,EAAE;MAC3B,OAAOA,KAAK;IAChB;IACA,MAAMC,MAAM,GAAG3D,KAAK,CAAC4D,KAAK,CAACF,KAAK,CAAC;IACjC,IAAI,OAAOC,MAAM,KAAK,QAAQ,EAAE;MAC5B,OAAOA,MAAM;IACjB;IACA,MAAME,WAAW,GAAG,OAAOH,KAAK;IAChC,MAAM,IAAIzC,KAAK,CAAC,iCAAiC4C,WAAW,IAAI,CAAC;EACrE;EAEA,MAActC,UAAUA,CAACnB,MAAyB,EAAE;IAChD,MAAM;MAAEoB,MAAM;MAAEC,MAAM;MAAEN;IAAI,CAAC,GAAGf,MAAM;IACtC,MAAM0D,KAA6B,GAAG;MAClCC,MAAM,EAAEtC,MAAM;MACduC,GAAG,EAAE7C;IACT,CAAC;IACD,MAAM8C,OAAO,GAAG,IAAInE,iBAAiB,CAACgE,KAAK,CAAC;IAC5C,IAAI;MACA,MAAMI,MAAM,GAAG,MAAM1C,MAAM,CAAC2C,IAAI,CAACF,OAAO,CAAC;MACzC,IAAIC,MAAM,CAACE,SAAS,EAAEC,cAAc,KAAK,GAAG,EAAE;QAC1C,OAAOH,MAAM;MACjB;IACJ,CAAC,CAAC,OAAOd,EAAE,EAAE;MACT,IAAIA,EAAE,CAACkB,IAAI,KAAK,UAAU,IAAIlB,EAAE,CAACgB,SAAS,EAAEC,cAAc,KAAK,GAAG,EAAE;QAChE,OAAO,IAAI;MACf;MACAd,OAAO,CAACC,KAAK,CAAC,yBAAyBrC,GAAG,cAAcM,MAAM,GAAG,CAAC;MAClE8B,OAAO,CAACE,GAAG,CAACxD,gBAAgB,CAACmD,EAAE,CAAC,CAAC;IACrC;IACA,OAAO,IAAI;EACf;EAEA,MAAchB,qBAAqBA,CAAChC,MAAoC,EAAmB;IACvF,MAAM;MAAEiB,YAAY;MAAEgB;IAAU,CAAC,GAAGjC,MAAM;IAC1C,MAAM0D,KAAwC,GAAG;MAC7CC,MAAM,EAAE1C,YAAY;MACpB2C,GAAG,EAAE3B;IACT,CAAC;IACD,MAAM4B,OAAO,GAAG,IAAIpE,4BAA4B,CAACiE,KAAK,CAAC;IACvD,MAAMS,QAAQ,GAAG,MAAM,IAAI,CAAC1D,YAAY,CAACsD,IAAI,CAACF,OAAO,CAAC;IACtD,IAAI,CAACM,QAAQ,CAACC,QAAQ,EAAE;MACpB,MAAM,IAAIvD,KAAK,CAAC,oCAAoC,CAAC;IACzD;IACA,OAAOsD,QAAQ,CAACC,QAAQ;EAC5B;EAEA,MAAc3B,cAAcA,CAACzC,MAA6B,EAAiB;IACvE,MAAM;MACFgB,YAAY;MACZ0B,SAAS;MACTzB,YAAY;MACZgB,SAAS;MACTF,QAAQ;MACRI,UAAU;MACVC,KAAK;MACLG,GAAG;MACHL;IACJ,CAAC,GAAGlC,MAAM;IAEV,MAAM0D,KAAiC,GAAG;MACtCC,MAAM,EAAE1C,YAAY;MACpB2C,GAAG,EAAE3B,SAAS;MACdmC,QAAQ,EAAErC,QAAQ;MAClBsC,UAAU,EAAElC,UAAU;MACtBmC,UAAU,EAAEC,kBAAkB,CAAC,GAAGvD,YAAY,IAAI0B,SAAS,EAAE,CAAC;MAC9D8B,eAAe,EAAE,SAASpC,KAAK,IAAIG,GAAG;IAC1C,CAAC;IAED,MAAMsB,OAAO,GAAG,IAAIlE,qBAAqB,CAAC+D,KAAK,CAAC;IAChD,MAAMS,QAAQ,GAAG,MAAM,IAAI,CAAC1D,YAAY,CAACsD,IAAI,CAACF,OAAO,CAAC;IAEtD,IAAI,CAACM,QAAQ,CAACM,cAAc,EAAElD,IAAI,EAAE;MAChC,MAAM,IAAIV,KAAK,CAAC,kCAAkCsB,UAAU,EAAE,CAAC;IACnE;IAEAD,cAAc,CAACS,IAAI,CAAC;MAChBpB,IAAI,EAAE4C,QAAQ,CAACM,cAAc,CAAClD,IAAI;MAClC8C,UAAU,EAAElC;IAChB,CAAC,CAAC;EACN;EAEA,MAAcY,uBAAuBA,CAAC/C,MAAsC,EAAE;IAC1E,MAAM;MAAEiB,YAAY;MAAEgB,SAAS;MAAEF,QAAQ;MAAEG;IAAe,CAAC,GAAGlC,MAAM;IAEpE,MAAM0E,WAAW,GAAG,CAAC,GAAGxC,cAAc,CAAC,CAACyC,IAAI,CAAC,CAACC,CAAC,EAAEC,CAAC,KAAK;MACnD,OAAOD,CAAC,CAACP,UAAU,GAAIQ,CAAC,CAACR,UAAW;IACxC,CAAC,CAAC;IAEF,MAAMX,KAA0C,GAAG;MAC/CC,MAAM,EAAE1C,YAAY;MACpB2C,GAAG,EAAE3B,SAAS;MACdmC,QAAQ,EAAErC,QAAQ;MAClB+C,eAAe,EAAE;QAAEC,KAAK,EAAEL;MAAY;IAC1C,CAAC;IAED,MAAMb,OAAO,GAAG,IAAItE,8BAA8B,CAACmE,KAAK,CAAC;IACzD,OAAO,MAAM,IAAI,CAACjD,YAAY,CAACsD,IAAI,CAACF,OAAO,CAAC;EAChD;EAEA,MAAcZ,oBAAoBA,CAACjD,MAAmC,EAAE;IACpE,MAAM;MAAEiB,YAAY;MAAEgB,SAAS;MAAEF;IAAS,CAAC,GAAG/B,MAAM;IAEpD,MAAM0D,KAAuC,GAAG;MAC5CC,MAAM,EAAE1C,YAAY;MACpB2C,GAAG,EAAE3B,SAAS;MACdmC,QAAQ,EAAErC;IACd,CAAC;IACD,MAAM8B,OAAO,GAAG,IAAIvE,2BAA2B,CAACoE,KAAK,CAAC;IACtD,MAAM,IAAI,CAACjD,YAAY,CAACsD,IAAI,CAACF,OAAO,CAAC;EACzC;EAEA,MAAcX,YAAYA,CACtBlD,MAA2B,EACY;IACvC,MAAM;MAAEgB,YAAY;MAAEC,YAAY;MAAEF;IAAI,CAAC,GAAGf,MAAM;IAElD,MAAMgF,WAAW,GAAG,GAAGjE,GAAG,WAAW;IAErC,MAAM,CAACkE,cAAc,EAAEC,cAAc,CAAC,GAAG,MAAMtC,OAAO,CAACE,GAAG,CAAC,CACvD,IAAI,CAAC3B,UAAU,CAAC;MACZC,MAAM,EAAE,IAAI,CAACb,YAAY;MACzBc,MAAM,EAAEL,YAAY;MACpBD,GAAG,EAAEiE;IACT,CAAC,CAAC,CAACG,KAAK,CAAC,MAAM,IAAI,CAAC,EACpB,IAAI,CAAChE,UAAU,CAAC;MACZC,MAAM,EAAE,IAAI,CAACX,YAAY;MACzBY,MAAM,EAAEJ,YAAY;MACpBF,GAAG,EAAEiE;IACT,CAAC,CAAC,CAACG,KAAK,CAAC,MAAM,IAAI,CAAC,CACvB,CAAC;IACF;AACR;AACA;IACQ,IAAI,CAACF,cAAc,EAAE1D,IAAI,EAAE;MACvB,OAAO,IAAI;IACf;IACA;AACR;AACA;IACQ,IAAI2D,cAAc,EAAE;MAChB,OAAO,IAAI;IACf;IAEA,MAAMxB,KAA6B,GAAG;MAClCC,MAAM,EAAE1C,YAAY;MACpB2C,GAAG,EAAEoB,WAAW;MAChBV,UAAU,EAAEC,kBAAkB,CAAC,GAAGvD,YAAY,IAAIgE,WAAW,EAAE;IACnE,CAAC;IAED,MAAMnB,OAAO,GAAG,IAAIrE,iBAAiB,CAACkE,KAAK,CAAC;IAE5C,OAAO,MAAM,IAAI,CAACjD,YAAY,CAACsD,IAAI,CAACF,OAAO,CAAC;EAChD;AACJ","ignoreList":[]}
|
|
1
|
+
{"version":3,"file":"worker/actions/copyFile/CopyFile.js","sources":["../../../../src/worker/actions/copyFile/CopyFile.ts"],"sourcesContent":["import type {\n AbortMultipartUploadCommandInput,\n CompletedPart,\n CompleteMultipartUploadCommandInput,\n CopyObjectCommandInput,\n CopyObjectCommandOutput,\n CreateMultipartUploadCommandInput,\n HeadObjectCommandInput,\n S3Client,\n S3ClientConfig,\n UploadPartCopyCommandInput\n} from \"@webiny/aws-sdk/client-s3/index.js\";\nimport {\n AbortMultipartUploadCommand,\n CompleteMultipartUploadCommand,\n CopyObjectCommand,\n CreateMultipartUploadCommand,\n HeadObjectCommand,\n UploadPartCopyCommand\n} from \"@webiny/aws-sdk/client-s3/index.js\";\nimport bytes from \"bytes\";\nimport { convertException } from \"@webiny/utils\";\n\ntype ByteSize = number | `${number}B` | `${number}KB` | `${number}MB` | `${number}GB`;\n\nexport interface ICopyFileParams {\n createS3Client: (config: Partial<S3ClientConfig>) => Pick<S3Client, \"send\">;\n sourceRegion: string;\n targetRegion: string;\n maxConcurrency?: number;\n minPartSize?: ByteSize;\n maxPartSize?: ByteSize;\n}\n\nexport interface ICopyParams {\n key: string;\n sourceBucket: string;\n targetBucket: string;\n}\n\nexport interface IHeadObjectParams {\n client: Pick<S3Client, \"send\">;\n bucket: string;\n key: string;\n}\n\nexport interface ICreateMultipartUploadParams {\n targetBucket: string;\n targetKey: string;\n}\n\nexport interface IUploadPartCopyParams {\n sourceBucket: string;\n sourceKey: string;\n targetBucket: string;\n targetKey: string;\n uploadId: string;\n partNumber: number;\n start: number;\n end: number;\n completedParts: CompletedPart[];\n}\n\nexport interface ICompleteMultipartUploadParams {\n targetBucket: string;\n targetKey: string;\n uploadId: string;\n completedParts: CompletedPart[];\n}\n\nexport interface IAbortMultipartUploadParams {\n targetBucket: string;\n targetKey: string;\n uploadId: string;\n}\n\ninterface ICopyMetadataParams {\n sourceBucket: string;\n targetBucket: string;\n key: string;\n}\n\nexport class CopyFile {\n private readonly sourceClient: Pick<S3Client, \"send\">;\n private readonly targetClient: Pick<S3Client, \"send\">;\n private readonly maxConcurrency: number;\n private readonly minPartSizeBytes: number;\n private readonly maxPartSizeBytes: number;\n\n public constructor(params: ICopyFileParams) {\n const {\n createS3Client,\n sourceRegion,\n targetRegion,\n maxConcurrency = 100,\n minPartSize = \"5MB\",\n maxPartSize = \"1GB\"\n } = params;\n\n this.sourceClient = createS3Client({\n region: sourceRegion\n });\n this.targetClient = createS3Client({\n region: targetRegion\n });\n this.maxConcurrency = maxConcurrency;\n\n this.minPartSizeBytes = this.parseByteSize(minPartSize);\n this.maxPartSizeBytes = this.parseByteSize(maxPartSize);\n\n if (this.minPartSizeBytes < 5 * 1024 * 1024) {\n throw new Error(\"minPartSize must be at least 5MB.\");\n }\n }\n\n public async copy(params: ICopyParams): Promise<void> {\n const { key, sourceBucket, targetBucket } = params;\n\n const sourceHead = await this.headObject({\n client: this.sourceClient,\n bucket: sourceBucket,\n key\n });\n\n if (!sourceHead?.ContentLength || !sourceHead.ETag) {\n throw new Error(\"Source object metadata is invalid or missing.\");\n }\n /**\n * If the target object already exists, we can skip the copy operation.\n */\n const targetHead = await this.headObject({\n client: this.targetClient,\n bucket: targetBucket,\n key\n });\n\n if (targetHead) {\n return;\n }\n\n const totalSize = sourceHead.ContentLength;\n let partSize = Math.max(\n this.minPartSizeBytes,\n Math.min(this.maxPartSizeBytes, Math.ceil(totalSize / this.maxConcurrency))\n );\n partSize = Math.min(partSize, totalSize);\n\n const uploadId = await this.createMultipartUpload({\n targetBucket,\n targetKey: key\n });\n const completedParts: CompletedPart[] = [];\n\n try {\n let partNumber = 1;\n let start = 0;\n const promises: Promise<void>[] = [];\n let active = 0;\n\n while (start < totalSize) {\n const end = Math.min(start + partSize - 1, totalSize - 1);\n\n const copyPromise = this.uploadPartCopy({\n sourceBucket,\n sourceKey: key,\n targetBucket,\n targetKey: key,\n uploadId,\n partNumber,\n start,\n end,\n completedParts\n });\n\n promises.push(copyPromise);\n active++;\n\n if (active >= this.maxConcurrency) {\n await Promise.race(promises);\n active--;\n }\n\n partNumber++;\n start += partSize;\n }\n\n await Promise.all(promises);\n /**\n * Complete the multipart upload with all the completed parts.\n */\n await this.completeMultipartUpload({\n targetBucket,\n targetKey: key,\n uploadId,\n completedParts\n });\n } catch (ex) {\n await this.abortMultipartUpload({ targetBucket, targetKey: key, uploadId });\n throw ex;\n }\n /**\n * And then copy the metadata file.\n */\n\n try {\n await this.copyMetadata({\n sourceBucket,\n targetBucket,\n key\n });\n } catch (ex) {\n console.error(\n `Failed to copy metadata for ${key} from ${sourceBucket} to ${targetBucket}.`\n );\n console.log(convertException(ex));\n }\n }\n\n private parseByteSize(value: ByteSize): number {\n if (typeof value === \"number\") {\n return value;\n }\n const parsed = bytes.parse(value);\n if (typeof parsed === \"number\") {\n return parsed;\n }\n const typeOfValue = typeof value;\n throw new Error(`Invalid byte size value type \"${typeOfValue}\".`);\n }\n\n private async headObject(params: IHeadObjectParams) {\n const { client, bucket, key } = params;\n const input: HeadObjectCommandInput = {\n Bucket: bucket,\n Key: key\n };\n const command = new HeadObjectCommand(input);\n try {\n const result = await client.send(command);\n if (result.$metadata?.httpStatusCode === 200) {\n return result;\n }\n } catch (ex) {\n if (ex.name === \"NotFound\" || ex.$metadata?.httpStatusCode === 404) {\n return null;\n }\n console.error(`Failed to head object ${key} in bucket ${bucket}.`);\n console.log(convertException(ex));\n }\n return null;\n }\n\n private async createMultipartUpload(params: ICreateMultipartUploadParams): Promise<string> {\n const { targetBucket, targetKey } = params;\n const input: CreateMultipartUploadCommandInput = {\n Bucket: targetBucket,\n Key: targetKey\n };\n const command = new CreateMultipartUploadCommand(input);\n const response = await this.targetClient.send(command);\n if (!response.UploadId) {\n throw new Error(\"Failed to create multipart upload.\");\n }\n return response.UploadId;\n }\n\n private async uploadPartCopy(params: IUploadPartCopyParams): Promise<void> {\n const {\n sourceBucket,\n sourceKey,\n targetBucket,\n targetKey,\n uploadId,\n partNumber,\n start,\n end,\n completedParts\n } = params;\n\n const input: UploadPartCopyCommandInput = {\n Bucket: targetBucket,\n Key: targetKey,\n UploadId: uploadId,\n PartNumber: partNumber,\n CopySource: encodeURIComponent(`${sourceBucket}/${sourceKey}`),\n CopySourceRange: `bytes=${start}-${end}`\n };\n\n const command = new UploadPartCopyCommand(input);\n const response = await this.targetClient.send(command);\n\n if (!response.CopyPartResult?.ETag) {\n throw new Error(`UploadPartCopy failed for part ${partNumber}`);\n }\n\n completedParts.push({\n ETag: response.CopyPartResult.ETag,\n PartNumber: partNumber\n });\n }\n\n private async completeMultipartUpload(params: ICompleteMultipartUploadParams) {\n const { targetBucket, targetKey, uploadId, completedParts } = params;\n\n const sortedParts = [...completedParts].sort((a, b) => {\n return a.PartNumber! - b.PartNumber!;\n });\n\n const input: CompleteMultipartUploadCommandInput = {\n Bucket: targetBucket,\n Key: targetKey,\n UploadId: uploadId,\n MultipartUpload: { Parts: sortedParts }\n };\n\n const command = new CompleteMultipartUploadCommand(input);\n return await this.targetClient.send(command);\n }\n\n private async abortMultipartUpload(params: IAbortMultipartUploadParams) {\n const { targetBucket, targetKey, uploadId } = params;\n\n const input: AbortMultipartUploadCommandInput = {\n Bucket: targetBucket,\n Key: targetKey,\n UploadId: uploadId\n };\n const command = new AbortMultipartUploadCommand(input);\n await this.targetClient.send(command);\n }\n\n private async copyMetadata(\n params: ICopyMetadataParams\n ): Promise<CopyObjectCommandOutput | null> {\n const { sourceBucket, targetBucket, key } = params;\n\n const metadataKey = `${key}.metadata`;\n\n const [sourceMetadata, targetMetadata] = await Promise.all([\n this.headObject({\n client: this.sourceClient,\n bucket: sourceBucket,\n key: metadataKey\n }).catch(() => null),\n this.headObject({\n client: this.targetClient,\n bucket: targetBucket,\n key: metadataKey\n }).catch(() => null)\n ]);\n /**\n * If the source metadata does not exist, we cannot copy it.\n */\n if (!sourceMetadata?.ETag) {\n return null;\n }\n /**\n * If the target metadata already exists, we can skip copying it.\n */\n if (targetMetadata) {\n return null;\n }\n\n const input: CopyObjectCommandInput = {\n Bucket: targetBucket,\n Key: metadataKey,\n CopySource: encodeURIComponent(`${sourceBucket}/${metadataKey}`)\n };\n\n const command = new CopyObjectCommand(input);\n\n return await this.targetClient.send(command);\n }\n}\n"],"names":["CopyFile","params","createS3Client","sourceRegion","targetRegion","maxConcurrency","minPartSize","maxPartSize","Error","key","sourceBucket","targetBucket","sourceHead","targetHead","totalSize","partSize","Math","uploadId","completedParts","partNumber","start","promises","active","end","copyPromise","Promise","ex","console","convertException","value","parsed","bytes","typeOfValue","client","bucket","input","command","HeadObjectCommand","result","targetKey","CreateMultipartUploadCommand","response","sourceKey","encodeURIComponent","UploadPartCopyCommand","sortedParts","a","b","CompleteMultipartUploadCommand","AbortMultipartUploadCommand","metadataKey","sourceMetadata","targetMetadata","CopyObjectCommand"],"mappings":";;;AAkFO,MAAMA;IAOT,YAAmBC,MAAuB,CAAE;QACxC,MAAM,EACFC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,iBAAiB,GAAG,EACpBC,cAAc,KAAK,EACnBC,cAAc,KAAK,EACtB,GAAGN;QAEJ,IAAI,CAAC,YAAY,GAAGC,eAAe;YAC/B,QAAQC;QACZ;QACA,IAAI,CAAC,YAAY,GAAGD,eAAe;YAC/B,QAAQE;QACZ;QACA,IAAI,CAAC,cAAc,GAAGC;QAEtB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAACC;QAC3C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAACC;QAE3C,IAAI,IAAI,CAAC,gBAAgB,GAAG,SACxB,MAAM,IAAIC,MAAM;IAExB;IAEA,MAAa,KAAKP,MAAmB,EAAiB;QAClD,MAAM,EAAEQ,GAAG,EAAEC,YAAY,EAAEC,YAAY,EAAE,GAAGV;QAE5C,MAAMW,aAAa,MAAM,IAAI,CAAC,UAAU,CAAC;YACrC,QAAQ,IAAI,CAAC,YAAY;YACzB,QAAQF;YACRD;QACJ;QAEA,IAAI,CAACG,YAAY,iBAAiB,CAACA,WAAW,IAAI,EAC9C,MAAM,IAAIJ,MAAM;QAKpB,MAAMK,aAAa,MAAM,IAAI,CAAC,UAAU,CAAC;YACrC,QAAQ,IAAI,CAAC,YAAY;YACzB,QAAQF;YACRF;QACJ;QAEA,IAAII,YACA;QAGJ,MAAMC,YAAYF,WAAW,aAAa;QAC1C,IAAIG,WAAWC,KAAK,GAAG,CACnB,IAAI,CAAC,gBAAgB,EACrBA,KAAK,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAEA,KAAK,IAAI,CAACF,YAAY,IAAI,CAAC,cAAc;QAE7EC,WAAWC,KAAK,GAAG,CAACD,UAAUD;QAE9B,MAAMG,WAAW,MAAM,IAAI,CAAC,qBAAqB,CAAC;YAC9CN;YACA,WAAWF;QACf;QACA,MAAMS,iBAAkC,EAAE;QAE1C,IAAI;YACA,IAAIC,aAAa;YACjB,IAAIC,QAAQ;YACZ,MAAMC,WAA4B,EAAE;YACpC,IAAIC,SAAS;YAEb,MAAOF,QAAQN,UAAW;gBACtB,MAAMS,MAAMP,KAAK,GAAG,CAACI,QAAQL,WAAW,GAAGD,YAAY;gBAEvD,MAAMU,cAAc,IAAI,CAAC,cAAc,CAAC;oBACpCd;oBACA,WAAWD;oBACXE;oBACA,WAAWF;oBACXQ;oBACAE;oBACAC;oBACAG;oBACAL;gBACJ;gBAEAG,SAAS,IAAI,CAACG;gBACdF;gBAEA,IAAIA,UAAU,IAAI,CAAC,cAAc,EAAE;oBAC/B,MAAMG,QAAQ,IAAI,CAACJ;oBACnBC;gBACJ;gBAEAH;gBACAC,SAASL;YACb;YAEA,MAAMU,QAAQ,GAAG,CAACJ;YAIlB,MAAM,IAAI,CAAC,uBAAuB,CAAC;gBAC/BV;gBACA,WAAWF;gBACXQ;gBACAC;YACJ;QACJ,EAAE,OAAOQ,IAAI;YACT,MAAM,IAAI,CAAC,oBAAoB,CAAC;gBAAEf;gBAAc,WAAWF;gBAAKQ;YAAS;YACzE,MAAMS;QACV;QAKA,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC;gBACpBhB;gBACAC;gBACAF;YACJ;QACJ,EAAE,OAAOiB,IAAI;YACTC,QAAQ,KAAK,CACT,CAAC,4BAA4B,EAAElB,IAAI,MAAM,EAAEC,aAAa,IAAI,EAAEC,aAAa,CAAC,CAAC;YAEjFgB,QAAQ,GAAG,CAACC,iBAAiBF;QACjC;IACJ;IAEQ,cAAcG,KAAe,EAAU;QAC3C,IAAI,AAAiB,YAAjB,OAAOA,OACP,OAAOA;QAEX,MAAMC,SAASC,MAAM,KAAK,CAACF;QAC3B,IAAI,AAAkB,YAAlB,OAAOC,QACP,OAAOA;QAEX,MAAME,cAAc,OAAOH;QAC3B,MAAM,IAAIrB,MAAM,CAAC,8BAA8B,EAAEwB,YAAY,EAAE,CAAC;IACpE;IAEA,MAAc,WAAW/B,MAAyB,EAAE;QAChD,MAAM,EAAEgC,MAAM,EAAEC,MAAM,EAAEzB,GAAG,EAAE,GAAGR;QAChC,MAAMkC,QAAgC;YAClC,QAAQD;YACR,KAAKzB;QACT;QACA,MAAM2B,UAAU,IAAIC,kBAAkBF;QACtC,IAAI;YACA,MAAMG,SAAS,MAAML,OAAO,IAAI,CAACG;YACjC,IAAIE,OAAO,SAAS,EAAE,mBAAmB,KACrC,OAAOA;QAEf,EAAE,OAAOZ,IAAI;YACT,IAAIA,AAAY,eAAZA,GAAG,IAAI,IAAmBA,GAAG,SAAS,EAAE,mBAAmB,KAC3D,OAAO;YAEXC,QAAQ,KAAK,CAAC,CAAC,sBAAsB,EAAElB,IAAI,WAAW,EAAEyB,OAAO,CAAC,CAAC;YACjEP,QAAQ,GAAG,CAACC,iBAAiBF;QACjC;QACA,OAAO;IACX;IAEA,MAAc,sBAAsBzB,MAAoC,EAAmB;QACvF,MAAM,EAAEU,YAAY,EAAE4B,SAAS,EAAE,GAAGtC;QACpC,MAAMkC,QAA2C;YAC7C,QAAQxB;YACR,KAAK4B;QACT;QACA,MAAMH,UAAU,IAAII,6BAA6BL;QACjD,MAAMM,WAAW,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAACL;QAC9C,IAAI,CAACK,SAAS,QAAQ,EAClB,MAAM,IAAIjC,MAAM;QAEpB,OAAOiC,SAAS,QAAQ;IAC5B;IAEA,MAAc,eAAexC,MAA6B,EAAiB;QACvE,MAAM,EACFS,YAAY,EACZgC,SAAS,EACT/B,YAAY,EACZ4B,SAAS,EACTtB,QAAQ,EACRE,UAAU,EACVC,KAAK,EACLG,GAAG,EACHL,cAAc,EACjB,GAAGjB;QAEJ,MAAMkC,QAAoC;YACtC,QAAQxB;YACR,KAAK4B;YACL,UAAUtB;YACV,YAAYE;YACZ,YAAYwB,mBAAmB,GAAGjC,aAAa,CAAC,EAAEgC,WAAW;YAC7D,iBAAiB,CAAC,MAAM,EAAEtB,MAAM,CAAC,EAAEG,KAAK;QAC5C;QAEA,MAAMa,UAAU,IAAIQ,sBAAsBT;QAC1C,MAAMM,WAAW,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAACL;QAE9C,IAAI,CAACK,SAAS,cAAc,EAAE,MAC1B,MAAM,IAAIjC,MAAM,CAAC,+BAA+B,EAAEW,YAAY;QAGlED,eAAe,IAAI,CAAC;YAChB,MAAMuB,SAAS,cAAc,CAAC,IAAI;YAClC,YAAYtB;QAChB;IACJ;IAEA,MAAc,wBAAwBlB,MAAsC,EAAE;QAC1E,MAAM,EAAEU,YAAY,EAAE4B,SAAS,EAAEtB,QAAQ,EAAEC,cAAc,EAAE,GAAGjB;QAE9D,MAAM4C,cAAc;eAAI3B;SAAe,CAAC,IAAI,CAAC,CAAC4B,GAAGC,IACtCD,EAAE,UAAU,GAAIC,EAAE,UAAU;QAGvC,MAAMZ,QAA6C;YAC/C,QAAQxB;YACR,KAAK4B;YACL,UAAUtB;YACV,iBAAiB;gBAAE,OAAO4B;YAAY;QAC1C;QAEA,MAAMT,UAAU,IAAIY,+BAA+Bb;QACnD,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAACC;IACxC;IAEA,MAAc,qBAAqBnC,MAAmC,EAAE;QACpE,MAAM,EAAEU,YAAY,EAAE4B,SAAS,EAAEtB,QAAQ,EAAE,GAAGhB;QAE9C,MAAMkC,QAA0C;YAC5C,QAAQxB;YACR,KAAK4B;YACL,UAAUtB;QACd;QACA,MAAMmB,UAAU,IAAIa,4BAA4Bd;QAChD,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAACC;IACjC;IAEA,MAAc,aACVnC,MAA2B,EACY;QACvC,MAAM,EAAES,YAAY,EAAEC,YAAY,EAAEF,GAAG,EAAE,GAAGR;QAE5C,MAAMiD,cAAc,GAAGzC,IAAI,SAAS,CAAC;QAErC,MAAM,CAAC0C,gBAAgBC,eAAe,GAAG,MAAM3B,QAAQ,GAAG,CAAC;YACvD,IAAI,CAAC,UAAU,CAAC;gBACZ,QAAQ,IAAI,CAAC,YAAY;gBACzB,QAAQf;gBACR,KAAKwC;YACT,GAAG,KAAK,CAAC,IAAM;YACf,IAAI,CAAC,UAAU,CAAC;gBACZ,QAAQ,IAAI,CAAC,YAAY;gBACzB,QAAQvC;gBACR,KAAKuC;YACT,GAAG,KAAK,CAAC,IAAM;SAClB;QAID,IAAI,CAACC,gBAAgB,MACjB,OAAO;QAKX,IAAIC,gBACA,OAAO;QAGX,MAAMjB,QAAgC;YAClC,QAAQxB;YACR,KAAKuC;YACL,YAAYP,mBAAmB,GAAGjC,aAAa,CAAC,EAAEwC,aAAa;QACnE;QAEA,MAAMd,UAAU,IAAIiB,kBAAkBlB;QAEtC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAACC;IACxC;AACJ"}
|
|
@@ -2,39 +2,31 @@ import { createWorkerActionPlugin } from "../../plugins/WorkerActionPlugin.js";
|
|
|
2
2
|
import { CopyFile } from "./CopyFile.js";
|
|
3
3
|
import { createCopyFileSchema } from "./copyFileSchema.js";
|
|
4
4
|
import { logValidationError } from "../logValidationError.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
return copyFile.copy({
|
|
32
|
-
key: data.key,
|
|
33
|
-
sourceBucket: data.source.bucket,
|
|
34
|
-
targetBucket: data.target.bucket
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
});
|
|
38
|
-
};
|
|
5
|
+
const createCopyFileAction = ({ createS3Client })=>createWorkerActionPlugin({
|
|
6
|
+
name: "sync.worker.action.copyFile",
|
|
7
|
+
parse (input) {
|
|
8
|
+
const schema = createCopyFileSchema();
|
|
9
|
+
const result = schema.safeParse(input);
|
|
10
|
+
if (!result.success || result.error) return void logValidationError(result.error);
|
|
11
|
+
return result.data;
|
|
12
|
+
},
|
|
13
|
+
async handle (params) {
|
|
14
|
+
const { data } = params;
|
|
15
|
+
const copyFile = new CopyFile({
|
|
16
|
+
createS3Client,
|
|
17
|
+
sourceRegion: data.source.region,
|
|
18
|
+
targetRegion: data.target.region,
|
|
19
|
+
maxConcurrency: void 0,
|
|
20
|
+
maxPartSize: void 0,
|
|
21
|
+
minPartSize: void 0
|
|
22
|
+
});
|
|
23
|
+
return copyFile.copy({
|
|
24
|
+
key: data.key,
|
|
25
|
+
sourceBucket: data.source.bucket,
|
|
26
|
+
targetBucket: data.target.bucket
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
export { createCopyFileAction };
|
|
39
31
|
|
|
40
32
|
//# sourceMappingURL=copyFileAction.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"worker/actions/copyFile/copyFileAction.js","sources":["../../../../src/worker/actions/copyFile/copyFileAction.ts"],"sourcesContent":["import { createWorkerActionPlugin } from \"~/worker/plugins/WorkerActionPlugin.js\";\nimport { CopyFile } from \"./CopyFile.js\";\nimport type { S3Client, S3ClientConfig } from \"@webiny/aws-sdk/client-s3/index.js\";\nimport type { ICopyFileActionEvent } from \"./types.js\";\nimport { createCopyFileSchema } from \"./copyFileSchema.js\";\nimport { logValidationError } from \"~/worker/actions/logValidationError.js\";\n\nexport interface ICreateCopyFileActionParams {\n createS3Client: (config: Partial<S3ClientConfig>) => Pick<S3Client, \"send\">;\n}\n\nexport const createCopyFileAction = ({ createS3Client }: ICreateCopyFileActionParams) => {\n return createWorkerActionPlugin<ICopyFileActionEvent>({\n name: \"sync.worker.action.copyFile\",\n parse(input) {\n const schema = createCopyFileSchema();\n const result = schema.safeParse(input);\n if (!result.success || result.error) {\n logValidationError(result.error);\n return undefined;\n }\n return result.data;\n },\n async handle(params) {\n const { data } = params;\n const copyFile = new CopyFile({\n createS3Client,\n sourceRegion: data.source.region,\n targetRegion: data.target.region,\n maxConcurrency: undefined,\n maxPartSize: undefined,\n minPartSize: undefined\n });\n return copyFile.copy({\n key: data.key,\n sourceBucket: data.source.bucket,\n targetBucket: data.target.bucket\n });\n }\n });\n};\n"],"names":["createCopyFileAction","createS3Client","createWorkerActionPlugin","input","schema","createCopyFileSchema","result","logValidationError","params","data","copyFile","CopyFile","undefined"],"mappings":";;;;AAWO,MAAMA,uBAAuB,CAAC,EAAEC,cAAc,EAA+B,GACzEC,yBAA+C;QAClD,MAAM;QACN,OAAMC,KAAK;YACP,MAAMC,SAASC;YACf,MAAMC,SAASF,OAAO,SAAS,CAACD;YAChC,IAAI,CAACG,OAAO,OAAO,IAAIA,OAAO,KAAK,EAAE,YACjCC,mBAAmBD,OAAO,KAAK;YAGnC,OAAOA,OAAO,IAAI;QACtB;QACA,MAAM,QAAOE,MAAM;YACf,MAAM,EAAEC,IAAI,EAAE,GAAGD;YACjB,MAAME,WAAW,IAAIC,SAAS;gBAC1BV;gBACA,cAAcQ,KAAK,MAAM,CAAC,MAAM;gBAChC,cAAcA,KAAK,MAAM,CAAC,MAAM;gBAChC,gBAAgBG;gBAChB,aAAaA;gBACb,aAAaA;YACjB;YACA,OAAOF,SAAS,IAAI,CAAC;gBACjB,KAAKD,KAAK,GAAG;gBACb,cAAcA,KAAK,MAAM,CAAC,MAAM;gBAChC,cAAcA,KAAK,MAAM,CAAC,MAAM;YACpC;QACJ;IACJ"}
|