@pi-r/aws 0.6.5 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -6
- package/client/index.d.ts +4 -17
- package/client/index.js +86 -157
- package/download/index.js +10 -11
- package/package.json +6 -5
- package/types/index.d.ts +7 -7
- package/upload/index.js +120 -53
package/README.md
CHANGED
package/client/index.d.ts
CHANGED
|
@@ -2,23 +2,14 @@ import type { ICloud, IModule } from '@e-mc/types/lib';
|
|
|
2
2
|
import type { BucketWebsiteOptions, CloudDatabase } from '@e-mc/types/lib/cloud';
|
|
3
3
|
import type { BatchQueryResult, QueryResult } from '@e-mc/types/lib/db';
|
|
4
4
|
|
|
5
|
-
import type {
|
|
5
|
+
import type { BucketCannedACL } from '@pi-r/aws-lib';
|
|
6
|
+
|
|
7
|
+
import type { AWSDatabaseCredential, AWSDatabaseQuery, AWSStorageCredential, ConfigureBucketOptions } from '../types';
|
|
6
8
|
|
|
7
|
-
import type { ConfigurationOptions } from 'aws-sdk/lib/core';
|
|
8
|
-
import type { ServiceConfigurationOptions } from 'aws-sdk/lib/service';
|
|
9
9
|
import type { AttributeValue, DocumentClient } from 'aws-sdk/clients/dynamodb';
|
|
10
|
-
import type {
|
|
11
|
-
import type { CreateBucketRequest, DefaultRetention } from 'aws-sdk/clients/s3';
|
|
10
|
+
import type { CreateBucketRequest } from 'aws-sdk/clients/s3';
|
|
12
11
|
|
|
13
12
|
declare namespace AWS {
|
|
14
|
-
function isAccessDefined(credential: Pick<ConfigurationOptions, "accessKeyId" | "secretAccessKey" | "sessionToken">): boolean;
|
|
15
|
-
function isEnvDefined(): boolean;
|
|
16
|
-
function isSharedCredentialsDefined(): boolean;
|
|
17
|
-
function isProviderChainDefined(): boolean;
|
|
18
|
-
function isDatabaseDefined(credential: AWSDatabaseCredential, data: CloudDatabase): boolean;
|
|
19
|
-
function getPublicReadPolicy(bucket: string, authenticated?: boolean, write?: boolean): string;
|
|
20
|
-
function getBucketPublicReadPolicy(bucket: string): string;
|
|
21
|
-
function getPrivatePolicy(bucket: string): string;
|
|
22
13
|
function validateStorage(credential: AWSStorageCredential): boolean;
|
|
23
14
|
function validateDatabase(credential: AWSDatabaseCredential, data: CloudDatabase): boolean;
|
|
24
15
|
function createStorageClient(this: IModule, credential: AWSStorageCredential, service?: string, sdk?: string): boolean;
|
|
@@ -31,11 +22,7 @@ declare namespace AWS {
|
|
|
31
22
|
function deleteObjectsV2(this: IModule, credential: AWSStorageCredential, Bucket: string, recursive?: boolean, service?: string, sdk?: string): Promise<void>;
|
|
32
23
|
function executeQuery(this: ICloud, credential: AWSDatabaseCredential, data: AWSDatabaseQuery, sessionKey?: string): Promise<QueryResult>;
|
|
33
24
|
function executeBatchQuery(this: ICloud, credential: AWSDatabaseCredential, batch: AWSDatabaseQuery[], sessionKey?: string): Promise<BatchQueryResult>;
|
|
34
|
-
function setDatabaseEndpoint(config: ServiceConfigurationOptions | DynamoDBClientConfig): void;
|
|
35
|
-
function checkBucketCannedACL(value: unknown): BucketCannedACL | undefined;
|
|
36
|
-
function writeMessageDefaultRetention(this: IModule, bucket: string, retention: DefaultRetention, service?: string): void;
|
|
37
25
|
function parseAttributeValue(value: unknown): AttributeValue;
|
|
38
|
-
function getBucketKey(credential: unknown, Bucket: string, acl: string | undefined, service: string, sdk: string): string;
|
|
39
26
|
function isNoSuchBucket(err: unknown): boolean;
|
|
40
27
|
}
|
|
41
28
|
|
package/client/index.js
CHANGED
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
|
|
3
|
-
exports.isNoSuchBucket = exports.getBucketKey = exports.parseAttributeValue = exports.writeMessageDefaultRetention = exports.checkBucketCannedACL = exports.setDatabaseEndpoint = exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.setBucketPolicy = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = exports.getPrivatePolicy = exports.getBucketPublicReadPolicy = exports.getPublicReadPolicy = exports.isDatabaseDefined = exports.isProviderChainDefined = exports.isEnvDefined = exports.isSharedCredentialsDefined = exports.isAccessDefined = void 0;
|
|
2
|
+
exports.CLOUD_UPLOAD_CHUNK = exports.CLOUD_UPLOAD_STREAM = exports.isNoSuchBucket = exports.parseAttributeValue = exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.setBucketTagging = exports.setBucketPolicy = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = void 0;
|
|
4
3
|
const aws = require("aws-sdk");
|
|
5
|
-
const types_1 = require("@e-mc/types");
|
|
6
|
-
const util_1 = require("@e-mc/cloud/util");
|
|
7
|
-
const Module = require("@e-mc/module");
|
|
8
4
|
const Cloud = require("@e-mc/cloud");
|
|
5
|
+
const util_1 = require("@e-mc/cloud/util");
|
|
6
|
+
const types_1 = require("@e-mc/types");
|
|
7
|
+
const aws_lib_1 = require("@pi-r/aws-lib");
|
|
9
8
|
const ACP_AUTHENTICATEDREAD = {
|
|
10
9
|
Grants: [{
|
|
11
10
|
Grantee: { Type: 'Group', URI: 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers' },
|
|
12
11
|
Permission: 'READ'
|
|
13
12
|
}]
|
|
14
13
|
};
|
|
15
|
-
async function setCannedAcl(S3, Bucket, ACL, service =
|
|
16
|
-
if (service ===
|
|
14
|
+
async function setCannedAcl(S3, Bucket, ACL, service = "aws", recursive) {
|
|
15
|
+
if (service === "oci") {
|
|
17
16
|
return;
|
|
18
17
|
}
|
|
19
18
|
const callback = (err) => {
|
|
@@ -31,114 +30,51 @@ async function setCannedAcl(S3, Bucket, ACL, service = 'aws', recursive) {
|
|
|
31
30
|
}
|
|
32
31
|
};
|
|
33
32
|
let promise;
|
|
34
|
-
if (service ===
|
|
33
|
+
if (service === "ibm") {
|
|
35
34
|
if (ACL === 1) {
|
|
36
|
-
ACL =
|
|
35
|
+
ACL = "public-read";
|
|
37
36
|
}
|
|
38
|
-
promise = S3.putBucketAcl(ACL ===
|
|
37
|
+
promise = S3.putBucketAcl(ACL === "authenticated-read" ? { Bucket, AccessControlPolicy: ACP_AUTHENTICATEDREAD } : { Bucket, ACL }).promise();
|
|
39
38
|
}
|
|
40
39
|
else {
|
|
41
40
|
switch (ACL) {
|
|
42
41
|
case 1:
|
|
43
|
-
promise = S3.putBucketPolicy({ Bucket, Policy: getBucketPublicReadPolicy(Bucket) }).promise();
|
|
42
|
+
promise = S3.putBucketPolicy({ Bucket, Policy: (0, aws_lib_1.getBucketPublicReadPolicy)(Bucket) }).promise();
|
|
44
43
|
break;
|
|
45
|
-
case
|
|
46
|
-
promise = S3.putBucketPolicy({ Bucket, Policy: getPublicReadPolicy(Bucket) }).promise();
|
|
44
|
+
case "public-read":
|
|
45
|
+
promise = S3.putBucketPolicy({ Bucket, Policy: (0, aws_lib_1.getPublicReadPolicy)(Bucket) }).promise();
|
|
47
46
|
break;
|
|
48
|
-
case
|
|
49
|
-
promise = S3.putBucketPolicy({ Bucket, Policy: getPublicReadPolicy(Bucket, false, true) }).promise();
|
|
47
|
+
case "public-read-write":
|
|
48
|
+
promise = S3.putBucketPolicy({ Bucket, Policy: (0, aws_lib_1.getPublicReadPolicy)(Bucket, false, true) }).promise();
|
|
50
49
|
break;
|
|
51
|
-
case
|
|
52
|
-
promise = S3.putBucketPolicy({ Bucket, Policy: getPublicReadPolicy(Bucket, true) }).promise();
|
|
50
|
+
case "authenticated-read":
|
|
51
|
+
promise = S3.putBucketPolicy({ Bucket, Policy: (0, aws_lib_1.getPublicReadPolicy)(Bucket, true) }).promise();
|
|
53
52
|
break;
|
|
54
53
|
default:
|
|
55
|
-
promise = S3.putBucketPolicy({ Bucket, Policy: getPrivatePolicy(Bucket) }).promise();
|
|
54
|
+
promise = S3.putBucketPolicy({ Bucket, Policy: (0, aws_lib_1.getPrivatePolicy)(Bucket) }).promise();
|
|
56
55
|
break;
|
|
57
56
|
}
|
|
58
57
|
}
|
|
59
|
-
return promise.then(() => callback(null)).catch(err => callback(err));
|
|
60
|
-
}
|
|
61
|
-
function isAccessDefined(credential) {
|
|
62
|
-
return !!(credential.accessKeyId && credential.secretAccessKey || credential.sessionToken);
|
|
63
|
-
}
|
|
64
|
-
exports.isAccessDefined = isAccessDefined;
|
|
65
|
-
function isSharedCredentialsDefined() {
|
|
66
|
-
return !!process.env.AWS_SDK_LOAD_CONFIG;
|
|
67
|
-
}
|
|
68
|
-
exports.isSharedCredentialsDefined = isSharedCredentialsDefined;
|
|
69
|
-
function isEnvDefined() {
|
|
70
|
-
return !!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY);
|
|
71
|
-
}
|
|
72
|
-
exports.isEnvDefined = isEnvDefined;
|
|
73
|
-
function isProviderChainDefined() {
|
|
74
|
-
return isSharedCredentialsDefined() || !!(process.env.AWS_WEB_IDENTITY_TOKEN_FILE || process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI || process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI);
|
|
75
|
-
}
|
|
76
|
-
exports.isProviderChainDefined = isProviderChainDefined;
|
|
77
|
-
function isDatabaseDefined(credential, data) {
|
|
78
|
-
return !!(data.table && (credential.region || credential.endpoint || process.env.AWS_DEFAULT_REGION || process.env.AWS_REGION));
|
|
79
|
-
}
|
|
80
|
-
exports.isDatabaseDefined = isDatabaseDefined;
|
|
81
|
-
function getPublicReadPolicy(bucket, authenticated, write) {
|
|
82
|
-
const Action = ["s3:GetObject", "s3:GetObjectVersion"];
|
|
83
|
-
if (write) {
|
|
84
|
-
Action.push("s3:PutObject", "s3:DeleteObjectVersion");
|
|
85
|
-
}
|
|
86
|
-
return JSON.stringify({
|
|
87
|
-
"Version": "2012-10-17",
|
|
88
|
-
"Statement": [{
|
|
89
|
-
"Sid": (authenticated ? "AuthenticatedRead" : "PublicRead") + (write ? "Write" : ""),
|
|
90
|
-
"Effect": "Allow",
|
|
91
|
-
"Principal": authenticated ? { "AWS": "*" } : "*",
|
|
92
|
-
"Action": Action,
|
|
93
|
-
"Resource": [`arn:aws:s3:::${bucket}/*`]
|
|
94
|
-
}]
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
exports.getPublicReadPolicy = getPublicReadPolicy;
|
|
98
|
-
function getBucketPublicReadPolicy(bucket) {
|
|
99
|
-
return JSON.stringify({
|
|
100
|
-
"Version": "2012-10-17",
|
|
101
|
-
"Statement": [{
|
|
102
|
-
"Sid": "BucketPublicRead",
|
|
103
|
-
"Effect": "Allow",
|
|
104
|
-
"Principal": "*",
|
|
105
|
-
"Action": ["s3:ListBucket", "s3:ListBucketVersions", "s3:ListBucketMultipartUploads"],
|
|
106
|
-
"Resource": [`arn:aws:s3:::${bucket}`]
|
|
107
|
-
}]
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
|
-
exports.getBucketPublicReadPolicy = getBucketPublicReadPolicy;
|
|
111
|
-
function getPrivatePolicy(bucket) {
|
|
112
|
-
return JSON.stringify({
|
|
113
|
-
"Version": "2012-10-17",
|
|
114
|
-
"Statement": [{
|
|
115
|
-
"Sid": "Private",
|
|
116
|
-
"Effect": "Deny",
|
|
117
|
-
"Principal": "*",
|
|
118
|
-
"Action": "*",
|
|
119
|
-
"Resource": [`arn:aws:s3:::${bucket}`, `arn:aws:s3:::${bucket}/*`]
|
|
120
|
-
}]
|
|
121
|
-
});
|
|
58
|
+
return promise.then(() => callback(null)).catch((err) => callback(err));
|
|
122
59
|
}
|
|
123
|
-
exports.getPrivatePolicy = getPrivatePolicy;
|
|
124
60
|
function validateStorage(credential) {
|
|
125
|
-
return !!(isAccessDefined(credential) || isEnvDefined() || credential.fromPath || credential.profile || isProviderChainDefined());
|
|
61
|
+
return !!((0, aws_lib_1.isAccessDefined)(credential) || (0, aws_lib_1.isEnvDefined)() || credential.fromPath || credential.profile || (0, aws_lib_1.isProviderChainDefined)());
|
|
126
62
|
}
|
|
127
63
|
exports.validateStorage = validateStorage;
|
|
128
64
|
function validateDatabase(credential, data) {
|
|
129
|
-
return isDatabaseDefined(credential, data) && validateStorage(credential);
|
|
65
|
+
return (0, aws_lib_1.isDatabaseDefined)(credential, data) && validateStorage(credential);
|
|
130
66
|
}
|
|
131
67
|
exports.validateDatabase = validateDatabase;
|
|
132
|
-
function createStorageClient(credential, service =
|
|
68
|
+
function createStorageClient(credential, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
133
69
|
try {
|
|
134
|
-
if (service ===
|
|
135
|
-
const {
|
|
136
|
-
if (fromPath) {
|
|
70
|
+
if (service === "aws") {
|
|
71
|
+
const { fromPath, profile } = credential;
|
|
72
|
+
if (fromPath && this.canRead(fromPath, { ownPermissionOnly: true })) {
|
|
137
73
|
const client = new aws.S3();
|
|
138
74
|
client.config.loadFromPath(fromPath);
|
|
139
75
|
return client;
|
|
140
76
|
}
|
|
141
|
-
if (profile || isSharedCredentialsDefined() && !isAccessDefined(credential) && !isEnvDefined()) {
|
|
77
|
+
if (profile || (0, aws_lib_1.isSharedCredentialsDefined)() && !(0, aws_lib_1.isAccessDefined)(credential) && !(0, aws_lib_1.isEnvDefined)()) {
|
|
142
78
|
credential = new aws.SharedIniFileCredentials({ profile });
|
|
143
79
|
}
|
|
144
80
|
return new aws.S3(credential);
|
|
@@ -155,23 +91,30 @@ function createDatabaseClient(credential) {
|
|
|
155
91
|
const { profile, fromPath } = credential;
|
|
156
92
|
let options;
|
|
157
93
|
if (fromPath) {
|
|
158
|
-
|
|
94
|
+
try {
|
|
95
|
+
options = JSON.parse(this.readFile(fromPath, { encoding: 'utf-8', ownPermissionOnly: true, cache: false }));
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
}
|
|
159
99
|
}
|
|
160
|
-
else if (profile || isSharedCredentialsDefined() && !isAccessDefined(credential) && !isEnvDefined()) {
|
|
100
|
+
else if (profile || (0, aws_lib_1.isSharedCredentialsDefined)() && !(0, aws_lib_1.isAccessDefined)(credential) && !(0, aws_lib_1.isEnvDefined)()) {
|
|
161
101
|
options = new aws.SharedIniFileCredentials({ profile });
|
|
162
102
|
}
|
|
163
103
|
else {
|
|
164
104
|
options = credential;
|
|
165
105
|
}
|
|
106
|
+
if (options?.endpoint && !options.region) {
|
|
107
|
+
(0, aws_lib_1.setDatabaseEndpoint)(options);
|
|
108
|
+
}
|
|
166
109
|
return new aws.DynamoDB.DocumentClient(options);
|
|
167
110
|
}
|
|
168
111
|
exports.createDatabaseClient = createDatabaseClient;
|
|
169
|
-
async function createBucket(credential, Bucket, publicRead, service =
|
|
170
|
-
return createBucketV2.call(this, credential, Bucket, publicRead ?
|
|
112
|
+
async function createBucket(credential, Bucket, publicRead, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
113
|
+
return createBucketV2.call(this, credential, Bucket, publicRead ? "public-read" : undefined, undefined, service, sdk);
|
|
171
114
|
}
|
|
172
115
|
exports.createBucket = createBucket;
|
|
173
|
-
async function createBucketV2(credential, Bucket, ACL, options, service =
|
|
174
|
-
ACL = ACL === 1 ? 1 : checkBucketCannedACL(ACL);
|
|
116
|
+
async function createBucketV2(credential, Bucket, ACL, options, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
117
|
+
ACL = ACL === 1 ? 1 : (0, aws_lib_1.checkBucketCannedACL)(ACL);
|
|
175
118
|
const S3 = createStorageClient.call(this, credential, service, sdk);
|
|
176
119
|
return S3.headBucket({ Bucket }).promise()
|
|
177
120
|
.then(async () => {
|
|
@@ -183,8 +126,8 @@ async function createBucketV2(credential, Bucket, ACL, options, service = 'aws',
|
|
|
183
126
|
.catch(async () => {
|
|
184
127
|
const input = { ...options, Bucket };
|
|
185
128
|
const region = credential.region;
|
|
186
|
-
if (region && (region !==
|
|
187
|
-
input.CreateBucketConfiguration
|
|
129
|
+
if (region && (region !== "us-east-1" || service !== "aws")) {
|
|
130
|
+
input.CreateBucketConfiguration ||= { LocationConstraint: region };
|
|
188
131
|
}
|
|
189
132
|
return S3.createBucket(input).promise()
|
|
190
133
|
.then(async () => {
|
|
@@ -210,11 +153,11 @@ async function createBucketV2(credential, Bucket, ACL, options, service = 'aws',
|
|
|
210
153
|
});
|
|
211
154
|
}
|
|
212
155
|
exports.createBucketV2 = createBucketV2;
|
|
213
|
-
async function setBucketPolicy(credential, Bucket, options, service =
|
|
214
|
-
const ibm = service ===
|
|
156
|
+
async function setBucketPolicy(credential, Bucket, options, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
157
|
+
const ibm = service === "ibm";
|
|
215
158
|
const S3 = createStorageClient.call(this, credential, service, sdk);
|
|
216
159
|
options.Bucket = Bucket;
|
|
217
|
-
if (ibm && 'ACL' in options && options.ACL ===
|
|
160
|
+
if (ibm && 'ACL' in options && options.ACL === "authenticated-read") {
|
|
218
161
|
options.AccessControlPolicy = ACP_AUTHENTICATEDREAD;
|
|
219
162
|
delete options.ACL;
|
|
220
163
|
}
|
|
@@ -223,7 +166,7 @@ async function setBucketPolicy(credential, Bucket, options, service = 'aws', sdk
|
|
|
223
166
|
this.formatMessage(64, service, ["Bucket policy configured", Bucket], '', { ...Cloud.LOG_CLOUD_COMMAND });
|
|
224
167
|
return true;
|
|
225
168
|
})
|
|
226
|
-
.catch(err => {
|
|
169
|
+
.catch((err) => {
|
|
227
170
|
if (!(0, exports.isNoSuchBucket)(err)) {
|
|
228
171
|
this.formatFail(64, service, ["Unable to update bucket policy", Bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
|
|
229
172
|
}
|
|
@@ -231,7 +174,30 @@ async function setBucketPolicy(credential, Bucket, options, service = 'aws', sdk
|
|
|
231
174
|
});
|
|
232
175
|
}
|
|
233
176
|
exports.setBucketPolicy = setBucketPolicy;
|
|
234
|
-
async function
|
|
177
|
+
async function setBucketTagging(credential, Bucket, options, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
178
|
+
if (!(0, types_1.isPlainObject)(options) || !Array.isArray(options.Tagging?.TagSet)) {
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
const S3 = createStorageClient.call(this, credential, service, sdk);
|
|
182
|
+
const deleting = options.Tagging.TagSet.length === 0;
|
|
183
|
+
const command = () => {
|
|
184
|
+
this.formatMessage(64, service, [deleting ? "Tags deleted" : "Tags created", Bucket], null, { ...Cloud.LOG_CLOUD_COMMAND });
|
|
185
|
+
return true;
|
|
186
|
+
};
|
|
187
|
+
const error = (err) => {
|
|
188
|
+
if (!(0, exports.isNoSuchBucket)(err)) {
|
|
189
|
+
this.formatFail(64, service, ["Unable to update bucket tagging", Bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
|
|
190
|
+
}
|
|
191
|
+
return false;
|
|
192
|
+
};
|
|
193
|
+
if (deleting) {
|
|
194
|
+
return S3.deleteBucketTagging({ Bucket, ExpectedBucketOwner: options.ExpectedBucketOwner }).promise().then(command).catch((err) => error(err));
|
|
195
|
+
}
|
|
196
|
+
options.Bucket = Bucket;
|
|
197
|
+
return S3.putBucketTagging(options).promise().then(command).catch((err) => error(err));
|
|
198
|
+
}
|
|
199
|
+
exports.setBucketTagging = setBucketTagging;
|
|
200
|
+
async function setBucketWebsite(credential, Bucket, options, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
235
201
|
const S3 = createStorageClient.call(this, credential, service, sdk);
|
|
236
202
|
const WebsiteConfiguration = {};
|
|
237
203
|
const { indexPage: Suffix, errorPage: Key } = options;
|
|
@@ -243,22 +209,22 @@ async function setBucketWebsite(credential, Bucket, options, service = 'aws', sd
|
|
|
243
209
|
}
|
|
244
210
|
return S3.putBucketWebsite({ Bucket, WebsiteConfiguration }).promise()
|
|
245
211
|
.then(() => {
|
|
246
|
-
this.formatMessage(64, service, ["Bucket configured", Bucket], WebsiteConfiguration, { ...Cloud.LOG_CLOUD_COMMAND });
|
|
212
|
+
this.formatMessage(64, service, ["Bucket website configured", Bucket], WebsiteConfiguration, { ...Cloud.LOG_CLOUD_COMMAND });
|
|
247
213
|
return true;
|
|
248
214
|
})
|
|
249
|
-
.catch(err => {
|
|
215
|
+
.catch((err) => {
|
|
250
216
|
if (!(0, exports.isNoSuchBucket)(err)) {
|
|
251
|
-
this.formatFail(64, service, ["Unable to
|
|
217
|
+
this.formatFail(64, service, ["Unable to set bucket website", Bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
|
|
252
218
|
}
|
|
253
219
|
return false;
|
|
254
220
|
});
|
|
255
221
|
}
|
|
256
222
|
exports.setBucketWebsite = setBucketWebsite;
|
|
257
|
-
async function deleteObjects(credential, Bucket, service =
|
|
223
|
+
async function deleteObjects(credential, Bucket, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
258
224
|
return deleteObjectsV2.call(this, credential, Bucket, true, service, sdk);
|
|
259
225
|
}
|
|
260
226
|
exports.deleteObjects = deleteObjects;
|
|
261
|
-
async function deleteObjectsV2(credential, Bucket, recursive = true, service =
|
|
227
|
+
async function deleteObjectsV2(credential, Bucket, recursive = true, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
262
228
|
const S3 = createStorageClient.call(this, credential, service, sdk);
|
|
263
229
|
return S3.listObjects({ Bucket }).promise()
|
|
264
230
|
.then(async ({ Contents }) => {
|
|
@@ -275,12 +241,12 @@ async function deleteObjectsV2(credential, Bucket, recursive = true, service = '
|
|
|
275
241
|
this.formatMessage(64, service, ["Bucket emptied" + ` (${recursive ? 'recursive' : files})`, Bucket], recursive ? files : '', { ...Cloud.LOG_CLOUD_COMMAND });
|
|
276
242
|
}
|
|
277
243
|
})
|
|
278
|
-
.catch(err => {
|
|
244
|
+
.catch((err) => {
|
|
279
245
|
this.formatFail(64, service, ["Unable to empty bucket", Bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
|
|
280
246
|
});
|
|
281
247
|
}
|
|
282
248
|
})
|
|
283
|
-
.catch(err => {
|
|
249
|
+
.catch((err) => {
|
|
284
250
|
if (!(0, exports.isNoSuchBucket)(err)) {
|
|
285
251
|
this.formatFail(64, service, ["Unable to list bucket", Bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
|
|
286
252
|
}
|
|
@@ -292,17 +258,16 @@ async function executeQuery(credential, data, sessionKey) {
|
|
|
292
258
|
}
|
|
293
259
|
exports.executeQuery = executeQuery;
|
|
294
260
|
async function executeBatchQuery(credential, batch, sessionKey) {
|
|
295
|
-
var _a;
|
|
296
261
|
const length = batch.length;
|
|
297
262
|
const result = new Array(length);
|
|
298
263
|
const caching = length > 0 && this.hasCache(batch[0].service, sessionKey);
|
|
299
264
|
const cacheValue = { value: this.valueOfKey(credential, 'cache'), sessionKey };
|
|
300
265
|
let client;
|
|
301
|
-
const createClient = () => client
|
|
302
|
-
setDatabaseEndpoint(credential);
|
|
266
|
+
const createClient = () => client ||= createDatabaseClient.call(this, credential);
|
|
267
|
+
(0, aws_lib_1.setDatabaseEndpoint)(credential);
|
|
303
268
|
for (let i = 0; i < length; ++i) {
|
|
304
269
|
const item = batch[i];
|
|
305
|
-
let { service, table: TableName, id, query,
|
|
270
|
+
let { service, table: TableName, id, query, key, limit = 0, update, ignoreCache } = item;
|
|
306
271
|
const useCache = caching && ignoreCache !== true;
|
|
307
272
|
const getCache = (value) => {
|
|
308
273
|
if (ignoreCache !== 1) {
|
|
@@ -317,7 +282,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
|
|
|
317
282
|
throw (0, util_1.formatError)(item, "Missing database table");
|
|
318
283
|
}
|
|
319
284
|
if (useCache) {
|
|
320
|
-
queryString = TableName + '_' +
|
|
285
|
+
queryString = TableName + '_' + Cloud.asString(key, true) + (id !== undefined ? '_' + Cloud.asString(id, true) : '');
|
|
321
286
|
if (!update && (rows = getCache(queryString))) {
|
|
322
287
|
result[i] = rows;
|
|
323
288
|
continue;
|
|
@@ -344,7 +309,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
|
|
|
344
309
|
if (limit > 0) {
|
|
345
310
|
query.Limit = limit;
|
|
346
311
|
}
|
|
347
|
-
if (useCache && (rows = getCache(queryString =
|
|
312
|
+
if (useCache && (rows = getCache(queryString = Cloud.asString(query, true)))) {
|
|
348
313
|
result[i] = rows;
|
|
349
314
|
continue;
|
|
350
315
|
}
|
|
@@ -358,14 +323,14 @@ async function executeBatchQuery(credential, batch, sessionKey) {
|
|
|
358
323
|
if (!(0, types_1.isPlainObject)(params.RequestItems)) {
|
|
359
324
|
params.RequestItems = {};
|
|
360
325
|
}
|
|
361
|
-
TableName
|
|
326
|
+
TableName ||= Object.keys(params.RequestItems)[0];
|
|
362
327
|
if (!TableName) {
|
|
363
328
|
throw (0, util_1.formatError)(item, "Missing database table");
|
|
364
329
|
}
|
|
365
|
-
const Item =
|
|
330
|
+
const Item = params.RequestItems[TableName] ||= {};
|
|
366
331
|
Item.Keys = query;
|
|
367
332
|
params = { RequestItems: { [TableName]: Item } };
|
|
368
|
-
if (useCache && (rows = getCache(queryString =
|
|
333
|
+
if (useCache && (rows = getCache(queryString = Cloud.asString(params, true)))) {
|
|
369
334
|
result[i] = rows;
|
|
370
335
|
continue;
|
|
371
336
|
}
|
|
@@ -382,7 +347,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
|
|
|
382
347
|
else {
|
|
383
348
|
params = { TableName };
|
|
384
349
|
}
|
|
385
|
-
if (useCache && (rows = getCache(queryString =
|
|
350
|
+
if (useCache && (rows = getCache(queryString = Cloud.asString(params, true)))) {
|
|
386
351
|
result[i] = rows;
|
|
387
352
|
continue;
|
|
388
353
|
}
|
|
@@ -399,42 +364,6 @@ async function executeBatchQuery(credential, batch, sessionKey) {
|
|
|
399
364
|
return result;
|
|
400
365
|
}
|
|
401
366
|
exports.executeBatchQuery = executeBatchQuery;
|
|
402
|
-
function setDatabaseEndpoint(config) {
|
|
403
|
-
let region = config.region;
|
|
404
|
-
const endpoint = config.endpoint || (config.endpoint = `https://dynamodb.${(0, types_1.isString)(region) ? region : process.env.AWS_DEFAULT_REGION || process.env.AWS_REGION || 'us-east-1'}.amazonaws.com`);
|
|
405
|
-
if (!region) {
|
|
406
|
-
region = ((0, types_1.isString)(endpoint) && /\bdynamodb\.([^.]+)\./i.exec(endpoint)?.[1] || process.env.AWS_DEFAULT_REGION || process.env.AWS_REGION)?.toLowerCase();
|
|
407
|
-
if (region && region !== 'us-east-1') {
|
|
408
|
-
config.region = region;
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
}
|
|
412
|
-
exports.setDatabaseEndpoint = setDatabaseEndpoint;
|
|
413
|
-
function checkBucketCannedACL(value) {
|
|
414
|
-
switch (value) {
|
|
415
|
-
case 'private':
|
|
416
|
-
case 'public-read':
|
|
417
|
-
case 'public-read-write':
|
|
418
|
-
case 'authenticated-read':
|
|
419
|
-
return value;
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
exports.checkBucketCannedACL = checkBucketCannedACL;
|
|
423
|
-
function writeMessageDefaultRetention(bucket, retention, service = 'aws') {
|
|
424
|
-
const { Years = 0, Days = 0, Mode } = retention;
|
|
425
|
-
const status = [];
|
|
426
|
-
if (Years > 0) {
|
|
427
|
-
status.push(Years.toString() + 'Y');
|
|
428
|
-
}
|
|
429
|
-
if (Days > 0) {
|
|
430
|
-
status.push(Days.toString() + 'D');
|
|
431
|
-
}
|
|
432
|
-
if (Mode) {
|
|
433
|
-
status.push(status.length ? `(${Mode})` : Mode);
|
|
434
|
-
}
|
|
435
|
-
this.formatMessage(64, service, ["Bucket configured" + ' (Retention Policy)', bucket], status.join(' '), { ...Cloud.LOG_CLOUD_COMMAND });
|
|
436
|
-
}
|
|
437
|
-
exports.writeMessageDefaultRetention = writeMessageDefaultRetention;
|
|
438
367
|
function parseAttributeValue(value) {
|
|
439
368
|
switch (typeof value) {
|
|
440
369
|
case 'string':
|
|
@@ -465,7 +394,7 @@ function parseAttributeValue(value) {
|
|
|
465
394
|
return { NULL: true };
|
|
466
395
|
}
|
|
467
396
|
exports.parseAttributeValue = parseAttributeValue;
|
|
468
|
-
const getBucketKey = (credential, Bucket, acl, service, sdk) => Module.asString(credential, true) + Bucket + '_' + (acl || '') + service + sdk;
|
|
469
|
-
exports.getBucketKey = getBucketKey;
|
|
470
397
|
const isNoSuchBucket = (err) => err instanceof Error && err.code === 'NoSuchBucket';
|
|
471
398
|
exports.isNoSuchBucket = isNoSuchBucket;
|
|
399
|
+
exports.CLOUD_UPLOAD_STREAM = true;
|
|
400
|
+
exports.CLOUD_UPLOAD_CHUNK = true;
|
package/download/index.js
CHANGED
|
@@ -1,25 +1,26 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
const types_1 = require("@e-mc/types");
|
|
3
|
-
const Module = require("@e-mc/module");
|
|
4
3
|
const Cloud = require("@e-mc/cloud");
|
|
5
4
|
const client_1 = require("@pi-r/aws");
|
|
6
|
-
function download(credential, service =
|
|
5
|
+
module.exports = function download(credential, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
7
6
|
const s3 = client_1.createStorageClient.call(this, credential, service, sdk);
|
|
8
7
|
return (data, callback) => {
|
|
9
8
|
const { bucket: Bucket, download: target } = data;
|
|
10
|
-
const Key = target.filename;
|
|
9
|
+
const Key = target.keyname || target.filename;
|
|
11
10
|
if (!Bucket || !Key) {
|
|
12
11
|
callback((0, types_1.errorValue)('Missing property', !Bucket ? 'Bucket' : 'Key'));
|
|
13
12
|
return;
|
|
14
13
|
}
|
|
15
|
-
const location =
|
|
16
|
-
|
|
17
|
-
s3.getObject(params, (err, result) => {
|
|
14
|
+
const location = Cloud.joinPath(Bucket, Key);
|
|
15
|
+
s3.getObject({ ...target.options, Bucket, Key, VersionId: target.versionId }, (err, result) => {
|
|
18
16
|
if (!err) {
|
|
19
17
|
callback(null, result.Body);
|
|
20
|
-
|
|
18
|
+
let deleteObject = target.deleteObject;
|
|
21
19
|
if (deleteObject) {
|
|
22
|
-
|
|
20
|
+
if (!(0, types_1.isPlainObject)(deleteObject)) {
|
|
21
|
+
deleteObject = undefined;
|
|
22
|
+
}
|
|
23
|
+
s3.deleteObject({ ...deleteObject, Bucket, Key, VersionId: target.versionId }, error => {
|
|
23
24
|
if (!error) {
|
|
24
25
|
this.formatMessage(64, service, "Delete success", location, { ...Cloud.LOG_CLOUD_DELETE });
|
|
25
26
|
}
|
|
@@ -34,6 +35,4 @@ function download(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
34
35
|
}
|
|
35
36
|
});
|
|
36
37
|
};
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
module.exports = download;
|
|
38
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-r/aws",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "AWS V2 cloud functions for E-mc.",
|
|
5
5
|
"main": "client/index.js",
|
|
6
6
|
"types": "client/index.d.ts",
|
|
@@ -21,9 +21,10 @@
|
|
|
21
21
|
"license": "MIT",
|
|
22
22
|
"homepage": "https://github.com/anpham6/pi-r#readme",
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@e-mc/cloud": "^0.
|
|
25
|
-
"@e-mc/module": "^0.
|
|
26
|
-
"@e-mc/types": "^0.
|
|
27
|
-
"aws-
|
|
24
|
+
"@e-mc/cloud": "^0.9.0",
|
|
25
|
+
"@e-mc/module": "^0.9.0",
|
|
26
|
+
"@e-mc/types": "^0.9.0",
|
|
27
|
+
"@pi-r/aws-lib": "^0.7.0",
|
|
28
|
+
"aws-sdk": "^2.1607.0"
|
|
28
29
|
}
|
|
29
30
|
}
|
package/types/index.d.ts
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
import type { CloudDatabase } from '@e-mc/types/lib/cloud';
|
|
1
|
+
import type { CloudDatabase, CloudStorage } from '@e-mc/types/lib/cloud';
|
|
3
2
|
|
|
4
3
|
import type { ConfigurationOptions } from 'aws-sdk/lib/core';
|
|
5
4
|
import type { ServiceConfigurationOptions } from 'aws-sdk/lib/service';
|
|
6
5
|
import type { DocumentClient } from 'aws-sdk/clients/dynamodb';
|
|
7
6
|
import type { PutBucketAclRequest, PutBucketPolicyRequest, PutPublicAccessBlockRequest } from 'aws-sdk/clients/s3';
|
|
8
7
|
|
|
9
|
-
export type BucketCannedACL = "authenticated-read" | "private" | "public-read" | "public-read-write";
|
|
10
|
-
export type ObjectCannedACL = BucketCannedACL | "aws-exec-read" | "bucket-owner-full-control" | "bucket-owner-read";
|
|
11
8
|
export type ConfigureBucketOptions = PutBucketAclRequest | PutBucketPolicyRequest | PutPublicAccessBlockRequest;
|
|
12
9
|
export type Key = DocumentClient.Key;
|
|
13
10
|
export type GetItemInput = DocumentClient.GetItemInput;
|
|
@@ -16,14 +13,17 @@ export type UpdateItemInput = DocumentClient.UpdateItemInput;
|
|
|
16
13
|
export type ScanInput = DocumentClient.ScanInput;
|
|
17
14
|
export type BatchGetItemInput = DocumentClient.BatchGetItemInput;
|
|
18
15
|
|
|
16
|
+
export type AWSStorage = CloudStorage<AWSStorageCredential, "aws">;
|
|
17
|
+
|
|
19
18
|
export interface AWSStorageCredential extends ConfigurationOptions {
|
|
20
19
|
profile?: string;
|
|
21
20
|
fromPath?: string;
|
|
22
21
|
}
|
|
23
22
|
|
|
24
|
-
export interface AWSDatabaseQuery extends CloudDatabase<QueryInput | Key[],
|
|
23
|
+
export interface AWSDatabaseQuery extends CloudDatabase<QueryInput | Key[], Record<string, unknown>, UpdateItemInput, BatchGetItemInput | ScanInput, AWSDatabaseCredential> {
|
|
24
|
+
source: "cloud";
|
|
25
|
+
service: "aws";
|
|
25
26
|
key?: string | Key;
|
|
26
|
-
partitionKey?: string | Key;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
export interface AWSDatabaseCredential extends AWSStorageCredential, ServiceConfigurationOptions {}
|
|
29
|
+
export interface AWSDatabaseCredential extends AWSStorageCredential, ServiceConfigurationOptions {}
|
package/upload/index.js
CHANGED
|
@@ -1,22 +1,24 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
const path = require("path");
|
|
3
|
+
const fs = require("fs");
|
|
4
|
+
const stream = require("stream");
|
|
5
|
+
const aws = require("aws-sdk");
|
|
6
|
+
const Cloud = require("@e-mc/cloud");
|
|
3
7
|
const util_1 = require("@e-mc/cloud/util");
|
|
4
8
|
const types_1 = require("@e-mc/types");
|
|
5
|
-
const
|
|
6
|
-
const Cloud = require("@e-mc/cloud");
|
|
9
|
+
const aws_lib_1 = require("@pi-r/aws-lib");
|
|
7
10
|
const client_1 = require("@pi-r/aws");
|
|
8
11
|
const BUCKET_SESSION = new Set();
|
|
9
12
|
const BUCKET_RESPONSE = {};
|
|
10
|
-
function upload(credential, service =
|
|
13
|
+
module.exports = function upload(credential, service = "aws", sdk = "aws-sdk/clients/s3") {
|
|
11
14
|
const s3 = client_1.createStorageClient.call(this, credential, service, sdk);
|
|
12
15
|
return async (data, callback) => {
|
|
13
|
-
var _a;
|
|
14
16
|
const { bucket: Bucket, localUri } = data;
|
|
15
|
-
const { pathname = '', fileGroup, contentType, metadata, tags, endpoint, active, publicRead, acl, admin = {}, overwrite, options } = data.upload;
|
|
17
|
+
const { pathname = '', flags = 0, fileGroup, contentType, metadata, tags: Tags, endpoint, active, publicRead, acl, admin = {}, overwrite, options } = data.upload;
|
|
16
18
|
let filename = data.upload.filename || path.basename(localUri), bucketKey;
|
|
17
19
|
const cleanup = () => {
|
|
18
20
|
BUCKET_SESSION.delete(service + Bucket);
|
|
19
|
-
if (bucketKey
|
|
21
|
+
if (bucketKey) {
|
|
20
22
|
delete BUCKET_RESPONSE[bucketKey];
|
|
21
23
|
}
|
|
22
24
|
};
|
|
@@ -27,25 +29,73 @@ function upload(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
27
29
|
const addLog = (err) => err instanceof Error && this.addLog(this.statusType.WARN, err, service, Bucket);
|
|
28
30
|
const configBucket = admin.configBucket;
|
|
29
31
|
if (!BUCKET_SESSION.has(service + Bucket)) {
|
|
30
|
-
const bucketAcl = admin.publicRead ?
|
|
31
|
-
const response = BUCKET_RESPONSE[
|
|
32
|
+
const bucketAcl = admin.publicRead ? "public-read" : admin.acl;
|
|
33
|
+
const response = BUCKET_RESPONSE[bucketKey = (0, aws_lib_1.getBucketKey)(credential, Bucket, bucketAcl, service, sdk)] ||= client_1.createBucketV2.call(this, credential, Bucket, bucketAcl, configBucket?.create, service, sdk);
|
|
32
34
|
if (!await response) {
|
|
33
35
|
errorResponse(null);
|
|
34
36
|
return;
|
|
35
37
|
}
|
|
36
38
|
BUCKET_SESSION.add(service + Bucket);
|
|
37
39
|
}
|
|
38
|
-
if (
|
|
39
|
-
const DefaultRetention = configBucket
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
40
|
+
if (configBucket && service !== "oci") {
|
|
41
|
+
const { cors: CORSConfiguration, lifecycle: LifecycleConfiguration, retentionPolicy: DefaultRetention } = configBucket;
|
|
42
|
+
const ExpectedBucketOwner = options?.ExpectedBucketOwner;
|
|
43
|
+
const commandMessage = (feature, message) => this.formatMessage(64, service, ["Bucket configured" + ` (${feature})`, Bucket], message || ExpectedBucketOwner, { ...Cloud[message === 'delete' ? 'LOG_CLOUD_WARN' : 'LOG_CLOUD_COMMAND'] });
|
|
44
|
+
if (service !== "ibm") {
|
|
45
|
+
if ((0, types_1.isPlainObject)(DefaultRetention)) {
|
|
46
|
+
s3.putObjectLockConfiguration({ Bucket, ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled', Rule: { DefaultRetention } }, ExpectedBucketOwner, RequestPayer: options?.RequestPayer }, err => {
|
|
47
|
+
if (!err) {
|
|
48
|
+
this.formatMessage(64, service, ["Bucket configured" + ' (Retention Policy)', Bucket], (0, aws_lib_1.formatDefaultRetention)(DefaultRetention), { ...Cloud.LOG_CLOUD_COMMAND });
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
addLog(err);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (CORSConfiguration && Array.isArray(CORSConfiguration.CORSRules)) {
|
|
57
|
+
if (CORSConfiguration.CORSRules.length === 0) {
|
|
58
|
+
s3.deleteBucketCors({ Bucket, ExpectedBucketOwner }, err => {
|
|
59
|
+
if (!err) {
|
|
60
|
+
commandMessage('CORS', 'delete');
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
addLog(err);
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
s3.putBucketCors({ Bucket, CORSConfiguration, ExpectedBucketOwner }, err => {
|
|
69
|
+
if (!err) {
|
|
70
|
+
commandMessage('CORS');
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
addLog(err);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (LifecycleConfiguration && Array.isArray(LifecycleConfiguration.Rules)) {
|
|
79
|
+
if (LifecycleConfiguration.Rules.length === 0) {
|
|
80
|
+
s3.deleteBucketLifecycle({ Bucket, ExpectedBucketOwner }, err => {
|
|
81
|
+
if (!err) {
|
|
82
|
+
commandMessage('Lifecycle', 'delete');
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
addLog(err);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
s3.putBucketLifecycleConfiguration({ Bucket, LifecycleConfiguration, ExpectedBucketOwner }, err => {
|
|
91
|
+
if (!err) {
|
|
92
|
+
commandMessage('Lifecycle');
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
addLog(err);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
}
|
|
49
99
|
}
|
|
50
100
|
}
|
|
51
101
|
if (!overwrite) {
|
|
@@ -59,7 +109,7 @@ function upload(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
59
109
|
break;
|
|
60
110
|
}
|
|
61
111
|
}
|
|
62
|
-
exists = await s3.headObject({ Bucket, Key: pathname ?
|
|
112
|
+
exists = await s3.headObject({ Bucket, Key: pathname ? Cloud.joinPath(pathname, filename) : filename }).promise()
|
|
63
113
|
.then(() => true)
|
|
64
114
|
.catch((err) => {
|
|
65
115
|
if (err instanceof Error && err.code !== 'NotFound') {
|
|
@@ -75,14 +125,39 @@ function upload(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
75
125
|
if (pathname) {
|
|
76
126
|
await s3.putObject({ Bucket, Key: pathname, Body: Buffer.from(''), ContentLength: 0 }).promise().catch(() => { });
|
|
77
127
|
}
|
|
128
|
+
const partSize = flags & 4 ? (0, types_1.alignSize)(data.upload.chunkSize, 1024) : 0;
|
|
78
129
|
const Key = [filename];
|
|
79
130
|
const Body = [data.buffer];
|
|
131
|
+
const Stream = [];
|
|
80
132
|
const ContentType = [contentType];
|
|
133
|
+
const getPartSize = () => partSize > 0 ? Math.max(partSize, aws.S3.ManagedUpload.minPartSize) : undefined;
|
|
134
|
+
if (Body[0].length === 0) {
|
|
135
|
+
try {
|
|
136
|
+
if ((flags & 2) || partSize > 0) {
|
|
137
|
+
Stream[0] = fs.createReadStream(localUri, { highWaterMark: getPartSize(), signal: this.signal });
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
Body[0] = fs.readFileSync(localUri);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
errorResponse(err);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
else if ((flags & 2) || partSize > 0) {
|
|
149
|
+
Stream[0] = stream.Readable.from(Body[0], { highWaterMark: getPartSize() });
|
|
150
|
+
}
|
|
81
151
|
if (fileGroup) {
|
|
82
|
-
const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, addLog);
|
|
152
|
+
const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, 0, addLog, flags);
|
|
83
153
|
Key.push(...key);
|
|
84
|
-
Body.push(...body);
|
|
85
154
|
ContentType.push(...type);
|
|
155
|
+
if (flags & 2) {
|
|
156
|
+
Stream.push(...body);
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
Body.push(...body);
|
|
160
|
+
}
|
|
86
161
|
}
|
|
87
162
|
for (let i = 0; i < Key.length; ++i) {
|
|
88
163
|
const first = i === 0;
|
|
@@ -92,12 +167,13 @@ function upload(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
92
167
|
}
|
|
93
168
|
return;
|
|
94
169
|
}
|
|
95
|
-
const params = { ...options, Bucket, Key: pathname + Key[i], Body: Body[i] };
|
|
170
|
+
const params = { ...options, Bucket, Key: pathname + Key[i], Body: Stream[i] || Body[i] };
|
|
96
171
|
const readable = publicRead || active && publicRead !== false && !acl;
|
|
172
|
+
let tags, length = -1;
|
|
97
173
|
if (first) {
|
|
98
|
-
params.ContentType
|
|
174
|
+
params.ContentType ||= ContentType[i];
|
|
99
175
|
if (readable) {
|
|
100
|
-
params.ACL =
|
|
176
|
+
params.ACL = "public-read";
|
|
101
177
|
}
|
|
102
178
|
else if (acl) {
|
|
103
179
|
params.ACL = acl;
|
|
@@ -105,19 +181,30 @@ function upload(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
105
181
|
if (metadata) {
|
|
106
182
|
params.Metadata = metadata;
|
|
107
183
|
}
|
|
184
|
+
if (service !== "oci") {
|
|
185
|
+
if ((0, types_1.isPlainObject)(Tags) && (length = Object.keys(Tags).length) > 0) {
|
|
186
|
+
tags = [];
|
|
187
|
+
for (const name in Tags) {
|
|
188
|
+
tags.push({ Key: name, Value: Tags[name] });
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
else if (Tags === false) {
|
|
192
|
+
length = 0;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
108
195
|
}
|
|
109
196
|
else {
|
|
110
197
|
params.ContentType = ContentType[i];
|
|
111
198
|
if (!params.ACL) {
|
|
112
199
|
if (readable) {
|
|
113
|
-
params.ACL =
|
|
200
|
+
params.ACL = "public-read";
|
|
114
201
|
}
|
|
115
202
|
else if (acl) {
|
|
116
203
|
params.ACL = acl;
|
|
117
204
|
}
|
|
118
205
|
}
|
|
119
206
|
}
|
|
120
|
-
|
|
207
|
+
new aws.S3.ManagedUpload({ service: s3, params, tags, partSize: getPartSize(), queueSize: partSize > 0 ? data.upload.chunkLimit : undefined }).send((err, result) => {
|
|
121
208
|
if (err) {
|
|
122
209
|
if (first) {
|
|
123
210
|
errorResponse(err);
|
|
@@ -127,28 +214,10 @@ function upload(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
127
214
|
}
|
|
128
215
|
return;
|
|
129
216
|
}
|
|
130
|
-
const url = endpoint ?
|
|
217
|
+
const url = endpoint ? Cloud.joinPath(endpoint, result.Key) : result.Location;
|
|
131
218
|
this.formatMessage(64, service, "Upload success", url, { ...Cloud.LOG_CLOUD_UPLOAD });
|
|
132
|
-
if (
|
|
133
|
-
|
|
134
|
-
}
|
|
135
|
-
if (service !== 'oci') {
|
|
136
|
-
let length = -1;
|
|
137
|
-
if ((0, types_1.isPlainObject)(tags) && (length = Object.keys(tags).length) > 0) {
|
|
138
|
-
const TagSet = [];
|
|
139
|
-
for (const name in tags) {
|
|
140
|
-
TagSet.push({ Key: name, Value: tags[name] });
|
|
141
|
-
}
|
|
142
|
-
s3.putObjectTagging({ Bucket, Key: result.Key, Tagging: { TagSet }, ExpectedBucketOwner: params.ExpectedBucketOwner, RequestPayer: params.RequestPayer }, error => {
|
|
143
|
-
if (!error) {
|
|
144
|
-
this.formatMessage(64, service, ["Tags created", Bucket], result.Key, { ...Cloud.LOG_CLOUD_COMMAND });
|
|
145
|
-
}
|
|
146
|
-
else {
|
|
147
|
-
addLog(error);
|
|
148
|
-
}
|
|
149
|
-
});
|
|
150
|
-
}
|
|
151
|
-
else if (tags === false || length === 0) {
|
|
219
|
+
if (first) {
|
|
220
|
+
if (length === 0) {
|
|
152
221
|
s3.deleteObjectTagging({ Bucket, Key: result.Key, ExpectedBucketOwner: params.ExpectedBucketOwner }, error => {
|
|
153
222
|
if (!error) {
|
|
154
223
|
this.formatMessage(64, service, ["Tags deleted", Bucket], result.Key, { ...Cloud.LOG_CLOUD_COMMAND });
|
|
@@ -158,12 +227,10 @@ function upload(credential, service = 'aws', sdk = 'aws-sdk/clients/s3') {
|
|
|
158
227
|
}
|
|
159
228
|
});
|
|
160
229
|
}
|
|
230
|
+
cleanup();
|
|
231
|
+
callback(null, url);
|
|
161
232
|
}
|
|
162
|
-
cleanup();
|
|
163
|
-
callback(null, url);
|
|
164
233
|
});
|
|
165
234
|
}
|
|
166
235
|
};
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
module.exports = upload;
|
|
236
|
+
};
|