mobiletto-driver-s3 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +312 -3
- package/package.json +3 -3
- package/drivers/s3/index.js +0 -314
package/index.js
CHANGED
|
@@ -1,5 +1,314 @@
|
|
|
1
|
-
const
|
|
1
|
+
const {
|
|
2
|
+
M_DIR, M_FILE,
|
|
3
|
+
MobilettoError, MobilettoNotFoundError, readStream
|
|
4
|
+
} = require('mobiletto-common')
|
|
2
5
|
|
|
3
|
-
|
|
4
|
-
|
|
6
|
+
const { dirname } = require('path')
|
|
7
|
+
|
|
8
|
+
const {
|
|
9
|
+
S3Client,
|
|
10
|
+
ListObjectsCommand,
|
|
11
|
+
HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey
|
|
12
|
+
} = require('@aws-sdk/client-s3')
|
|
13
|
+
|
|
14
|
+
const { Readable } = require('stream')
|
|
15
|
+
const {Upload} = require("@aws-sdk/lib-storage")
|
|
16
|
+
|
|
17
|
+
const DEFAULT_REGION = 'us-east-1'
|
|
18
|
+
const DEFAULT_PREFIX = ''
|
|
19
|
+
const DEFAULT_DELIMITER = '/'
|
|
20
|
+
|
|
21
|
+
const DELETE_OBJECTS_MAX_KEYS = 1000
|
|
22
|
+
|
|
23
|
+
class StorageClient {
|
|
24
|
+
client
|
|
25
|
+
region
|
|
26
|
+
bucket
|
|
27
|
+
prefix
|
|
28
|
+
delimiter
|
|
29
|
+
normalizeRegex
|
|
30
|
+
constructor(key, secret, opts) {
|
|
31
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
32
|
+
throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
|
|
33
|
+
}
|
|
34
|
+
this.bucket = opts.bucket
|
|
35
|
+
const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
|
|
36
|
+
this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, 'g')
|
|
37
|
+
this.prefix = opts.prefix || DEFAULT_PREFIX
|
|
38
|
+
? opts.prefix.endsWith(delim)
|
|
39
|
+
? opts.prefix
|
|
40
|
+
: opts.prefix + delim
|
|
41
|
+
: ''
|
|
42
|
+
this.region = opts.region || DEFAULT_REGION
|
|
43
|
+
const credentials = {
|
|
44
|
+
accessKeyId: key,
|
|
45
|
+
secretAccessKey: secret
|
|
46
|
+
}
|
|
47
|
+
this.client = new S3Client({region: this.region, credentials})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
51
|
+
testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
|
|
52
|
+
|
|
53
|
+
stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
|
|
54
|
+
|
|
55
|
+
nameToObj = (name) => {
|
|
56
|
+
const relName = this.stripPrefix(name)
|
|
57
|
+
return {
|
|
58
|
+
name: relName,
|
|
59
|
+
type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async list (path = '', recursive = false, visitor = null) {
|
|
64
|
+
try {
|
|
65
|
+
return await this._list(path, recursive, visitor)
|
|
66
|
+
} catch (e) {
|
|
67
|
+
if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
|
|
68
|
+
// are we trying to list a single file?
|
|
69
|
+
const objects = await this._list(dirname(path), false)
|
|
70
|
+
const found = objects.find(o => o.name === path)
|
|
71
|
+
if (found) {
|
|
72
|
+
return [found]
|
|
73
|
+
}
|
|
74
|
+
throw e
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
80
|
+
const logPrefix = `_list(path=${path})`
|
|
81
|
+
|
|
82
|
+
// Declare truncated as a flag that the while loop is based on.
|
|
83
|
+
let truncated = true
|
|
84
|
+
|
|
85
|
+
const Prefix = this.prefix +
|
|
86
|
+
(path.startsWith(this.delimiter) ? path.substring(0) : path) +
|
|
87
|
+
(path.length === 0 || path.endsWith(this.delimiter) ? '' : this.delimiter)
|
|
88
|
+
const bucketParams = Object.assign({}, params, {
|
|
89
|
+
Region: this.region,
|
|
90
|
+
Bucket: this.bucket,
|
|
91
|
+
Prefix
|
|
92
|
+
})
|
|
93
|
+
if (!recursive) {
|
|
94
|
+
bucketParams.Delimiter = this.delimiter
|
|
95
|
+
}
|
|
96
|
+
const objects = []
|
|
97
|
+
let objectCount = 0
|
|
98
|
+
logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
|
|
99
|
+
|
|
100
|
+
// while loop that runs until 'response.truncated' is false.
|
|
101
|
+
while (truncated) {
|
|
102
|
+
try {
|
|
103
|
+
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
104
|
+
const hasContents = typeof response.Contents !== 'undefined'
|
|
105
|
+
if (hasContents) {
|
|
106
|
+
for (const item of response.Contents) {
|
|
107
|
+
const obj = this.nameToObj(item.Key);
|
|
108
|
+
if (visitor) {
|
|
109
|
+
await visitor(obj)
|
|
110
|
+
}
|
|
111
|
+
objects.push(obj)
|
|
112
|
+
objectCount++
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
116
|
+
if (hasCommonPrefixes) {
|
|
117
|
+
for (const item of response.CommonPrefixes) {
|
|
118
|
+
const obj = this.nameToObj(item.Prefix)
|
|
119
|
+
if (visitor) {
|
|
120
|
+
await visitor(obj)
|
|
121
|
+
}
|
|
122
|
+
objects.push(obj)
|
|
123
|
+
objectCount++
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
truncated = response.IsTruncated
|
|
127
|
+
// If truncated is true, advance the marker
|
|
128
|
+
if (truncated) {
|
|
129
|
+
bucketParams.Marker = response.NextMarker
|
|
130
|
+
} else if (!hasContents && !hasCommonPrefixes) {
|
|
131
|
+
if (path === '') {
|
|
132
|
+
break
|
|
133
|
+
}
|
|
134
|
+
throw new MobilettoNotFoundError(path)
|
|
135
|
+
}
|
|
136
|
+
} catch (err) {
|
|
137
|
+
if (err instanceof MobilettoNotFoundError) {
|
|
138
|
+
throw err
|
|
139
|
+
}
|
|
140
|
+
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
if (recursive && objectCount === 0 && path !== '') {
|
|
144
|
+
throw new MobilettoNotFoundError(path)
|
|
145
|
+
}
|
|
146
|
+
const filtered = objects.filter(o => o.name !== path)
|
|
147
|
+
return filtered
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
normalizeKey = (path) => {
|
|
151
|
+
const p = (path.startsWith(this.prefix)
|
|
152
|
+
? path
|
|
153
|
+
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
|
|
154
|
+
return p.replaceAll(this.normalizeRegex, this.delimiter)
|
|
155
|
+
}
|
|
156
|
+
denormalizeKey = (key) => {
|
|
157
|
+
return key.startsWith(this.prefix)
|
|
158
|
+
? key.substring(this.prefix.length)
|
|
159
|
+
: key
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
s3error (err, key, path, method) {
|
|
163
|
+
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
164
|
+
? err
|
|
165
|
+
: (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
|
|
166
|
+
? new MobilettoNotFoundError(this.denormalizeKey(key))
|
|
167
|
+
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async metadata (path) {
|
|
171
|
+
const Key = this.normalizeKey(path)
|
|
172
|
+
const bucketParams = {
|
|
173
|
+
Region: this.region,
|
|
174
|
+
Bucket: this.bucket,
|
|
175
|
+
Key,
|
|
176
|
+
Delimiter: this.delimiter
|
|
177
|
+
}
|
|
178
|
+
try {
|
|
179
|
+
const head = await this.client.send(new HeadObjectCommand(bucketParams))
|
|
180
|
+
const meta = {
|
|
181
|
+
name: this.stripPrefix(path),
|
|
182
|
+
size: head.ContentLength,
|
|
183
|
+
type: path.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
184
|
+
}
|
|
185
|
+
if (head.LastModified) {
|
|
186
|
+
meta.mtime = Date.parse(head.LastModified)
|
|
187
|
+
}
|
|
188
|
+
return meta
|
|
189
|
+
} catch (err) {
|
|
190
|
+
throw this.s3error(err, Key, path, 'metadata')
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
async write (path, generator) {
|
|
195
|
+
const Key = this.normalizeKey(path)
|
|
196
|
+
const bucketParams = {
|
|
197
|
+
Region: this.region,
|
|
198
|
+
Bucket: this.bucket,
|
|
199
|
+
Key,
|
|
200
|
+
Body: Readable.from(generator),
|
|
201
|
+
Delimiter: this.delimiter
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const uploader = new Upload({
|
|
205
|
+
client: this.client,
|
|
206
|
+
params: bucketParams,
|
|
207
|
+
queueSize: 1, // optional concurrency configuration
|
|
208
|
+
partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
|
|
209
|
+
leavePartsOnError: false // optional manually handle dropped parts
|
|
210
|
+
})
|
|
211
|
+
let total = 0
|
|
212
|
+
uploader.on('httpUploadProgress', (progress) => {
|
|
213
|
+
logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`)
|
|
214
|
+
total += progress.loaded
|
|
215
|
+
})
|
|
216
|
+
const response = await uploader.done()
|
|
217
|
+
if (response.Key !== Key) {
|
|
218
|
+
throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`)
|
|
219
|
+
}
|
|
220
|
+
return total
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async read (path, callback, endCallback = null) {
|
|
224
|
+
const Key = this.normalizeKey(path)
|
|
225
|
+
logger.debug(`read: reading Key: ${path} - ${Key}`)
|
|
226
|
+
const bucketParams = {
|
|
227
|
+
Region: this.region,
|
|
228
|
+
Bucket: this.bucket,
|
|
229
|
+
Key,
|
|
230
|
+
Delimiter: this.delimiter
|
|
231
|
+
}
|
|
232
|
+
try {
|
|
233
|
+
const data = await this.client.send(new GetObjectCommand(bucketParams))
|
|
234
|
+
return await readStream(data.Body, callback, endCallback)
|
|
235
|
+
} catch (err) {
|
|
236
|
+
throw this.s3error(err, Key, path, 'read')
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async remove (path, recursive, quiet) {
|
|
241
|
+
if (recursive) {
|
|
242
|
+
const removed = []
|
|
243
|
+
let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
|
|
244
|
+
while (objects && objects.length > 0) {
|
|
245
|
+
const Delete = {
|
|
246
|
+
Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
|
|
247
|
+
}
|
|
248
|
+
if (quiet) {
|
|
249
|
+
Delete.Quiet = true
|
|
250
|
+
}
|
|
251
|
+
const bucketParams = {
|
|
252
|
+
Region: this.region,
|
|
253
|
+
Bucket: this.bucket,
|
|
254
|
+
Delete
|
|
255
|
+
}
|
|
256
|
+
logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`)
|
|
257
|
+
const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
|
|
258
|
+
let statusCode = response.$metadata.httpStatusCode
|
|
259
|
+
let statusClass = Math.floor(statusCode / 100)
|
|
260
|
+
if (statusClass !== 2) {
|
|
261
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`)
|
|
262
|
+
}
|
|
263
|
+
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
264
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
265
|
+
}
|
|
266
|
+
if (response.Deleted) {
|
|
267
|
+
removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
|
|
268
|
+
}
|
|
269
|
+
try {
|
|
270
|
+
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
271
|
+
} catch (e) {
|
|
272
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
273
|
+
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`)
|
|
274
|
+
}
|
|
275
|
+
objects = null
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return removed
|
|
279
|
+
} else {
|
|
280
|
+
const Key = this.normalizeKey(path)
|
|
281
|
+
const bucketParams = {
|
|
282
|
+
Region: this.region,
|
|
283
|
+
Bucket: this.bucket,
|
|
284
|
+
Key
|
|
285
|
+
}
|
|
286
|
+
try {
|
|
287
|
+
// DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
|
|
288
|
+
// Thus, if quiet is false, we must check metadata explicitly, which will fail with
|
|
289
|
+
// MobilettoNotFoundError, which is the correct behavior
|
|
290
|
+
if (!quiet) {
|
|
291
|
+
await this.metadata(path)
|
|
292
|
+
}
|
|
293
|
+
const response = await this.client.send(new DeleteObjectCommand(bucketParams))
|
|
294
|
+
let statusCode = response.$metadata.httpStatusCode
|
|
295
|
+
let statusClass = Math.floor(statusCode / 100)
|
|
296
|
+
if (statusClass !== 2) {
|
|
297
|
+
throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`)
|
|
298
|
+
}
|
|
299
|
+
} catch (err) {
|
|
300
|
+
throw this.s3error(err, Key, path, 'remove')
|
|
301
|
+
}
|
|
302
|
+
return path
|
|
303
|
+
}
|
|
304
|
+
}
|
|
5
305
|
}
|
|
306
|
+
|
|
307
|
+
function storageClient (key, secret, opts) {
|
|
308
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
309
|
+
throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
|
|
310
|
+
}
|
|
311
|
+
return new StorageClient(key, secret, opts)
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
module.exports = { storageClient }
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto-driver-s3",
|
|
3
|
-
"version": "1.0.
|
|
4
|
-
"description": "The
|
|
3
|
+
"version": "1.0.4",
|
|
4
|
+
"description": "The Amazon S3 driver for mobiletto. This package is intended to be used with mobiletto-base, which does not include any drivers",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"storage",
|
|
@@ -29,6 +29,6 @@
|
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@aws-sdk/client-s3": "^3.359.0",
|
|
31
31
|
"@aws-sdk/lib-storage": "^3.359.0",
|
|
32
|
-
"mobiletto-common": "^1.0.
|
|
32
|
+
"mobiletto-common": "^1.0.2"
|
|
33
33
|
}
|
|
34
34
|
}
|
package/drivers/s3/index.js
DELETED
|
@@ -1,314 +0,0 @@
|
|
|
1
|
-
const {
|
|
2
|
-
M_DIR, M_FILE,
|
|
3
|
-
MobilettoError, MobilettoNotFoundError, readStream
|
|
4
|
-
} = require('mobiletto-common')
|
|
5
|
-
|
|
6
|
-
const { dirname } = require('path')
|
|
7
|
-
|
|
8
|
-
const {
|
|
9
|
-
S3Client,
|
|
10
|
-
ListObjectsCommand,
|
|
11
|
-
HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey
|
|
12
|
-
} = require('@aws-sdk/client-s3')
|
|
13
|
-
|
|
14
|
-
const { Readable } = require('stream')
|
|
15
|
-
const {Upload} = require("@aws-sdk/lib-storage")
|
|
16
|
-
|
|
17
|
-
const DEFAULT_REGION = 'us-east-1'
|
|
18
|
-
const DEFAULT_PREFIX = ''
|
|
19
|
-
const DEFAULT_DELIMITER = '/'
|
|
20
|
-
|
|
21
|
-
const DELETE_OBJECTS_MAX_KEYS = 1000
|
|
22
|
-
|
|
23
|
-
class StorageClient {
|
|
24
|
-
client
|
|
25
|
-
region
|
|
26
|
-
bucket
|
|
27
|
-
prefix
|
|
28
|
-
delimiter
|
|
29
|
-
normalizeRegex
|
|
30
|
-
constructor(key, secret, opts) {
|
|
31
|
-
if (!key || !secret || !opts || !opts.bucket) {
|
|
32
|
-
throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
|
|
33
|
-
}
|
|
34
|
-
this.bucket = opts.bucket
|
|
35
|
-
const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
|
|
36
|
-
this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, 'g')
|
|
37
|
-
this.prefix = opts.prefix || DEFAULT_PREFIX
|
|
38
|
-
? opts.prefix.endsWith(delim)
|
|
39
|
-
? opts.prefix
|
|
40
|
-
: opts.prefix + delim
|
|
41
|
-
: ''
|
|
42
|
-
this.region = opts.region || DEFAULT_REGION
|
|
43
|
-
const credentials = {
|
|
44
|
-
accessKeyId: key,
|
|
45
|
-
secretAccessKey: secret
|
|
46
|
-
}
|
|
47
|
-
this.client = new S3Client({region: this.region, credentials})
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
51
|
-
testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
|
|
52
|
-
|
|
53
|
-
stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
|
|
54
|
-
|
|
55
|
-
nameToObj = (name) => {
|
|
56
|
-
const relName = this.stripPrefix(name)
|
|
57
|
-
return {
|
|
58
|
-
name: relName,
|
|
59
|
-
type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
async list (path = '', recursive = false, visitor = null) {
|
|
64
|
-
try {
|
|
65
|
-
return await this._list(path, recursive, visitor)
|
|
66
|
-
} catch (e) {
|
|
67
|
-
if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
|
|
68
|
-
// are we trying to list a single file?
|
|
69
|
-
const objects = await this._list(dirname(path), false)
|
|
70
|
-
const found = objects.find(o => o.name === path)
|
|
71
|
-
if (found) {
|
|
72
|
-
return [found]
|
|
73
|
-
}
|
|
74
|
-
throw e
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
80
|
-
const logPrefix = `_list(path=${path})`
|
|
81
|
-
|
|
82
|
-
// Declare truncated as a flag that the while loop is based on.
|
|
83
|
-
let truncated = true
|
|
84
|
-
|
|
85
|
-
const Prefix = this.prefix +
|
|
86
|
-
(path.startsWith(this.delimiter) ? path.substring(0) : path) +
|
|
87
|
-
(path.length === 0 || path.endsWith(this.delimiter) ? '' : this.delimiter)
|
|
88
|
-
const bucketParams = Object.assign({}, params, {
|
|
89
|
-
Region: this.region,
|
|
90
|
-
Bucket: this.bucket,
|
|
91
|
-
Prefix
|
|
92
|
-
})
|
|
93
|
-
if (!recursive) {
|
|
94
|
-
bucketParams.Delimiter = this.delimiter
|
|
95
|
-
}
|
|
96
|
-
const objects = []
|
|
97
|
-
let objectCount = 0
|
|
98
|
-
logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
|
|
99
|
-
|
|
100
|
-
// while loop that runs until 'response.truncated' is false.
|
|
101
|
-
while (truncated) {
|
|
102
|
-
try {
|
|
103
|
-
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
104
|
-
const hasContents = typeof response.Contents !== 'undefined'
|
|
105
|
-
if (hasContents) {
|
|
106
|
-
for (const item of response.Contents) {
|
|
107
|
-
const obj = this.nameToObj(item.Key);
|
|
108
|
-
if (visitor) {
|
|
109
|
-
await visitor(obj)
|
|
110
|
-
}
|
|
111
|
-
objects.push(obj)
|
|
112
|
-
objectCount++
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
116
|
-
if (hasCommonPrefixes) {
|
|
117
|
-
for (const item of response.CommonPrefixes) {
|
|
118
|
-
const obj = this.nameToObj(item.Prefix)
|
|
119
|
-
if (visitor) {
|
|
120
|
-
await visitor(obj)
|
|
121
|
-
}
|
|
122
|
-
objects.push(obj)
|
|
123
|
-
objectCount++
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
truncated = response.IsTruncated
|
|
127
|
-
// If truncated is true, advance the marker
|
|
128
|
-
if (truncated) {
|
|
129
|
-
bucketParams.Marker = response.NextMarker
|
|
130
|
-
} else if (!hasContents && !hasCommonPrefixes) {
|
|
131
|
-
if (path === '') {
|
|
132
|
-
break
|
|
133
|
-
}
|
|
134
|
-
throw new MobilettoNotFoundError(path)
|
|
135
|
-
}
|
|
136
|
-
} catch (err) {
|
|
137
|
-
if (err instanceof MobilettoNotFoundError) {
|
|
138
|
-
throw err
|
|
139
|
-
}
|
|
140
|
-
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
if (recursive && objectCount === 0 && path !== '') {
|
|
144
|
-
throw new MobilettoNotFoundError(path)
|
|
145
|
-
}
|
|
146
|
-
const filtered = objects.filter(o => o.name !== path)
|
|
147
|
-
return filtered
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
normalizeKey = (path) => {
|
|
151
|
-
const p = (path.startsWith(this.prefix)
|
|
152
|
-
? path
|
|
153
|
-
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
|
|
154
|
-
return p.replaceAll(this.normalizeRegex, this.delimiter)
|
|
155
|
-
}
|
|
156
|
-
denormalizeKey = (key) => {
|
|
157
|
-
return key.startsWith(this.prefix)
|
|
158
|
-
? key.substring(this.prefix.length)
|
|
159
|
-
: key
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
s3error (err, key, path, method) {
|
|
163
|
-
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
164
|
-
? err
|
|
165
|
-
: (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
|
|
166
|
-
? new MobilettoNotFoundError(this.denormalizeKey(key))
|
|
167
|
-
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
async metadata (path) {
|
|
171
|
-
const Key = this.normalizeKey(path)
|
|
172
|
-
const bucketParams = {
|
|
173
|
-
Region: this.region,
|
|
174
|
-
Bucket: this.bucket,
|
|
175
|
-
Key,
|
|
176
|
-
Delimiter: this.delimiter
|
|
177
|
-
}
|
|
178
|
-
try {
|
|
179
|
-
const head = await this.client.send(new HeadObjectCommand(bucketParams))
|
|
180
|
-
const meta = {
|
|
181
|
-
name: this.stripPrefix(path),
|
|
182
|
-
size: head.ContentLength,
|
|
183
|
-
type: path.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
184
|
-
}
|
|
185
|
-
if (head.LastModified) {
|
|
186
|
-
meta.mtime = Date.parse(head.LastModified)
|
|
187
|
-
}
|
|
188
|
-
return meta
|
|
189
|
-
} catch (err) {
|
|
190
|
-
throw this.s3error(err, Key, path, 'metadata')
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
async write (path, generator) {
|
|
195
|
-
const Key = this.normalizeKey(path)
|
|
196
|
-
const bucketParams = {
|
|
197
|
-
Region: this.region,
|
|
198
|
-
Bucket: this.bucket,
|
|
199
|
-
Key,
|
|
200
|
-
Body: Readable.from(generator),
|
|
201
|
-
Delimiter: this.delimiter
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
const uploader = new Upload({
|
|
205
|
-
client: this.client,
|
|
206
|
-
params: bucketParams,
|
|
207
|
-
queueSize: 1, // optional concurrency configuration
|
|
208
|
-
partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
|
|
209
|
-
leavePartsOnError: false // optional manually handle dropped parts
|
|
210
|
-
})
|
|
211
|
-
let total = 0
|
|
212
|
-
uploader.on('httpUploadProgress', (progress) => {
|
|
213
|
-
logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`)
|
|
214
|
-
total += progress.loaded
|
|
215
|
-
})
|
|
216
|
-
const response = await uploader.done()
|
|
217
|
-
if (response.Key !== Key) {
|
|
218
|
-
throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`)
|
|
219
|
-
}
|
|
220
|
-
return total
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
async read (path, callback, endCallback = null) {
|
|
224
|
-
const Key = this.normalizeKey(path)
|
|
225
|
-
logger.debug(`read: reading Key: ${path} - ${Key}`)
|
|
226
|
-
const bucketParams = {
|
|
227
|
-
Region: this.region,
|
|
228
|
-
Bucket: this.bucket,
|
|
229
|
-
Key,
|
|
230
|
-
Delimiter: this.delimiter
|
|
231
|
-
}
|
|
232
|
-
try {
|
|
233
|
-
const data = await this.client.send(new GetObjectCommand(bucketParams))
|
|
234
|
-
return await readStream(data.Body, callback, endCallback)
|
|
235
|
-
} catch (err) {
|
|
236
|
-
throw this.s3error(err, Key, path, 'read')
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
async remove (path, recursive, quiet) {
|
|
241
|
-
if (recursive) {
|
|
242
|
-
const removed = []
|
|
243
|
-
let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
|
|
244
|
-
while (objects && objects.length > 0) {
|
|
245
|
-
const Delete = {
|
|
246
|
-
Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
|
|
247
|
-
}
|
|
248
|
-
if (quiet) {
|
|
249
|
-
Delete.Quiet = true
|
|
250
|
-
}
|
|
251
|
-
const bucketParams = {
|
|
252
|
-
Region: this.region,
|
|
253
|
-
Bucket: this.bucket,
|
|
254
|
-
Delete
|
|
255
|
-
}
|
|
256
|
-
logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`)
|
|
257
|
-
const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
|
|
258
|
-
let statusCode = response.$metadata.httpStatusCode
|
|
259
|
-
let statusClass = Math.floor(statusCode / 100)
|
|
260
|
-
if (statusClass !== 2) {
|
|
261
|
-
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`)
|
|
262
|
-
}
|
|
263
|
-
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
264
|
-
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
265
|
-
}
|
|
266
|
-
if (response.Deleted) {
|
|
267
|
-
removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
|
|
268
|
-
}
|
|
269
|
-
try {
|
|
270
|
-
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
271
|
-
} catch (e) {
|
|
272
|
-
if (!(e instanceof MobilettoNotFoundError)) {
|
|
273
|
-
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`)
|
|
274
|
-
}
|
|
275
|
-
objects = null
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
return removed
|
|
279
|
-
} else {
|
|
280
|
-
const Key = this.normalizeKey(path)
|
|
281
|
-
const bucketParams = {
|
|
282
|
-
Region: this.region,
|
|
283
|
-
Bucket: this.bucket,
|
|
284
|
-
Key
|
|
285
|
-
}
|
|
286
|
-
try {
|
|
287
|
-
// DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
|
|
288
|
-
// Thus, if quiet is false, we must check metadata explicitly, which will fail with
|
|
289
|
-
// MobilettoNotFoundError, which is the correct behavior
|
|
290
|
-
if (!quiet) {
|
|
291
|
-
await this.metadata(path)
|
|
292
|
-
}
|
|
293
|
-
const response = await this.client.send(new DeleteObjectCommand(bucketParams))
|
|
294
|
-
let statusCode = response.$metadata.httpStatusCode
|
|
295
|
-
let statusClass = Math.floor(statusCode / 100)
|
|
296
|
-
if (statusClass !== 2) {
|
|
297
|
-
throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`)
|
|
298
|
-
}
|
|
299
|
-
} catch (err) {
|
|
300
|
-
throw this.s3error(err, Key, path, 'remove')
|
|
301
|
-
}
|
|
302
|
-
return path
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
function storageClient (key, secret, opts) {
|
|
308
|
-
if (!key || !secret || !opts || !opts.bucket) {
|
|
309
|
-
throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
|
|
310
|
-
}
|
|
311
|
-
return new StorageClient(key, secret, opts)
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
module.exports = { storageClient }
|