mobiletto-driver-s3 1.0.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/index.d.ts +30 -0
- package/lib/cjs/index.js +316 -0
- package/lib/cjs/package.json +16 -0
- package/lib/cjs/package.json~ +16 -0
- package/lib/esm/index.d.ts +30 -0
- package/lib/esm/index.js +312 -0
- package/package.json +34 -9
- package/.babelrc +0 -5
- package/.eslintrc.js +0 -14
- package/index.js +0 -314
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { MobilettoError, MobilettoNotFoundError, MobilettoOptions, MobilettoVisitor, MobilettoMetadata, MobilettoWriteSource, MobilettoListOptions, MobilettoRemoveOptions } from "mobiletto-base";
|
|
2
|
+
export type S3Options = MobilettoOptions & {
|
|
3
|
+
bucket: string;
|
|
4
|
+
prefix?: string;
|
|
5
|
+
delimiter?: string;
|
|
6
|
+
region?: string;
|
|
7
|
+
};
|
|
8
|
+
declare class StorageClient {
|
|
9
|
+
private client;
|
|
10
|
+
private region;
|
|
11
|
+
private bucket;
|
|
12
|
+
private prefix;
|
|
13
|
+
private delimiter;
|
|
14
|
+
private normalizeRegex;
|
|
15
|
+
constructor(key: string, secret: string, opts: S3Options);
|
|
16
|
+
testConfig: () => Promise<MobilettoMetadata[]>;
|
|
17
|
+
stripPrefix: (name: string) => string;
|
|
18
|
+
nameToObj: (name: string) => MobilettoMetadata;
|
|
19
|
+
list(path: string | undefined, recursiveOrOpts: MobilettoListOptions | boolean, visitor?: MobilettoVisitor): Promise<MobilettoMetadata[] | undefined>;
|
|
20
|
+
_list(path: string, recursive?: boolean, visitor?: MobilettoVisitor, params?: {}): Promise<MobilettoMetadata[]>;
|
|
21
|
+
normalizeKey: (path: string) => string;
|
|
22
|
+
denormalizeKey: (key: string) => string;
|
|
23
|
+
s3error(err: any, key: string, path: string, method: string): MobilettoError | MobilettoNotFoundError;
|
|
24
|
+
metadata(path: string): Promise<MobilettoMetadata>;
|
|
25
|
+
write(path: string, generator: MobilettoWriteSource): Promise<number>;
|
|
26
|
+
read(path: string, callback: (data: any) => void, endCallback?: () => unknown): Promise<number>;
|
|
27
|
+
remove(path: string, optsOrRecursive?: MobilettoRemoveOptions | boolean, quiet?: boolean): Promise<string | string[]>;
|
|
28
|
+
}
|
|
29
|
+
export declare const storageClient: (key: string, secret: string, opts: S3Options) => StorageClient;
|
|
30
|
+
export {};
|
package/lib/cjs/index.js
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.storageClient = void 0;
|
|
13
|
+
const mobiletto_base_1 = require("mobiletto-base");
|
|
14
|
+
const path_1 = require("path");
|
|
15
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
16
|
+
const stream_1 = require("stream");
|
|
17
|
+
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
|
18
|
+
const DEFAULT_REGION = "us-east-1";
|
|
19
|
+
const DEFAULT_PREFIX = "";
|
|
20
|
+
const DEFAULT_DELIMITER = "/";
|
|
21
|
+
const DELETE_OBJECTS_MAX_KEYS = 1000;
|
|
22
|
+
class StorageClient {
|
|
23
|
+
constructor(key, secret, opts) {
|
|
24
|
+
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
25
|
+
this.testConfig = () => __awaiter(this, void 0, void 0, function* () { return yield this._list("", false, undefined, { MaxKeys: 1 }); });
|
|
26
|
+
this.stripPrefix = (name) => (name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name);
|
|
27
|
+
this.nameToObj = (name) => {
|
|
28
|
+
const relName = this.stripPrefix(name);
|
|
29
|
+
return {
|
|
30
|
+
name: relName,
|
|
31
|
+
type: relName.endsWith(this.delimiter) ? mobiletto_base_1.M_DIR : mobiletto_base_1.M_FILE,
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
this.normalizeKey = (path) => {
|
|
35
|
+
const p = path.startsWith(this.prefix)
|
|
36
|
+
? path
|
|
37
|
+
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path);
|
|
38
|
+
return p.replace(this.normalizeRegex, this.delimiter);
|
|
39
|
+
};
|
|
40
|
+
this.denormalizeKey = (key) => {
|
|
41
|
+
return key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key;
|
|
42
|
+
};
|
|
43
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
44
|
+
throw new mobiletto_base_1.MobilettoError("s3.StorageClient: key, secret, and opts.bucket are required");
|
|
45
|
+
}
|
|
46
|
+
this.bucket = opts.bucket;
|
|
47
|
+
const delim = (this.delimiter = opts.delimiter || DEFAULT_DELIMITER);
|
|
48
|
+
this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, "g");
|
|
49
|
+
this.prefix = opts.prefix || DEFAULT_PREFIX;
|
|
50
|
+
if (!this.prefix.endsWith(delim)) {
|
|
51
|
+
this.prefix += delim;
|
|
52
|
+
}
|
|
53
|
+
this.region = opts.region || DEFAULT_REGION;
|
|
54
|
+
const credentials = {
|
|
55
|
+
accessKeyId: key,
|
|
56
|
+
secretAccessKey: secret,
|
|
57
|
+
};
|
|
58
|
+
this.client = new client_s3_1.S3Client({ region: this.region, credentials });
|
|
59
|
+
}
|
|
60
|
+
list(path = "", recursiveOrOpts, visitor) {
|
|
61
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
62
|
+
const recursive = recursiveOrOpts === true || (typeof recursiveOrOpts === "object" && recursiveOrOpts.recursive);
|
|
63
|
+
try {
|
|
64
|
+
return yield this._list(path, recursive, visitor);
|
|
65
|
+
}
|
|
66
|
+
catch (e) {
|
|
67
|
+
if (e instanceof mobiletto_base_1.MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
|
|
68
|
+
// are we trying to list a single file?
|
|
69
|
+
const objects = yield this._list((0, path_1.dirname)(path), false);
|
|
70
|
+
const found = objects.find((o) => o.name === path);
|
|
71
|
+
if (found) {
|
|
72
|
+
return [found];
|
|
73
|
+
}
|
|
74
|
+
throw e;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
_list(path, recursive = false, visitor, params = {}) {
|
|
80
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
81
|
+
const logPrefix = `_list(path=${path})`;
|
|
82
|
+
// Declare truncated as a flag that the while loop is based on.
|
|
83
|
+
let truncated = true;
|
|
84
|
+
const Prefix = this.prefix +
|
|
85
|
+
(path.startsWith(this.delimiter) ? path.substring(0) : path) +
|
|
86
|
+
(path.length === 0 || path.endsWith(this.delimiter) ? "" : this.delimiter);
|
|
87
|
+
const bucketParams = Object.assign({}, params, {
|
|
88
|
+
Region: this.region,
|
|
89
|
+
Bucket: this.bucket,
|
|
90
|
+
Prefix,
|
|
91
|
+
});
|
|
92
|
+
if (!recursive) {
|
|
93
|
+
bucketParams.Delimiter = this.delimiter;
|
|
94
|
+
}
|
|
95
|
+
const objects = [];
|
|
96
|
+
let objectCount = 0;
|
|
97
|
+
mobiletto_base_1.logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`);
|
|
98
|
+
// while loop that runs until 'response.truncated' is false.
|
|
99
|
+
while (truncated) {
|
|
100
|
+
try {
|
|
101
|
+
const response = yield this.client.send(new client_s3_1.ListObjectsCommand(bucketParams));
|
|
102
|
+
const hasContents = typeof response.Contents !== "undefined";
|
|
103
|
+
if (hasContents) {
|
|
104
|
+
for (const item of response.Contents || []) {
|
|
105
|
+
if (!item.Key)
|
|
106
|
+
continue;
|
|
107
|
+
const obj = this.nameToObj(item.Key);
|
|
108
|
+
if (visitor) {
|
|
109
|
+
yield visitor(obj);
|
|
110
|
+
}
|
|
111
|
+
objects.push(obj);
|
|
112
|
+
objectCount++;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const hasCommonPrefixes = typeof response.CommonPrefixes !== "undefined";
|
|
116
|
+
if (hasCommonPrefixes) {
|
|
117
|
+
for (const item of response.CommonPrefixes || []) {
|
|
118
|
+
if (!item.Prefix)
|
|
119
|
+
continue;
|
|
120
|
+
const obj = this.nameToObj(item.Prefix);
|
|
121
|
+
if (visitor) {
|
|
122
|
+
yield visitor(obj);
|
|
123
|
+
}
|
|
124
|
+
objects.push(obj);
|
|
125
|
+
objectCount++;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
truncated = response.IsTruncated || false;
|
|
129
|
+
// If truncated is true, advance the marker
|
|
130
|
+
if (truncated) {
|
|
131
|
+
bucketParams.Marker = response.NextMarker;
|
|
132
|
+
}
|
|
133
|
+
else if (!hasContents && !hasCommonPrefixes) {
|
|
134
|
+
if (path === "") {
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
throw new mobiletto_base_1.MobilettoNotFoundError(path);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
catch (err) {
|
|
141
|
+
if (err instanceof mobiletto_base_1.MobilettoNotFoundError) {
|
|
142
|
+
throw err;
|
|
143
|
+
}
|
|
144
|
+
throw new mobiletto_base_1.MobilettoError(`${logPrefix} Error: ${err}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (recursive && objectCount === 0 && path !== "") {
|
|
148
|
+
throw new mobiletto_base_1.MobilettoNotFoundError(path);
|
|
149
|
+
}
|
|
150
|
+
const filtered = objects.filter((o) => o.name !== path);
|
|
151
|
+
return filtered;
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
s3error(err, key, path, method) {
|
|
155
|
+
return err instanceof mobiletto_base_1.MobilettoError || err instanceof mobiletto_base_1.MobilettoNotFoundError
|
|
156
|
+
? err
|
|
157
|
+
: err instanceof client_s3_1.NoSuchKey || (err.name && err.name === "NotFound")
|
|
158
|
+
? new mobiletto_base_1.MobilettoNotFoundError(this.denormalizeKey(key))
|
|
159
|
+
: new mobiletto_base_1.MobilettoError(`${method}(${path}) error: ${err}`, err);
|
|
160
|
+
}
|
|
161
|
+
metadata(path) {
|
|
162
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
163
|
+
const Key = this.normalizeKey(path);
|
|
164
|
+
const bucketParams = {
|
|
165
|
+
Region: this.region,
|
|
166
|
+
Bucket: this.bucket,
|
|
167
|
+
Key,
|
|
168
|
+
Delimiter: this.delimiter,
|
|
169
|
+
};
|
|
170
|
+
try {
|
|
171
|
+
const head = yield this.client.send(new client_s3_1.HeadObjectCommand(bucketParams));
|
|
172
|
+
const meta = {
|
|
173
|
+
name: this.stripPrefix(path),
|
|
174
|
+
size: head.ContentLength,
|
|
175
|
+
type: path.endsWith(this.delimiter) ? mobiletto_base_1.M_DIR : mobiletto_base_1.M_FILE,
|
|
176
|
+
};
|
|
177
|
+
if (head.LastModified) {
|
|
178
|
+
meta.mtime = Date.parse(head.LastModified.toString());
|
|
179
|
+
}
|
|
180
|
+
return meta;
|
|
181
|
+
}
|
|
182
|
+
catch (err) {
|
|
183
|
+
throw this.s3error(err, Key, path, "metadata");
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
write(path, generator) {
|
|
188
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
189
|
+
const Key = this.normalizeKey(path);
|
|
190
|
+
const bucketParams = {
|
|
191
|
+
Region: this.region,
|
|
192
|
+
Bucket: this.bucket,
|
|
193
|
+
Key,
|
|
194
|
+
Body: stream_1.Readable.from(generator),
|
|
195
|
+
Delimiter: this.delimiter,
|
|
196
|
+
};
|
|
197
|
+
const uploader = new lib_storage_1.Upload({
|
|
198
|
+
client: this.client,
|
|
199
|
+
params: bucketParams,
|
|
200
|
+
queueSize: 1,
|
|
201
|
+
partSize: 1024 * 1024 * 5,
|
|
202
|
+
leavePartsOnError: false, // optional manually handle dropped parts
|
|
203
|
+
});
|
|
204
|
+
let total = 0;
|
|
205
|
+
uploader.on("httpUploadProgress", (progress) => {
|
|
206
|
+
mobiletto_base_1.logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`);
|
|
207
|
+
total += progress.loaded || 0;
|
|
208
|
+
});
|
|
209
|
+
const response = yield uploader.done();
|
|
210
|
+
if (response.Key === Key) {
|
|
211
|
+
return total;
|
|
212
|
+
}
|
|
213
|
+
throw new mobiletto_base_1.MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`);
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
read(path, callback, endCallback) {
|
|
217
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
218
|
+
const Key = this.normalizeKey(path);
|
|
219
|
+
mobiletto_base_1.logger.debug(`read: reading Key: ${path} - ${Key}`);
|
|
220
|
+
const bucketParams = {
|
|
221
|
+
Region: this.region,
|
|
222
|
+
Bucket: this.bucket,
|
|
223
|
+
Key,
|
|
224
|
+
Delimiter: this.delimiter,
|
|
225
|
+
};
|
|
226
|
+
try {
|
|
227
|
+
const data = yield this.client.send(new client_s3_1.GetObjectCommand(bucketParams));
|
|
228
|
+
return yield (0, mobiletto_base_1.readStream)(data.Body, callback, endCallback);
|
|
229
|
+
}
|
|
230
|
+
catch (err) {
|
|
231
|
+
throw this.s3error(err, Key, path, "read");
|
|
232
|
+
}
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
remove(path, optsOrRecursive, quiet) {
|
|
236
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
237
|
+
const recursive = optsOrRecursive === true || (optsOrRecursive && optsOrRecursive.recursive);
|
|
238
|
+
if (recursive) {
|
|
239
|
+
const removed = [];
|
|
240
|
+
let objects = yield this._list(path, true, undefined, {
|
|
241
|
+
MaxKeys: DELETE_OBJECTS_MAX_KEYS,
|
|
242
|
+
});
|
|
243
|
+
while (objects && objects.length > 0) {
|
|
244
|
+
const Delete = {
|
|
245
|
+
Objects: objects.map((obj) => {
|
|
246
|
+
return { Key: this.normalizeKey(obj.name) };
|
|
247
|
+
}),
|
|
248
|
+
Quiet: quiet || false,
|
|
249
|
+
};
|
|
250
|
+
const bucketParams = {
|
|
251
|
+
Bucket: this.bucket,
|
|
252
|
+
Delete,
|
|
253
|
+
};
|
|
254
|
+
mobiletto_base_1.logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`);
|
|
255
|
+
const response = yield this.client.send(new client_s3_1.DeleteObjectsCommand(bucketParams));
|
|
256
|
+
const statusCode = response.$metadata.httpStatusCode;
|
|
257
|
+
const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
|
|
258
|
+
if (statusClass !== 2) {
|
|
259
|
+
throw new mobiletto_base_1.MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`);
|
|
260
|
+
}
|
|
261
|
+
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
262
|
+
throw new mobiletto_base_1.MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`);
|
|
263
|
+
}
|
|
264
|
+
if (response.Deleted) {
|
|
265
|
+
removed.push(...response.Deleted.map((del) => del.Key ? this.denormalizeKey(del.Key) : "?del.Key undefined?"));
|
|
266
|
+
}
|
|
267
|
+
try {
|
|
268
|
+
objects = yield this._list(path, true, undefined, { MaxKeys: DELETE_OBJECTS_MAX_KEYS });
|
|
269
|
+
}
|
|
270
|
+
catch (e) {
|
|
271
|
+
if (!(e instanceof mobiletto_base_1.MobilettoNotFoundError)) {
|
|
272
|
+
throw e instanceof mobiletto_base_1.MobilettoError
|
|
273
|
+
? e
|
|
274
|
+
: new mobiletto_base_1.MobilettoError(`remove(${path}) error listing: ${e}`);
|
|
275
|
+
}
|
|
276
|
+
objects = null;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return removed;
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
const Key = this.normalizeKey(path);
|
|
283
|
+
const bucketParams = {
|
|
284
|
+
Region: this.region,
|
|
285
|
+
Bucket: this.bucket,
|
|
286
|
+
Key,
|
|
287
|
+
};
|
|
288
|
+
try {
|
|
289
|
+
// DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
|
|
290
|
+
// Thus, if quiet is false, we must check metadata explicitly, which will fail with
|
|
291
|
+
// MobilettoNotFoundError, which is the correct behavior
|
|
292
|
+
if (!quiet) {
|
|
293
|
+
yield this.metadata(path);
|
|
294
|
+
}
|
|
295
|
+
const response = yield this.client.send(new client_s3_1.DeleteObjectCommand(bucketParams));
|
|
296
|
+
const statusCode = response.$metadata.httpStatusCode;
|
|
297
|
+
const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
|
|
298
|
+
if (statusClass !== 2) {
|
|
299
|
+
throw new mobiletto_base_1.MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
catch (err) {
|
|
303
|
+
throw this.s3error(err, Key, path, "remove");
|
|
304
|
+
}
|
|
305
|
+
return path;
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
const storageClient = (key, secret, opts) => {
|
|
311
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
312
|
+
throw new mobiletto_base_1.MobilettoError("s3.storageClient: key, secret, and opts.bucket are required");
|
|
313
|
+
}
|
|
314
|
+
return new StorageClient(key, secret, opts);
|
|
315
|
+
};
|
|
316
|
+
exports.storageClient = storageClient;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "mobiletto-driver-s3",
|
|
3
|
+
"type": "commonjs",
|
|
4
|
+
"main": "./index.js",
|
|
5
|
+
"homepage": "https://github.com/cobbzilla/mobiletto",
|
|
6
|
+
"author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
|
|
7
|
+
"funding": {
|
|
8
|
+
"type": "patreon",
|
|
9
|
+
"url": "https://www.patreon.com/cobbzilla"
|
|
10
|
+
},
|
|
11
|
+
"repository": {
|
|
12
|
+
"type": "git",
|
|
13
|
+
"url": "https://github.com/cobbzilla/mobiletto-driver-s3"
|
|
14
|
+
},
|
|
15
|
+
"license": "Apache-2.0"
|
|
16
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "mobiletto-common",
|
|
3
|
+
"type": "commonjs",
|
|
4
|
+
"main": "./index.js",
|
|
5
|
+
"homepage": "https://github.com/cobbzilla/mobiletto",
|
|
6
|
+
"author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
|
|
7
|
+
"funding": {
|
|
8
|
+
"type": "patreon",
|
|
9
|
+
"url": "https://www.patreon.com/cobbzilla"
|
|
10
|
+
},
|
|
11
|
+
"repository": {
|
|
12
|
+
"type": "git",
|
|
13
|
+
"url": "https://github.com/cobbzilla/mobiletto-common"
|
|
14
|
+
},
|
|
15
|
+
"license": "Apache-2.0"
|
|
16
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { MobilettoError, MobilettoNotFoundError, MobilettoOptions, MobilettoVisitor, MobilettoMetadata, MobilettoWriteSource, MobilettoListOptions, MobilettoRemoveOptions } from "mobiletto-base";
|
|
2
|
+
export type S3Options = MobilettoOptions & {
|
|
3
|
+
bucket: string;
|
|
4
|
+
prefix?: string;
|
|
5
|
+
delimiter?: string;
|
|
6
|
+
region?: string;
|
|
7
|
+
};
|
|
8
|
+
declare class StorageClient {
|
|
9
|
+
private client;
|
|
10
|
+
private region;
|
|
11
|
+
private bucket;
|
|
12
|
+
private prefix;
|
|
13
|
+
private delimiter;
|
|
14
|
+
private normalizeRegex;
|
|
15
|
+
constructor(key: string, secret: string, opts: S3Options);
|
|
16
|
+
testConfig: () => Promise<MobilettoMetadata[]>;
|
|
17
|
+
stripPrefix: (name: string) => string;
|
|
18
|
+
nameToObj: (name: string) => MobilettoMetadata;
|
|
19
|
+
list(path: string | undefined, recursiveOrOpts: MobilettoListOptions | boolean, visitor?: MobilettoVisitor): Promise<MobilettoMetadata[] | undefined>;
|
|
20
|
+
_list(path: string, recursive?: boolean, visitor?: MobilettoVisitor, params?: {}): Promise<MobilettoMetadata[]>;
|
|
21
|
+
normalizeKey: (path: string) => string;
|
|
22
|
+
denormalizeKey: (key: string) => string;
|
|
23
|
+
s3error(err: any, key: string, path: string, method: string): MobilettoError | MobilettoNotFoundError;
|
|
24
|
+
metadata(path: string): Promise<MobilettoMetadata>;
|
|
25
|
+
write(path: string, generator: MobilettoWriteSource): Promise<number>;
|
|
26
|
+
read(path: string, callback: (data: any) => void, endCallback?: () => unknown): Promise<number>;
|
|
27
|
+
remove(path: string, optsOrRecursive?: MobilettoRemoveOptions | boolean, quiet?: boolean): Promise<string | string[]>;
|
|
28
|
+
}
|
|
29
|
+
export declare const storageClient: (key: string, secret: string, opts: S3Options) => StorageClient;
|
|
30
|
+
export {};
|
package/lib/esm/index.js
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { M_DIR, M_FILE, logger, MobilettoError, MobilettoNotFoundError, readStream, } from "mobiletto-base";
|
|
11
|
+
import { dirname } from "path";
|
|
12
|
+
import { S3Client, ListObjectsCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey, } from "@aws-sdk/client-s3";
|
|
13
|
+
import { Readable } from "stream";
|
|
14
|
+
import { Upload } from "@aws-sdk/lib-storage";
|
|
15
|
+
const DEFAULT_REGION = "us-east-1";
|
|
16
|
+
const DEFAULT_PREFIX = "";
|
|
17
|
+
const DEFAULT_DELIMITER = "/";
|
|
18
|
+
const DELETE_OBJECTS_MAX_KEYS = 1000;
|
|
19
|
+
class StorageClient {
|
|
20
|
+
constructor(key, secret, opts) {
|
|
21
|
+
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
22
|
+
this.testConfig = () => __awaiter(this, void 0, void 0, function* () { return yield this._list("", false, undefined, { MaxKeys: 1 }); });
|
|
23
|
+
this.stripPrefix = (name) => (name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name);
|
|
24
|
+
this.nameToObj = (name) => {
|
|
25
|
+
const relName = this.stripPrefix(name);
|
|
26
|
+
return {
|
|
27
|
+
name: relName,
|
|
28
|
+
type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE,
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
this.normalizeKey = (path) => {
|
|
32
|
+
const p = path.startsWith(this.prefix)
|
|
33
|
+
? path
|
|
34
|
+
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path);
|
|
35
|
+
return p.replace(this.normalizeRegex, this.delimiter);
|
|
36
|
+
};
|
|
37
|
+
this.denormalizeKey = (key) => {
|
|
38
|
+
return key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key;
|
|
39
|
+
};
|
|
40
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
41
|
+
throw new MobilettoError("s3.StorageClient: key, secret, and opts.bucket are required");
|
|
42
|
+
}
|
|
43
|
+
this.bucket = opts.bucket;
|
|
44
|
+
const delim = (this.delimiter = opts.delimiter || DEFAULT_DELIMITER);
|
|
45
|
+
this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, "g");
|
|
46
|
+
this.prefix = opts.prefix || DEFAULT_PREFIX;
|
|
47
|
+
if (!this.prefix.endsWith(delim)) {
|
|
48
|
+
this.prefix += delim;
|
|
49
|
+
}
|
|
50
|
+
this.region = opts.region || DEFAULT_REGION;
|
|
51
|
+
const credentials = {
|
|
52
|
+
accessKeyId: key,
|
|
53
|
+
secretAccessKey: secret,
|
|
54
|
+
};
|
|
55
|
+
this.client = new S3Client({ region: this.region, credentials });
|
|
56
|
+
}
|
|
57
|
+
list(path = "", recursiveOrOpts, visitor) {
|
|
58
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
59
|
+
const recursive = recursiveOrOpts === true || (typeof recursiveOrOpts === "object" && recursiveOrOpts.recursive);
|
|
60
|
+
try {
|
|
61
|
+
return yield this._list(path, recursive, visitor);
|
|
62
|
+
}
|
|
63
|
+
catch (e) {
|
|
64
|
+
if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
|
|
65
|
+
// are we trying to list a single file?
|
|
66
|
+
const objects = yield this._list(dirname(path), false);
|
|
67
|
+
const found = objects.find((o) => o.name === path);
|
|
68
|
+
if (found) {
|
|
69
|
+
return [found];
|
|
70
|
+
}
|
|
71
|
+
throw e;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
_list(path, recursive = false, visitor, params = {}) {
|
|
77
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
78
|
+
const logPrefix = `_list(path=${path})`;
|
|
79
|
+
// Declare truncated as a flag that the while loop is based on.
|
|
80
|
+
let truncated = true;
|
|
81
|
+
const Prefix = this.prefix +
|
|
82
|
+
(path.startsWith(this.delimiter) ? path.substring(0) : path) +
|
|
83
|
+
(path.length === 0 || path.endsWith(this.delimiter) ? "" : this.delimiter);
|
|
84
|
+
const bucketParams = Object.assign({}, params, {
|
|
85
|
+
Region: this.region,
|
|
86
|
+
Bucket: this.bucket,
|
|
87
|
+
Prefix,
|
|
88
|
+
});
|
|
89
|
+
if (!recursive) {
|
|
90
|
+
bucketParams.Delimiter = this.delimiter;
|
|
91
|
+
}
|
|
92
|
+
const objects = [];
|
|
93
|
+
let objectCount = 0;
|
|
94
|
+
logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`);
|
|
95
|
+
// while loop that runs until 'response.truncated' is false.
|
|
96
|
+
while (truncated) {
|
|
97
|
+
try {
|
|
98
|
+
const response = yield this.client.send(new ListObjectsCommand(bucketParams));
|
|
99
|
+
const hasContents = typeof response.Contents !== "undefined";
|
|
100
|
+
if (hasContents) {
|
|
101
|
+
for (const item of response.Contents || []) {
|
|
102
|
+
if (!item.Key)
|
|
103
|
+
continue;
|
|
104
|
+
const obj = this.nameToObj(item.Key);
|
|
105
|
+
if (visitor) {
|
|
106
|
+
yield visitor(obj);
|
|
107
|
+
}
|
|
108
|
+
objects.push(obj);
|
|
109
|
+
objectCount++;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const hasCommonPrefixes = typeof response.CommonPrefixes !== "undefined";
|
|
113
|
+
if (hasCommonPrefixes) {
|
|
114
|
+
for (const item of response.CommonPrefixes || []) {
|
|
115
|
+
if (!item.Prefix)
|
|
116
|
+
continue;
|
|
117
|
+
const obj = this.nameToObj(item.Prefix);
|
|
118
|
+
if (visitor) {
|
|
119
|
+
yield visitor(obj);
|
|
120
|
+
}
|
|
121
|
+
objects.push(obj);
|
|
122
|
+
objectCount++;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
truncated = response.IsTruncated || false;
|
|
126
|
+
// If truncated is true, advance the marker
|
|
127
|
+
if (truncated) {
|
|
128
|
+
bucketParams.Marker = response.NextMarker;
|
|
129
|
+
}
|
|
130
|
+
else if (!hasContents && !hasCommonPrefixes) {
|
|
131
|
+
if (path === "") {
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
throw new MobilettoNotFoundError(path);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch (err) {
|
|
138
|
+
if (err instanceof MobilettoNotFoundError) {
|
|
139
|
+
throw err;
|
|
140
|
+
}
|
|
141
|
+
throw new MobilettoError(`${logPrefix} Error: ${err}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (recursive && objectCount === 0 && path !== "") {
|
|
145
|
+
throw new MobilettoNotFoundError(path);
|
|
146
|
+
}
|
|
147
|
+
const filtered = objects.filter((o) => o.name !== path);
|
|
148
|
+
return filtered;
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
s3error(err, key, path, method) {
|
|
152
|
+
return err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
153
|
+
? err
|
|
154
|
+
: err instanceof NoSuchKey || (err.name && err.name === "NotFound")
|
|
155
|
+
? new MobilettoNotFoundError(this.denormalizeKey(key))
|
|
156
|
+
: new MobilettoError(`${method}(${path}) error: ${err}`, err);
|
|
157
|
+
}
|
|
158
|
+
metadata(path) {
|
|
159
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
160
|
+
const Key = this.normalizeKey(path);
|
|
161
|
+
const bucketParams = {
|
|
162
|
+
Region: this.region,
|
|
163
|
+
Bucket: this.bucket,
|
|
164
|
+
Key,
|
|
165
|
+
Delimiter: this.delimiter,
|
|
166
|
+
};
|
|
167
|
+
try {
|
|
168
|
+
const head = yield this.client.send(new HeadObjectCommand(bucketParams));
|
|
169
|
+
const meta = {
|
|
170
|
+
name: this.stripPrefix(path),
|
|
171
|
+
size: head.ContentLength,
|
|
172
|
+
type: path.endsWith(this.delimiter) ? M_DIR : M_FILE,
|
|
173
|
+
};
|
|
174
|
+
if (head.LastModified) {
|
|
175
|
+
meta.mtime = Date.parse(head.LastModified.toString());
|
|
176
|
+
}
|
|
177
|
+
return meta;
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
throw this.s3error(err, Key, path, "metadata");
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
write(path, generator) {
|
|
185
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
186
|
+
const Key = this.normalizeKey(path);
|
|
187
|
+
const bucketParams = {
|
|
188
|
+
Region: this.region,
|
|
189
|
+
Bucket: this.bucket,
|
|
190
|
+
Key,
|
|
191
|
+
Body: Readable.from(generator),
|
|
192
|
+
Delimiter: this.delimiter,
|
|
193
|
+
};
|
|
194
|
+
const uploader = new Upload({
|
|
195
|
+
client: this.client,
|
|
196
|
+
params: bucketParams,
|
|
197
|
+
queueSize: 1,
|
|
198
|
+
partSize: 1024 * 1024 * 5,
|
|
199
|
+
leavePartsOnError: false, // optional manually handle dropped parts
|
|
200
|
+
});
|
|
201
|
+
let total = 0;
|
|
202
|
+
uploader.on("httpUploadProgress", (progress) => {
|
|
203
|
+
logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`);
|
|
204
|
+
total += progress.loaded || 0;
|
|
205
|
+
});
|
|
206
|
+
const response = yield uploader.done();
|
|
207
|
+
if (response.Key === Key) {
|
|
208
|
+
return total;
|
|
209
|
+
}
|
|
210
|
+
throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`);
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
read(path, callback, endCallback) {
|
|
214
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
215
|
+
const Key = this.normalizeKey(path);
|
|
216
|
+
logger.debug(`read: reading Key: ${path} - ${Key}`);
|
|
217
|
+
const bucketParams = {
|
|
218
|
+
Region: this.region,
|
|
219
|
+
Bucket: this.bucket,
|
|
220
|
+
Key,
|
|
221
|
+
Delimiter: this.delimiter,
|
|
222
|
+
};
|
|
223
|
+
try {
|
|
224
|
+
const data = yield this.client.send(new GetObjectCommand(bucketParams));
|
|
225
|
+
return yield readStream(data.Body, callback, endCallback);
|
|
226
|
+
}
|
|
227
|
+
catch (err) {
|
|
228
|
+
throw this.s3error(err, Key, path, "read");
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
remove(path, optsOrRecursive, quiet) {
|
|
233
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
234
|
+
const recursive = optsOrRecursive === true || (optsOrRecursive && optsOrRecursive.recursive);
|
|
235
|
+
if (recursive) {
|
|
236
|
+
const removed = [];
|
|
237
|
+
let objects = yield this._list(path, true, undefined, {
|
|
238
|
+
MaxKeys: DELETE_OBJECTS_MAX_KEYS,
|
|
239
|
+
});
|
|
240
|
+
while (objects && objects.length > 0) {
|
|
241
|
+
const Delete = {
|
|
242
|
+
Objects: objects.map((obj) => {
|
|
243
|
+
return { Key: this.normalizeKey(obj.name) };
|
|
244
|
+
}),
|
|
245
|
+
Quiet: quiet || false,
|
|
246
|
+
};
|
|
247
|
+
const bucketParams = {
|
|
248
|
+
Bucket: this.bucket,
|
|
249
|
+
Delete,
|
|
250
|
+
};
|
|
251
|
+
logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`);
|
|
252
|
+
const response = yield this.client.send(new DeleteObjectsCommand(bucketParams));
|
|
253
|
+
const statusCode = response.$metadata.httpStatusCode;
|
|
254
|
+
const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
|
|
255
|
+
if (statusClass !== 2) {
|
|
256
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`);
|
|
257
|
+
}
|
|
258
|
+
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
259
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`);
|
|
260
|
+
}
|
|
261
|
+
if (response.Deleted) {
|
|
262
|
+
removed.push(...response.Deleted.map((del) => del.Key ? this.denormalizeKey(del.Key) : "?del.Key undefined?"));
|
|
263
|
+
}
|
|
264
|
+
try {
|
|
265
|
+
objects = yield this._list(path, true, undefined, { MaxKeys: DELETE_OBJECTS_MAX_KEYS });
|
|
266
|
+
}
|
|
267
|
+
catch (e) {
|
|
268
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
269
|
+
throw e instanceof MobilettoError
|
|
270
|
+
? e
|
|
271
|
+
: new MobilettoError(`remove(${path}) error listing: ${e}`);
|
|
272
|
+
}
|
|
273
|
+
objects = null;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
return removed;
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
const Key = this.normalizeKey(path);
|
|
280
|
+
const bucketParams = {
|
|
281
|
+
Region: this.region,
|
|
282
|
+
Bucket: this.bucket,
|
|
283
|
+
Key,
|
|
284
|
+
};
|
|
285
|
+
try {
|
|
286
|
+
// DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
|
|
287
|
+
// Thus, if quiet is false, we must check metadata explicitly, which will fail with
|
|
288
|
+
// MobilettoNotFoundError, which is the correct behavior
|
|
289
|
+
if (!quiet) {
|
|
290
|
+
yield this.metadata(path);
|
|
291
|
+
}
|
|
292
|
+
const response = yield this.client.send(new DeleteObjectCommand(bucketParams));
|
|
293
|
+
const statusCode = response.$metadata.httpStatusCode;
|
|
294
|
+
const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
|
|
295
|
+
if (statusClass !== 2) {
|
|
296
|
+
throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
catch (err) {
|
|
300
|
+
throw this.s3error(err, Key, path, "remove");
|
|
301
|
+
}
|
|
302
|
+
return path;
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
export const storageClient = (key, secret, opts) => {
|
|
308
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
309
|
+
throw new MobilettoError("s3.storageClient: key, secret, and opts.bucket are required");
|
|
310
|
+
}
|
|
311
|
+
return new StorageClient(key, secret, opts);
|
|
312
|
+
};
|
package/package.json
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto-driver-s3",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"type": "module",
|
|
4
5
|
"description": "The Amazon S3 driver for mobiletto. This package is intended to be used with mobiletto-base, which does not include any drivers",
|
|
5
|
-
"main": "index.js",
|
|
6
|
+
"main": "./lib/cjs/index.js",
|
|
6
7
|
"keywords": [
|
|
7
8
|
"storage",
|
|
8
9
|
"s3",
|
|
@@ -18,17 +19,41 @@
|
|
|
18
19
|
},
|
|
19
20
|
"repository": {
|
|
20
21
|
"type": "git",
|
|
21
|
-
"url": "https://github.com/cobbzilla/mobiletto"
|
|
22
|
+
"url": "https://github.com/cobbzilla/mobiletto-driver-s3"
|
|
22
23
|
},
|
|
23
24
|
"license": "Apache-2.0",
|
|
25
|
+
"scripts": {
|
|
26
|
+
"test": "mocha --timeout 30000 ./test/*.spec.js",
|
|
27
|
+
"tsc": "tsc -p tsconfig.json && tsc -p tsconfig-cjs.json",
|
|
28
|
+
"lint": "npx eslint src",
|
|
29
|
+
"lint:fix": "npm run lint -- --fix",
|
|
30
|
+
"prettier": "npx prettier src --check",
|
|
31
|
+
"prettier:fix": "npm run prettier -- --write"
|
|
32
|
+
},
|
|
33
|
+
"module": "./lib/esm/index.js",
|
|
34
|
+
"files": [
|
|
35
|
+
"lib/"
|
|
36
|
+
],
|
|
24
37
|
"devDependencies": {
|
|
25
|
-
"@
|
|
26
|
-
"@
|
|
27
|
-
"@
|
|
38
|
+
"@types/chai": "^4.3.5",
|
|
39
|
+
"@types/mocha": "^10.0.1",
|
|
40
|
+
"@types/node": "^20.3.3",
|
|
41
|
+
"@typescript-eslint/eslint-plugin": "^5.61.0",
|
|
42
|
+
"@typescript-eslint/parser": "^5.61.0",
|
|
43
|
+
"chai": "^4.3.7",
|
|
44
|
+
"dotenv": "^16.3.1",
|
|
45
|
+
"eslint": "^8.44.0",
|
|
46
|
+
"eslint-config-prettier": "^8.8.0",
|
|
47
|
+
"mobiletto-base": "^2.0.16",
|
|
48
|
+
"mobiletto-driver-local": "^2.0.2",
|
|
49
|
+
"mocha": "^10.2.0",
|
|
50
|
+
"prettier": "^2.8.8",
|
|
51
|
+
"randomstring": "^1.3.0",
|
|
52
|
+
"tslint-config-prettier": "^1.18.0",
|
|
53
|
+
"typescript": "^5.1.6"
|
|
28
54
|
},
|
|
29
55
|
"dependencies": {
|
|
30
|
-
"@aws-sdk/client-s3": "^3.
|
|
31
|
-
"@aws-sdk/lib-storage": "^3.
|
|
32
|
-
"mobiletto-common": "^1.0.2"
|
|
56
|
+
"@aws-sdk/client-s3": "^3.367.0",
|
|
57
|
+
"@aws-sdk/lib-storage": "^3.367.0"
|
|
33
58
|
}
|
|
34
59
|
}
|
package/.eslintrc.js
DELETED
package/index.js
DELETED
|
@@ -1,314 +0,0 @@
|
|
|
1
|
-
const {
|
|
2
|
-
M_DIR, M_FILE,
|
|
3
|
-
MobilettoError, MobilettoNotFoundError, readStream
|
|
4
|
-
} = require('mobiletto-common')
|
|
5
|
-
|
|
6
|
-
const { dirname } = require('path')
|
|
7
|
-
|
|
8
|
-
const {
|
|
9
|
-
S3Client,
|
|
10
|
-
ListObjectsCommand,
|
|
11
|
-
HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey
|
|
12
|
-
} = require('@aws-sdk/client-s3')
|
|
13
|
-
|
|
14
|
-
const { Readable } = require('stream')
|
|
15
|
-
const {Upload} = require("@aws-sdk/lib-storage")
|
|
16
|
-
|
|
17
|
-
const DEFAULT_REGION = 'us-east-1'
|
|
18
|
-
const DEFAULT_PREFIX = ''
|
|
19
|
-
const DEFAULT_DELIMITER = '/'
|
|
20
|
-
|
|
21
|
-
const DELETE_OBJECTS_MAX_KEYS = 1000
|
|
22
|
-
|
|
23
|
-
class StorageClient {
|
|
24
|
-
client
|
|
25
|
-
region
|
|
26
|
-
bucket
|
|
27
|
-
prefix
|
|
28
|
-
delimiter
|
|
29
|
-
normalizeRegex
|
|
30
|
-
constructor(key, secret, opts) {
|
|
31
|
-
if (!key || !secret || !opts || !opts.bucket) {
|
|
32
|
-
throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
|
|
33
|
-
}
|
|
34
|
-
this.bucket = opts.bucket
|
|
35
|
-
const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
|
|
36
|
-
this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, 'g')
|
|
37
|
-
this.prefix = opts.prefix || DEFAULT_PREFIX
|
|
38
|
-
? opts.prefix.endsWith(delim)
|
|
39
|
-
? opts.prefix
|
|
40
|
-
: opts.prefix + delim
|
|
41
|
-
: ''
|
|
42
|
-
this.region = opts.region || DEFAULT_REGION
|
|
43
|
-
const credentials = {
|
|
44
|
-
accessKeyId: key,
|
|
45
|
-
secretAccessKey: secret
|
|
46
|
-
}
|
|
47
|
-
this.client = new S3Client({region: this.region, credentials})
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
51
|
-
testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
|
|
52
|
-
|
|
53
|
-
stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
|
|
54
|
-
|
|
55
|
-
nameToObj = (name) => {
|
|
56
|
-
const relName = this.stripPrefix(name)
|
|
57
|
-
return {
|
|
58
|
-
name: relName,
|
|
59
|
-
type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
async list (path = '', recursive = false, visitor = null) {
|
|
64
|
-
try {
|
|
65
|
-
return await this._list(path, recursive, visitor)
|
|
66
|
-
} catch (e) {
|
|
67
|
-
if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
|
|
68
|
-
// are we trying to list a single file?
|
|
69
|
-
const objects = await this._list(dirname(path), false)
|
|
70
|
-
const found = objects.find(o => o.name === path)
|
|
71
|
-
if (found) {
|
|
72
|
-
return [found]
|
|
73
|
-
}
|
|
74
|
-
throw e
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
80
|
-
const logPrefix = `_list(path=${path})`
|
|
81
|
-
|
|
82
|
-
// Declare truncated as a flag that the while loop is based on.
|
|
83
|
-
let truncated = true
|
|
84
|
-
|
|
85
|
-
const Prefix = this.prefix +
|
|
86
|
-
(path.startsWith(this.delimiter) ? path.substring(0) : path) +
|
|
87
|
-
(path.length === 0 || path.endsWith(this.delimiter) ? '' : this.delimiter)
|
|
88
|
-
const bucketParams = Object.assign({}, params, {
|
|
89
|
-
Region: this.region,
|
|
90
|
-
Bucket: this.bucket,
|
|
91
|
-
Prefix
|
|
92
|
-
})
|
|
93
|
-
if (!recursive) {
|
|
94
|
-
bucketParams.Delimiter = this.delimiter
|
|
95
|
-
}
|
|
96
|
-
const objects = []
|
|
97
|
-
let objectCount = 0
|
|
98
|
-
logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
|
|
99
|
-
|
|
100
|
-
// while loop that runs until 'response.truncated' is false.
|
|
101
|
-
while (truncated) {
|
|
102
|
-
try {
|
|
103
|
-
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
104
|
-
const hasContents = typeof response.Contents !== 'undefined'
|
|
105
|
-
if (hasContents) {
|
|
106
|
-
for (const item of response.Contents) {
|
|
107
|
-
const obj = this.nameToObj(item.Key);
|
|
108
|
-
if (visitor) {
|
|
109
|
-
await visitor(obj)
|
|
110
|
-
}
|
|
111
|
-
objects.push(obj)
|
|
112
|
-
objectCount++
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
116
|
-
if (hasCommonPrefixes) {
|
|
117
|
-
for (const item of response.CommonPrefixes) {
|
|
118
|
-
const obj = this.nameToObj(item.Prefix)
|
|
119
|
-
if (visitor) {
|
|
120
|
-
await visitor(obj)
|
|
121
|
-
}
|
|
122
|
-
objects.push(obj)
|
|
123
|
-
objectCount++
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
truncated = response.IsTruncated
|
|
127
|
-
// If truncated is true, advance the marker
|
|
128
|
-
if (truncated) {
|
|
129
|
-
bucketParams.Marker = response.NextMarker
|
|
130
|
-
} else if (!hasContents && !hasCommonPrefixes) {
|
|
131
|
-
if (path === '') {
|
|
132
|
-
break
|
|
133
|
-
}
|
|
134
|
-
throw new MobilettoNotFoundError(path)
|
|
135
|
-
}
|
|
136
|
-
} catch (err) {
|
|
137
|
-
if (err instanceof MobilettoNotFoundError) {
|
|
138
|
-
throw err
|
|
139
|
-
}
|
|
140
|
-
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
if (recursive && objectCount === 0 && path !== '') {
|
|
144
|
-
throw new MobilettoNotFoundError(path)
|
|
145
|
-
}
|
|
146
|
-
const filtered = objects.filter(o => o.name !== path)
|
|
147
|
-
return filtered
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
normalizeKey = (path) => {
|
|
151
|
-
const p = (path.startsWith(this.prefix)
|
|
152
|
-
? path
|
|
153
|
-
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
|
|
154
|
-
return p.replaceAll(this.normalizeRegex, this.delimiter)
|
|
155
|
-
}
|
|
156
|
-
denormalizeKey = (key) => {
|
|
157
|
-
return key.startsWith(this.prefix)
|
|
158
|
-
? key.substring(this.prefix.length)
|
|
159
|
-
: key
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
s3error (err, key, path, method) {
|
|
163
|
-
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
164
|
-
? err
|
|
165
|
-
: (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
|
|
166
|
-
? new MobilettoNotFoundError(this.denormalizeKey(key))
|
|
167
|
-
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
async metadata (path) {
|
|
171
|
-
const Key = this.normalizeKey(path)
|
|
172
|
-
const bucketParams = {
|
|
173
|
-
Region: this.region,
|
|
174
|
-
Bucket: this.bucket,
|
|
175
|
-
Key,
|
|
176
|
-
Delimiter: this.delimiter
|
|
177
|
-
}
|
|
178
|
-
try {
|
|
179
|
-
const head = await this.client.send(new HeadObjectCommand(bucketParams))
|
|
180
|
-
const meta = {
|
|
181
|
-
name: this.stripPrefix(path),
|
|
182
|
-
size: head.ContentLength,
|
|
183
|
-
type: path.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
184
|
-
}
|
|
185
|
-
if (head.LastModified) {
|
|
186
|
-
meta.mtime = Date.parse(head.LastModified)
|
|
187
|
-
}
|
|
188
|
-
return meta
|
|
189
|
-
} catch (err) {
|
|
190
|
-
throw this.s3error(err, Key, path, 'metadata')
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
async write (path, generator) {
|
|
195
|
-
const Key = this.normalizeKey(path)
|
|
196
|
-
const bucketParams = {
|
|
197
|
-
Region: this.region,
|
|
198
|
-
Bucket: this.bucket,
|
|
199
|
-
Key,
|
|
200
|
-
Body: Readable.from(generator),
|
|
201
|
-
Delimiter: this.delimiter
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
const uploader = new Upload({
|
|
205
|
-
client: this.client,
|
|
206
|
-
params: bucketParams,
|
|
207
|
-
queueSize: 1, // optional concurrency configuration
|
|
208
|
-
partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
|
|
209
|
-
leavePartsOnError: false // optional manually handle dropped parts
|
|
210
|
-
})
|
|
211
|
-
let total = 0
|
|
212
|
-
uploader.on('httpUploadProgress', (progress) => {
|
|
213
|
-
logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`)
|
|
214
|
-
total += progress.loaded
|
|
215
|
-
})
|
|
216
|
-
const response = await uploader.done()
|
|
217
|
-
if (response.Key !== Key) {
|
|
218
|
-
throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`)
|
|
219
|
-
}
|
|
220
|
-
return total
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
async read (path, callback, endCallback = null) {
|
|
224
|
-
const Key = this.normalizeKey(path)
|
|
225
|
-
logger.debug(`read: reading Key: ${path} - ${Key}`)
|
|
226
|
-
const bucketParams = {
|
|
227
|
-
Region: this.region,
|
|
228
|
-
Bucket: this.bucket,
|
|
229
|
-
Key,
|
|
230
|
-
Delimiter: this.delimiter
|
|
231
|
-
}
|
|
232
|
-
try {
|
|
233
|
-
const data = await this.client.send(new GetObjectCommand(bucketParams))
|
|
234
|
-
return await readStream(data.Body, callback, endCallback)
|
|
235
|
-
} catch (err) {
|
|
236
|
-
throw this.s3error(err, Key, path, 'read')
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
async remove (path, recursive, quiet) {
|
|
241
|
-
if (recursive) {
|
|
242
|
-
const removed = []
|
|
243
|
-
let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
|
|
244
|
-
while (objects && objects.length > 0) {
|
|
245
|
-
const Delete = {
|
|
246
|
-
Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
|
|
247
|
-
}
|
|
248
|
-
if (quiet) {
|
|
249
|
-
Delete.Quiet = true
|
|
250
|
-
}
|
|
251
|
-
const bucketParams = {
|
|
252
|
-
Region: this.region,
|
|
253
|
-
Bucket: this.bucket,
|
|
254
|
-
Delete
|
|
255
|
-
}
|
|
256
|
-
logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`)
|
|
257
|
-
const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
|
|
258
|
-
let statusCode = response.$metadata.httpStatusCode
|
|
259
|
-
let statusClass = Math.floor(statusCode / 100)
|
|
260
|
-
if (statusClass !== 2) {
|
|
261
|
-
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`)
|
|
262
|
-
}
|
|
263
|
-
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
264
|
-
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
265
|
-
}
|
|
266
|
-
if (response.Deleted) {
|
|
267
|
-
removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
|
|
268
|
-
}
|
|
269
|
-
try {
|
|
270
|
-
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
271
|
-
} catch (e) {
|
|
272
|
-
if (!(e instanceof MobilettoNotFoundError)) {
|
|
273
|
-
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`)
|
|
274
|
-
}
|
|
275
|
-
objects = null
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
return removed
|
|
279
|
-
} else {
|
|
280
|
-
const Key = this.normalizeKey(path)
|
|
281
|
-
const bucketParams = {
|
|
282
|
-
Region: this.region,
|
|
283
|
-
Bucket: this.bucket,
|
|
284
|
-
Key
|
|
285
|
-
}
|
|
286
|
-
try {
|
|
287
|
-
// DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
|
|
288
|
-
// Thus, if quiet is false, we must check metadata explicitly, which will fail with
|
|
289
|
-
// MobilettoNotFoundError, which is the correct behavior
|
|
290
|
-
if (!quiet) {
|
|
291
|
-
await this.metadata(path)
|
|
292
|
-
}
|
|
293
|
-
const response = await this.client.send(new DeleteObjectCommand(bucketParams))
|
|
294
|
-
let statusCode = response.$metadata.httpStatusCode
|
|
295
|
-
let statusClass = Math.floor(statusCode / 100)
|
|
296
|
-
if (statusClass !== 2) {
|
|
297
|
-
throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`)
|
|
298
|
-
}
|
|
299
|
-
} catch (err) {
|
|
300
|
-
throw this.s3error(err, Key, path, 'remove')
|
|
301
|
-
}
|
|
302
|
-
return path
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
function storageClient (key, secret, opts) {
|
|
308
|
-
if (!key || !secret || !opts || !opts.bucket) {
|
|
309
|
-
throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
|
|
310
|
-
}
|
|
311
|
-
return new StorageClient(key, secret, opts)
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
module.exports = { storageClient }
|