mobiletto-driver-s3 1.0.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,30 @@
1
+ import { MobilettoError, MobilettoNotFoundError, MobilettoOptions, MobilettoVisitor, MobilettoMetadata, MobilettoWriteSource, MobilettoListOptions, MobilettoRemoveOptions } from "mobiletto-base";
2
+ export type S3Options = MobilettoOptions & {
3
+ bucket: string;
4
+ prefix?: string;
5
+ delimiter?: string;
6
+ region?: string;
7
+ };
8
+ declare class StorageClient {
9
+ private client;
10
+ private region;
11
+ private bucket;
12
+ private prefix;
13
+ private delimiter;
14
+ private normalizeRegex;
15
+ constructor(key: string, secret: string, opts: S3Options);
16
+ testConfig: () => Promise<MobilettoMetadata[]>;
17
+ stripPrefix: (name: string) => string;
18
+ nameToObj: (name: string) => MobilettoMetadata;
19
+ list(path: string | undefined, recursiveOrOpts: MobilettoListOptions | boolean, visitor?: MobilettoVisitor): Promise<MobilettoMetadata[] | undefined>;
20
+ _list(path: string, recursive?: boolean, visitor?: MobilettoVisitor, params?: {}): Promise<MobilettoMetadata[]>;
21
+ normalizeKey: (path: string) => string;
22
+ denormalizeKey: (key: string) => string;
23
+ s3error(err: any, key: string, path: string, method: string): MobilettoError | MobilettoNotFoundError;
24
+ metadata(path: string): Promise<MobilettoMetadata>;
25
+ write(path: string, generator: MobilettoWriteSource): Promise<number>;
26
+ read(path: string, callback: (data: any) => void, endCallback?: () => unknown): Promise<number>;
27
+ remove(path: string, optsOrRecursive?: MobilettoRemoveOptions | boolean, quiet?: boolean): Promise<string | string[]>;
28
+ }
29
+ export declare const storageClient: (key: string, secret: string, opts: S3Options) => StorageClient;
30
+ export {};
@@ -0,0 +1,316 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.storageClient = void 0;
13
+ const mobiletto_base_1 = require("mobiletto-base");
14
+ const path_1 = require("path");
15
+ const client_s3_1 = require("@aws-sdk/client-s3");
16
+ const stream_1 = require("stream");
17
+ const lib_storage_1 = require("@aws-sdk/lib-storage");
18
+ const DEFAULT_REGION = "us-east-1";
19
+ const DEFAULT_PREFIX = "";
20
+ const DEFAULT_DELIMITER = "/";
21
+ const DELETE_OBJECTS_MAX_KEYS = 1000;
22
+ class StorageClient {
23
+ constructor(key, secret, opts) {
24
+ // noinspection JSUnusedGlobalSymbols -- called by driver init
25
+ this.testConfig = () => __awaiter(this, void 0, void 0, function* () { return yield this._list("", false, undefined, { MaxKeys: 1 }); });
26
+ this.stripPrefix = (name) => (name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name);
27
+ this.nameToObj = (name) => {
28
+ const relName = this.stripPrefix(name);
29
+ return {
30
+ name: relName,
31
+ type: relName.endsWith(this.delimiter) ? mobiletto_base_1.M_DIR : mobiletto_base_1.M_FILE,
32
+ };
33
+ };
34
+ this.normalizeKey = (path) => {
35
+ const p = path.startsWith(this.prefix)
36
+ ? path
37
+ : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path);
38
+ return p.replace(this.normalizeRegex, this.delimiter);
39
+ };
40
+ this.denormalizeKey = (key) => {
41
+ return key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key;
42
+ };
43
+ if (!key || !secret || !opts || !opts.bucket) {
44
+ throw new mobiletto_base_1.MobilettoError("s3.StorageClient: key, secret, and opts.bucket are required");
45
+ }
46
+ this.bucket = opts.bucket;
47
+ const delim = (this.delimiter = opts.delimiter || DEFAULT_DELIMITER);
48
+ this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, "g");
49
+ this.prefix = opts.prefix || DEFAULT_PREFIX;
50
+ if (!this.prefix.endsWith(delim)) {
51
+ this.prefix += delim;
52
+ }
53
+ this.region = opts.region || DEFAULT_REGION;
54
+ const credentials = {
55
+ accessKeyId: key,
56
+ secretAccessKey: secret,
57
+ };
58
+ this.client = new client_s3_1.S3Client({ region: this.region, credentials });
59
+ }
60
+ list(path = "", recursiveOrOpts, visitor) {
61
+ return __awaiter(this, void 0, void 0, function* () {
62
+ const recursive = recursiveOrOpts === true || (typeof recursiveOrOpts === "object" && recursiveOrOpts.recursive);
63
+ try {
64
+ return yield this._list(path, recursive, visitor);
65
+ }
66
+ catch (e) {
67
+ if (e instanceof mobiletto_base_1.MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
68
+ // are we trying to list a single file?
69
+ const objects = yield this._list((0, path_1.dirname)(path), false);
70
+ const found = objects.find((o) => o.name === path);
71
+ if (found) {
72
+ return [found];
73
+ }
74
+ throw e;
75
+ }
76
+ }
77
+ });
78
+ }
79
+ _list(path, recursive = false, visitor, params = {}) {
80
+ return __awaiter(this, void 0, void 0, function* () {
81
+ const logPrefix = `_list(path=${path})`;
82
+ // Declare truncated as a flag that the while loop is based on.
83
+ let truncated = true;
84
+ const Prefix = this.prefix +
85
+ (path.startsWith(this.delimiter) ? path.substring(0) : path) +
86
+ (path.length === 0 || path.endsWith(this.delimiter) ? "" : this.delimiter);
87
+ const bucketParams = Object.assign({}, params, {
88
+ Region: this.region,
89
+ Bucket: this.bucket,
90
+ Prefix,
91
+ });
92
+ if (!recursive) {
93
+ bucketParams.Delimiter = this.delimiter;
94
+ }
95
+ const objects = [];
96
+ let objectCount = 0;
97
+ mobiletto_base_1.logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`);
98
+ // while loop that runs until 'response.truncated' is false.
99
+ while (truncated) {
100
+ try {
101
+ const response = yield this.client.send(new client_s3_1.ListObjectsCommand(bucketParams));
102
+ const hasContents = typeof response.Contents !== "undefined";
103
+ if (hasContents) {
104
+ for (const item of response.Contents || []) {
105
+ if (!item.Key)
106
+ continue;
107
+ const obj = this.nameToObj(item.Key);
108
+ if (visitor) {
109
+ yield visitor(obj);
110
+ }
111
+ objects.push(obj);
112
+ objectCount++;
113
+ }
114
+ }
115
+ const hasCommonPrefixes = typeof response.CommonPrefixes !== "undefined";
116
+ if (hasCommonPrefixes) {
117
+ for (const item of response.CommonPrefixes || []) {
118
+ if (!item.Prefix)
119
+ continue;
120
+ const obj = this.nameToObj(item.Prefix);
121
+ if (visitor) {
122
+ yield visitor(obj);
123
+ }
124
+ objects.push(obj);
125
+ objectCount++;
126
+ }
127
+ }
128
+ truncated = response.IsTruncated || false;
129
+ // If truncated is true, advance the marker
130
+ if (truncated) {
131
+ bucketParams.Marker = response.NextMarker;
132
+ }
133
+ else if (!hasContents && !hasCommonPrefixes) {
134
+ if (path === "") {
135
+ break;
136
+ }
137
+ throw new mobiletto_base_1.MobilettoNotFoundError(path);
138
+ }
139
+ }
140
+ catch (err) {
141
+ if (err instanceof mobiletto_base_1.MobilettoNotFoundError) {
142
+ throw err;
143
+ }
144
+ throw new mobiletto_base_1.MobilettoError(`${logPrefix} Error: ${err}`);
145
+ }
146
+ }
147
+ if (recursive && objectCount === 0 && path !== "") {
148
+ throw new mobiletto_base_1.MobilettoNotFoundError(path);
149
+ }
150
+ const filtered = objects.filter((o) => o.name !== path);
151
+ return filtered;
152
+ });
153
+ }
154
+ s3error(err, key, path, method) {
155
+ return err instanceof mobiletto_base_1.MobilettoError || err instanceof mobiletto_base_1.MobilettoNotFoundError
156
+ ? err
157
+ : err instanceof client_s3_1.NoSuchKey || (err.name && err.name === "NotFound")
158
+ ? new mobiletto_base_1.MobilettoNotFoundError(this.denormalizeKey(key))
159
+ : new mobiletto_base_1.MobilettoError(`${method}(${path}) error: ${err}`, err);
160
+ }
161
+ metadata(path) {
162
+ return __awaiter(this, void 0, void 0, function* () {
163
+ const Key = this.normalizeKey(path);
164
+ const bucketParams = {
165
+ Region: this.region,
166
+ Bucket: this.bucket,
167
+ Key,
168
+ Delimiter: this.delimiter,
169
+ };
170
+ try {
171
+ const head = yield this.client.send(new client_s3_1.HeadObjectCommand(bucketParams));
172
+ const meta = {
173
+ name: this.stripPrefix(path),
174
+ size: head.ContentLength,
175
+ type: path.endsWith(this.delimiter) ? mobiletto_base_1.M_DIR : mobiletto_base_1.M_FILE,
176
+ };
177
+ if (head.LastModified) {
178
+ meta.mtime = Date.parse(head.LastModified.toString());
179
+ }
180
+ return meta;
181
+ }
182
+ catch (err) {
183
+ throw this.s3error(err, Key, path, "metadata");
184
+ }
185
+ });
186
+ }
187
+ write(path, generator) {
188
+ return __awaiter(this, void 0, void 0, function* () {
189
+ const Key = this.normalizeKey(path);
190
+ const bucketParams = {
191
+ Region: this.region,
192
+ Bucket: this.bucket,
193
+ Key,
194
+ Body: stream_1.Readable.from(generator),
195
+ Delimiter: this.delimiter,
196
+ };
197
+ const uploader = new lib_storage_1.Upload({
198
+ client: this.client,
199
+ params: bucketParams,
200
+ queueSize: 1,
201
+ partSize: 1024 * 1024 * 5,
202
+ leavePartsOnError: false, // optional manually handle dropped parts
203
+ });
204
+ let total = 0;
205
+ uploader.on("httpUploadProgress", (progress) => {
206
+ mobiletto_base_1.logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`);
207
+ total += progress.loaded || 0;
208
+ });
209
+ const response = yield uploader.done();
210
+ if (response.Key === Key) {
211
+ return total;
212
+ }
213
+ throw new mobiletto_base_1.MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`);
214
+ });
215
+ }
216
+ read(path, callback, endCallback) {
217
+ return __awaiter(this, void 0, void 0, function* () {
218
+ const Key = this.normalizeKey(path);
219
+ mobiletto_base_1.logger.debug(`read: reading Key: ${path} - ${Key}`);
220
+ const bucketParams = {
221
+ Region: this.region,
222
+ Bucket: this.bucket,
223
+ Key,
224
+ Delimiter: this.delimiter,
225
+ };
226
+ try {
227
+ const data = yield this.client.send(new client_s3_1.GetObjectCommand(bucketParams));
228
+ return yield (0, mobiletto_base_1.readStream)(data.Body, callback, endCallback);
229
+ }
230
+ catch (err) {
231
+ throw this.s3error(err, Key, path, "read");
232
+ }
233
+ });
234
+ }
235
+ remove(path, optsOrRecursive, quiet) {
236
+ return __awaiter(this, void 0, void 0, function* () {
237
+ const recursive = optsOrRecursive === true || (optsOrRecursive && optsOrRecursive.recursive);
238
+ if (recursive) {
239
+ const removed = [];
240
+ let objects = yield this._list(path, true, undefined, {
241
+ MaxKeys: DELETE_OBJECTS_MAX_KEYS,
242
+ });
243
+ while (objects && objects.length > 0) {
244
+ const Delete = {
245
+ Objects: objects.map((obj) => {
246
+ return { Key: this.normalizeKey(obj.name) };
247
+ }),
248
+ Quiet: quiet || false,
249
+ };
250
+ const bucketParams = {
251
+ Bucket: this.bucket,
252
+ Delete,
253
+ };
254
+ mobiletto_base_1.logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`);
255
+ const response = yield this.client.send(new client_s3_1.DeleteObjectsCommand(bucketParams));
256
+ const statusCode = response.$metadata.httpStatusCode;
257
+ const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
258
+ if (statusClass !== 2) {
259
+ throw new mobiletto_base_1.MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`);
260
+ }
261
+ if (!quiet && response.Errors && response.Errors.length > 0) {
262
+ throw new mobiletto_base_1.MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`);
263
+ }
264
+ if (response.Deleted) {
265
+ removed.push(...response.Deleted.map((del) => del.Key ? this.denormalizeKey(del.Key) : "?del.Key undefined?"));
266
+ }
267
+ try {
268
+ objects = yield this._list(path, true, undefined, { MaxKeys: DELETE_OBJECTS_MAX_KEYS });
269
+ }
270
+ catch (e) {
271
+ if (!(e instanceof mobiletto_base_1.MobilettoNotFoundError)) {
272
+ throw e instanceof mobiletto_base_1.MobilettoError
273
+ ? e
274
+ : new mobiletto_base_1.MobilettoError(`remove(${path}) error listing: ${e}`);
275
+ }
276
+ objects = null;
277
+ }
278
+ }
279
+ return removed;
280
+ }
281
+ else {
282
+ const Key = this.normalizeKey(path);
283
+ const bucketParams = {
284
+ Region: this.region,
285
+ Bucket: this.bucket,
286
+ Key,
287
+ };
288
+ try {
289
+ // DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
290
+ // Thus, if quiet is false, we must check metadata explicitly, which will fail with
291
+ // MobilettoNotFoundError, which is the correct behavior
292
+ if (!quiet) {
293
+ yield this.metadata(path);
294
+ }
295
+ const response = yield this.client.send(new client_s3_1.DeleteObjectCommand(bucketParams));
296
+ const statusCode = response.$metadata.httpStatusCode;
297
+ const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
298
+ if (statusClass !== 2) {
299
+ throw new mobiletto_base_1.MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`);
300
+ }
301
+ }
302
+ catch (err) {
303
+ throw this.s3error(err, Key, path, "remove");
304
+ }
305
+ return path;
306
+ }
307
+ });
308
+ }
309
+ }
310
+ const storageClient = (key, secret, opts) => {
311
+ if (!key || !secret || !opts || !opts.bucket) {
312
+ throw new mobiletto_base_1.MobilettoError("s3.storageClient: key, secret, and opts.bucket are required");
313
+ }
314
+ return new StorageClient(key, secret, opts);
315
+ };
316
+ exports.storageClient = storageClient;
@@ -0,0 +1,16 @@
1
+ {
2
+ "name": "mobiletto-driver-s3",
3
+ "type": "commonjs",
4
+ "main": "./index.js",
5
+ "homepage": "https://github.com/cobbzilla/mobiletto",
6
+ "author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
7
+ "funding": {
8
+ "type": "patreon",
9
+ "url": "https://www.patreon.com/cobbzilla"
10
+ },
11
+ "repository": {
12
+ "type": "git",
13
+ "url": "https://github.com/cobbzilla/mobiletto-driver-s3"
14
+ },
15
+ "license": "Apache-2.0"
16
+ }
@@ -0,0 +1,16 @@
1
+ {
2
+ "name": "mobiletto-common",
3
+ "type": "commonjs",
4
+ "main": "./index.js",
5
+ "homepage": "https://github.com/cobbzilla/mobiletto",
6
+ "author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
7
+ "funding": {
8
+ "type": "patreon",
9
+ "url": "https://www.patreon.com/cobbzilla"
10
+ },
11
+ "repository": {
12
+ "type": "git",
13
+ "url": "https://github.com/cobbzilla/mobiletto-common"
14
+ },
15
+ "license": "Apache-2.0"
16
+ }
@@ -0,0 +1,30 @@
1
+ import { MobilettoError, MobilettoNotFoundError, MobilettoOptions, MobilettoVisitor, MobilettoMetadata, MobilettoWriteSource, MobilettoListOptions, MobilettoRemoveOptions } from "mobiletto-base";
2
+ export type S3Options = MobilettoOptions & {
3
+ bucket: string;
4
+ prefix?: string;
5
+ delimiter?: string;
6
+ region?: string;
7
+ };
8
+ declare class StorageClient {
9
+ private client;
10
+ private region;
11
+ private bucket;
12
+ private prefix;
13
+ private delimiter;
14
+ private normalizeRegex;
15
+ constructor(key: string, secret: string, opts: S3Options);
16
+ testConfig: () => Promise<MobilettoMetadata[]>;
17
+ stripPrefix: (name: string) => string;
18
+ nameToObj: (name: string) => MobilettoMetadata;
19
+ list(path: string | undefined, recursiveOrOpts: MobilettoListOptions | boolean, visitor?: MobilettoVisitor): Promise<MobilettoMetadata[] | undefined>;
20
+ _list(path: string, recursive?: boolean, visitor?: MobilettoVisitor, params?: {}): Promise<MobilettoMetadata[]>;
21
+ normalizeKey: (path: string) => string;
22
+ denormalizeKey: (key: string) => string;
23
+ s3error(err: any, key: string, path: string, method: string): MobilettoError | MobilettoNotFoundError;
24
+ metadata(path: string): Promise<MobilettoMetadata>;
25
+ write(path: string, generator: MobilettoWriteSource): Promise<number>;
26
+ read(path: string, callback: (data: any) => void, endCallback?: () => unknown): Promise<number>;
27
+ remove(path: string, optsOrRecursive?: MobilettoRemoveOptions | boolean, quiet?: boolean): Promise<string | string[]>;
28
+ }
29
+ export declare const storageClient: (key: string, secret: string, opts: S3Options) => StorageClient;
30
+ export {};
@@ -0,0 +1,312 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { M_DIR, M_FILE, logger, MobilettoError, MobilettoNotFoundError, readStream, } from "mobiletto-base";
11
+ import { dirname } from "path";
12
+ import { S3Client, ListObjectsCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey, } from "@aws-sdk/client-s3";
13
+ import { Readable } from "stream";
14
+ import { Upload } from "@aws-sdk/lib-storage";
15
+ const DEFAULT_REGION = "us-east-1";
16
+ const DEFAULT_PREFIX = "";
17
+ const DEFAULT_DELIMITER = "/";
18
+ const DELETE_OBJECTS_MAX_KEYS = 1000;
19
+ class StorageClient {
20
+ constructor(key, secret, opts) {
21
+ // noinspection JSUnusedGlobalSymbols -- called by driver init
22
+ this.testConfig = () => __awaiter(this, void 0, void 0, function* () { return yield this._list("", false, undefined, { MaxKeys: 1 }); });
23
+ this.stripPrefix = (name) => (name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name);
24
+ this.nameToObj = (name) => {
25
+ const relName = this.stripPrefix(name);
26
+ return {
27
+ name: relName,
28
+ type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE,
29
+ };
30
+ };
31
+ this.normalizeKey = (path) => {
32
+ const p = path.startsWith(this.prefix)
33
+ ? path
34
+ : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path);
35
+ return p.replace(this.normalizeRegex, this.delimiter);
36
+ };
37
+ this.denormalizeKey = (key) => {
38
+ return key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key;
39
+ };
40
+ if (!key || !secret || !opts || !opts.bucket) {
41
+ throw new MobilettoError("s3.StorageClient: key, secret, and opts.bucket are required");
42
+ }
43
+ this.bucket = opts.bucket;
44
+ const delim = (this.delimiter = opts.delimiter || DEFAULT_DELIMITER);
45
+ this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, "g");
46
+ this.prefix = opts.prefix || DEFAULT_PREFIX;
47
+ if (!this.prefix.endsWith(delim)) {
48
+ this.prefix += delim;
49
+ }
50
+ this.region = opts.region || DEFAULT_REGION;
51
+ const credentials = {
52
+ accessKeyId: key,
53
+ secretAccessKey: secret,
54
+ };
55
+ this.client = new S3Client({ region: this.region, credentials });
56
+ }
57
+ list(path = "", recursiveOrOpts, visitor) {
58
+ return __awaiter(this, void 0, void 0, function* () {
59
+ const recursive = recursiveOrOpts === true || (typeof recursiveOrOpts === "object" && recursiveOrOpts.recursive);
60
+ try {
61
+ return yield this._list(path, recursive, visitor);
62
+ }
63
+ catch (e) {
64
+ if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
65
+ // are we trying to list a single file?
66
+ const objects = yield this._list(dirname(path), false);
67
+ const found = objects.find((o) => o.name === path);
68
+ if (found) {
69
+ return [found];
70
+ }
71
+ throw e;
72
+ }
73
+ }
74
+ });
75
+ }
76
+ _list(path, recursive = false, visitor, params = {}) {
77
+ return __awaiter(this, void 0, void 0, function* () {
78
+ const logPrefix = `_list(path=${path})`;
79
+ // Declare truncated as a flag that the while loop is based on.
80
+ let truncated = true;
81
+ const Prefix = this.prefix +
82
+ (path.startsWith(this.delimiter) ? path.substring(0) : path) +
83
+ (path.length === 0 || path.endsWith(this.delimiter) ? "" : this.delimiter);
84
+ const bucketParams = Object.assign({}, params, {
85
+ Region: this.region,
86
+ Bucket: this.bucket,
87
+ Prefix,
88
+ });
89
+ if (!recursive) {
90
+ bucketParams.Delimiter = this.delimiter;
91
+ }
92
+ const objects = [];
93
+ let objectCount = 0;
94
+ logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`);
95
+ // while loop that runs until 'response.truncated' is false.
96
+ while (truncated) {
97
+ try {
98
+ const response = yield this.client.send(new ListObjectsCommand(bucketParams));
99
+ const hasContents = typeof response.Contents !== "undefined";
100
+ if (hasContents) {
101
+ for (const item of response.Contents || []) {
102
+ if (!item.Key)
103
+ continue;
104
+ const obj = this.nameToObj(item.Key);
105
+ if (visitor) {
106
+ yield visitor(obj);
107
+ }
108
+ objects.push(obj);
109
+ objectCount++;
110
+ }
111
+ }
112
+ const hasCommonPrefixes = typeof response.CommonPrefixes !== "undefined";
113
+ if (hasCommonPrefixes) {
114
+ for (const item of response.CommonPrefixes || []) {
115
+ if (!item.Prefix)
116
+ continue;
117
+ const obj = this.nameToObj(item.Prefix);
118
+ if (visitor) {
119
+ yield visitor(obj);
120
+ }
121
+ objects.push(obj);
122
+ objectCount++;
123
+ }
124
+ }
125
+ truncated = response.IsTruncated || false;
126
+ // If truncated is true, advance the marker
127
+ if (truncated) {
128
+ bucketParams.Marker = response.NextMarker;
129
+ }
130
+ else if (!hasContents && !hasCommonPrefixes) {
131
+ if (path === "") {
132
+ break;
133
+ }
134
+ throw new MobilettoNotFoundError(path);
135
+ }
136
+ }
137
+ catch (err) {
138
+ if (err instanceof MobilettoNotFoundError) {
139
+ throw err;
140
+ }
141
+ throw new MobilettoError(`${logPrefix} Error: ${err}`);
142
+ }
143
+ }
144
+ if (recursive && objectCount === 0 && path !== "") {
145
+ throw new MobilettoNotFoundError(path);
146
+ }
147
+ const filtered = objects.filter((o) => o.name !== path);
148
+ return filtered;
149
+ });
150
+ }
151
+ s3error(err, key, path, method) {
152
+ return err instanceof MobilettoError || err instanceof MobilettoNotFoundError
153
+ ? err
154
+ : err instanceof NoSuchKey || (err.name && err.name === "NotFound")
155
+ ? new MobilettoNotFoundError(this.denormalizeKey(key))
156
+ : new MobilettoError(`${method}(${path}) error: ${err}`, err);
157
+ }
158
+ metadata(path) {
159
+ return __awaiter(this, void 0, void 0, function* () {
160
+ const Key = this.normalizeKey(path);
161
+ const bucketParams = {
162
+ Region: this.region,
163
+ Bucket: this.bucket,
164
+ Key,
165
+ Delimiter: this.delimiter,
166
+ };
167
+ try {
168
+ const head = yield this.client.send(new HeadObjectCommand(bucketParams));
169
+ const meta = {
170
+ name: this.stripPrefix(path),
171
+ size: head.ContentLength,
172
+ type: path.endsWith(this.delimiter) ? M_DIR : M_FILE,
173
+ };
174
+ if (head.LastModified) {
175
+ meta.mtime = Date.parse(head.LastModified.toString());
176
+ }
177
+ return meta;
178
+ }
179
+ catch (err) {
180
+ throw this.s3error(err, Key, path, "metadata");
181
+ }
182
+ });
183
+ }
184
+ write(path, generator) {
185
+ return __awaiter(this, void 0, void 0, function* () {
186
+ const Key = this.normalizeKey(path);
187
+ const bucketParams = {
188
+ Region: this.region,
189
+ Bucket: this.bucket,
190
+ Key,
191
+ Body: Readable.from(generator),
192
+ Delimiter: this.delimiter,
193
+ };
194
+ const uploader = new Upload({
195
+ client: this.client,
196
+ params: bucketParams,
197
+ queueSize: 1,
198
+ partSize: 1024 * 1024 * 5,
199
+ leavePartsOnError: false, // optional manually handle dropped parts
200
+ });
201
+ let total = 0;
202
+ uploader.on("httpUploadProgress", (progress) => {
203
+ logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`);
204
+ total += progress.loaded || 0;
205
+ });
206
+ const response = yield uploader.done();
207
+ if (response.Key === Key) {
208
+ return total;
209
+ }
210
+ throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`);
211
+ });
212
+ }
213
+ read(path, callback, endCallback) {
214
+ return __awaiter(this, void 0, void 0, function* () {
215
+ const Key = this.normalizeKey(path);
216
+ logger.debug(`read: reading Key: ${path} - ${Key}`);
217
+ const bucketParams = {
218
+ Region: this.region,
219
+ Bucket: this.bucket,
220
+ Key,
221
+ Delimiter: this.delimiter,
222
+ };
223
+ try {
224
+ const data = yield this.client.send(new GetObjectCommand(bucketParams));
225
+ return yield readStream(data.Body, callback, endCallback);
226
+ }
227
+ catch (err) {
228
+ throw this.s3error(err, Key, path, "read");
229
+ }
230
+ });
231
+ }
232
+ remove(path, optsOrRecursive, quiet) {
233
+ return __awaiter(this, void 0, void 0, function* () {
234
+ const recursive = optsOrRecursive === true || (optsOrRecursive && optsOrRecursive.recursive);
235
+ if (recursive) {
236
+ const removed = [];
237
+ let objects = yield this._list(path, true, undefined, {
238
+ MaxKeys: DELETE_OBJECTS_MAX_KEYS,
239
+ });
240
+ while (objects && objects.length > 0) {
241
+ const Delete = {
242
+ Objects: objects.map((obj) => {
243
+ return { Key: this.normalizeKey(obj.name) };
244
+ }),
245
+ Quiet: quiet || false,
246
+ };
247
+ const bucketParams = {
248
+ Bucket: this.bucket,
249
+ Delete,
250
+ };
251
+ logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`);
252
+ const response = yield this.client.send(new DeleteObjectsCommand(bucketParams));
253
+ const statusCode = response.$metadata.httpStatusCode;
254
+ const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
255
+ if (statusClass !== 2) {
256
+ throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`);
257
+ }
258
+ if (!quiet && response.Errors && response.Errors.length > 0) {
259
+ throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`);
260
+ }
261
+ if (response.Deleted) {
262
+ removed.push(...response.Deleted.map((del) => del.Key ? this.denormalizeKey(del.Key) : "?del.Key undefined?"));
263
+ }
264
+ try {
265
+ objects = yield this._list(path, true, undefined, { MaxKeys: DELETE_OBJECTS_MAX_KEYS });
266
+ }
267
+ catch (e) {
268
+ if (!(e instanceof MobilettoNotFoundError)) {
269
+ throw e instanceof MobilettoError
270
+ ? e
271
+ : new MobilettoError(`remove(${path}) error listing: ${e}`);
272
+ }
273
+ objects = null;
274
+ }
275
+ }
276
+ return removed;
277
+ }
278
+ else {
279
+ const Key = this.normalizeKey(path);
280
+ const bucketParams = {
281
+ Region: this.region,
282
+ Bucket: this.bucket,
283
+ Key,
284
+ };
285
+ try {
286
+ // DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
287
+ // Thus, if quiet is false, we must check metadata explicitly, which will fail with
288
+ // MobilettoNotFoundError, which is the correct behavior
289
+ if (!quiet) {
290
+ yield this.metadata(path);
291
+ }
292
+ const response = yield this.client.send(new DeleteObjectCommand(bucketParams));
293
+ const statusCode = response.$metadata.httpStatusCode;
294
+ const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
295
+ if (statusClass !== 2) {
296
+ throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`);
297
+ }
298
+ }
299
+ catch (err) {
300
+ throw this.s3error(err, Key, path, "remove");
301
+ }
302
+ return path;
303
+ }
304
+ });
305
+ }
306
+ }
307
+ export const storageClient = (key, secret, opts) => {
308
+ if (!key || !secret || !opts || !opts.bucket) {
309
+ throw new MobilettoError("s3.storageClient: key, secret, and opts.bucket are required");
310
+ }
311
+ return new StorageClient(key, secret, opts);
312
+ };
package/package.json CHANGED
@@ -1,8 +1,9 @@
1
1
  {
2
2
  "name": "mobiletto-driver-s3",
3
- "version": "1.0.4",
3
+ "version": "2.0.0",
4
+ "type": "module",
4
5
  "description": "The Amazon S3 driver for mobiletto. This package is intended to be used with mobiletto-base, which does not include any drivers",
5
- "main": "index.js",
6
+ "main": "./lib/cjs/index.js",
6
7
  "keywords": [
7
8
  "storage",
8
9
  "s3",
@@ -18,17 +19,41 @@
18
19
  },
19
20
  "repository": {
20
21
  "type": "git",
21
- "url": "https://github.com/cobbzilla/mobiletto"
22
+ "url": "https://github.com/cobbzilla/mobiletto-driver-s3"
22
23
  },
23
24
  "license": "Apache-2.0",
25
+ "scripts": {
26
+ "test": "mocha --timeout 30000 ./test/*.spec.js",
27
+ "tsc": "tsc -p tsconfig.json && tsc -p tsconfig-cjs.json",
28
+ "lint": "npx eslint src",
29
+ "lint:fix": "npm run lint -- --fix",
30
+ "prettier": "npx prettier src --check",
31
+ "prettier:fix": "npm run prettier -- --write"
32
+ },
33
+ "module": "./lib/esm/index.js",
34
+ "files": [
35
+ "lib/"
36
+ ],
24
37
  "devDependencies": {
25
- "@babel/core": "^7.22.5",
26
- "@babel/node": "^7.22.5",
27
- "@babel/preset-env": "^7.22.5"
38
+ "@types/chai": "^4.3.5",
39
+ "@types/mocha": "^10.0.1",
40
+ "@types/node": "^20.3.3",
41
+ "@typescript-eslint/eslint-plugin": "^5.61.0",
42
+ "@typescript-eslint/parser": "^5.61.0",
43
+ "chai": "^4.3.7",
44
+ "dotenv": "^16.3.1",
45
+ "eslint": "^8.44.0",
46
+ "eslint-config-prettier": "^8.8.0",
47
+ "mobiletto-base": "^2.0.16",
48
+ "mobiletto-driver-local": "^2.0.2",
49
+ "mocha": "^10.2.0",
50
+ "prettier": "^2.8.8",
51
+ "randomstring": "^1.3.0",
52
+ "tslint-config-prettier": "^1.18.0",
53
+ "typescript": "^5.1.6"
28
54
  },
29
55
  "dependencies": {
30
- "@aws-sdk/client-s3": "^3.359.0",
31
- "@aws-sdk/lib-storage": "^3.359.0",
32
- "mobiletto-common": "^1.0.2"
56
+ "@aws-sdk/client-s3": "^3.367.0",
57
+ "@aws-sdk/lib-storage": "^3.367.0"
33
58
  }
34
59
  }
package/.babelrc DELETED
@@ -1,5 +0,0 @@
1
- {
2
- "presets": [
3
- "@babel/preset-env"
4
- ]
5
- }
package/.eslintrc.js DELETED
@@ -1,14 +0,0 @@
1
- module.exports = {
2
- root: true,
3
- env: {
4
- browser: true,
5
- node: true
6
- },
7
- parserOptions: {
8
- parser: '@babel/eslint-parser',
9
- requireConfigFile: false
10
- },
11
- plugins: [],
12
- // add your custom rules here
13
- rules: {}
14
- }
package/index.js DELETED
@@ -1,314 +0,0 @@
1
- const {
2
- M_DIR, M_FILE,
3
- MobilettoError, MobilettoNotFoundError, readStream
4
- } = require('mobiletto-common')
5
-
6
- const { dirname } = require('path')
7
-
8
- const {
9
- S3Client,
10
- ListObjectsCommand,
11
- HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey
12
- } = require('@aws-sdk/client-s3')
13
-
14
- const { Readable } = require('stream')
15
- const {Upload} = require("@aws-sdk/lib-storage")
16
-
17
- const DEFAULT_REGION = 'us-east-1'
18
- const DEFAULT_PREFIX = ''
19
- const DEFAULT_DELIMITER = '/'
20
-
21
- const DELETE_OBJECTS_MAX_KEYS = 1000
22
-
23
- class StorageClient {
24
- client
25
- region
26
- bucket
27
- prefix
28
- delimiter
29
- normalizeRegex
30
- constructor(key, secret, opts) {
31
- if (!key || !secret || !opts || !opts.bucket) {
32
- throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
33
- }
34
- this.bucket = opts.bucket
35
- const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
36
- this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, 'g')
37
- this.prefix = opts.prefix || DEFAULT_PREFIX
38
- ? opts.prefix.endsWith(delim)
39
- ? opts.prefix
40
- : opts.prefix + delim
41
- : ''
42
- this.region = opts.region || DEFAULT_REGION
43
- const credentials = {
44
- accessKeyId: key,
45
- secretAccessKey: secret
46
- }
47
- this.client = new S3Client({region: this.region, credentials})
48
- }
49
-
50
- // noinspection JSUnusedGlobalSymbols -- called by driver init
51
- testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
52
-
53
- stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
54
-
55
- nameToObj = (name) => {
56
- const relName = this.stripPrefix(name)
57
- return {
58
- name: relName,
59
- type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
60
- }
61
- }
62
-
63
- async list (path = '', recursive = false, visitor = null) {
64
- try {
65
- return await this._list(path, recursive, visitor)
66
- } catch (e) {
67
- if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
68
- // are we trying to list a single file?
69
- const objects = await this._list(dirname(path), false)
70
- const found = objects.find(o => o.name === path)
71
- if (found) {
72
- return [found]
73
- }
74
- throw e
75
- }
76
- }
77
- }
78
-
79
- async _list (path, recursive = false, visitor = null, params = {}) {
80
- const logPrefix = `_list(path=${path})`
81
-
82
- // Declare truncated as a flag that the while loop is based on.
83
- let truncated = true
84
-
85
- const Prefix = this.prefix +
86
- (path.startsWith(this.delimiter) ? path.substring(0) : path) +
87
- (path.length === 0 || path.endsWith(this.delimiter) ? '' : this.delimiter)
88
- const bucketParams = Object.assign({}, params, {
89
- Region: this.region,
90
- Bucket: this.bucket,
91
- Prefix
92
- })
93
- if (!recursive) {
94
- bucketParams.Delimiter = this.delimiter
95
- }
96
- const objects = []
97
- let objectCount = 0
98
- logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
99
-
100
- // while loop that runs until 'response.truncated' is false.
101
- while (truncated) {
102
- try {
103
- const response = await this.client.send(new ListObjectsCommand(bucketParams))
104
- const hasContents = typeof response.Contents !== 'undefined'
105
- if (hasContents) {
106
- for (const item of response.Contents) {
107
- const obj = this.nameToObj(item.Key);
108
- if (visitor) {
109
- await visitor(obj)
110
- }
111
- objects.push(obj)
112
- objectCount++
113
- }
114
- }
115
- const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
116
- if (hasCommonPrefixes) {
117
- for (const item of response.CommonPrefixes) {
118
- const obj = this.nameToObj(item.Prefix)
119
- if (visitor) {
120
- await visitor(obj)
121
- }
122
- objects.push(obj)
123
- objectCount++
124
- }
125
- }
126
- truncated = response.IsTruncated
127
- // If truncated is true, advance the marker
128
- if (truncated) {
129
- bucketParams.Marker = response.NextMarker
130
- } else if (!hasContents && !hasCommonPrefixes) {
131
- if (path === '') {
132
- break
133
- }
134
- throw new MobilettoNotFoundError(path)
135
- }
136
- } catch (err) {
137
- if (err instanceof MobilettoNotFoundError) {
138
- throw err
139
- }
140
- throw new MobilettoError(`${logPrefix} Error: ${err}`)
141
- }
142
- }
143
- if (recursive && objectCount === 0 && path !== '') {
144
- throw new MobilettoNotFoundError(path)
145
- }
146
- const filtered = objects.filter(o => o.name !== path)
147
- return filtered
148
- }
149
-
150
- normalizeKey = (path) => {
151
- const p = (path.startsWith(this.prefix)
152
- ? path
153
- : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
154
- return p.replaceAll(this.normalizeRegex, this.delimiter)
155
- }
156
- denormalizeKey = (key) => {
157
- return key.startsWith(this.prefix)
158
- ? key.substring(this.prefix.length)
159
- : key
160
- }
161
-
162
- s3error (err, key, path, method) {
163
- return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
164
- ? err
165
- : (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
166
- ? new MobilettoNotFoundError(this.denormalizeKey(key))
167
- : new MobilettoError(`${method}(${path}) error: ${err}`, err)
168
- }
169
-
170
- async metadata (path) {
171
- const Key = this.normalizeKey(path)
172
- const bucketParams = {
173
- Region: this.region,
174
- Bucket: this.bucket,
175
- Key,
176
- Delimiter: this.delimiter
177
- }
178
- try {
179
- const head = await this.client.send(new HeadObjectCommand(bucketParams))
180
- const meta = {
181
- name: this.stripPrefix(path),
182
- size: head.ContentLength,
183
- type: path.endsWith(this.delimiter) ? M_DIR : M_FILE
184
- }
185
- if (head.LastModified) {
186
- meta.mtime = Date.parse(head.LastModified)
187
- }
188
- return meta
189
- } catch (err) {
190
- throw this.s3error(err, Key, path, 'metadata')
191
- }
192
- }
193
-
194
- async write (path, generator) {
195
- const Key = this.normalizeKey(path)
196
- const bucketParams = {
197
- Region: this.region,
198
- Bucket: this.bucket,
199
- Key,
200
- Body: Readable.from(generator),
201
- Delimiter: this.delimiter
202
- }
203
-
204
- const uploader = new Upload({
205
- client: this.client,
206
- params: bucketParams,
207
- queueSize: 1, // optional concurrency configuration
208
- partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
209
- leavePartsOnError: false // optional manually handle dropped parts
210
- })
211
- let total = 0
212
- uploader.on('httpUploadProgress', (progress) => {
213
- logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`)
214
- total += progress.loaded
215
- })
216
- const response = await uploader.done()
217
- if (response.Key !== Key) {
218
- throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`)
219
- }
220
- return total
221
- }
222
-
223
- async read (path, callback, endCallback = null) {
224
- const Key = this.normalizeKey(path)
225
- logger.debug(`read: reading Key: ${path} - ${Key}`)
226
- const bucketParams = {
227
- Region: this.region,
228
- Bucket: this.bucket,
229
- Key,
230
- Delimiter: this.delimiter
231
- }
232
- try {
233
- const data = await this.client.send(new GetObjectCommand(bucketParams))
234
- return await readStream(data.Body, callback, endCallback)
235
- } catch (err) {
236
- throw this.s3error(err, Key, path, 'read')
237
- }
238
- }
239
-
240
- async remove (path, recursive, quiet) {
241
- if (recursive) {
242
- const removed = []
243
- let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
244
- while (objects && objects.length > 0) {
245
- const Delete = {
246
- Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
247
- }
248
- if (quiet) {
249
- Delete.Quiet = true
250
- }
251
- const bucketParams = {
252
- Region: this.region,
253
- Bucket: this.bucket,
254
- Delete
255
- }
256
- logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`)
257
- const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
258
- let statusCode = response.$metadata.httpStatusCode
259
- let statusClass = Math.floor(statusCode / 100)
260
- if (statusClass !== 2) {
261
- throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`)
262
- }
263
- if (!quiet && response.Errors && response.Errors.length > 0) {
264
- throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
265
- }
266
- if (response.Deleted) {
267
- removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
268
- }
269
- try {
270
- objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
271
- } catch (e) {
272
- if (!(e instanceof MobilettoNotFoundError)) {
273
- throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`)
274
- }
275
- objects = null
276
- }
277
- }
278
- return removed
279
- } else {
280
- const Key = this.normalizeKey(path)
281
- const bucketParams = {
282
- Region: this.region,
283
- Bucket: this.bucket,
284
- Key
285
- }
286
- try {
287
- // DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
288
- // Thus, if quiet is false, we must check metadata explicitly, which will fail with
289
- // MobilettoNotFoundError, which is the correct behavior
290
- if (!quiet) {
291
- await this.metadata(path)
292
- }
293
- const response = await this.client.send(new DeleteObjectCommand(bucketParams))
294
- let statusCode = response.$metadata.httpStatusCode
295
- let statusClass = Math.floor(statusCode / 100)
296
- if (statusClass !== 2) {
297
- throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`)
298
- }
299
- } catch (err) {
300
- throw this.s3error(err, Key, path, 'remove')
301
- }
302
- return path
303
- }
304
- }
305
- }
306
-
307
- function storageClient (key, secret, opts) {
308
- if (!key || !secret || !opts || !opts.bucket) {
309
- throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
310
- }
311
- return new StorageClient(key, secret, opts)
312
- }
313
-
314
- module.exports = { storageClient }