mobiletto-driver-s3 2.0.14 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,36 +1,17 @@
1
- import { MobilettoError, MobilettoNotFoundError, MobilettoOptions, MobilettoVisitor, MobilettoMetadata, MobilettoWriteSource, MobilettoListOptions, MobilettoRemoveOptions, MobilettoDriverInfo, MobilettoDriverScope } from "mobiletto-base";
1
+ import { MobilettoDriverParameter, MobilettoDriverScope, MobilettoOptions } from "mobiletto-base";
2
2
  export type S3Options = MobilettoOptions & {
3
3
  bucket?: string;
4
4
  prefix?: string;
5
5
  delimiter?: string;
6
6
  region?: string;
7
+ /** custom endpoint for S3-compatible stores (MinIO, R2, ...) */
8
+ endpoint?: string;
9
+ /** path-style addressing, usually required for custom endpoints */
10
+ forcePathStyle?: boolean;
7
11
  };
8
12
  export type S3InfoType = {
9
13
  driver: string;
10
14
  scope: MobilettoDriverScope;
11
15
  };
12
16
  export declare const S3Info: S3InfoType;
13
- declare class StorageClient {
14
- private client;
15
- private region;
16
- private bucket;
17
- private prefix;
18
- private delimiter;
19
- private normalizeRegex;
20
- constructor(key: string, secret: string, opts: S3Options);
21
- testConfig: () => Promise<MobilettoMetadata[]>;
22
- info: () => MobilettoDriverInfo;
23
- stripPrefix: (name: string) => string;
24
- nameToObj: (name: string) => MobilettoMetadata;
25
- list(path: string | undefined, recursiveOrOpts: MobilettoListOptions | boolean, visitor?: MobilettoVisitor): Promise<MobilettoMetadata[] | undefined>;
26
- _list(path: string, recursive?: boolean, visitor?: MobilettoVisitor, params?: {}): Promise<MobilettoMetadata[]>;
27
- normalizeKey: (path: string) => string;
28
- denormalizeKey: (key: string) => string;
29
- s3error(err: any, key: string, path: string, method: string): MobilettoError | MobilettoNotFoundError;
30
- metadata(path: string): Promise<MobilettoMetadata>;
31
- write(path: string, generator: MobilettoWriteSource): Promise<number>;
32
- read(path: string, callback: (data: any) => void, endCallback?: () => unknown): Promise<number>;
33
- remove(path: string, optsOrRecursive?: MobilettoRemoveOptions | boolean, quiet?: boolean): Promise<string | string[]>;
34
- }
35
- export declare const storageClient: (key: string, secret: string, opts: S3Options) => StorageClient;
36
- export {};
17
+ export declare const storageClient: MobilettoDriverParameter;
package/lib/esm/index.js CHANGED
@@ -1,30 +1,27 @@
1
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
- return new (P || (P = Promise))(function (resolve, reject) {
4
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
- step((generator = generator.apply(thisArg, _arguments || [])).next());
8
- });
9
- };
10
- import { M_DIR, M_FILE, logger, MobilettoError, MobilettoNotFoundError, readStream, } from "mobiletto-base";
11
- import { dirname } from "path";
12
- import { S3Client, ListObjectsCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey, } from "@aws-sdk/client-s3";
13
- import { Readable } from "stream";
1
+ import { logger, M_DIR, M_FILE, MobilettoDriverScope, MobilettoError, MobilettoNotFoundError, readStream, } from "mobiletto-base";
2
+ import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, NoSuchKey, S3Client, } from "@aws-sdk/client-s3";
3
+ import { Readable, Transform } from "stream";
14
4
  import { Upload } from "@aws-sdk/lib-storage";
15
5
  const DEFAULT_REGION = "us-east-1";
16
6
  const DEFAULT_PREFIX = "";
17
7
  const DEFAULT_DELIMITER = "/";
18
8
  const DELETE_OBJECTS_MAX_KEYS = 1000;
9
+ const S3_MAX_KEYS_PER_PAGE = 1000; // S3 silently caps MaxKeys at 1000 per page; we page past it
19
10
  export const S3Info = {
20
11
  driver: "s3",
21
- scope: "global",
12
+ scope: MobilettoDriverScope.global,
22
13
  };
14
+ const escapeRegExpChars = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
23
15
  class StorageClient {
24
16
  constructor(key, secret, opts) {
25
17
  // noinspection JSUnusedGlobalSymbols -- called by driver init
26
- this.testConfig = () => __awaiter(this, void 0, void 0, function* () { return yield this._list("", false, undefined, { MaxKeys: 1 }); });
27
- this.info = () => (Object.assign({ canonicalName: () => `s3:${this.bucket}` }, S3Info));
18
+ // NB: an empty bucket returns {entries: []}, which is truthy — exactly what
19
+ // mobiletto()'s testConfig truthiness check requires. Do not return entries here.
20
+ this.testConfig = async () => await this._list("", { recursive: false, limit: 1 });
21
+ this.info = () => ({
22
+ canonicalName: () => `s3:${this.bucket}`,
23
+ ...S3Info,
24
+ });
28
25
  this.stripPrefix = (name) => (name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name);
29
26
  this.nameToObj = (name) => {
30
27
  const relName = this.stripPrefix(name);
@@ -36,7 +33,7 @@ class StorageClient {
36
33
  this.normalizeKey = (path) => {
37
34
  const p = path.startsWith(this.prefix)
38
35
  ? path
39
- : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path);
36
+ : this.prefix + (path.startsWith(this.delimiter) ? path.substring(this.delimiter.length) : path);
40
37
  return p.replace(this.normalizeRegex, this.delimiter);
41
38
  };
42
39
  this.denormalizeKey = (key) => {
@@ -47,7 +44,7 @@ class StorageClient {
47
44
  }
48
45
  this.bucket = opts.bucket;
49
46
  const delim = (this.delimiter = opts.delimiter || DEFAULT_DELIMITER);
50
- this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, "g");
47
+ this.normalizeRegex = new RegExp(`${escapeRegExpChars(this.delimiter)}{2,}`, "g");
51
48
  this.prefix = opts.prefix || DEFAULT_PREFIX;
52
49
  if (!this.prefix.endsWith(delim)) {
53
50
  this.prefix += delim;
@@ -57,256 +54,330 @@ class StorageClient {
57
54
  accessKeyId: key,
58
55
  secretAccessKey: secret,
59
56
  };
60
- this.client = new S3Client({ region: this.region, credentials });
57
+ const clientConfig = { region: this.region, credentials };
58
+ if (opts.endpoint)
59
+ clientConfig.endpoint = opts.endpoint;
60
+ if (typeof opts.forcePathStyle === "boolean")
61
+ clientConfig.forcePathStyle = opts.forcePathStyle;
62
+ this.client = new S3Client(clientConfig);
61
63
  }
62
- list(path = "", recursiveOrOpts, visitor) {
63
- return __awaiter(this, void 0, void 0, function* () {
64
- const recursive = recursiveOrOpts === true || (typeof recursiveOrOpts === "object" && recursiveOrOpts.recursive);
65
- try {
66
- return yield this._list(path, recursive, visitor);
67
- }
68
- catch (e) {
69
- if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
70
- // are we trying to list a single file?
71
- const objects = yield this._list(dirname(path), false);
72
- const found = objects.find((o) => o.name === path);
73
- if (found) {
74
- return [found];
75
- }
76
- throw e;
64
+ async list(path = "", opts) {
65
+ const recursive = opts?.recursive || false;
66
+ try {
67
+ return await this._list(path, opts);
68
+ }
69
+ catch (e) {
70
+ if (e instanceof MobilettoNotFoundError && !recursive) {
71
+ // are we trying to list a single file? Ask S3 for the exact key instead of
72
+ // paging the parent (whose first page may not contain the file at all)
73
+ try {
74
+ const meta = await this.metadata(path);
75
+ return { entries: [meta] };
76
+ }
77
+ catch (metaErr) {
78
+ if (metaErr instanceof MobilettoNotFoundError)
79
+ throw e;
80
+ throw metaErr;
77
81
  }
78
82
  }
79
- });
83
+ throw e;
84
+ }
80
85
  }
81
- _list(path, recursive = false, visitor, params = {}) {
82
- return __awaiter(this, void 0, void 0, function* () {
83
- const logPrefix = `_list(path=${path})`;
84
- // Declare truncated as a flag that the while loop is based on.
85
- let truncated = true;
86
- const Prefix = this.prefix +
87
- (path.startsWith(this.delimiter) ? path.substring(0) : path) +
88
- (path.length === 0 || path.endsWith(this.delimiter) ? "" : this.delimiter);
89
- const bucketParams = Object.assign({}, params, {
90
- Region: this.region,
91
- Bucket: this.bucket,
92
- Prefix,
93
- });
94
- if (!recursive) {
95
- bucketParams.Delimiter = this.delimiter;
96
- }
97
- const objects = [];
98
- let objectCount = 0;
99
- logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`);
100
- // while loop that runs until 'response.truncated' is false.
101
- while (truncated) {
102
- try {
103
- const response = yield this.client.send(new ListObjectsCommand(bucketParams));
104
- const hasContents = typeof response.Contents !== "undefined";
105
- if (hasContents) {
106
- for (const item of response.Contents || []) {
107
- if (!item.Key)
108
- continue;
109
- const obj = this.nameToObj(item.Key);
110
- if (visitor) {
111
- yield visitor(obj);
112
- }
113
- objects.push(obj);
114
- objectCount++;
115
- }
116
- }
117
- const hasCommonPrefixes = typeof response.CommonPrefixes !== "undefined";
118
- if (hasCommonPrefixes) {
119
- for (const item of response.CommonPrefixes || []) {
120
- if (!item.Prefix)
121
- continue;
122
- const obj = this.nameToObj(item.Prefix);
123
- if (visitor) {
124
- yield visitor(obj);
125
- }
126
- objects.push(obj);
127
- objectCount++;
128
- }
129
- }
130
- truncated = response.IsTruncated || false;
131
- // If truncated is true, advance the marker
132
- if (truncated) {
133
- bucketParams.Marker = response.NextMarker;
86
+ async _list(path, opts) {
87
+ const logPrefix = `_list(path=${path})`;
88
+ const Prefix = this.prefix +
89
+ (path.startsWith(this.delimiter) ? path.substring(this.delimiter.length) : path) +
90
+ (path.length === 0 || path.endsWith(this.delimiter) ? "" : this.delimiter);
91
+ const limit = opts?.limit ? opts.limit : undefined;
92
+ const recursive = opts?.recursive || false;
93
+ const visitor = opts?.visitor || undefined;
94
+ const cursor = typeof opts?.cursor === "string" ? opts.cursor : undefined;
95
+ if (cursor === "")
96
+ return { entries: [], cursor };
97
+ const params = {
98
+ Bucket: this.bucket,
99
+ Prefix,
100
+ MaxKeys: limit ? Math.min(limit, S3_MAX_KEYS_PER_PAGE) : undefined,
101
+ };
102
+ // external cursors are name-based (prefix + last entry name); V2 resumes from
103
+ // them via StartAfter on the first page, then pages with ContinuationToken
104
+ if (cursor)
105
+ params.StartAfter = cursor;
106
+ if (!recursive) {
107
+ params.Delimiter = this.delimiter;
108
+ }
109
+ const objects = [];
110
+ let sawAnything = false;
111
+ let exhausted;
112
+ logger.debug(`${logPrefix} params=${JSON.stringify(params)}`);
113
+ try {
114
+ while (true) {
115
+ const response = await this.client.send(new ListObjectsV2Command(params));
116
+ const contents = response.Contents || [];
117
+ const commonPrefixes = response.CommonPrefixes || [];
118
+ sawAnything = sawAnything || contents.length > 0 || commonPrefixes.length > 0;
119
+ let pageDone = false;
120
+ for (const item of contents) {
121
+ if (!item.Key)
122
+ continue;
123
+ const obj = this.nameToObj(item.Key);
124
+ if (obj.name === path)
125
+ continue; // self/marker entry: filter BEFORE visiting
126
+ if (limit && objects.length >= limit) {
127
+ pageDone = true;
128
+ break;
134
129
  }
135
- else if (!hasContents && !hasCommonPrefixes) {
136
- if (path === "") {
130
+ if (visitor)
131
+ await visitor(obj);
132
+ objects.push(obj);
133
+ }
134
+ if (!pageDone) {
135
+ for (const item of commonPrefixes) {
136
+ if (!item.Prefix)
137
+ continue;
138
+ const obj = this.nameToObj(item.Prefix);
139
+ if (obj.name === path)
140
+ continue;
141
+ if (limit && objects.length >= limit) {
142
+ pageDone = true;
137
143
  break;
138
144
  }
139
- throw new MobilettoNotFoundError(path);
145
+ if (visitor)
146
+ await visitor(obj);
147
+ objects.push(obj);
140
148
  }
141
149
  }
142
- catch (err) {
143
- if (err instanceof MobilettoNotFoundError) {
144
- throw err;
145
- }
146
- throw new MobilettoError(`${logPrefix} Error: ${err}`);
150
+ const truncated = response.IsTruncated || false;
151
+ if (pageDone || (limit && objects.length >= limit)) {
152
+ // stopped early: more entries may exist (either later in this page or on
153
+ // further pages)
154
+ exhausted = !truncated && !pageDone;
155
+ break;
156
+ }
157
+ if (!truncated) {
158
+ exhausted = true;
159
+ break;
147
160
  }
161
+ // advance to the next page
162
+ params.ContinuationToken = response.NextContinuationToken;
163
+ delete params.StartAfter;
148
164
  }
149
- if (recursive && objectCount === 0 && path !== "") {
150
- throw new MobilettoNotFoundError(path);
165
+ }
166
+ catch (err) {
167
+ if (err instanceof MobilettoNotFoundError) {
168
+ throw err;
151
169
  }
152
- const filtered = objects.filter((o) => o.name !== path);
153
- return filtered;
154
- });
170
+ throw new MobilettoError(`${logPrefix} Error: ${err}`);
171
+ }
172
+ // mobiletto semantics: an empty (virtual) directory does not exist. A fresh listing
173
+ // (no cursor) of a non-root path that matches nothing is NotFound; an exhausted
174
+ // cursor-resumed listing legitimately returns empty.
175
+ if (!sawAnything && !cursor && path !== "") {
176
+ throw new MobilettoNotFoundError(path);
177
+ }
178
+ if (recursive && objects.length === 0 && !cursor && path !== "") {
179
+ throw new MobilettoNotFoundError(path);
180
+ }
181
+ let nextCursor = undefined;
182
+ if (limit || cursor) {
183
+ nextCursor = exhausted || objects.length === 0 ? "" : this.prefix + objects[objects.length - 1].name;
184
+ }
185
+ return { entries: objects, cursor: nextCursor };
155
186
  }
187
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
156
188
  s3error(err, key, path, method) {
157
189
  return err instanceof MobilettoError || err instanceof MobilettoNotFoundError
158
190
  ? err
159
- : err instanceof NoSuchKey || (err.name && err.name === "NotFound")
191
+ : err instanceof NoSuchKey ||
192
+ (err?.name && err.name === "NotFound") ||
193
+ err?.$metadata?.httpStatusCode === 404
160
194
  ? new MobilettoNotFoundError(this.denormalizeKey(key))
161
195
  : new MobilettoError(`${method}(${path}) error: ${err}`, err);
162
196
  }
163
- metadata(path) {
164
- return __awaiter(this, void 0, void 0, function* () {
165
- const Key = this.normalizeKey(path);
166
- const bucketParams = {
167
- Region: this.region,
168
- Bucket: this.bucket,
169
- Key,
170
- Delimiter: this.delimiter,
197
+ async metadata(path) {
198
+ const Key = this.normalizeKey(path);
199
+ const bucketParams = {
200
+ Bucket: this.bucket,
201
+ Key,
202
+ };
203
+ try {
204
+ const head = await this.client.send(new HeadObjectCommand(bucketParams));
205
+ const meta = {
206
+ name: this.stripPrefix(path),
207
+ size: head.ContentLength,
208
+ type: path.endsWith(this.delimiter) ? M_DIR : M_FILE,
171
209
  };
172
- try {
173
- const head = yield this.client.send(new HeadObjectCommand(bucketParams));
174
- const meta = {
175
- name: this.stripPrefix(path),
176
- size: head.ContentLength,
177
- type: path.endsWith(this.delimiter) ? M_DIR : M_FILE,
178
- };
179
- if (head.LastModified) {
180
- meta.mtime = Date.parse(head.LastModified.toString());
181
- }
182
- return meta;
210
+ if (head.LastModified) {
211
+ meta.mtime = Date.parse(head.LastModified.toString());
183
212
  }
184
- catch (err) {
185
- throw this.s3error(err, Key, path, "metadata");
213
+ return meta;
214
+ }
215
+ catch (err) {
216
+ const e = this.s3error(err, Key, path, "metadata");
217
+ if (e instanceof MobilettoNotFoundError) {
218
+ // S3 has no real directories: Head on a populated virtual dir 404s. Treat a
219
+ // prefix with at least one entry as an existing directory (size/mtime are
220
+ // unavailable for virtual dirs).
221
+ try {
222
+ const listing = await this._list(path, { limit: 1 });
223
+ if (listing.entries.length > 0) {
224
+ return { name: this.stripPrefix(path), type: M_DIR };
225
+ }
226
+ }
227
+ catch (listErr) {
228
+ if (!(listErr instanceof MobilettoNotFoundError))
229
+ throw listErr;
230
+ }
186
231
  }
187
- });
232
+ throw e;
233
+ }
188
234
  }
189
- write(path, generator) {
190
- return __awaiter(this, void 0, void 0, function* () {
191
- const Key = this.normalizeKey(path);
192
- const bucketParams = {
193
- Region: this.region,
194
- Bucket: this.bucket,
195
- Key,
196
- Body: Readable.from(generator),
197
- Delimiter: this.delimiter,
198
- };
199
- const uploader = new Upload({
200
- client: this.client,
201
- params: bucketParams,
202
- queueSize: 1,
203
- partSize: 1024 * 1024 * 5,
204
- leavePartsOnError: false, // optional manually handle dropped parts
205
- });
206
- let total = 0;
207
- uploader.on("httpUploadProgress", (progress) => {
208
- logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`);
209
- total += progress.loaded || 0;
210
- });
211
- const response = yield uploader.done();
212
- if (response.Key === Key) {
213
- return total;
214
- }
215
- throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`);
235
+ async write(path, generator) {
236
+ const Key = this.normalizeKey(path);
237
+ // a bare Buffer must not be chunked byte-by-byte (Readable.from(buffer) iterates
238
+ // single bytes as numbers); wrap scalars in an array
239
+ const source = Buffer.isBuffer(generator) || typeof generator === "string"
240
+ ? Readable.from([generator])
241
+ : // eslint-disable-next-line @typescript-eslint/no-explicit-any
242
+ Readable.from(generator);
243
+ // count the actual bytes flowing to the uploader: exact regardless of how many
244
+ // (cumulative) httpUploadProgress events fire
245
+ let total = 0;
246
+ const counter = new Transform({
247
+ transform(chunk, _encoding, cb) {
248
+ total += Buffer.isBuffer(chunk) ? chunk.length : Buffer.byteLength(chunk);
249
+ cb(null, chunk);
250
+ },
216
251
  });
217
- }
218
- read(path, callback, endCallback) {
219
- return __awaiter(this, void 0, void 0, function* () {
220
- const Key = this.normalizeKey(path);
221
- logger.debug(`read: reading Key: ${path} - ${Key}`);
222
- const bucketParams = {
223
- Region: this.region,
224
- Bucket: this.bucket,
225
- Key,
226
- Delimiter: this.delimiter,
227
- };
228
- try {
229
- const data = yield this.client.send(new GetObjectCommand(bucketParams));
230
- return yield readStream(data.Body, callback, endCallback);
231
- }
232
- catch (err) {
233
- throw this.s3error(err, Key, path, "read");
234
- }
252
+ const bucketParams = {
253
+ Bucket: this.bucket,
254
+ Key,
255
+ Body: source.pipe(counter),
256
+ };
257
+ const uploader = new Upload({
258
+ client: this.client,
259
+ params: bucketParams,
260
+ queueSize: 1, // optional concurrency configuration
261
+ partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
262
+ leavePartsOnError: false, // optional manually handle dropped parts
235
263
  });
264
+ uploader.on("httpUploadProgress", (progress) => {
265
+ logger.debug(`write(${Key}) ${JSON.stringify(progress)}`);
266
+ });
267
+ const response = await uploader.done();
268
+ if (response.Key === Key) {
269
+ return total;
270
+ }
271
+ throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`);
236
272
  }
237
- remove(path, optsOrRecursive, quiet) {
238
- return __awaiter(this, void 0, void 0, function* () {
239
- const recursive = optsOrRecursive === true || (optsOrRecursive && optsOrRecursive.recursive);
240
- if (recursive) {
241
- const removed = [];
242
- let objects = yield this._list(path, true, undefined, {
243
- MaxKeys: DELETE_OBJECTS_MAX_KEYS,
244
- });
245
- while (objects && objects.length > 0) {
246
- const Delete = {
247
- Objects: objects.map((obj) => {
248
- return { Key: this.normalizeKey(obj.name) };
249
- }),
250
- Quiet: quiet || false,
251
- };
252
- const bucketParams = {
253
- Bucket: this.bucket,
254
- Delete,
255
- };
256
- logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`);
257
- const response = yield this.client.send(new DeleteObjectsCommand(bucketParams));
258
- const statusCode = response.$metadata.httpStatusCode;
259
- const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
260
- if (statusClass !== 2) {
261
- throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`);
262
- }
263
- if (!quiet && response.Errors && response.Errors.length > 0) {
264
- throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`);
265
- }
266
- if (response.Deleted) {
267
- removed.push(...response.Deleted.map((del) => del.Key ? this.denormalizeKey(del.Key) : "?del.Key undefined?"));
268
- }
269
- try {
270
- objects = yield this._list(path, true, undefined, { MaxKeys: DELETE_OBJECTS_MAX_KEYS });
271
- }
272
- catch (e) {
273
- if (!(e instanceof MobilettoNotFoundError)) {
274
- throw e instanceof MobilettoError
275
- ? e
276
- : new MobilettoError(`remove(${path}) error listing: ${e}`);
277
- }
278
- objects = null;
279
- }
273
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
274
+ async read(path, callback, endCallback) {
275
+ const Key = this.normalizeKey(path);
276
+ logger.debug(`read: reading Key: ${path} - ${Key}`);
277
+ const bucketParams = {
278
+ Bucket: this.bucket,
279
+ Key,
280
+ };
281
+ try {
282
+ const data = await this.client.send(new GetObjectCommand(bucketParams));
283
+ return await readStream(data.Body, callback, endCallback);
284
+ }
285
+ catch (err) {
286
+ throw this.s3error(err, Key, path, "read");
287
+ }
288
+ }
289
+ async remove(path, opts) {
290
+ const recursive = opts?.recursive || false;
291
+ const quiet = opts?.quiet || false;
292
+ if (recursive) {
293
+ const removed = [];
294
+ const failedKeys = new Set();
295
+ let prevRoundKeys = null;
296
+ let listing = await this._listOrNull(path);
297
+ while (listing && listing.entries.length > 0) {
298
+ // skip keys that already failed to delete: retrying them forever would loop
299
+ const objects = listing.entries.filter((obj) => !failedKeys.has(this.normalizeKey(obj.name)));
300
+ if (objects.length === 0)
301
+ break;
302
+ const roundKeys = objects.map((obj) => this.normalizeKey(obj.name));
303
+ const roundSig = roundKeys.join("\n");
304
+ if (roundSig === prevRoundKeys) {
305
+ throw new MobilettoError(`remove(${path}) no progress: the same ${roundKeys.length} keys remain after a delete round`);
280
306
  }
281
- return removed;
282
- }
283
- else {
284
- const Key = this.normalizeKey(path);
307
+ prevRoundKeys = roundSig;
308
+ const Delete = {
309
+ Objects: roundKeys.map((Key) => ({ Key })),
310
+ // always request verbose results: Quiet omits the Deleted list and hides
311
+ // per-object errors, which the loop below depends on
312
+ Quiet: false,
313
+ };
285
314
  const bucketParams = {
286
- Region: this.region,
287
315
  Bucket: this.bucket,
288
- Key,
316
+ Delete,
289
317
  };
290
- try {
291
- // DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
292
- // Thus, if quiet is false, we must check metadata explicitly, which will fail with
293
- // MobilettoNotFoundError, which is the correct behavior
294
- if (!quiet) {
295
- yield this.metadata(path);
318
+ logger.debug(`remove(${path}) deleting ${roundKeys.length} objects`);
319
+ const response = await this.client.send(new DeleteObjectsCommand(bucketParams));
320
+ const statusCode = response.$metadata.httpStatusCode;
321
+ const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
322
+ if (statusClass !== 2) {
323
+ throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`);
324
+ }
325
+ const errored = new Set();
326
+ if (response.Errors && response.Errors.length > 0) {
327
+ for (const e of response.Errors) {
328
+ if (e.Key)
329
+ errored.add(e.Key);
296
330
  }
297
- const response = yield this.client.send(new DeleteObjectCommand(bucketParams));
298
- const statusCode = response.$metadata.httpStatusCode;
299
- const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
300
- if (statusClass !== 2) {
301
- throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`);
331
+ if (!quiet) {
332
+ throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`);
302
333
  }
334
+ if (logger.isWarningEnabled())
335
+ logger.warn(`remove(${path}) ${errored.size} objects failed to delete (quiet mode)`);
336
+ for (const k of errored)
337
+ failedKeys.add(k);
338
+ }
339
+ // removed = keys we sent minus keys that errored (correct in all modes;
340
+ // response.Deleted would be empty under S3's Quiet)
341
+ removed.push(...roundKeys.filter((k) => !errored.has(k)).map((k) => this.denormalizeKey(k)));
342
+ listing = await this._listOrNull(path);
343
+ }
344
+ return removed;
345
+ }
346
+ else {
347
+ const Key = this.normalizeKey(path);
348
+ const bucketParams = {
349
+ Bucket: this.bucket,
350
+ Key,
351
+ };
352
+ try {
353
+ // DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
354
+ // Thus, if quiet is false, we must check metadata explicitly, which will fail with
355
+ // MobilettoNotFoundError, which is the correct behavior
356
+ if (!quiet) {
357
+ await this.metadata(path);
303
358
  }
304
- catch (err) {
305
- throw this.s3error(err, Key, path, "remove");
359
+ const response = await this.client.send(new DeleteObjectCommand(bucketParams));
360
+ const statusCode = response.$metadata.httpStatusCode;
361
+ const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
362
+ if (statusClass !== 2) {
363
+ throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`);
306
364
  }
307
- return path;
308
365
  }
309
- });
366
+ catch (err) {
367
+ throw this.s3error(err, Key, path, "remove");
368
+ }
369
+ return path;
370
+ }
371
+ }
372
+ async _listOrNull(path) {
373
+ try {
374
+ return await this._list(path, { recursive: true, limit: DELETE_OBJECTS_MAX_KEYS });
375
+ }
376
+ catch (e) {
377
+ if (e instanceof MobilettoNotFoundError)
378
+ return null;
379
+ throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`);
380
+ }
310
381
  }
311
382
  }
312
383
  export const storageClient = (key, secret, opts) => {
@@ -315,3 +386,4 @@ export const storageClient = (key, secret, opts) => {
315
386
  }
316
387
  return new StorageClient(key, secret, opts);
317
388
  };
389
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,MAAM,EACN,KAAK,EACL,MAAM,EAGN,oBAAoB,EACpB,cAAc,EAId,sBAAsB,EAItB,UAAU,GACb,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAEH,mBAAmB,EACnB,oBAAoB,EAEpB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAGpB,SAAS,EACT,QAAQ,GACX,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC7C,OAAO,EAAY,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAExD,MAAM,cAAc,GAAG,WAAW,CAAC;AACnC,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,oBAAoB,GAAG,IAAI,CAAC,CAAC,6DAA6D;AAkBhG,MAAM,CAAC,MAAM,MAAM,GAAe;IAC9B,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,oBAAoB,CAAC,MAAM;CACrC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AAE1F,MAAM,aAAa;IAOf,YAAY,GAAW,EAAE,MAAc,EAAE,IAAe;QAsBxD,8DAA8D;QAC9D,4EAA4E;QAC5E,kFAAkF;QAClF,eAAU,GAAG,KAAK,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9E,SAAI,GAAG,GAAwB,EAAE,CAAC,CAAC;YAC/B,aAAa,EAAE,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE;YACxC,GAAG,MAAM;SACZ,CAAC,CAAC;QAEH,gBAAW,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE3G,cAAS,GAAG,CAAC,IAAY,EAAqB,EAAE;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACvC,OAAO;gBACH,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM;aAC1D,CAAC;QACN,CAAC,CAAC;QA6HF,iBAAY,GAAG,CAAC,IAAY,EAAE,EAAE;YAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;gBAClC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACrG,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1D,CAAC,CAAC;QACF,mBAAc,GAAG,CAAC,GAAW,EAAE,EAAE;YAC7B,OAAO,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACjF,CAAC,CAAC;QA5KE,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,cAAc,CAAC,6DAA6D,CAAC,CAAC;QAC5F,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAC,CAAC;QACrE,IAAI,CAAC,cAAc,GAAG,IAAI,MAAM,CAAC,GAAG,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,cAAc,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,cAAc,CAAC;QAC5C,MAAM,WAAW,GAAG;YAChB,WAAW,EAAE,GAAG;YAChB,eAAe,EAAE,MAAM;SAC1B,CAAC;QACF,MAAM,YAAY,GAAmB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;QAC1E,IAAI,IAAI,CAAC,QAAQ;YAAE,YAAY,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QACzD,IAAI,OAAO,IAAI,CAAC,cAAc,KAAK,SAAS;YAAE,YAAY,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAChG,IAAI,CAAC,MAAM,GAAG,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC7C,CAAC;IAsBD,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,EAAE,IAA2B;QAC7C,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,KAAK,CAAC;QAC3C,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,sBAAsB,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpD,2EAA2E;gBAC3E,uEAAuE;gBACvE,IAAI,CAAC;oBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBACvC,OAAO,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,CAAC;gBAAC,OAAO,OAAO,EAAE,CAAC;oBACf,IAAI,OAAO,YAAY,sBAAsB;wBAAE,MAAM,CAAC,CAAC;oBACvD,MAAM,OAAO,CAAC;gBAClB,CAAC;YACL,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,IAA2B;QACjD,MAAM,SAAS,GAAG,cAAc,IAAI,GAAG,CAAC;QAExC,MAAM,MAAM,GACR,IAAI,CAAC,MAAM;YACX,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAChF,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE/E,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACnD,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,KAAK,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,SAAS,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,IAAI,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,IAAI,MAAM,KAAK,EAAE;YAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;QAElD,MAAM,MAAM,GAAyB;YACjC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;SACrE,CAAC;QACF,8EAA8E;QAC9E,2EAA2E;QAC3E,IAAI,MAAM;YAAE,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC;QACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACtC,CAAC;QAED,MAAM,OAAO,GAAwB,EAAE,CAAC;QACxC,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,SAAkB,CAAC;QACvB,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,WAAW,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC;YACD,OAAO,IAAI,EAAE,CAAC;gBACV,MAAM,QAAQ,GAA+B,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtG,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;gBACzC,MAAM,cAAc,GAAG,QAAQ,CAAC,cAAc,IAAI,EAAE,CAAC;gBACrD,WAAW,GAAG,WAAW,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;gBAE9E,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,GAAG;wBAAE,SAAS;oBACxB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACrC,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI;wBAAE,SAAS,CAAC,4CAA4C;oBAC7E,IAAI,KAAK,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;wBACnC,QAAQ,GAAG,IAAI,CAAC;wBAChB,MAAM;oBACV,CAAC;oBACD,IAAI,OAAO;wBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACtB,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACZ,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;wBAChC,IAAI,CAAC,IAAI,CAAC,MAAM;4BAAE,SAAS;wBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACxC,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI;4BAAE,SAAS;wBAChC,IAAI,KAAK,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;4BACnC,QAAQ,GAAG,IAAI,CAAC;4BAChB,MAAM;wBACV,CAAC;wBACD,IAAI,OAAO;4BAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;wBAChC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACtB,CAAC;gBACL,CAAC;gBAED,MAAM,SAAS,GAAG,QAAQ,CAAC,WAAW,IAAI,KAAK,CAAC;gBAChD,IAAI,QAAQ,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC;oBACjD,yEAAyE;oBACzE,iBAAiB;oBACjB,SAAS,GAAG,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC;oBACpC,MAAM;gBACV,CAAC;gBACD,IAAI,CAAC,SAAS,EAAE,CAAC;oBACb,SAAS,GAAG,IAAI,CAAC;oBACjB,MAAM;gBACV,CAAC;gBACD,2BAA2B;gBAC3B,MAAM,CAAC,iBAAiB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;gBAC1D,OAAO,MAAM,CAAC,UAAU,CAAC;YAC7B,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,sBAAsB,EAAE,CAAC;gBACxC,MAAM,GAAG,CAAC;YACd,CAAC;YACD,MAAM,IAAI,cAAc,CAAC,GAAG,SAAS,WAAW,GAAG,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,oFAAoF;QACpF,gFAAgF;QAChF,qDAAqD;QACrD,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACzC,MAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YAC9D,MAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,UAAU,GAAuB,SAAS,CAAC;QAC/C,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAClB,UAAU,GAAG,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IACpD,CAAC;IAYD,8DAA8D;IAC9D,OAAO,CAAC,GAAQ,EAAE,GAAW,EAAE,IAAY,EAAE,MAAc;QACvD,OAAO,GAAG,YAAY,cAAc,IAAI,GAAG,YAAY,sBAAsB;YACzE,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,GAAG,YAAY,SAAS;gBACtB,CAAC,GAAG,EAAE,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,CAAC;gBACtC,GAAG,EAAE,SAAS,EAAE,cAAc,KAAK,GAAG;gBACxC,CAAC,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBACtD,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,MAAM,IAAI,IAAI,YAAY,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,YAAY,GAAG;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG;SACN,CAAC;QACF,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;YACzE,MAAM,IAAI,GAAsB;gBAC5B,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,IAAI,CAAC,aAAa;gBACxB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM;aACvD,CAAC;YACF,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YACnD,IAAI,CAAC,YAAY,sBAAsB,EAAE,CAAC;gBACtC,4EAA4E;gBAC5E,0EAA0E;gBAC1E,iCAAiC;gBACjC,IAAI,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;oBACrD,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC7B,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;oBACzD,CAAC;gBACL,CAAC;gBAAC,OAAO,OAAO,EAAE,CAAC;oBACf,IAAI,CAAC,CAAC,OAAO,YAAY,sBAAsB,CAAC;wBAAE,MAAM,OAAO,CAAC;gBACpE,CAAC;YACL,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,SAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACpC,iFAAiF;QACjF,qDAAqD;QACrD,MAAM,MAAM,GACR,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,SAAS,KAAK,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC,CAAC,8DAA8D;gBAC9D,QAAQ,CAAC,IAAI,CAAC,SAA+C,CAAC,CAAC;QACzE,+EAA+E;QAC/E,8CAA8C;QAC9C,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC;YAC1B,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;gBAC1B,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC1E,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACpB,CAAC;SACJ,CAAC,CAAC;QACH,MAAM,YAAY,GAAG;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG;YACH,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;SAC7B,CAAC;QAEF,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,YAAY;YACpB,SAAS,EAAE,CAAC,EAAE,qCAAqC;YACnD,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,CAAC,EAAE,qDAAqD;YAChF,iBAAiB,EAAE,KAAK,EAAE,yCAAyC;SACtE,CAAC,CAAC;QACH,QAAQ,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,QAAkB,EAAE,EAAE;YACrD,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAyC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC7E,IAAI,QAAQ,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;YACvB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,IAAI,cAAc,CAAC,yCAAyC,GAAG,2BAA2B,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;IACpH,CAAC;IAED,8DAA8D;IAC9D,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,QAA6B,EAAE,WAA2B;QAC/E,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,sBAAsB,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG;SACN,CAAC;QACF,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;YACxE,OAAO,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,IAA6B;QACpD,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,KAAK,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,KAAK,CAAC;QACnC,IAAI,SAAS,EAAE,CAAC;YACZ,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;YACrC,IAAI,aAAa,GAAkB,IAAI,CAAC;YACxC,IAAI,OAAO,GAAgC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACxE,OAAO,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3C,4EAA4E;gBAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC9F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM;gBAChC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;gBACpE,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,QAAQ,KAAK,aAAa,EAAE,CAAC;oBAC7B,MAAM,IAAI,cAAc,CACpB,UAAU,IAAI,2BAA2B,SAAS,CAAC,MAAM,mCAAmC,CAC/F,CAAC;gBACN,CAAC;gBACD,aAAa,GAAG,QAAQ,CAAC;gBACzB,MAAM,MAAM,GAAG;oBACX,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC1C,yEAAyE;oBACzE,qDAAqD;oBACrD,KAAK,EAAE,KAAK;iBACf,CAAC;gBACF,MAAM,YAAY,GAAyB;oBACvC,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,MAAM;iBACT,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,cAAc,SAAS,CAAC,MAAM,UAAU,CAAC,CAAC;gBACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,YAAY,CAAC,CAAC,CAAC;gBAChF,MAAM,UAAU,GAAG,QAAQ,CAAC,SAAS,CAAC,cAAc,CAAC;gBACrD,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;oBACpB,MAAM,IAAI,cAAc,CAAC,UAAU,IAAI,+CAA+C,UAAU,EAAE,CAAC,CAAC;gBACxG,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;gBAClC,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChD,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAC9B,IAAI,CAAC,CAAC,GAAG;4BAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAClC,CAAC;oBACD,IAAI,CAAC,KAAK,EAAE,CAAC;wBACT,MAAM,IAAI,cAAc,CACpB,UAAU,IAAI,2CAA2C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAC7F,CAAC;oBACN,CAAC;oBACD,IAAI,MAAM,CAAC,gBAAgB,EAAE;wBACzB,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,KAAK,OAAO,CAAC,IAAI,wCAAwC,CAAC,CAAC;oBACzF,KAAK,MAAM,CAAC,IAAI,OAAO;wBAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC/C,CAAC;gBACD,wEAAwE;gBACxE,oDAAoD;gBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7F,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;YACD,OAAO,OAAO,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,YAAY,GAAG;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG;aACN,CAAC;YACF,IAAI,CAAC;gBACD,wFAAwF;gBACxF,mFAAmF;gBACnF,wDAAwD;gBACxD,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;gBAC/E,MAAM,UAAU,GAAG,QAAQ,CAAC,SAAS,CAAC,cAAc,CAAC;gBACrD,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;oBACpB,MAAM,IAAI,cAAc,CAAC,oDAAoD,UAAU,EAAE,CAAC,CAAC;gBAC/F,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;YACjD,CAAC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAY;QAClC,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;QACvF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,sBAAsB;gBAAE,OAAO,IAAI,CAAC;YACrD,MAAM,CAAC,YAAY,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,UAAU,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;QACtG,CAAC;IACL,CAAC;CACJ;AAED,MAAM,CAAC,MAAM,aAAa,GAA6B,CAAC,GAAW,EAAE,MAAe,EAAE,IAAgB,EAAE,EAAE;IACtG,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3C,MAAM,IAAI,cAAc,CAAC,6DAA6D,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,IAAI,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AAChD,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,9 +1,8 @@
1
1
  {
2
2
  "name": "mobiletto-driver-s3",
3
- "version": "2.0.14",
3
+ "version": "2.3.0",
4
4
  "type": "module",
5
5
  "description": "The Amazon S3 driver for mobiletto. This package is intended to be used with mobiletto-base, which does not include any drivers",
6
- "main": "./lib/cjs/index.js",
7
6
  "keywords": [
8
7
  "storage",
9
8
  "s3",
@@ -22,37 +21,48 @@
22
21
  "url": "https://github.com/cobbzilla/mobiletto-driver-s3"
23
22
  },
24
23
  "license": "Apache-2.0",
25
- "scripts": {
26
- "test": "mocha --timeout 30000 ./test/*.spec.js",
27
- "tsc": "tsc -p tsconfig.json && tsc -p tsconfig-cjs.json",
28
- "lint": "npx eslint src test",
29
- "lint:fix": "npm run lint -- --fix",
30
- "prettier": "npx prettier src test --check",
31
- "prettier:fix": "npm run prettier -- --write"
32
- },
24
+ "main": "./lib/esm/index.js",
33
25
  "module": "./lib/esm/index.js",
34
26
  "files": [
35
27
  "lib/"
36
28
  ],
37
29
  "devDependencies": {
38
- "@types/chai": "^4.3.5",
39
- "@types/mocha": "^10.0.1",
40
- "@types/node": "^20.3.3",
41
- "@typescript-eslint/eslint-plugin": "^5.61.0",
42
- "@typescript-eslint/parser": "^5.61.0",
43
- "chai": "^4.3.7",
44
- "dotenv": "^16.3.1",
45
- "eslint": "^8.44.0",
46
- "eslint-config-prettier": "^8.8.0",
47
- "mobiletto-base": "^2.1.1",
48
- "mobiletto-driver-local": "^2.0.14",
49
- "mocha": "^10.2.0",
50
- "prettier": "^2.8.8",
51
- "tslint-config-prettier": "^1.18.0",
52
- "typescript": "^5.1.6"
30
+ "@eslint/eslintrc": "^3.3.7",
31
+ "@eslint/js": "^10.0.1",
32
+ "@types/chai": "^5.2.3",
33
+ "@types/mocha": "^10.0.10",
34
+ "@types/node": "^26.6.6",
35
+ "@typescript-eslint/eslint-plugin": "^8.73.0",
36
+ "@typescript-eslint/parser": "^8.73.0",
37
+ "aws-sdk-client-mock": "^4.1.0",
38
+ "chai": "^6.3.0",
39
+ "dotenv": "^18.0.10",
40
+ "eslint": "^10.15.0",
41
+ "eslint-config-prettier": "^10.1.8",
42
+ "globals": "^17.14.0",
43
+ "mobiletto-base": "^2.3.2",
44
+ "mobiletto-driver-local": "file:../mobiletto-driver-local",
45
+ "mobiletto-driver-test": "file:../mobiletto-driver-test",
46
+ "mocha": "^12.0.3",
47
+ "prettier": "^3.10.1",
48
+ "tsx": "^4.23.15",
49
+ "typescript": "^6.0.3",
50
+ "zilla-util": "^1.3.0"
53
51
  },
54
52
  "dependencies": {
55
- "@aws-sdk/client-s3": "^3.367.0",
56
- "@aws-sdk/lib-storage": "^3.367.0"
53
+ "@aws-sdk/client-s3": "^3.1175.0",
54
+ "@aws-sdk/lib-storage": "^3.1175.0"
55
+ },
56
+ "peerDependencies": {
57
+ "mobiletto-base": "^2.3.2"
58
+ },
59
+ "scripts": {
60
+ "tsc": "tsc -p tsconfig.json",
61
+ "lint": "npx eslint src test",
62
+ "lint:fix": "npm run lint -- --fix",
63
+ "prettier": "npx prettier src test --check",
64
+ "prettier:fix": "npm run prettier -- --write",
65
+ "test": "if [[ -z \"$SKIP_TESTS\" ]] ; then mocha --timeout 60000 ./test/*.spec.ts; else echo 'Skipping tests due to SKIP_TESTS env variable'; fi",
66
+ "release": "pnpm tsc && pnpm prettier && pnpm lint && pnpm test"
57
67
  }
58
- }
68
+ }
@@ -1,36 +0,0 @@
1
- import { MobilettoError, MobilettoNotFoundError, MobilettoOptions, MobilettoVisitor, MobilettoMetadata, MobilettoWriteSource, MobilettoListOptions, MobilettoRemoveOptions, MobilettoDriverInfo, MobilettoDriverScope } from "mobiletto-base";
2
- export type S3Options = MobilettoOptions & {
3
- bucket?: string;
4
- prefix?: string;
5
- delimiter?: string;
6
- region?: string;
7
- };
8
- export type S3InfoType = {
9
- driver: string;
10
- scope: MobilettoDriverScope;
11
- };
12
- export declare const S3Info: S3InfoType;
13
- declare class StorageClient {
14
- private client;
15
- private region;
16
- private bucket;
17
- private prefix;
18
- private delimiter;
19
- private normalizeRegex;
20
- constructor(key: string, secret: string, opts: S3Options);
21
- testConfig: () => Promise<MobilettoMetadata[]>;
22
- info: () => MobilettoDriverInfo;
23
- stripPrefix: (name: string) => string;
24
- nameToObj: (name: string) => MobilettoMetadata;
25
- list(path: string | undefined, recursiveOrOpts: MobilettoListOptions | boolean, visitor?: MobilettoVisitor): Promise<MobilettoMetadata[] | undefined>;
26
- _list(path: string, recursive?: boolean, visitor?: MobilettoVisitor, params?: {}): Promise<MobilettoMetadata[]>;
27
- normalizeKey: (path: string) => string;
28
- denormalizeKey: (key: string) => string;
29
- s3error(err: any, key: string, path: string, method: string): MobilettoError | MobilettoNotFoundError;
30
- metadata(path: string): Promise<MobilettoMetadata>;
31
- write(path: string, generator: MobilettoWriteSource): Promise<number>;
32
- read(path: string, callback: (data: any) => void, endCallback?: () => unknown): Promise<number>;
33
- remove(path: string, optsOrRecursive?: MobilettoRemoveOptions | boolean, quiet?: boolean): Promise<string | string[]>;
34
- }
35
- export declare const storageClient: (key: string, secret: string, opts: S3Options) => StorageClient;
36
- export {};
package/lib/cjs/index.js DELETED
@@ -1,321 +0,0 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.storageClient = exports.S3Info = void 0;
13
- const mobiletto_base_1 = require("mobiletto-base");
14
- const path_1 = require("path");
15
- const client_s3_1 = require("@aws-sdk/client-s3");
16
- const stream_1 = require("stream");
17
- const lib_storage_1 = require("@aws-sdk/lib-storage");
18
- const DEFAULT_REGION = "us-east-1";
19
- const DEFAULT_PREFIX = "";
20
- const DEFAULT_DELIMITER = "/";
21
- const DELETE_OBJECTS_MAX_KEYS = 1000;
22
- exports.S3Info = {
23
- driver: "s3",
24
- scope: "global",
25
- };
26
- class StorageClient {
27
- constructor(key, secret, opts) {
28
- // noinspection JSUnusedGlobalSymbols -- called by driver init
29
- this.testConfig = () => __awaiter(this, void 0, void 0, function* () { return yield this._list("", false, undefined, { MaxKeys: 1 }); });
30
- this.info = () => (Object.assign({ canonicalName: () => `s3:${this.bucket}` }, exports.S3Info));
31
- this.stripPrefix = (name) => (name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name);
32
- this.nameToObj = (name) => {
33
- const relName = this.stripPrefix(name);
34
- return {
35
- name: relName,
36
- type: relName.endsWith(this.delimiter) ? mobiletto_base_1.M_DIR : mobiletto_base_1.M_FILE,
37
- };
38
- };
39
- this.normalizeKey = (path) => {
40
- const p = path.startsWith(this.prefix)
41
- ? path
42
- : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path);
43
- return p.replace(this.normalizeRegex, this.delimiter);
44
- };
45
- this.denormalizeKey = (key) => {
46
- return key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key;
47
- };
48
- if (!key || !secret || !opts || !opts.bucket) {
49
- throw new mobiletto_base_1.MobilettoError("s3.StorageClient: key, secret, and opts.bucket are required");
50
- }
51
- this.bucket = opts.bucket;
52
- const delim = (this.delimiter = opts.delimiter || DEFAULT_DELIMITER);
53
- this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, "g");
54
- this.prefix = opts.prefix || DEFAULT_PREFIX;
55
- if (!this.prefix.endsWith(delim)) {
56
- this.prefix += delim;
57
- }
58
- this.region = opts.region || DEFAULT_REGION;
59
- const credentials = {
60
- accessKeyId: key,
61
- secretAccessKey: secret,
62
- };
63
- this.client = new client_s3_1.S3Client({ region: this.region, credentials });
64
- }
65
- list(path = "", recursiveOrOpts, visitor) {
66
- return __awaiter(this, void 0, void 0, function* () {
67
- const recursive = recursiveOrOpts === true || (typeof recursiveOrOpts === "object" && recursiveOrOpts.recursive);
68
- try {
69
- return yield this._list(path, recursive, visitor);
70
- }
71
- catch (e) {
72
- if (e instanceof mobiletto_base_1.MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
73
- // are we trying to list a single file?
74
- const objects = yield this._list((0, path_1.dirname)(path), false);
75
- const found = objects.find((o) => o.name === path);
76
- if (found) {
77
- return [found];
78
- }
79
- throw e;
80
- }
81
- }
82
- });
83
- }
84
- _list(path, recursive = false, visitor, params = {}) {
85
- return __awaiter(this, void 0, void 0, function* () {
86
- const logPrefix = `_list(path=${path})`;
87
- // Declare truncated as a flag that the while loop is based on.
88
- let truncated = true;
89
- const Prefix = this.prefix +
90
- (path.startsWith(this.delimiter) ? path.substring(0) : path) +
91
- (path.length === 0 || path.endsWith(this.delimiter) ? "" : this.delimiter);
92
- const bucketParams = Object.assign({}, params, {
93
- Region: this.region,
94
- Bucket: this.bucket,
95
- Prefix,
96
- });
97
- if (!recursive) {
98
- bucketParams.Delimiter = this.delimiter;
99
- }
100
- const objects = [];
101
- let objectCount = 0;
102
- mobiletto_base_1.logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`);
103
- // while loop that runs until 'response.truncated' is false.
104
- while (truncated) {
105
- try {
106
- const response = yield this.client.send(new client_s3_1.ListObjectsCommand(bucketParams));
107
- const hasContents = typeof response.Contents !== "undefined";
108
- if (hasContents) {
109
- for (const item of response.Contents || []) {
110
- if (!item.Key)
111
- continue;
112
- const obj = this.nameToObj(item.Key);
113
- if (visitor) {
114
- yield visitor(obj);
115
- }
116
- objects.push(obj);
117
- objectCount++;
118
- }
119
- }
120
- const hasCommonPrefixes = typeof response.CommonPrefixes !== "undefined";
121
- if (hasCommonPrefixes) {
122
- for (const item of response.CommonPrefixes || []) {
123
- if (!item.Prefix)
124
- continue;
125
- const obj = this.nameToObj(item.Prefix);
126
- if (visitor) {
127
- yield visitor(obj);
128
- }
129
- objects.push(obj);
130
- objectCount++;
131
- }
132
- }
133
- truncated = response.IsTruncated || false;
134
- // If truncated is true, advance the marker
135
- if (truncated) {
136
- bucketParams.Marker = response.NextMarker;
137
- }
138
- else if (!hasContents && !hasCommonPrefixes) {
139
- if (path === "") {
140
- break;
141
- }
142
- throw new mobiletto_base_1.MobilettoNotFoundError(path);
143
- }
144
- }
145
- catch (err) {
146
- if (err instanceof mobiletto_base_1.MobilettoNotFoundError) {
147
- throw err;
148
- }
149
- throw new mobiletto_base_1.MobilettoError(`${logPrefix} Error: ${err}`);
150
- }
151
- }
152
- if (recursive && objectCount === 0 && path !== "") {
153
- throw new mobiletto_base_1.MobilettoNotFoundError(path);
154
- }
155
- const filtered = objects.filter((o) => o.name !== path);
156
- return filtered;
157
- });
158
- }
159
- s3error(err, key, path, method) {
160
- return err instanceof mobiletto_base_1.MobilettoError || err instanceof mobiletto_base_1.MobilettoNotFoundError
161
- ? err
162
- : err instanceof client_s3_1.NoSuchKey || (err.name && err.name === "NotFound")
163
- ? new mobiletto_base_1.MobilettoNotFoundError(this.denormalizeKey(key))
164
- : new mobiletto_base_1.MobilettoError(`${method}(${path}) error: ${err}`, err);
165
- }
166
- metadata(path) {
167
- return __awaiter(this, void 0, void 0, function* () {
168
- const Key = this.normalizeKey(path);
169
- const bucketParams = {
170
- Region: this.region,
171
- Bucket: this.bucket,
172
- Key,
173
- Delimiter: this.delimiter,
174
- };
175
- try {
176
- const head = yield this.client.send(new client_s3_1.HeadObjectCommand(bucketParams));
177
- const meta = {
178
- name: this.stripPrefix(path),
179
- size: head.ContentLength,
180
- type: path.endsWith(this.delimiter) ? mobiletto_base_1.M_DIR : mobiletto_base_1.M_FILE,
181
- };
182
- if (head.LastModified) {
183
- meta.mtime = Date.parse(head.LastModified.toString());
184
- }
185
- return meta;
186
- }
187
- catch (err) {
188
- throw this.s3error(err, Key, path, "metadata");
189
- }
190
- });
191
- }
192
- write(path, generator) {
193
- return __awaiter(this, void 0, void 0, function* () {
194
- const Key = this.normalizeKey(path);
195
- const bucketParams = {
196
- Region: this.region,
197
- Bucket: this.bucket,
198
- Key,
199
- Body: stream_1.Readable.from(generator),
200
- Delimiter: this.delimiter,
201
- };
202
- const uploader = new lib_storage_1.Upload({
203
- client: this.client,
204
- params: bucketParams,
205
- queueSize: 1,
206
- partSize: 1024 * 1024 * 5,
207
- leavePartsOnError: false, // optional manually handle dropped parts
208
- });
209
- let total = 0;
210
- uploader.on("httpUploadProgress", (progress) => {
211
- mobiletto_base_1.logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`);
212
- total += progress.loaded || 0;
213
- });
214
- const response = yield uploader.done();
215
- if (response.Key === Key) {
216
- return total;
217
- }
218
- throw new mobiletto_base_1.MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`);
219
- });
220
- }
221
- read(path, callback, endCallback) {
222
- return __awaiter(this, void 0, void 0, function* () {
223
- const Key = this.normalizeKey(path);
224
- mobiletto_base_1.logger.debug(`read: reading Key: ${path} - ${Key}`);
225
- const bucketParams = {
226
- Region: this.region,
227
- Bucket: this.bucket,
228
- Key,
229
- Delimiter: this.delimiter,
230
- };
231
- try {
232
- const data = yield this.client.send(new client_s3_1.GetObjectCommand(bucketParams));
233
- return yield (0, mobiletto_base_1.readStream)(data.Body, callback, endCallback);
234
- }
235
- catch (err) {
236
- throw this.s3error(err, Key, path, "read");
237
- }
238
- });
239
- }
240
- remove(path, optsOrRecursive, quiet) {
241
- return __awaiter(this, void 0, void 0, function* () {
242
- const recursive = optsOrRecursive === true || (optsOrRecursive && optsOrRecursive.recursive);
243
- if (recursive) {
244
- const removed = [];
245
- let objects = yield this._list(path, true, undefined, {
246
- MaxKeys: DELETE_OBJECTS_MAX_KEYS,
247
- });
248
- while (objects && objects.length > 0) {
249
- const Delete = {
250
- Objects: objects.map((obj) => {
251
- return { Key: this.normalizeKey(obj.name) };
252
- }),
253
- Quiet: quiet || false,
254
- };
255
- const bucketParams = {
256
- Bucket: this.bucket,
257
- Delete,
258
- };
259
- mobiletto_base_1.logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`);
260
- const response = yield this.client.send(new client_s3_1.DeleteObjectsCommand(bucketParams));
261
- const statusCode = response.$metadata.httpStatusCode;
262
- const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
263
- if (statusClass !== 2) {
264
- throw new mobiletto_base_1.MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`);
265
- }
266
- if (!quiet && response.Errors && response.Errors.length > 0) {
267
- throw new mobiletto_base_1.MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`);
268
- }
269
- if (response.Deleted) {
270
- removed.push(...response.Deleted.map((del) => del.Key ? this.denormalizeKey(del.Key) : "?del.Key undefined?"));
271
- }
272
- try {
273
- objects = yield this._list(path, true, undefined, { MaxKeys: DELETE_OBJECTS_MAX_KEYS });
274
- }
275
- catch (e) {
276
- if (!(e instanceof mobiletto_base_1.MobilettoNotFoundError)) {
277
- throw e instanceof mobiletto_base_1.MobilettoError
278
- ? e
279
- : new mobiletto_base_1.MobilettoError(`remove(${path}) error listing: ${e}`);
280
- }
281
- objects = null;
282
- }
283
- }
284
- return removed;
285
- }
286
- else {
287
- const Key = this.normalizeKey(path);
288
- const bucketParams = {
289
- Region: this.region,
290
- Bucket: this.bucket,
291
- Key,
292
- };
293
- try {
294
- // DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
295
- // Thus, if quiet is false, we must check metadata explicitly, which will fail with
296
- // MobilettoNotFoundError, which is the correct behavior
297
- if (!quiet) {
298
- yield this.metadata(path);
299
- }
300
- const response = yield this.client.send(new client_s3_1.DeleteObjectCommand(bucketParams));
301
- const statusCode = response.$metadata.httpStatusCode;
302
- const statusClass = statusCode ? Math.floor(statusCode / 100) : -1;
303
- if (statusClass !== 2) {
304
- throw new mobiletto_base_1.MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`);
305
- }
306
- }
307
- catch (err) {
308
- throw this.s3error(err, Key, path, "remove");
309
- }
310
- return path;
311
- }
312
- });
313
- }
314
- }
315
- const storageClient = (key, secret, opts) => {
316
- if (!key || !secret || !opts || !opts.bucket) {
317
- throw new mobiletto_base_1.MobilettoError("s3.storageClient: key, secret, and opts.bucket are required");
318
- }
319
- return new StorageClient(key, secret, opts);
320
- };
321
- exports.storageClient = storageClient;
@@ -1,16 +0,0 @@
1
- {
2
- "name": "mobiletto-driver-s3",
3
- "type": "commonjs",
4
- "main": "./index.js",
5
- "homepage": "https://github.com/cobbzilla/mobiletto",
6
- "author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
7
- "funding": {
8
- "type": "patreon",
9
- "url": "https://www.patreon.com/cobbzilla"
10
- },
11
- "repository": {
12
- "type": "git",
13
- "url": "https://github.com/cobbzilla/mobiletto-driver-s3"
14
- },
15
- "license": "Apache-2.0"
16
- }
@@ -1,16 +0,0 @@
1
- {
2
- "name": "mobiletto-common",
3
- "type": "commonjs",
4
- "main": "./index.js",
5
- "homepage": "https://github.com/cobbzilla/mobiletto",
6
- "author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
7
- "funding": {
8
- "type": "patreon",
9
- "url": "https://www.patreon.com/cobbzilla"
10
- },
11
- "repository": {
12
- "type": "git",
13
- "url": "https://github.com/cobbzilla/mobiletto-common"
14
- },
15
- "license": "Apache-2.0"
16
- }