mobiletto 1.0.46 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -3,28 +3,64 @@ Mobiletto
3
3
 
4
4
  Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
5
5
 
6
- It supports apps that are agnostic to where files are stored.
6
+ # Contents
7
+ * [Why Mobiletto?](#Why-Mobiletto?)
8
+ * [Quick start](#Quick-start)
9
+ * [Mobiletto CLI](#mobiletto-cli)
10
+ * [Source](#Source)
11
+ * [Basic usage](#Basic-usage)
12
+ * [Metadata](#Metadata)
13
+ * [Alternate import style](#Alternate-import-style)
14
+ * [Caching](#Caching)
15
+ * [Mirroring](#Mirroring)
16
+ * [Transparent encryption](#Transparent-encryption)
17
+ * [Key rotation](#Key-rotation)
18
+ * [Driver interface](#Driver-interface)
19
+ * [Logging](#Logging)
20
+
21
+ ## Why Mobiletto?
22
+
23
+ ### Goodbye vendor lock-in!
24
+ The various cloud storage providers have incompatible APIs. Even those that strive for "S3 compatibility"
25
+ have idiosyncratic behaviors.
26
+
27
+ When you choose a particular storage vendor for your app, if you code directly to their API, your app
28
+ is now dependent on that service. As time goes by and code accumulates, changing vendors becomes
29
+ increasingly untenable. Welcome to the fun world of vendor lock-in!
30
+
31
+ Mobiletto was designed to solve this problem. By coding your app to mobiletto's API, you can easily
32
+ change storage providers and know that your app's storage layer will behave identically.
7
33
 
34
+ ### Extensive testing
8
35
  All drivers are tested for identical behavior with over 55 tests for each driver.
36
+ Each driver test actually runs *four times*, with every (2x2) combination of:
37
+ * Encryption: both enabled and disabled
38
+ * Redis cache: both enabled and disabled
9
39
 
10
- Today the supported drivers are:
40
+ This approach gives us peace-of-mind that mobiletto will behave the same regardless of which driver you use,
41
+ and regardless of whether you enable caching and/or encryption.
11
42
 
12
- * `s3`: read/write an Amazon S3 bucket
13
- * `local`: read/write to local filesystem
43
+ ### Driver support
44
+ Current Mobiletto storage drivers:
45
+ * `s3`: Amazon S3
46
+ * `b2`: Backblaze B2
47
+ * `local`: local filesystem
14
48
 
15
- ### mobiletto-cli
49
+ *Contributions to support more cloud storage providers are very welcome!*
50
+
51
+ ## mobiletto-cli
16
52
  Mobiletto is intended to be used as a library by other JavaScript code.
17
53
 
18
54
  To work with mobiletto at the command-line, use [mobiletto-cli](https://www.npmjs.com/package/mobiletto-cli)
19
55
 
20
- ### Source
56
+ ## Source
21
57
  * [mobiletto on GitHub](https://github.com/cobbzilla/mobiletto)
22
58
  * [mobiletto on npm](https://www.npmjs.com/package/mobiletto)
23
59
 
24
- ## TLDR
60
+ ## Quick Start
25
61
  A simple example using the mobiletto `s3` driver.
26
62
 
27
- This code would run the same if the driver were `local`.
63
+ This code would run the same if the driver were `b2` or `local`.
28
64
 
29
65
  const storage = require('mobiletto')
30
66
  const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
@@ -46,20 +82,12 @@ This code would run the same if the driver were `local`.
46
82
  // read file as a stream using data callback
47
83
  const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
48
84
 
49
- // remove a file, returns true upon success
50
- const removed = await s3.remove('some/path')
85
+ // remove a file, returns the path removed
86
+ let removed = await s3.remove('some/path')
87
+
88
+ // remove a directory, returns array of paths removed
89
+ removed = await s3.remove('some/directory', {recursive: true})
51
90
 
52
- ----
53
- # Contents
54
- * [Basic usage](#Basic-usage)
55
- * [Metadata](#Metadata)
56
- * [Alternate import style](#Alternate-import-style)
57
- * [Caching](#Caching)
58
- * [Mirroring](#Mirroring)
59
- * [Transparent encryption](#Transparent-encryption)
60
- * [Key rotation](#Key-rotation)
61
- * [Driver interface](#Driver-interface)
62
- * [Logging](#Logging)
63
91
  ----
64
92
  ## Basic usage
65
93
  const { mobiletto } = require('mobiletto')
@@ -87,44 +115,50 @@ This code would run the same if the driver were `local`.
87
115
  // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
88
116
  const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
89
117
 
118
+ // To use 'b2' driver:
119
+ // * key: Backblaze Key ID
120
+ // * secret: Backblaze Application Key
121
+ // * opts object:
122
+ // * readOnly: optional, never change anything on the bucket; default is false
123
+ // * bucket: required, the ID (**not the name**) of the B2 bucket
124
+ // * prefix: optional, all read/writes within the B2 bucket will be under this prefix
125
+ // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
126
+ // * partSize: optional, large files will be split into chunks of this size when uploading
127
+ const b3 = await mobiletto('b2', b2_key_id, b2_app_key, {bucket: 'bk', partSize: 10000000})
128
+
90
129
  // List files
91
- local.list() // --> returns an array of metadata objects
92
- s3.list() // --> returns an array of metadata objects
130
+ api.list() // --> returns an array of metadata objects
93
131
 
94
132
  // List files recursively
95
- local.list({ recursive: true })
96
- s3.list({ recursive: true })
133
+ api.list({ recursive: true })
97
134
 
98
135
  // List files in a directory
99
136
  const path = 'some/path'
100
- local.list(path)
101
- s3.list(path, { recursive: true }) // also supports recursive flag
137
+ api.list(path)
138
+ api.list(path, { recursive: true }) // also supports recursive flag
102
139
 
103
140
  // Visit files in a directory -- visitor function must be async
104
- local.list(path, { visitor: myAsyncFunc })
105
- s3.list(path, { visitor: myAsyncFunc, recursive: true })
141
+ api.list(path, { visitor: myAsyncFunc })
142
+ api.list(path, { visitor: myAsyncFunc, recursive: true })
106
143
 
107
144
  // The `list` method throws MobilettoNotFoundError if the path does not exist
108
145
  // When you call `safeList` on a non-existent path, it returns an empty array
109
- s3.safeList('/path/that/does/not/exist') // returns []
146
+ api.safeList('/path/that/does/not/exist') // returns []
110
147
 
111
148
  // Read metadata for a file
112
- local.metadata(path) // returns metadata object
113
- s3.metadata(path) // returns metadata object
149
+ api.metadata(path) // returns metadata object
114
150
 
115
151
  // The `metadata` method throws MobilettoNotFoundError if the path does not exist
116
152
  // When you call `safeMetadata` on a non-existent path, it returns null
117
- local.safeMetadata('/tmp/does_not_exist') // returns null
153
+ api.safeMetadata('/tmp/does_not_exist') // returns null
118
154
 
119
155
  // Read a file
120
156
  // Provide a callback that writes the data someplace
121
157
  const callback = (chunk) => { ... write chunk somewhere ... }
122
- local.read(path, callback) // returns count of bytes read
123
- s3.read(path, callback) // returns count of bytes read
158
+ api.read(path, callback) // returns count of bytes read
124
159
 
125
160
  // Read an entire file at once (convenience method)
126
- const localData = await local.readFile(path) // returns a byte Buffer of the file contents
127
- const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
161
+ const data = await api.readFile(path) // returns a byte Buffer of the file contents
128
162
 
129
163
  // Write a file
130
164
  // Provide a generator function that yields chunks of data
@@ -134,26 +168,22 @@ This code would run the same if the driver were `local`.
134
168
  yield data
135
169
  }
136
170
  }
137
- local.write(path, generator) // returns count of bytes written
138
- s3.write(path, generator) // returns count of bytes written
171
+ local.api(path, generator) // returns count of bytes written
139
172
 
140
173
  // Write an entire file at once (convenience method)
141
- await local.writeFile(path, bufferOrString) // returns count of bytes written
142
- await s3.writeFile(path, bufferOrString) // returns count of bytes written
174
+ await api.writeFile(path, bufferOrString) // returns count of bytes written
143
175
 
144
176
  // Delete a file
145
177
  // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
146
178
  // Always returns a value or throws an error.
147
179
  // Return value may be a single string of the file removed, or an array of all files removed (driver-dependent)
148
180
  const quiet = true
149
- local.remove(path, {quiet}) // returns single path removed
150
- s3.remove(path, {quiet}) // returns single path removed
181
+ api.remove(path, {quiet}) // returns single path removed
151
182
 
152
183
  // Recursively delete a directory and do it quietly (do not report errors)
153
184
  const recursive = true
154
185
  const quiet = true
155
- local.remove(path, {recursive, quiet}) // returns paths removed
156
- s3.remove(path, {recursive, quiet}) // returns array containing all paths removed
186
+ api.remove(path, {recursive, quiet}) // returns array of paths removed
157
187
 
158
188
  ## Metadata
159
189
  The `metadata` command returns metadata about a single filesystem entry.
@@ -194,15 +224,19 @@ You can override either of these:
194
224
  Mobiletto will store all of its redis keys with the prefix `_mobiletto__`. You can change this
195
225
  by setting the `MOBILETTO_REDIS_PREFIX` env var.
196
226
 
197
- ### Turn off redis caching
198
- To disable redis, you can do either of:
199
- * Not run a redis server on 127.0.0.1:6379 that allows unauthenticated local connections
200
- * Set `MOBILETTO_REDIS_HOST` to an empty string or anything that won't resolve as a host, for example:
201
- * `MOBILETTO_REDIS_HOST=`
202
- * `MOBILETTO_REDIS_HOST=/dev/null`
203
- * `MOBILETTO_REDIS_HOST=~invalid~hostname~`
204
- * `MOBILETTO_REDIS_HOST=invalid-hostname` *<-- not this!*
205
- * **^^^ NO! that last one is a valid hostname** and depending on the name resolution scheme, it might actually be valid and you'll connect to it!
227
+ You can also set per-connection caching with the `opts.redisConfig` object:
228
+
229
+ const redisConfig = {
230
+ enabled: true, // optional, default is true. if false other props are ignored
231
+ host: '127.0.0.1',
232
+ port: 6379,
233
+ prefix: '_mobiletto__'
234
+ }
235
+ const opts = { redisConfig, bucket: 'bk', region: 'us-east-1' }
236
+ const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
237
+
238
+ ### Don't want redis caching?
239
+ To disable: pass `enabled: false` in your `opts.redisConfig` object when you establish your connection.
206
240
 
207
241
  As discussed below, disabling caching will have an adverse effect on performance and incur more requests
208
242
  to storage that you really need to.
@@ -347,7 +381,7 @@ or any other connection configuration information.
347
381
  Use the `MOBILETTO_LOG_LEVEL` environment variable to set the log level, using one
348
382
  of the `npm` levels defined in [https://www.npmjs.com/package/winston#logging-levels](https://www.npmjs.com/package/winston#logging-levels)
349
383
 
350
- The default level is `warn`. The most verbose level is `silly`, although currently mobiletto
384
+ The default level is `error`. The most verbose level is `silly`, although currently mobiletto
351
385
  does not log at levels below `debug`
352
386
 
353
387
  MOBILETTO_LOG_LEVEL=silly # maximum logs!
@@ -0,0 +1,428 @@
1
+ const B2 = require('backblaze-b2')
2
+
3
+ // this library is a godsend for B2 uploads
4
+ B2.prototype.uploadAny = require('@gideo-llc/backblaze-b2-upload-any')
5
+
6
+ const { Readable } = require('stream')
7
+
8
+ const {
9
+ M_FILE, M_DIR,
10
+ readStream, isReadable, isAsyncGenerator,
11
+ MobilettoError, MobilettoNotFoundError
12
+ } = require('../../index')
13
+
14
+ const { logger } = require('../../util/logger')
15
+
16
+ // B2 tokens last 24 hours. Let's refresh ours after 23 hours and 50 minutes
17
+ const AUTH_REFRESH = (1000 * 60 * 60 * 24) - (1000 * 60 * 10)
18
+
19
+ class StorageClient {
20
+ b2
21
+ bucket
22
+ prefix
23
+ delimiter
24
+ opts
25
+ lastAuth
26
+ absoluteMinimumPartSize
27
+ recommendedPartSize
28
+ configuredPartSize
29
+ constructor(keyId, appKey, opts) {
30
+ if (!keyId || !appKey || !opts.bucket) throw new MobilettoError(`b2.StorageClient: key, secret and opts.bucket are required`)
31
+ this.b2 = new B2({
32
+ applicationKeyId: keyId,
33
+ applicationKey: appKey
34
+ })
35
+ this.opts = opts
36
+ this.configuredPartSize = opts.partSize ? opts.partSize : -1
37
+ this.bucket = opts.bucket
38
+ this.delimiter = opts.delimiter || '/'
39
+ this.prefix = opts.prefix
40
+ ? opts.prefix.endsWith(this.delimiter)
41
+ ? opts.prefix
42
+ : opts.prefix + this.delimiter
43
+ : ''
44
+ this.lastAuth = 0
45
+ }
46
+
47
+ auth = async () => {
48
+ if (Date.now() - this.lastAuth > AUTH_REFRESH) {
49
+ try {
50
+ const response = await this.b2.authorize({})
51
+ this.lastAuth = Date.now()
52
+ this.absoluteMinimumPartSize = response.data.absoluteMinimumPartSize
53
+ this.recommendedPartSize = response.data.recommendedPartSize
54
+ if (this.configuredPartSize === -1) {
55
+ logger.info(`partSize was not configured, using recommendedPartSize=${this.recommendedPartSize}`)
56
+ this.configuredPartSize = this.recommendedPartSize
57
+ } else if (this.configuredPartSize < this.absoluteMinimumPartSize) {
58
+ logger.warn(`partSize was too small (${this.configuredPartSize}), using absoluteMinimumPartSize=${this.absoluteMinimumPartSize}`)
59
+ this.configuredPartSize = this.absoluteMinimumPartSize
60
+ }
61
+ } catch (e) {
62
+ logger.error(`auth error: ${e}`)
63
+ throw e
64
+ }
65
+ }
66
+ }
67
+
68
+ testConfig = async () => await this.b2_list('', false, null, {max: 1})
69
+
70
+ normalizePath = (path) => path.length === 0 ? this.prefix :
71
+ path.startsWith(this.prefix)
72
+ ? path
73
+ : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path)
74
+ denormalizePath = (path) => {
75
+ const norm = path.startsWith(this.prefix) ? path.substring(this.prefix.length) : path
76
+ return norm.startsWith(this.delimiter) ? norm.substring(1) : norm
77
+ }
78
+ bbError = (err, path, method) => {
79
+ if (err.response && err.response.status && err.response.status === 404) {
80
+ return new MobilettoNotFoundError(path)
81
+ } else if (err.response && err.response.data && err.response.data.message) {
82
+ return new MobilettoError(`${method} error: ${err.response.data.message}`)
83
+ }
84
+ return err
85
+ }
86
+
87
+ b2type = f => f.fileName.endsWith(this.delimiter) && f.contentType === null ? M_DIR : M_FILE
88
+
89
+ file2object(f) {
90
+ return {
91
+ name: this.denormalizePath(f.fileName),
92
+ size: f.contentLength,
93
+ type: this.b2type(f),
94
+ bb2id: f.fileId
95
+ }
96
+ }
97
+
98
+ processList = async (path, response, visitor) => {
99
+ const files = response && response.data && response.data.files ? response.data.files : null
100
+ if (!files) {
101
+ return []
102
+ }
103
+ const objects = files.map((f) => {
104
+ logger.debug(`addListedFiles(${path}): adding file: ${f.fileName}`)
105
+ return this.file2object(f)
106
+ })
107
+ if (visitor) {
108
+ for (const obj of objects) { await visitor(obj) }
109
+ }
110
+ return objects
111
+ }
112
+
113
+ list = (pth = '', recursive = false, visitor = null) =>
114
+ this.b2_list(pth, recursive, visitor)
115
+
116
+ async b2_list (pth = '', recursive = false, visitor = null, opts = {}) {
117
+ try {
118
+ let optsDir = opts.dir || false
119
+ let dirDiscovered = null
120
+ let firstResult = null
121
+ const files = []
122
+ await this.auth()
123
+ const hasMax = opts && opts.max
124
+ const maxFileCount = hasMax ? opts.max : 100
125
+ const pfx = (this.prefix.length === 0 || this.prefix.endsWith(this.delimiter) ? this.prefix : this.prefix + this.delimiter) +
126
+ (pth.startsWith(this.delimiter) ? pth.substring(0) : pth)
127
+
128
+ let response = await this.b2.listFileNames({
129
+ bucketId: this.bucket,
130
+ maxFileCount,
131
+ delimiter: recursive ? '' : this.delimiter,
132
+ prefix: pfx
133
+ })
134
+ const handleResponse = async (files, pth, response, visitor) => {
135
+ const list = await this.processList(pth, response, visitor)
136
+ if (!optsDir && !firstResult && list.length > 0) {
137
+ firstResult = list[0]
138
+ if (list[0].name.endsWith('/')) {
139
+ dirDiscovered = true
140
+ } else if (!dirDiscovered && list.length > 1 && list[1].name.startsWith(list[0].name)) {
141
+ files.push(list[0])
142
+ return true
143
+ }
144
+ }
145
+ if (hasMax && files.length + list.length >= maxFileCount) {
146
+ files.push(...list.slice(0, (list.length - maxFileCount) + 1))
147
+ return true
148
+ } else {
149
+ files.push(...list)
150
+ return false
151
+ }
152
+ }
153
+ while (response.data.nextFileName) {
154
+ if (await handleResponse(files, pth, response, visitor)) {
155
+ return files
156
+ }
157
+ response = await this.b2.listFileNames({
158
+ bucketId: this.bucket,
159
+ startFileName: response.data.nextFileName,
160
+ maxFileCount,
161
+ delimiter: recursive ? '' : this.delimiter,
162
+ prefix: this.prefix
163
+ })
164
+ }
165
+ await handleResponse(files, pth, response, visitor)
166
+ if (dirDiscovered && !pth.endsWith(this.delimiter)) {
167
+ const subdirFiles = await this.b2_list(pth + this.delimiter, recursive, null, { max: 10000, dir: true })
168
+ files.shift()
169
+ files.push(...subdirFiles)
170
+ }
171
+ return files
172
+ } catch (err) {
173
+ throw this.bbError(err, pth, 'list')
174
+ }
175
+ }
176
+
177
+ b2_meta = (path) => (this.driver_metadata ? this.driver_metadata(path) : this.metadata(path))
178
+
179
+ async metadata (pth) {
180
+ await this.auth()
181
+ let files = null
182
+ const normPath = this.normalizePath(pth);
183
+ try {
184
+ files = (await this.b2.listFileNames({
185
+ bucketId: this.bucket,
186
+ startFileName: normPath,
187
+ maxFileCount: 1,
188
+ delimiter: '',
189
+ prefix: this.prefix
190
+ })).data.files
191
+ } catch (err) {
192
+ throw this.bbError(err, pth, 'metadata')
193
+ }
194
+ if (files.length === 0) {
195
+ throw new MobilettoNotFoundError(pth)
196
+ }
197
+ const found = files.filter(f => f.fileName === normPath)
198
+ if (!found || found.length === 0) {
199
+ // if this is a directory, we may have returned the first item in it, that's OK
200
+ if (files[0].fileName.startsWith(normPath.endsWith(this.delimiter) ? normPath : normPath + this.delimiter)) {
201
+ let obj = this.file2object(files[0]);
202
+ obj.name = pth
203
+ obj.type = M_DIR
204
+ return obj
205
+ }
206
+ throw new MobilettoNotFoundError(pth)
207
+ }
208
+ if (found.length > 1) {
209
+ throw new MobilettoError(`metadata(${pth}) multiple files matched: ${JSON.stringify(found)}`)
210
+ }
211
+ return this.file2object(files[0])
212
+ }
213
+
214
+ async write (path, generatorOrReadable) {
215
+ await this.auth()
216
+ logger.debug(`write: writing path ${path}`)
217
+ let stream = generatorOrReadable
218
+ if (!isReadable(generatorOrReadable)) {
219
+ stream = new Readable({
220
+ async read (size) {
221
+ let chunk = isAsyncGenerator(generatorOrReadable)
222
+ ? (await generatorOrReadable.next()).value
223
+ : generatorOrReadable.next().value
224
+ this.push(typeof chunk !== 'undefined' ? chunk : null)
225
+ }
226
+ })
227
+ }
228
+ let response = null
229
+ try {
230
+ response = await this.b2.uploadAny({
231
+ bucketId: this.bucket,
232
+ fileName: this.normalizePath(path),
233
+ partSize: this.configuredPartSize,
234
+ data: stream
235
+ });
236
+ // response is from either b2_upload_file or b2_finish_large_file
237
+ // It depends on how large the file was, see: https://github.com/gideo-llc/backblaze-b2-upload-any/blob/master/README.md#uploadanyoptions
238
+ if (response.contentLength) {
239
+ return response.contentLength
240
+ }
241
+ } catch (e) {
242
+ logger.error(`write error: ${e}`)
243
+ throw this.bbError(e, path, 'write')
244
+ }
245
+ if (response) {
246
+ throw new MobilettoError(`write error: response status ${response.status} with data ${JSON.stringify(response.data)}`)
247
+ }
248
+ }
249
+
250
+ async read (path, callback, endCallback = null) {
251
+ await this.auth()
252
+ const file = this.normalizePath(path)
253
+ logger.debug(`read: reading path: ${path}`)
254
+ let meta = null
255
+ try {
256
+ meta = await (this.b2_meta(file))
257
+ } catch (e) {
258
+ if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
259
+ throw e
260
+ }
261
+ throw new MobilettoError(`read: error reading metadata for path ${path}`, e)
262
+ }
263
+ if (meta.type === M_DIR) {
264
+ throw new MobilettoError(`read: cannot read directory: ${path}`)
265
+ }
266
+ try {
267
+ const stream = await this.b2.downloadFileById({
268
+ bucketId: this.bucket,
269
+ fileId: meta.bb2id,
270
+ responseType: 'stream'
271
+ })
272
+ return await readStream(stream.data, callback, endCallback)
273
+ } catch (err) {
274
+ throw this.bbError(err, path, 'read')
275
+ }
276
+ }
277
+
278
+ async deleteVersions (path, normPath, versions, quiet, errors = {}) {
279
+ const removed = []
280
+ for (const version of versions) {
281
+ logger.debug(`deleteVersions(${path}, ${versions.length} versions) pushing version to delete: fileName=${version.fileName} fileId=${version.fileId}`)
282
+ await this.auth()
283
+ try {
284
+ if (!version.fileName.startsWith(normPath)) {
285
+ logger.debug(`deleteVersions(${path}, ${versions.length} versions) ending iteration on mismatched file: ${version.fileName}`)
286
+ return removed
287
+ }
288
+ const delResponse = await this.b2.deleteFileVersion({
289
+ bucketId: this.bucket,
290
+ fileName: version.fileName,
291
+ fileId: version.fileId
292
+ })
293
+ if (delResponse && delResponse.status !== 200) {
294
+ const message = `deleteVersions(${path}) received HTTP status ${delResponse.status} trying to delete file version: ${JSON.stringify(version)}`
295
+ logger.error(message)
296
+ if (errors) {
297
+ errors[path] = message
298
+ }
299
+ } else if (removed) {
300
+ logger.debug(`deleteVersions(${path}, ${versions.length} versions) REMOVED: fileName=${version.fileName} fileId=${version.fileId}`)
301
+ removed.push(this.denormalizePath(version.fileName))
302
+ }
303
+ } catch (err) {
304
+ const message = `deleteVersions(${path}) error deleting version ${version.fileId}: ${err}`
305
+ logger.error(message)
306
+ if (errors) {
307
+ errors[path] = message
308
+ }
309
+ if (!quiet) throw new MobilettoError(message, err)
310
+ }
311
+ }
312
+ const errPaths = Object.keys(errors);
313
+ if (errPaths.length > 0) {
314
+ if (quiet) {
315
+ logger.info(`deleteVersions(${path}) had ${errPaths.length} errors but quiet=true, returning ${removed.length} removed paths`)
316
+ return removed
317
+ }
318
+ throw new MobilettoError(`deleteVersions(${path}) had errors: ${JSON.stringify(errors)}`)
319
+ }
320
+ return removed
321
+ }
322
+
323
+ async remove (path, recursive, quiet) {
324
+ logger.debug(`b2.remove(${path}, recursive=${recursive}) starting...`)
325
+ await this.auth()
326
+ let meta = null
327
+ const errors = {}
328
+ const normPath = this.normalizePath(path)
329
+ try {
330
+ logger.debug(`b2.remove(${path}) fetching metadata to determine file/directory`)
331
+ meta = await (this.b2_meta(path))
332
+ } catch (e) {
333
+ if (quiet) {
334
+ logger.debug(`b2.remove(${path}) quietly returning, not reporting error fetching metadata for ${path}: ${e}`)
335
+ return path
336
+ }
337
+ if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
338
+ throw e
339
+ }
340
+ throw new MobilettoError(`b2.remove(${path}) error reading metadata for path ${path}`, e)
341
+ }
342
+ if (meta.type === M_FILE) {
343
+ try {
344
+ const response = await this.b2.listFileVersions({
345
+ bucketId: this.bucket,
346
+ startFileName: normPath,
347
+ startFileId: meta.bb2id,
348
+ prefix: this.prefix,
349
+ delimiter: this.delimiter,
350
+ maxFileCount: 1000
351
+ })
352
+ const versions = response.data.files.filter(f => f.fileName === normPath)
353
+ await this.deleteVersions(path, normPath, versions, quiet, errors)
354
+ } catch (err) {
355
+ if (err instanceof MobilettoError) {
356
+ throw err
357
+ }
358
+ throw new MobilettoError(`b2.remove(${path}) error: ${err}`, err)
359
+ }
360
+ return path
361
+ } else if (meta.type === M_DIR) {
362
+ if (!recursive) {
363
+ if (quiet) {
364
+ logger.debug(`b2.remove(${path}) quietly returning, not deleting directory when recursive=false`)
365
+ return path
366
+ }
367
+ throw new MobilettoError(`b2.remove(${path}) cannot remove directory (set recursive=true to remove)`)
368
+ }
369
+ const removed = []
370
+ const errors = {}
371
+ let versionsResponse
372
+ try {
373
+ logger.debug(`b2.remove(${path})`)
374
+ let normPath = this.normalizePath(path);
375
+ versionsResponse = await this.b2.listFileVersions({
376
+ bucketId: this.bucket,
377
+ startFileName: normPath,
378
+ prefix: normPath,
379
+ delimiter: '',
380
+ maxFileCount: 1000
381
+ })
382
+ while (versionsResponse.data.nextFileName &&
383
+ versionsResponse.data.nextFileId &&
384
+ versionsResponse.data.nextFileName.startsWith(normPath)) {
385
+ logger.debug(`b2.remove(${path}) removing a batch of files then asking for more`)
386
+ removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
387
+ logger.debug(`b2.remove(${path})`)
388
+ versionsResponse = await this.b2.listFileVersions({
389
+ bucketId: this.bucket,
390
+ startFileName: versionsResponse.data.nextFileName,
391
+ startFileId: versionsResponse.data.nextFileId,
392
+ prefix: normPath,
393
+ delimiter: ''
394
+ })
395
+ }
396
+ logger.debug(`b2.remove(${path}) removing last batch of files`)
397
+ removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
398
+ } catch (e) {
399
+ const message = `b2.remove(${path}) error listing/deleting: ${e}`
400
+ if (quiet) {
401
+ logger.warn(message)
402
+ } else {
403
+ throw new MobilettoError(message)
404
+ }
405
+ }
406
+ if (Object.keys(errors).length > 0) {
407
+ if (quiet) {
408
+ logger.warn(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
409
+ } else {
410
+ throw new MobilettoError(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
411
+ }
412
+ }
413
+ return removed
414
+
415
+ } else {
416
+ throw new MobilettoError(`b2.remove(${path}) cannot remove file of type: ${meta.type}`)
417
+ }
418
+ }
419
+ }
420
+
421
+ function storageClient (key, secret, opts) {
422
+ if (!key || !secret || !opts || !opts.bucket) {
423
+ throw new MobilettoError('b2.storageClient: key, secret, and opts.bucket are required')
424
+ }
425
+ return new StorageClient(key, secret, opts)
426
+ }
427
+
428
+ module.exports = { storageClient }