mobiletto 1.0.47 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -3,28 +3,64 @@ Mobiletto
3
3
 
4
4
  Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
5
5
 
6
- It supports apps that are agnostic to where files are stored.
6
+ # Contents
7
+ * [Why Mobiletto?](#Why-Mobiletto?)
8
+ * [Quick start](#Quick-start)
9
+ * [Mobiletto CLI](#mobiletto-cli)
10
+ * [Source](#Source)
11
+ * [Basic usage](#Basic-usage)
12
+ * [Metadata](#Metadata)
13
+ * [Alternate import style](#Alternate-import-style)
14
+ * [Caching](#Caching)
15
+ * [Mirroring](#Mirroring)
16
+ * [Transparent encryption](#Transparent-encryption)
17
+ * [Key rotation](#Key-rotation)
18
+ * [Driver interface](#Driver-interface)
19
+ * [Logging](#Logging)
20
+
21
+ ## Why Mobiletto?
22
+
23
+ ### Goodbye vendor lock-in!
24
+ The various cloud storage providers have incompatible APIs. Even those that strive for "S3 compatibility"
25
+ have idiosyncratic behaviors.
7
26
 
27
+ When you choose a particular storage vendor for your app, if you code directly to their API, your app
28
+ is now dependent on that service. As time goes by and code accumulates, changing vendors becomes
29
+ increasingly untenable. Welcome to the fun world of vendor lock-in!
30
+
31
+ Mobiletto was designed to solve this problem. By coding your app to mobiletto's API, you can easily
32
+ change storage providers and know that your app's storage layer will behave identically.
33
+
34
+ ### Extensive testing
8
35
  All drivers are tested for identical behavior with over 55 tests for each driver.
36
+ Each driver test actually runs *four times*, with every (2x2) combination of:
37
+ * Encryption: both enabled and disabled
38
+ * Redis cache: both enabled and disabled
9
39
 
10
- Today the supported drivers are:
40
+ This approach gives us peace-of-mind that mobiletto will behave the same regardless of which driver you use,
41
+ and regardless of whether you enable caching and/or encryption.
11
42
 
12
- * `s3`: read/write an Amazon S3 bucket
13
- * `local`: read/write to local filesystem
43
+ ### Driver support
44
+ Current Mobiletto storage drivers:
45
+ * `s3`: Amazon S3
46
+ * `b2`: Backblaze B2
47
+ * `local`: local filesystem
14
48
 
15
- ### mobiletto-cli
49
+ *Contributions to support more cloud storage providers are very welcome!*
50
+
51
+ ## mobiletto-cli
16
52
  Mobiletto is intended to be used as a library by other JavaScript code.
17
53
 
18
54
  To work with mobiletto at the command-line, use [mobiletto-cli](https://www.npmjs.com/package/mobiletto-cli)
19
55
 
20
- ### Source
56
+ ## Source
21
57
  * [mobiletto on GitHub](https://github.com/cobbzilla/mobiletto)
22
58
  * [mobiletto on npm](https://www.npmjs.com/package/mobiletto)
23
59
 
24
- ## TLDR
60
+ ## Quick Start
25
61
  A simple example using the mobiletto `s3` driver.
26
62
 
27
- This code would run the same if the driver were `local`.
63
+ This code would run the same if the driver were `b2` or `local`.
28
64
 
29
65
  const storage = require('mobiletto')
30
66
  const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
@@ -46,20 +82,12 @@ This code would run the same if the driver were `local`.
46
82
  // read file as a stream using data callback
47
83
  const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
48
84
 
49
- // remove a file, returns true upon success
50
- const removed = await s3.remove('some/path')
85
+ // remove a file, returns the path removed
86
+ let removed = await s3.remove('some/path')
87
+
88
+ // remove a directory, returns array of paths removed
89
+ removed = await s3.remove('some/directory', {recursive: true})
51
90
 
52
- ----
53
- # Contents
54
- * [Basic usage](#Basic-usage)
55
- * [Metadata](#Metadata)
56
- * [Alternate import style](#Alternate-import-style)
57
- * [Caching](#Caching)
58
- * [Mirroring](#Mirroring)
59
- * [Transparent encryption](#Transparent-encryption)
60
- * [Key rotation](#Key-rotation)
61
- * [Driver interface](#Driver-interface)
62
- * [Logging](#Logging)
63
91
  ----
64
92
  ## Basic usage
65
93
  const { mobiletto } = require('mobiletto')
@@ -87,44 +115,50 @@ This code would run the same if the driver were `local`.
87
115
  // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
88
116
  const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
89
117
 
118
+ // To use 'b2' driver:
119
+ // * key: Backblaze Key ID
120
+ // * secret: Backblaze Application Key
121
+ // * opts object:
122
+ // * readOnly: optional, never change anything on the bucket; default is false
123
+ // * bucket: required, the ID (**not the name**) of the B2 bucket
124
+ // * prefix: optional, all read/writes within the B2 bucket will be under this prefix
125
+ // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
126
+ // * partSize: optional, large files will be split into chunks of this size when uploading
127
+ const b3 = await mobiletto('b2', b2_key_id, b2_app_key, {bucket: 'bk', partSize: 10000000})
128
+
90
129
  // List files
91
- local.list() // --> returns an array of metadata objects
92
- s3.list() // --> returns an array of metadata objects
130
+ api.list() // --> returns an array of metadata objects
93
131
 
94
132
  // List files recursively
95
- local.list({ recursive: true })
96
- s3.list({ recursive: true })
133
+ api.list({ recursive: true })
97
134
 
98
135
  // List files in a directory
99
136
  const path = 'some/path'
100
- local.list(path)
101
- s3.list(path, { recursive: true }) // also supports recursive flag
137
+ api.list(path)
138
+ api.list(path, { recursive: true }) // also supports recursive flag
102
139
 
103
140
  // Visit files in a directory -- visitor function must be async
104
- local.list(path, { visitor: myAsyncFunc })
105
- s3.list(path, { visitor: myAsyncFunc, recursive: true })
141
+ api.list(path, { visitor: myAsyncFunc })
142
+ api.list(path, { visitor: myAsyncFunc, recursive: true })
106
143
 
107
144
  // The `list` method throws MobilettoNotFoundError if the path does not exist
108
145
  // When you call `safeList` on a non-existent path, it returns an empty array
109
- s3.safeList('/path/that/does/not/exist') // returns []
146
+ api.safeList('/path/that/does/not/exist') // returns []
110
147
 
111
148
  // Read metadata for a file
112
- local.metadata(path) // returns metadata object
113
- s3.metadata(path) // returns metadata object
149
+ api.metadata(path) // returns metadata object
114
150
 
115
151
  // The `metadata` method throws MobilettoNotFoundError if the path does not exist
116
152
  // When you call `safeMetadata` on a non-existent path, it returns null
117
- local.safeMetadata('/tmp/does_not_exist') // returns null
153
+ api.safeMetadata('/tmp/does_not_exist') // returns null
118
154
 
119
155
  // Read a file
120
156
  // Provide a callback that writes the data someplace
121
157
  const callback = (chunk) => { ... write chunk somewhere ... }
122
- local.read(path, callback) // returns count of bytes read
123
- s3.read(path, callback) // returns count of bytes read
158
+ api.read(path, callback) // returns count of bytes read
124
159
 
125
160
  // Read an entire file at once (convenience method)
126
- const localData = await local.readFile(path) // returns a byte Buffer of the file contents
127
- const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
161
+ const data = await api.readFile(path) // returns a byte Buffer of the file contents
128
162
 
129
163
  // Write a file
130
164
  // Provide a generator function that yields chunks of data
@@ -134,26 +168,22 @@ This code would run the same if the driver were `local`.
134
168
  yield data
135
169
  }
136
170
  }
137
- local.write(path, generator) // returns count of bytes written
138
- s3.write(path, generator) // returns count of bytes written
171
+ local.api(path, generator) // returns count of bytes written
139
172
 
140
173
  // Write an entire file at once (convenience method)
141
- await local.writeFile(path, bufferOrString) // returns count of bytes written
142
- await s3.writeFile(path, bufferOrString) // returns count of bytes written
174
+ await api.writeFile(path, bufferOrString) // returns count of bytes written
143
175
 
144
176
  // Delete a file
145
177
  // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
146
178
  // Always returns a value or throws an error.
147
179
  // Return value may be a single string of the file removed, or an array of all files removed (driver-dependent)
148
180
  const quiet = true
149
- local.remove(path, {quiet}) // returns single path removed
150
- s3.remove(path, {quiet}) // returns single path removed
181
+ api.remove(path, {quiet}) // returns single path removed
151
182
 
152
183
  // Recursively delete a directory and do it quietly (do not report errors)
153
184
  const recursive = true
154
185
  const quiet = true
155
- local.remove(path, {recursive, quiet}) // returns paths removed
156
- s3.remove(path, {recursive, quiet}) // returns array containing all paths removed
186
+ api.remove(path, {recursive, quiet}) // returns array of paths removed
157
187
 
158
188
  ## Metadata
159
189
  The `metadata` command returns metadata about a single filesystem entry.
@@ -351,7 +381,7 @@ or any other connection configuration information.
351
381
  Use the `MOBILETTO_LOG_LEVEL` environment variable to set the log level, using one
352
382
  of the `npm` levels defined in [https://www.npmjs.com/package/winston#logging-levels](https://www.npmjs.com/package/winston#logging-levels)
353
383
 
354
- The default level is `warn`. The most verbose level is `silly`, although currently mobiletto
384
+ The default level is `error`. The most verbose level is `silly`, although currently mobiletto
355
385
  does not log at levels below `debug`
356
386
 
357
387
  MOBILETTO_LOG_LEVEL=silly # maximum logs!
@@ -0,0 +1,428 @@
1
+ const B2 = require('backblaze-b2')
2
+
3
+ // this library is a godsend for B2 uploads
4
+ B2.prototype.uploadAny = require('@gideo-llc/backblaze-b2-upload-any')
5
+
6
+ const { Readable } = require('stream')
7
+
8
+ const {
9
+ M_FILE, M_DIR,
10
+ readStream, isReadable, isAsyncGenerator,
11
+ MobilettoError, MobilettoNotFoundError
12
+ } = require('../../index')
13
+
14
+ const { logger } = require('../../util/logger')
15
+
16
+ // B2 tokens last 24 hours. Let's refresh ours after 23 hours and 50 minutes
17
+ const AUTH_REFRESH = (1000 * 60 * 60 * 24) - (1000 * 60 * 10)
18
+
19
+ class StorageClient {
20
+ b2
21
+ bucket
22
+ prefix
23
+ delimiter
24
+ opts
25
+ lastAuth
26
+ absoluteMinimumPartSize
27
+ recommendedPartSize
28
+ configuredPartSize
29
+ constructor(keyId, appKey, opts) {
30
+ if (!keyId || !appKey || !opts.bucket) throw new MobilettoError(`b2.StorageClient: key, secret and opts.bucket are required`)
31
+ this.b2 = new B2({
32
+ applicationKeyId: keyId,
33
+ applicationKey: appKey
34
+ })
35
+ this.opts = opts
36
+ this.configuredPartSize = opts.partSize ? opts.partSize : -1
37
+ this.bucket = opts.bucket
38
+ this.delimiter = opts.delimiter || '/'
39
+ this.prefix = opts.prefix
40
+ ? opts.prefix.endsWith(this.delimiter)
41
+ ? opts.prefix
42
+ : opts.prefix + this.delimiter
43
+ : ''
44
+ this.lastAuth = 0
45
+ }
46
+
47
+ auth = async () => {
48
+ if (Date.now() - this.lastAuth > AUTH_REFRESH) {
49
+ try {
50
+ const response = await this.b2.authorize({})
51
+ this.lastAuth = Date.now()
52
+ this.absoluteMinimumPartSize = response.data.absoluteMinimumPartSize
53
+ this.recommendedPartSize = response.data.recommendedPartSize
54
+ if (this.configuredPartSize === -1) {
55
+ logger.info(`partSize was not configured, using recommendedPartSize=${this.recommendedPartSize}`)
56
+ this.configuredPartSize = this.recommendedPartSize
57
+ } else if (this.configuredPartSize < this.absoluteMinimumPartSize) {
58
+ logger.warn(`partSize was too small (${this.configuredPartSize}), using absoluteMinimumPartSize=${this.absoluteMinimumPartSize}`)
59
+ this.configuredPartSize = this.absoluteMinimumPartSize
60
+ }
61
+ } catch (e) {
62
+ logger.error(`auth error: ${e}`)
63
+ throw e
64
+ }
65
+ }
66
+ }
67
+
68
+ testConfig = async () => await this.b2_list('', false, null, {max: 1})
69
+
70
+ normalizePath = (path) => path.length === 0 ? this.prefix :
71
+ path.startsWith(this.prefix)
72
+ ? path
73
+ : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path)
74
+ denormalizePath = (path) => {
75
+ const norm = path.startsWith(this.prefix) ? path.substring(this.prefix.length) : path
76
+ return norm.startsWith(this.delimiter) ? norm.substring(1) : norm
77
+ }
78
+ bbError = (err, path, method) => {
79
+ if (err.response && err.response.status && err.response.status === 404) {
80
+ return new MobilettoNotFoundError(path)
81
+ } else if (err.response && err.response.data && err.response.data.message) {
82
+ return new MobilettoError(`${method} error: ${err.response.data.message}`)
83
+ }
84
+ return err
85
+ }
86
+
87
+ b2type = f => f.fileName.endsWith(this.delimiter) && f.contentType === null ? M_DIR : M_FILE
88
+
89
+ file2object(f) {
90
+ return {
91
+ name: this.denormalizePath(f.fileName),
92
+ size: f.contentLength,
93
+ type: this.b2type(f),
94
+ bb2id: f.fileId
95
+ }
96
+ }
97
+
98
+ processList = async (path, response, visitor) => {
99
+ const files = response && response.data && response.data.files ? response.data.files : null
100
+ if (!files) {
101
+ return []
102
+ }
103
+ const objects = files.map((f) => {
104
+ logger.debug(`addListedFiles(${path}): adding file: ${f.fileName}`)
105
+ return this.file2object(f)
106
+ })
107
+ if (visitor) {
108
+ for (const obj of objects) { await visitor(obj) }
109
+ }
110
+ return objects
111
+ }
112
+
113
+ list = (pth = '', recursive = false, visitor = null) =>
114
+ this.b2_list(pth, recursive, visitor)
115
+
116
+ async b2_list (pth = '', recursive = false, visitor = null, opts = {}) {
117
+ try {
118
+ let optsDir = opts.dir || false
119
+ let dirDiscovered = null
120
+ let firstResult = null
121
+ const files = []
122
+ await this.auth()
123
+ const hasMax = opts && opts.max
124
+ const maxFileCount = hasMax ? opts.max : 100
125
+ const pfx = (this.prefix.length === 0 || this.prefix.endsWith(this.delimiter) ? this.prefix : this.prefix + this.delimiter) +
126
+ (pth.startsWith(this.delimiter) ? pth.substring(0) : pth)
127
+
128
+ let response = await this.b2.listFileNames({
129
+ bucketId: this.bucket,
130
+ maxFileCount,
131
+ delimiter: recursive ? '' : this.delimiter,
132
+ prefix: pfx
133
+ })
134
+ const handleResponse = async (files, pth, response, visitor) => {
135
+ const list = await this.processList(pth, response, visitor)
136
+ if (!optsDir && !firstResult && list.length > 0) {
137
+ firstResult = list[0]
138
+ if (list[0].name.endsWith('/')) {
139
+ dirDiscovered = true
140
+ } else if (!dirDiscovered && list.length > 1 && list[1].name.startsWith(list[0].name)) {
141
+ files.push(list[0])
142
+ return true
143
+ }
144
+ }
145
+ if (hasMax && files.length + list.length >= maxFileCount) {
146
+ files.push(...list.slice(0, (list.length - maxFileCount) + 1))
147
+ return true
148
+ } else {
149
+ files.push(...list)
150
+ return false
151
+ }
152
+ }
153
+ while (response.data.nextFileName) {
154
+ if (await handleResponse(files, pth, response, visitor)) {
155
+ return files
156
+ }
157
+ response = await this.b2.listFileNames({
158
+ bucketId: this.bucket,
159
+ startFileName: response.data.nextFileName,
160
+ maxFileCount,
161
+ delimiter: recursive ? '' : this.delimiter,
162
+ prefix: this.prefix
163
+ })
164
+ }
165
+ await handleResponse(files, pth, response, visitor)
166
+ if (dirDiscovered && !pth.endsWith(this.delimiter)) {
167
+ const subdirFiles = await this.b2_list(pth + this.delimiter, recursive, null, { max: 10000, dir: true })
168
+ files.shift()
169
+ files.push(...subdirFiles)
170
+ }
171
+ return files
172
+ } catch (err) {
173
+ throw this.bbError(err, pth, 'list')
174
+ }
175
+ }
176
+
177
+ b2_meta = (path) => (this.driver_metadata ? this.driver_metadata(path) : this.metadata(path))
178
+
179
+ async metadata (pth) {
180
+ await this.auth()
181
+ let files = null
182
+ const normPath = this.normalizePath(pth);
183
+ try {
184
+ files = (await this.b2.listFileNames({
185
+ bucketId: this.bucket,
186
+ startFileName: normPath,
187
+ maxFileCount: 1,
188
+ delimiter: '',
189
+ prefix: this.prefix
190
+ })).data.files
191
+ } catch (err) {
192
+ throw this.bbError(err, pth, 'metadata')
193
+ }
194
+ if (files.length === 0) {
195
+ throw new MobilettoNotFoundError(pth)
196
+ }
197
+ const found = files.filter(f => f.fileName === normPath)
198
+ if (!found || found.length === 0) {
199
+ // if this is a directory, we may have returned the first item in it, that's OK
200
+ if (files[0].fileName.startsWith(normPath.endsWith(this.delimiter) ? normPath : normPath + this.delimiter)) {
201
+ let obj = this.file2object(files[0]);
202
+ obj.name = pth
203
+ obj.type = M_DIR
204
+ return obj
205
+ }
206
+ throw new MobilettoNotFoundError(pth)
207
+ }
208
+ if (found.length > 1) {
209
+ throw new MobilettoError(`metadata(${pth}) multiple files matched: ${JSON.stringify(found)}`)
210
+ }
211
+ return this.file2object(files[0])
212
+ }
213
+
214
+ async write (path, generatorOrReadable) {
215
+ await this.auth()
216
+ logger.debug(`write: writing path ${path}`)
217
+ let stream = generatorOrReadable
218
+ if (!isReadable(generatorOrReadable)) {
219
+ stream = new Readable({
220
+ async read (size) {
221
+ let chunk = isAsyncGenerator(generatorOrReadable)
222
+ ? (await generatorOrReadable.next()).value
223
+ : generatorOrReadable.next().value
224
+ this.push(typeof chunk !== 'undefined' ? chunk : null)
225
+ }
226
+ })
227
+ }
228
+ let response = null
229
+ try {
230
+ response = await this.b2.uploadAny({
231
+ bucketId: this.bucket,
232
+ fileName: this.normalizePath(path),
233
+ partSize: this.configuredPartSize,
234
+ data: stream
235
+ });
236
+ // response is from either b2_upload_file or b2_finish_large_file
237
+ // It depends on how large the file was, see: https://github.com/gideo-llc/backblaze-b2-upload-any/blob/master/README.md#uploadanyoptions
238
+ if (response.contentLength) {
239
+ return response.contentLength
240
+ }
241
+ } catch (e) {
242
+ logger.error(`write error: ${e}`)
243
+ throw this.bbError(e, path, 'write')
244
+ }
245
+ if (response) {
246
+ throw new MobilettoError(`write error: response status ${response.status} with data ${JSON.stringify(response.data)}`)
247
+ }
248
+ }
249
+
250
+ async read (path, callback, endCallback = null) {
251
+ await this.auth()
252
+ const file = this.normalizePath(path)
253
+ logger.debug(`read: reading path: ${path}`)
254
+ let meta = null
255
+ try {
256
+ meta = await (this.b2_meta(file))
257
+ } catch (e) {
258
+ if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
259
+ throw e
260
+ }
261
+ throw new MobilettoError(`read: error reading metadata for path ${path}`, e)
262
+ }
263
+ if (meta.type === M_DIR) {
264
+ throw new MobilettoError(`read: cannot read directory: ${path}`)
265
+ }
266
+ try {
267
+ const stream = await this.b2.downloadFileById({
268
+ bucketId: this.bucket,
269
+ fileId: meta.bb2id,
270
+ responseType: 'stream'
271
+ })
272
+ return await readStream(stream.data, callback, endCallback)
273
+ } catch (err) {
274
+ throw this.bbError(err, path, 'read')
275
+ }
276
+ }
277
+
278
+ async deleteVersions (path, normPath, versions, quiet, errors = {}) {
279
+ const removed = []
280
+ for (const version of versions) {
281
+ logger.debug(`deleteVersions(${path}, ${versions.length} versions) pushing version to delete: fileName=${version.fileName} fileId=${version.fileId}`)
282
+ await this.auth()
283
+ try {
284
+ if (!version.fileName.startsWith(normPath)) {
285
+ logger.debug(`deleteVersions(${path}, ${versions.length} versions) ending iteration on mismatched file: ${version.fileName}`)
286
+ return removed
287
+ }
288
+ const delResponse = await this.b2.deleteFileVersion({
289
+ bucketId: this.bucket,
290
+ fileName: version.fileName,
291
+ fileId: version.fileId
292
+ })
293
+ if (delResponse && delResponse.status !== 200) {
294
+ const message = `deleteVersions(${path}) received HTTP status ${delResponse.status} trying to delete file version: ${JSON.stringify(version)}`
295
+ logger.error(message)
296
+ if (errors) {
297
+ errors[path] = message
298
+ }
299
+ } else if (removed) {
300
+ logger.debug(`deleteVersions(${path}, ${versions.length} versions) REMOVED: fileName=${version.fileName} fileId=${version.fileId}`)
301
+ removed.push(this.denormalizePath(version.fileName))
302
+ }
303
+ } catch (err) {
304
+ const message = `deleteVersions(${path}) error deleting version ${version.fileId}: ${err}`
305
+ logger.error(message)
306
+ if (errors) {
307
+ errors[path] = message
308
+ }
309
+ if (!quiet) throw new MobilettoError(message, err)
310
+ }
311
+ }
312
+ const errPaths = Object.keys(errors);
313
+ if (errPaths.length > 0) {
314
+ if (quiet) {
315
+ logger.info(`deleteVersions(${path}) had ${errPaths.length} errors but quiet=true, returning ${removed.length} removed paths`)
316
+ return removed
317
+ }
318
+ throw new MobilettoError(`deleteVersions(${path}) had errors: ${JSON.stringify(errors)}`)
319
+ }
320
+ return removed
321
+ }
322
+
323
+ async remove (path, recursive, quiet) {
324
+ logger.debug(`b2.remove(${path}, recursive=${recursive}) starting...`)
325
+ await this.auth()
326
+ let meta = null
327
+ const errors = {}
328
+ const normPath = this.normalizePath(path)
329
+ try {
330
+ logger.debug(`b2.remove(${path}) fetching metadata to determine file/directory`)
331
+ meta = await (this.b2_meta(path))
332
+ } catch (e) {
333
+ if (quiet) {
334
+ logger.debug(`b2.remove(${path}) quietly returning, not reporting error fetching metadata for ${path}: ${e}`)
335
+ return path
336
+ }
337
+ if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
338
+ throw e
339
+ }
340
+ throw new MobilettoError(`b2.remove(${path}) error reading metadata for path ${path}`, e)
341
+ }
342
+ if (meta.type === M_FILE) {
343
+ try {
344
+ const response = await this.b2.listFileVersions({
345
+ bucketId: this.bucket,
346
+ startFileName: normPath,
347
+ startFileId: meta.bb2id,
348
+ prefix: this.prefix,
349
+ delimiter: this.delimiter,
350
+ maxFileCount: 1000
351
+ })
352
+ const versions = response.data.files.filter(f => f.fileName === normPath)
353
+ await this.deleteVersions(path, normPath, versions, quiet, errors)
354
+ } catch (err) {
355
+ if (err instanceof MobilettoError) {
356
+ throw err
357
+ }
358
+ throw new MobilettoError(`b2.remove(${path}) error: ${err}`, err)
359
+ }
360
+ return path
361
+ } else if (meta.type === M_DIR) {
362
+ if (!recursive) {
363
+ if (quiet) {
364
+ logger.debug(`b2.remove(${path}) quietly returning, not deleting directory when recursive=false`)
365
+ return path
366
+ }
367
+ throw new MobilettoError(`b2.remove(${path}) cannot remove directory (set recursive=true to remove)`)
368
+ }
369
+ const removed = []
370
+ const errors = {}
371
+ let versionsResponse
372
+ try {
373
+ logger.debug(`b2.remove(${path})`)
374
+ let normPath = this.normalizePath(path);
375
+ versionsResponse = await this.b2.listFileVersions({
376
+ bucketId: this.bucket,
377
+ startFileName: normPath,
378
+ prefix: normPath,
379
+ delimiter: '',
380
+ maxFileCount: 1000
381
+ })
382
+ while (versionsResponse.data.nextFileName &&
383
+ versionsResponse.data.nextFileId &&
384
+ versionsResponse.data.nextFileName.startsWith(normPath)) {
385
+ logger.debug(`b2.remove(${path}) removing a batch of files then asking for more`)
386
+ removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
387
+ logger.debug(`b2.remove(${path})`)
388
+ versionsResponse = await this.b2.listFileVersions({
389
+ bucketId: this.bucket,
390
+ startFileName: versionsResponse.data.nextFileName,
391
+ startFileId: versionsResponse.data.nextFileId,
392
+ prefix: normPath,
393
+ delimiter: ''
394
+ })
395
+ }
396
+ logger.debug(`b2.remove(${path}) removing last batch of files`)
397
+ removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
398
+ } catch (e) {
399
+ const message = `b2.remove(${path}) error listing/deleting: ${e}`
400
+ if (quiet) {
401
+ logger.warn(message)
402
+ } else {
403
+ throw new MobilettoError(message)
404
+ }
405
+ }
406
+ if (Object.keys(errors).length > 0) {
407
+ if (quiet) {
408
+ logger.warn(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
409
+ } else {
410
+ throw new MobilettoError(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
411
+ }
412
+ }
413
+ return removed
414
+
415
+ } else {
416
+ throw new MobilettoError(`b2.remove(${path}) cannot remove file of type: ${meta.type}`)
417
+ }
418
+ }
419
+ }
420
+
421
+ function storageClient (key, secret, opts) {
422
+ if (!key || !secret || !opts || !opts.bucket) {
423
+ throw new MobilettoError('b2.storageClient: key, secret, and opts.bucket are required')
424
+ }
425
+ return new StorageClient(key, secret, opts)
426
+ }
427
+
428
+ module.exports = { storageClient }
@@ -32,10 +32,13 @@ class StorageClient {
32
32
  fileMode
33
33
  dirMode
34
34
  constructor(baseDir, opts) {
35
+ if (!baseDir) {
36
+ throw new MobilettoError('local.StorageClient: key (baseDir) is required')
37
+ }
35
38
  const resolved = this.resolveSymlinks(baseDir)
36
39
  if (!resolved.stat.isDirectory()) {
37
40
  const dest = resolved.path === baseDir ? null : resolved.path
38
- throw new MobilettoError(`local.storageClient: not a directory: ${baseDir}${dest ? ` (resolved to ${dest})` : ''}`)
41
+ throw new MobilettoError(`local.StorageClient: not a directory: ${baseDir}${dest ? ` (resolved to ${dest})` : ''}`)
39
42
  }
40
43
  const dir = resolved.path
41
44
  this.baseDir = dir.endsWith('/') ? dir : dir + '/'
@@ -109,7 +112,7 @@ class StorageClient {
109
112
  }
110
113
  return files
111
114
  } catch (e) {
112
- logger.error(`dirFiles: ${e}`)
115
+ logger.warn(`dirFiles: ${e}`)
113
116
  throw this.ioError(e, norm, 'dirFiles')
114
117
  }
115
118
  }
@@ -283,4 +286,4 @@ function storageClient (key, secret, opts) {
283
286
  return new StorageClient(key, opts)
284
287
  }
285
288
 
286
- module.exports = {storageClient}
289
+ module.exports = { storageClient }
@@ -30,7 +30,7 @@ class StorageClient {
30
30
  delimiter
31
31
  constructor(key, secret, opts) {
32
32
  if (!key || !secret || !opts || !opts.bucket) {
33
- throw new MobilettoError('key, secret, and opts.bucket are required')
33
+ throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
34
34
  }
35
35
  this.bucket = opts.bucket
36
36
  const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
@@ -305,9 +305,9 @@ class StorageClient {
305
305
 
306
306
  function storageClient (key, secret, opts) {
307
307
  if (!key || !secret || !opts || !opts.bucket) {
308
- throw new MobilettoError('key, secret, and opts.bucket are required')
308
+ throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
309
309
  }
310
310
  return new StorageClient(key, secret, opts)
311
311
  }
312
312
 
313
- module.exports = {storageClient}
313
+ module.exports = { storageClient }
package/index.js CHANGED
@@ -38,6 +38,7 @@ function MobilettoError (message, err) {
38
38
  this.stack = (new Error(this.message)).stack
39
39
  }
40
40
  }
41
+ MobilettoError.prototype.toString = () => JSON.stringify(this)
41
42
 
42
43
  // adapted from https://stackoverflow.com/a/27724419
43
44
  function MobilettoNotFoundError (message) {
@@ -50,6 +51,7 @@ function MobilettoNotFoundError (message) {
50
51
  this.stack = (new Error(this.message)).stack
51
52
  }
52
53
  }
54
+ MobilettoNotFoundError.prototype.toString = () => JSON.stringify(this)
53
55
 
54
56
  const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
55
57
 
@@ -173,6 +175,7 @@ const UTILITY_FUNCTIONS = {
173
175
  },
174
176
  remove: (client) => async (path, opts) => {
175
177
  const recursive = opts && opts.recursive ? opts.recursive : false
178
+ // noinspection JSUnresolvedVariable
176
179
  const quiet = opts && opts.quiet ? opts.quiet : false
177
180
  // noinspection JSUnresolvedFunction
178
181
  const result = await client.driver_remove(path, recursive, quiet)
@@ -197,9 +200,11 @@ const UTILITY_FUNCTIONS = {
197
200
  return data
198
201
  },
199
202
  write: (client) => async (path, data) => {
203
+ logger.debug(`util.write(${path}) starting ...`)
200
204
  // noinspection JSUnresolvedFunction
201
205
  const bytesWritten = await client.driver_write(path, data)
202
206
  await client.flush()
207
+ logger.debug(`util.write(${path}) wrote ${bytesWritten} bytes`)
203
208
  return bytesWritten
204
209
  },
205
210
  writeFile: (client) => async (path, data) => {
@@ -220,7 +225,8 @@ const UTILITY_FUNCTIONS = {
220
225
  const destName = obj.name.startsWith(sourcePath)
221
226
  ? obj.name.substring(sourcePath.length)
222
227
  : obj.name
223
- const destFullPath = clientPath + destName;
228
+ const destFullPath = (clientPath.endsWith('/') ? clientPath : clientPath + '/') +
229
+ (destName.startsWith('/') ? destName.substring(1) : destName)
224
230
  try {
225
231
  // if dest already exists and is the same size, don't copy it again
226
232
  let srcSize = null
@@ -338,8 +344,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
338
344
  logger.info(`mobiletto: connecting with driver ${driverPath}`)
339
345
  const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
340
346
  const client = driver.storageClient(key, secret, opts)
341
- if (!(await client.testConfig())) {
342
- const message = `mobiletto(${driverPath}) error: test API call failed`
347
+ const configValue = await client.testConfig();
348
+ if (!configValue) {
349
+ const message = `mobiletto(${driverPath}) error: test API call failed: ${configValue}`
343
350
  logger.error(message)
344
351
  throw new MobilettoError(message)
345
352
  }
@@ -457,8 +464,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
457
464
  const df = direntFile(direntDir(dirname(path)), path);
458
465
  const recursive = false
459
466
  const quiet = true
467
+
468
+ logger.debug(`removeDirentFile(${path}) removing df=${df}`)
460
469
  await client.remove(df, recursive, quiet)
470
+
471
+ logger.debug(`removeDirentFile(${path}) removing encryptPath(path)=${encryptPath(path)}`)
461
472
  await client.remove(encryptPath(path), recursive, quiet)
473
+
474
+ logger.debug(`removeDirentFile(${path}) removing metaPath(path)=${metaPath(path)}`)
462
475
  await client.remove(metaPath(path), recursive, quiet)
463
476
  }
464
477
 
@@ -468,6 +481,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
468
481
  }
469
482
  }
470
483
 
484
+ // noinspection JSUnusedGlobalSymbols
471
485
  const encClient = {
472
486
  id: internalIdForDriver(),
473
487
  redisConfig: client.redisConfig,
@@ -490,6 +504,23 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
490
504
  return thing
491
505
  }
492
506
 
507
+ async function tryParentDirForSingleFile (path, visitor, e) {
508
+ // it might be a single file, try listing the parent dir
509
+ const parentDirent = direntDir(path.dirname(p));
510
+ entries = await client.list(parentDirent, false)
511
+ const objects = await _loadMeta(parentDirent, entries)
512
+ const found = objects.find(o => o.name === p)
513
+ if (found) {
514
+ if (visitor) {
515
+ await visitor(found)
516
+ }
517
+ logger.debug(`tryParentDirForSingleFile(${path}) found ${found.name}`)
518
+ return cacheAndReturn([found])
519
+ }
520
+ logger.debug(`tryParentDirForSingleFile(${path}) nothing found! e=${e}`)
521
+ throw e === null ? new MobilettoNotFoundError(path) : e
522
+ }
523
+
493
524
  if (cached) {
494
525
  entries = cached
495
526
  } else {
@@ -497,18 +528,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
497
528
  entries = await client.list(dirent, recursive)
498
529
  } catch (e) {
499
530
  if (e instanceof MobilettoNotFoundError && p.includes('/')) {
500
- // it might be a single file, try listing the parent dir
501
- const parentDirent = direntDir(path.dirname(p));
502
- entries = await client.list(parentDirent, false)
503
- const objects = await _loadMeta(parentDirent, entries)
504
- const found = objects.find(o => o.name === p)
505
- if (found) {
506
- return cacheAndReturn([found])
507
- }
508
- throw e
531
+ return await tryParentDirForSingleFile(path, visitor, e)
509
532
  }
510
533
  }
511
534
  }
535
+ if (entries.length === 0 && p.includes('/')) {
536
+ return await tryParentDirForSingleFile(path, visitor, null)
537
+ }
512
538
  if (cache) {
513
539
  cache.set(cacheKey, entries).then(
514
540
  () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
@@ -592,12 +618,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
592
618
  await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
593
619
  return generatorBytes
594
620
  },
595
- // todo: remove should return an array of the paths that were actually removed
621
+
596
622
  remove: async (path, options) => {
623
+ logger.debug(`enc.remove(${path}) starting`)
597
624
  const recursive = options === true || (options && options.recursive) || false
598
625
  // const quiet = (options && options.quiet) || false
599
626
  if (recursive) {
600
627
  // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
628
+ const removed = []
601
629
  async function recRm (path) {
602
630
  const dirent = direntDir(path)
603
631
  let entries
@@ -616,13 +644,15 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
616
644
  await recRm(f.name)
617
645
  }
618
646
  }
619
- await removeDirentFile(path);
647
+ await removeDirentFile(path)
648
+ removed.push(path)
620
649
  }
621
650
  await recRm(path)
651
+ return removed
622
652
  }
623
653
 
624
654
  // remove the single file/dir
625
- await removeDirentFile(path);
655
+ await removeDirentFile(path)
626
656
 
627
657
  // if we were the last dirent file, then also remove dirent directory, and recursively upwards
628
658
  let parent = path
@@ -647,7 +677,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
647
677
  parent = dirname(parent)
648
678
  dirent = direntDir(parent)
649
679
  }
650
- return true
680
+ return path
651
681
  }
652
682
  }
653
683
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
package/package.json CHANGED
@@ -1,11 +1,12 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.47",
4
- "description": "A storage abstraction layer",
3
+ "version": "1.1.2",
4
+ "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
7
7
  "storage",
8
8
  "s3",
9
+ "b2",
9
10
  "simple",
10
11
  "filesystem"
11
12
  ],
@@ -21,7 +22,7 @@
21
22
  },
22
23
  "license": "Apache-2.0",
23
24
  "scripts": {
24
- "test": "mocha --timeout 10000"
25
+ "test": "mocha --timeout 60000 ./test/*.spec.js"
25
26
  },
26
27
  "devDependencies": {
27
28
  "@babel/core": "^7.18.10",
@@ -35,6 +36,8 @@
35
36
  "dependencies": {
36
37
  "@aws-sdk/client-s3": "^3.150.0",
37
38
  "@aws-sdk/lib-storage": "^3.150.0",
39
+ "@gideo-llc/backblaze-b2-upload-any": "^0.1.4",
40
+ "backblaze-b2": "^1.7.0",
38
41
  "ioredis": "^5.2.3",
39
42
  "lru-cache": "^7.14.0",
40
43
  "node-fetch-commonjs": "^3.1.1",
@@ -1,8 +1,8 @@
1
1
  // To run the tests, you need a .env file one level above this directory
2
- // that contains env vars for all the process.env stuff in DRIVER_CONFIG below
2
+ require('dotenv').config()
3
+
3
4
  const fs = require('fs')
4
5
  const { basename, dirname } = require('path')
5
- require('dotenv').config()
6
6
 
7
7
  const randomstring = require('randomstring')
8
8
  const crypto = require('crypto')
@@ -15,25 +15,13 @@ const {
15
15
  mobiletto, connect, MobilettoNotFoundError, readStream
16
16
  } = require("../index")
17
17
 
18
- const {
19
- DEFAULT_CRYPT_ALGO,
20
- normalizeKey, normalizeIV,
21
- encrypt, decrypt,
22
- getCipher, getDecipher
23
- } = require("../util/crypt")
24
-
25
- const { logger } = require("../util/logger");
18
+ const { logger } = require("../util/logger")
26
19
 
27
20
  // chunk size used by generator function, used by driver's 'write' function
28
21
  // the temp file is also TEMP_SZ_MULTIPLE of this number
29
22
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
30
23
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
31
24
 
32
- // encoded bytes written will differ from actual bytes provided
33
- // likewise, plaintext bytes read will differ from encoded bytes read
34
- // our tests give some leeway when filesize differences
35
- const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
36
-
37
25
  DRIVER_CONFIG = {
38
26
  local: {
39
27
  key: process.env.LOCAL_STORAGE || '/tmp'
@@ -43,9 +31,17 @@ DRIVER_CONFIG = {
43
31
  secret: process.env.S3_SECRET,
44
32
  opts: {
45
33
  bucket: process.env.S3_BUCKET,
46
- prefix: process.env.S3_PREFIX,
34
+ prefix: process.env.S3_PREFIX || 'mobiletto_test',
47
35
  region: process.env.S3_REGION
48
36
  }
37
+ },
38
+ b2: {
39
+ key: process.env.B2_KEY_ID,
40
+ secret: process.env.B2_KEY_SECRET,
41
+ opts: {
42
+ bucket: process.env.B2_BUCKET,
43
+ prefix: process.env.B2_PREFIX || 'mobiletto_test'
44
+ }
49
45
  }
50
46
  }
51
47
  DRIVER_NAMES = Object.keys(DRIVER_CONFIG)
@@ -103,62 +99,9 @@ async function writeRandomFile(fixture, size) {
103
99
  return await fixture.api.write(fixture.name, dataGenerator())
104
100
  }
105
101
 
106
- describe('crypto test', () => {
107
- const enc = {
108
- key: normalizeKey(randomstring.generate(32)),
109
- iv: normalizeIV(randomstring.generate(16)),
110
- algo: DEFAULT_CRYPT_ALGO
111
- }
112
- it("should encrypt and decrypt a data buffer successfully", async () => {
113
- const plaintext = randomstring.generate((64 * 1024) + Math.floor(1024*Math.random()))
114
- const ciphertext = encrypt(plaintext, enc)
115
- const decrypted = decrypt(ciphertext, enc)
116
- expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
117
- })
118
- it("should encrypt and decrypt a stream successfully", async () => {
119
- const plaintext = crypto.randomBytes(81920)
120
- const cipher = getCipher(enc)
121
- const updateData = cipher.update(plaintext)
122
- const finalData = cipher.final()
123
- const encrypted = Buffer.concat([updateData, finalData])
124
-
125
- const decipher = getDecipher(enc)
126
- const decryptUpdate = decipher.update(encrypted)
127
- const decipherFinal = decipher.final()
128
- const decrypted = Buffer.concat([decryptUpdate, decipherFinal])
129
- expect(decrypted.length).to.equal(plaintext.length)
130
- for (let i = 0; i < decrypted.length; i++ ) {
131
- expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
132
- }
133
- })
134
- it("should encrypt and decrypt a stream successfully using readStream", async () => {
135
- const plaintext = crypto.randomBytes(81920)
136
- const cipher = getCipher(enc)
137
- const updateData = cipher.update(plaintext)
138
- const finalData = cipher.final()
139
- const encrypted = Buffer.concat([updateData, finalData])
140
-
141
- const tempFile = `/tmp/stream_foo_${Date.now}.tmp`
142
- fs.writeFileSync(tempFile, encrypted)
143
-
144
- const rs = fs.createReadStream(tempFile)
145
- const decipher = getDecipher(enc)
146
- const buffers = []
147
- await readStream(rs, (chunk) => {
148
- buffers.push(decipher.update(chunk))
149
- }, () => {
150
- buffers.push(decipher.final())
151
- })
152
- const decrypted = Buffer.concat(buffers)
153
- expect(decrypted.length).to.equal(plaintext.length)
154
- for (let i = 0; i < decrypted.length; i++ ) {
155
- expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
156
- }
157
- })
158
-
159
- })
160
-
161
102
  const encryptionTests = () => [null, { key: rand(32) }]
103
+ // const encryptionTests = () => [null]
104
+ // const encryptionTests = () => [{ key: rand(32) }]
162
105
 
163
106
  const REDIS_ENABLED = {
164
107
  name: 'redis-enabled',
@@ -178,26 +121,13 @@ const REDIS_DISABLED = {
178
121
  }
179
122
  const redisTests = () => [REDIS_ENABLED, REDIS_DISABLED]
180
123
 
181
- // For testing single drivers
182
- // Comment out the for loop for driver tests, and hard-code one of the drivers below
183
- // const driverName = 'local'
184
- // const driverName = 's3'
185
-
186
- // For testing single redis setups
187
- // Comment out the for loop for redis test, and hard-code one of the setups below
188
- // const redisSetup = REDIS_ENABLED
189
- // const redisSetup = REDIS_DISABLED
190
-
191
- // When you're NOT testing a single driver, it's important that *all* tests remain
192
- // within the various 'for' loops below. This ensures that all drivers pass exactly
193
- // the same set of tests, with exactly the same inputs/outputs, both with and without
194
- // redis caching enabled, both with and without encryption enabled.
195
- // This greatly supports developer sanity
196
-
197
124
  for (const redisSetup of redisTests()) {
198
125
  if (!redisSetup.enabled) continue
199
126
 
200
127
  for (const driverName of DRIVER_NAMES) {
128
+ // For testing single drivers
129
+ if (driverName !== 'b2') continue
130
+
201
131
  const driverTest = `${driverName} [${redisSetup.name}]`
202
132
  const config = redisSetup.config(DRIVER_CONFIG[driverName])
203
133
  const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
@@ -219,7 +149,8 @@ for (const driverName of DRIVER_NAMES) {
219
149
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
220
150
  should().exist(api, 'expected API object to exist')
221
151
  } catch (e) {
222
- logger.error(`wtf: ${e}`)
152
+ logger.error(`error creating API: ${e} ${JSON.stringify(e)}`)
153
+ throw e
223
154
  }
224
155
  })
225
156
  })
@@ -249,12 +180,12 @@ for (const driverName of DRIVER_NAMES) {
249
180
  expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
250
181
  })
251
182
  it("should read the file we just wrote", async () => {
252
- const data = await fixture.api.readFile(fixture.name);
183
+ const data = await fixture.api.readFile(fixture.name)
253
184
  expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
254
185
  expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
255
186
  })
256
187
  it("should load metadata on the file we just wrote", async () => {
257
- await assertMeta (fixture.api, fixture.name, fixture.randomData.length)
188
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
258
189
  })
259
190
  it("should delete the file we just wrote", async () => {
260
191
  const removed = await fixture.api.remove(fixture.name)
@@ -331,16 +262,16 @@ for (const driverName of DRIVER_NAMES) {
331
262
  .finally(done)
332
263
  })
333
264
  it("should write an encrypted file", async () => {
334
- const bytesWritten = await writeRandomFile(fixture, size);
265
+ const bytesWritten = await writeRandomFile(fixture, size)
335
266
  expect(bytesWritten).is.equal(size, 'expected write API to return correct number of bytes written')
336
267
  })
337
268
  it("should read the encrypted file we just wrote", async () => {
338
- const data = await fixture.api.readFile(fixture.name);
269
+ const data = await fixture.api.readFile(fixture.name)
339
270
  expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
340
271
  expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
341
272
  })
342
273
  it("should load metadata on the encrypted file we just wrote", async () => {
343
- await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
274
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
344
275
  })
345
276
  it("should delete the encrypted file we just wrote", async () => {
346
277
  const removed = await fixture.api.remove(fixture.name)
@@ -353,7 +284,8 @@ for (const driverName of DRIVER_NAMES) {
353
284
 
354
285
  for (const encryption of encryptionTests()) {
355
286
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
356
- const mirrorDriver = pickAnotherDriver(driverName)
287
+ // const mirrorDriver = pickAnotherDriver(driverName)
288
+ const mirrorDriver = 'local' // for hard-coding the mirror driver
357
289
  const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
358
290
  const mirrorDest = `mirrorDest_${rand(5)}/`
359
291
  describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
@@ -365,7 +297,7 @@ for (const driverName of DRIVER_NAMES) {
365
297
  const subdirName = `subdir_` + Date.now()
366
298
  const fullSubdirPath = `${randomParent}/${subdirName}`
367
299
  const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
368
- const randomStreamFileSize = READ_SZ * 10;
300
+ const randomStreamFileSize = READ_SZ * 10
369
301
  const randomStreamFileData = crypto.randomBytes(randomStreamFileSize)
370
302
  //const fileCount = 3 + Math.floor(Math.random() * 10)
371
303
  const fileCount = 2
@@ -396,7 +328,7 @@ for (const driverName of DRIVER_NAMES) {
396
328
  })
397
329
  it('should write a file in a new directory using a stream', async () => {
398
330
  // create a random temp file, write it
399
- const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`;
331
+ const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`
400
332
  fs.writeFileSync(tempFile, randomStreamFileData)
401
333
  const reader = fs.createReadStream(tempFile)
402
334
  const streamFile = fixture.name + '_stream'
@@ -404,13 +336,14 @@ for (const driverName of DRIVER_NAMES) {
404
336
  expect(bytesWritten).to.equal(randomStreamFileSize, 'expected write API to return correct number of bytes written for STREAM')
405
337
  })
406
338
  it("should successfully read the file back that was stream-written", async () => {
407
- const streamFile = fixture.name + '_stream';
339
+ const streamFile = fixture.name + '_stream'
408
340
  const singleFile = (await fixture.api.readFile(streamFile))
409
341
  expect(singleFile).to.have.lengthOf(randomStreamFileSize)
410
342
  for (let i = 0; i < singleFile.length; i++) {
411
343
  expect(singleFile[i]).to.equal(randomStreamFileData[i])
412
344
  }
413
- expect(!!fixture.api.remove(streamFile)).to.be.true
345
+ const removal = await fixture.api.remove(streamFile)
346
+ expect(removal).to.equal(streamFile, `expected stream file to be deleted, but was: ${removal}`)
414
347
  })
415
348
  it("should successfully list a file individually", async () => {
416
349
  const singleFile = await fixture.api.list(fixture.name)
@@ -420,7 +353,7 @@ for (const driverName of DRIVER_NAMES) {
420
353
  it("should successfully list files in the base directory", async () => {
421
354
  const baseListing = await fixture.api.list()
422
355
  expect(baseListing).to.have.lengthOf.greaterThanOrEqual(1)
423
- expect(baseListing.find(o => o.name === dirname(randomParent))).to.not.be.null
356
+ expect(baseListing.find(o => o.name === dirname(randomParent) || o.name === dirname(randomParent)+'/')).to.not.be.null
424
357
  })
425
358
  it("should successfully list files in a subdirectory, with or without a trailing slash", async () => {
426
359
  const noSlash = await fixture.api.list(randomParent)
@@ -435,7 +368,7 @@ for (const driverName of DRIVER_NAMES) {
435
368
  }
436
369
  })
437
370
  it("should load metadata for one of the new files", async () => {
438
- await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
371
+ await assertMeta(fixture.api, fixture.name, READ_SZ)
439
372
  })
440
373
  it("should see correct types on objects returned from a listing of the new directory", async () => {
441
374
  const objects = await fixture.api.list(randomParent)
@@ -541,6 +474,7 @@ for (const driverName of DRIVER_NAMES) {
541
474
  }
542
475
  })
543
476
  })
477
+
544
478
  })
545
479
  }
546
480
  }
@@ -0,0 +1,67 @@
1
+ const fs = require('fs')
2
+ const { expect } = require('chai')
3
+ const randomstring = require('randomstring')
4
+ const crypto = require('crypto')
5
+
6
+ const {
7
+ DEFAULT_CRYPT_ALGO,
8
+ normalizeKey, normalizeIV,
9
+ encrypt, decrypt,
10
+ getCipher, getDecipher
11
+ } = require("../util/crypt")
12
+
13
+ const { readStream } = require("../index")
14
+
15
+ describe('crypto test', () => {
16
+ const enc = {
17
+ key: normalizeKey(randomstring.generate(32)),
18
+ iv: normalizeIV(randomstring.generate(16)),
19
+ algo: DEFAULT_CRYPT_ALGO
20
+ }
21
+ it("should encrypt and decrypt a data buffer successfully", async () => {
22
+ const plaintext = randomstring.generate((64 * 1024) + Math.floor(1024*Math.random()))
23
+ const ciphertext = encrypt(plaintext, enc)
24
+ const decrypted = decrypt(ciphertext, enc)
25
+ expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
26
+ })
27
+ it("should encrypt and decrypt a stream successfully", async () => {
28
+ const plaintext = crypto.randomBytes(81920)
29
+ const cipher = getCipher(enc)
30
+ const updateData = cipher.update(plaintext)
31
+ const finalData = cipher.final()
32
+ const encrypted = Buffer.concat([updateData, finalData])
33
+
34
+ const decipher = getDecipher(enc)
35
+ const decryptUpdate = decipher.update(encrypted)
36
+ const decipherFinal = decipher.final()
37
+ const decrypted = Buffer.concat([decryptUpdate, decipherFinal])
38
+ expect(decrypted.length).to.equal(plaintext.length)
39
+ for (let i = 0; i < decrypted.length; i++ ) {
40
+ expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
41
+ }
42
+ })
43
+ it("should encrypt and decrypt a stream successfully using readStream", async () => {
44
+ const plaintext = crypto.randomBytes(81920)
45
+ const cipher = getCipher(enc)
46
+ const updateData = cipher.update(plaintext)
47
+ const finalData = cipher.final()
48
+ const encrypted = Buffer.concat([updateData, finalData])
49
+
50
+ const tempFile = `/tmp/stream_foo_${Date.now}.tmp`
51
+ fs.writeFileSync(tempFile, encrypted)
52
+
53
+ const rs = fs.createReadStream(tempFile)
54
+ const decipher = getDecipher(enc)
55
+ const buffers = []
56
+ await readStream(rs, (chunk) => {
57
+ buffers.push(decipher.update(chunk))
58
+ }, () => {
59
+ buffers.push(decipher.final())
60
+ })
61
+ const decrypted = Buffer.concat(buffers)
62
+ expect(decrypted.length).to.equal(plaintext.length)
63
+ for (let i = 0; i < decrypted.length; i++ ) {
64
+ expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
65
+ }
66
+ })
67
+ })
package/util/logger.js CHANGED
@@ -1,7 +1,7 @@
1
1
  const winston = require('winston')
2
2
 
3
3
  const logger = winston.createLogger({
4
- level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
4
+ level: process.env.MOBILETTO_LOG_LEVEL || 'debug',
5
5
  format: winston.format.simple(),
6
6
  transports: process.env.MOBILETTO_LOG_FILE
7
7
  ? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
package/util/redis.js CHANGED
@@ -49,10 +49,16 @@ class MobilettoCache {
49
49
  this.redis = null
50
50
  }
51
51
  this.prefix = prefix
52
- this.flush().then(
53
- () => { logger.debug(`redis(${name}): successfully flushed`) },
54
- (e) => { logger.error(`redis(${name}): error flushing: ${e}`) }
55
- )
52
+ if (this.redis) {
53
+ // test connection by flushing
54
+ this.flush().then(
55
+ () => { logger.debug(`redis(${name}): successfully flushed`) },
56
+ (e) => {
57
+ logger.error(`redis(${name}): error flushing: ${e}, not using redis!`)
58
+ this.redis = null
59
+ }
60
+ )
61
+ }
56
62
  }
57
63
 
58
64
  stats = () => this.counters
@@ -81,11 +87,11 @@ class MobilettoCache {
81
87
  }
82
88
  }
83
89
 
84
- get = key => {
90
+ get = async key => {
85
91
  this.counters.get++
86
- logger.debug(`redis(${this.name}): get(${key}) starting`)
87
- const val = this.doRedis(r => r.get(this.pfx(key)), null)
88
- logger.debug(`redis(${this.name}): get(${key}) found value: ${val}`)
92
+ logger.silly(`redis(${this.name}): get(${key}) starting`)
93
+ const val = await this.doRedis(r => r.get(this.pfx(key)), null)
94
+ logger.silly(`redis(${this.name}): get(${key}) found value: ${val}`)
89
95
  if (val) {
90
96
  this.counters.hit++
91
97
  } else {
@@ -100,9 +106,9 @@ class MobilettoCache {
100
106
 
101
107
  set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
102
108
  this.counters.set++
103
- logger.debug(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) starting`)
109
+ logger.silly(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) starting`)
104
110
  await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
105
- logger.debug(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) finished`)
111
+ logger.silly(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) finished`)
106
112
  }
107
113
 
108
114
  del = async key => {