mobiletto 1.0.48 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -46
- package/drivers/b2/index.js +428 -0
- package/drivers/local/index.js +6 -3
- package/drivers/s3/index.js +3 -3
- package/index.js +51 -18
- package/package.json +6 -3
- package/test/{index.spec.js → api.spec.js} +36 -102
- package/test/crypt.spec.js +67 -0
- package/util/logger.js +1 -1
package/README.md
CHANGED
|
@@ -3,28 +3,64 @@ Mobiletto
|
|
|
3
3
|
|
|
4
4
|
Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# Contents
|
|
7
|
+
* [Why Mobiletto?](#Why-Mobiletto?)
|
|
8
|
+
* [Quick start](#Quick-start)
|
|
9
|
+
* [Mobiletto CLI](#mobiletto-cli)
|
|
10
|
+
* [Source](#Source)
|
|
11
|
+
* [Basic usage](#Basic-usage)
|
|
12
|
+
* [Metadata](#Metadata)
|
|
13
|
+
* [Alternate import style](#Alternate-import-style)
|
|
14
|
+
* [Caching](#Caching)
|
|
15
|
+
* [Mirroring](#Mirroring)
|
|
16
|
+
* [Transparent encryption](#Transparent-encryption)
|
|
17
|
+
* [Key rotation](#Key-rotation)
|
|
18
|
+
* [Driver interface](#Driver-interface)
|
|
19
|
+
* [Logging](#Logging)
|
|
20
|
+
|
|
21
|
+
## Why Mobiletto?
|
|
22
|
+
|
|
23
|
+
### Goodbye vendor lock-in!
|
|
24
|
+
The various cloud storage providers have incompatible APIs. Even those that strive for "S3 compatibility"
|
|
25
|
+
have idiosyncratic behaviors.
|
|
7
26
|
|
|
27
|
+
When you choose a particular storage vendor for your app, if you code directly to their API, your app
|
|
28
|
+
is now dependent on that service. As time goes by and code accumulates, changing vendors becomes
|
|
29
|
+
increasingly untenable. Welcome to the fun world of vendor lock-in!
|
|
30
|
+
|
|
31
|
+
Mobiletto was designed to solve this problem. By coding your app to mobiletto's API, you can easily
|
|
32
|
+
change storage providers and know that your app's storage layer will behave identically.
|
|
33
|
+
|
|
34
|
+
### Extensive testing
|
|
8
35
|
All drivers are tested for identical behavior with over 55 tests for each driver.
|
|
36
|
+
Each driver test actually runs *four times*, with every (2x2) combination of:
|
|
37
|
+
* Encryption: both enabled and disabled
|
|
38
|
+
* Redis cache: both enabled and disabled
|
|
9
39
|
|
|
10
|
-
|
|
40
|
+
This approach gives us peace-of-mind that mobiletto will behave the same regardless of which driver you use,
|
|
41
|
+
and regardless of whether you enable caching and/or encryption.
|
|
11
42
|
|
|
12
|
-
|
|
13
|
-
|
|
43
|
+
### Driver support
|
|
44
|
+
Current Mobiletto storage drivers:
|
|
45
|
+
* `s3`: Amazon S3
|
|
46
|
+
* `b2`: Backblaze B2
|
|
47
|
+
* `local`: local filesystem
|
|
14
48
|
|
|
15
|
-
|
|
49
|
+
*Contributions to support more cloud storage providers are very welcome!*
|
|
50
|
+
|
|
51
|
+
## mobiletto-cli
|
|
16
52
|
Mobiletto is intended to be used as a library by other JavaScript code.
|
|
17
53
|
|
|
18
54
|
To work with mobiletto at the command-line, use [mobiletto-cli](https://www.npmjs.com/package/mobiletto-cli)
|
|
19
55
|
|
|
20
|
-
|
|
56
|
+
## Source
|
|
21
57
|
* [mobiletto on GitHub](https://github.com/cobbzilla/mobiletto)
|
|
22
58
|
* [mobiletto on npm](https://www.npmjs.com/package/mobiletto)
|
|
23
59
|
|
|
24
|
-
##
|
|
60
|
+
## Quick Start
|
|
25
61
|
A simple example using the mobiletto `s3` driver.
|
|
26
62
|
|
|
27
|
-
This code would run the same if the driver were `local`.
|
|
63
|
+
This code would run the same if the driver were `b2` or `local`.
|
|
28
64
|
|
|
29
65
|
const storage = require('mobiletto')
|
|
30
66
|
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
|
|
@@ -46,20 +82,12 @@ This code would run the same if the driver were `local`.
|
|
|
46
82
|
// read file as a stream using data callback
|
|
47
83
|
const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
|
|
48
84
|
|
|
49
|
-
// remove a file, returns
|
|
50
|
-
|
|
85
|
+
// remove a file, returns the path removed
|
|
86
|
+
let removed = await s3.remove('some/path')
|
|
87
|
+
|
|
88
|
+
// remove a directory, returns array of paths removed
|
|
89
|
+
removed = await s3.remove('some/directory', {recursive: true})
|
|
51
90
|
|
|
52
|
-
----
|
|
53
|
-
# Contents
|
|
54
|
-
* [Basic usage](#Basic-usage)
|
|
55
|
-
* [Metadata](#Metadata)
|
|
56
|
-
* [Alternate import style](#Alternate-import-style)
|
|
57
|
-
* [Caching](#Caching)
|
|
58
|
-
* [Mirroring](#Mirroring)
|
|
59
|
-
* [Transparent encryption](#Transparent-encryption)
|
|
60
|
-
* [Key rotation](#Key-rotation)
|
|
61
|
-
* [Driver interface](#Driver-interface)
|
|
62
|
-
* [Logging](#Logging)
|
|
63
91
|
----
|
|
64
92
|
## Basic usage
|
|
65
93
|
const { mobiletto } = require('mobiletto')
|
|
@@ -87,44 +115,50 @@ This code would run the same if the driver were `local`.
|
|
|
87
115
|
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
88
116
|
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
89
117
|
|
|
118
|
+
// To use 'b2' driver:
|
|
119
|
+
// * key: Backblaze Key ID
|
|
120
|
+
// * secret: Backblaze Application Key
|
|
121
|
+
// * opts object:
|
|
122
|
+
// * readOnly: optional, never change anything on the bucket; default is false
|
|
123
|
+
// * bucket: required, the ID (**not the name**) of the B2 bucket
|
|
124
|
+
// * prefix: optional, all read/writes within the B2 bucket will be under this prefix
|
|
125
|
+
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
126
|
+
// * partSize: optional, large files will be split into chunks of this size when uploading
|
|
127
|
+
const b3 = await mobiletto('b2', b2_key_id, b2_app_key, {bucket: 'bk', partSize: 10000000})
|
|
128
|
+
|
|
90
129
|
// List files
|
|
91
|
-
|
|
92
|
-
s3.list() // --> returns an array of metadata objects
|
|
130
|
+
api.list() // --> returns an array of metadata objects
|
|
93
131
|
|
|
94
132
|
// List files recursively
|
|
95
|
-
|
|
96
|
-
s3.list({ recursive: true })
|
|
133
|
+
api.list({ recursive: true })
|
|
97
134
|
|
|
98
135
|
// List files in a directory
|
|
99
136
|
const path = 'some/path'
|
|
100
|
-
|
|
101
|
-
|
|
137
|
+
api.list(path)
|
|
138
|
+
api.list(path, { recursive: true }) // also supports recursive flag
|
|
102
139
|
|
|
103
140
|
// Visit files in a directory -- visitor function must be async
|
|
104
|
-
|
|
105
|
-
|
|
141
|
+
api.list(path, { visitor: myAsyncFunc })
|
|
142
|
+
api.list(path, { visitor: myAsyncFunc, recursive: true })
|
|
106
143
|
|
|
107
144
|
// The `list` method throws MobilettoNotFoundError if the path does not exist
|
|
108
145
|
// When you call `safeList` on a non-existent path, it returns an empty array
|
|
109
|
-
|
|
146
|
+
api.safeList('/path/that/does/not/exist') // returns []
|
|
110
147
|
|
|
111
148
|
// Read metadata for a file
|
|
112
|
-
|
|
113
|
-
s3.metadata(path) // returns metadata object
|
|
149
|
+
api.metadata(path) // returns metadata object
|
|
114
150
|
|
|
115
151
|
// The `metadata` method throws MobilettoNotFoundError if the path does not exist
|
|
116
152
|
// When you call `safeMetadata` on a non-existent path, it returns null
|
|
117
|
-
|
|
153
|
+
api.safeMetadata('/tmp/does_not_exist') // returns null
|
|
118
154
|
|
|
119
155
|
// Read a file
|
|
120
156
|
// Provide a callback that writes the data someplace
|
|
121
157
|
const callback = (chunk) => { ... write chunk somewhere ... }
|
|
122
|
-
|
|
123
|
-
s3.read(path, callback) // returns count of bytes read
|
|
158
|
+
api.read(path, callback) // returns count of bytes read
|
|
124
159
|
|
|
125
160
|
// Read an entire file at once (convenience method)
|
|
126
|
-
const
|
|
127
|
-
const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
|
|
161
|
+
const data = await api.readFile(path) // returns a byte Buffer of the file contents
|
|
128
162
|
|
|
129
163
|
// Write a file
|
|
130
164
|
// Provide a generator function that yields chunks of data
|
|
@@ -134,26 +168,22 @@ This code would run the same if the driver were `local`.
|
|
|
134
168
|
yield data
|
|
135
169
|
}
|
|
136
170
|
}
|
|
137
|
-
local.
|
|
138
|
-
s3.write(path, generator) // returns count of bytes written
|
|
171
|
+
local.api(path, generator) // returns count of bytes written
|
|
139
172
|
|
|
140
173
|
// Write an entire file at once (convenience method)
|
|
141
|
-
await
|
|
142
|
-
await s3.writeFile(path, bufferOrString) // returns count of bytes written
|
|
174
|
+
await api.writeFile(path, bufferOrString) // returns count of bytes written
|
|
143
175
|
|
|
144
176
|
// Delete a file
|
|
145
177
|
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
146
178
|
// Always returns a value or throws an error.
|
|
147
179
|
// Return value may be a single string of the file removed, or an array of all files removed (driver-dependent)
|
|
148
180
|
const quiet = true
|
|
149
|
-
|
|
150
|
-
s3.remove(path, {quiet}) // returns single path removed
|
|
181
|
+
api.remove(path, {quiet}) // returns single path removed
|
|
151
182
|
|
|
152
183
|
// Recursively delete a directory and do it quietly (do not report errors)
|
|
153
184
|
const recursive = true
|
|
154
185
|
const quiet = true
|
|
155
|
-
|
|
156
|
-
s3.remove(path, {recursive, quiet}) // returns array containing all paths removed
|
|
186
|
+
api.remove(path, {recursive, quiet}) // returns array of paths removed
|
|
157
187
|
|
|
158
188
|
## Metadata
|
|
159
189
|
The `metadata` command returns metadata about a single filesystem entry.
|
|
@@ -351,7 +381,7 @@ or any other connection configuration information.
|
|
|
351
381
|
Use the `MOBILETTO_LOG_LEVEL` environment variable to set the log level, using one
|
|
352
382
|
of the `npm` levels defined in [https://www.npmjs.com/package/winston#logging-levels](https://www.npmjs.com/package/winston#logging-levels)
|
|
353
383
|
|
|
354
|
-
The default level is `
|
|
384
|
+
The default level is `error`. The most verbose level is `silly`, although currently mobiletto
|
|
355
385
|
does not log at levels below `debug`
|
|
356
386
|
|
|
357
387
|
MOBILETTO_LOG_LEVEL=silly # maximum logs!
|
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
const B2 = require('backblaze-b2')
|
|
2
|
+
|
|
3
|
+
// this library is a godsend for B2 uploads
|
|
4
|
+
B2.prototype.uploadAny = require('@gideo-llc/backblaze-b2-upload-any')
|
|
5
|
+
|
|
6
|
+
const { Readable } = require('stream')
|
|
7
|
+
|
|
8
|
+
const {
|
|
9
|
+
M_FILE, M_DIR,
|
|
10
|
+
readStream, isReadable, isAsyncGenerator,
|
|
11
|
+
MobilettoError, MobilettoNotFoundError
|
|
12
|
+
} = require('../../index')
|
|
13
|
+
|
|
14
|
+
const { logger } = require('../../util/logger')
|
|
15
|
+
|
|
16
|
+
// B2 tokens last 24 hours. Let's refresh ours after 23 hours and 50 minutes
|
|
17
|
+
const AUTH_REFRESH = (1000 * 60 * 60 * 24) - (1000 * 60 * 10)
|
|
18
|
+
|
|
19
|
+
class StorageClient {
|
|
20
|
+
b2
|
|
21
|
+
bucket
|
|
22
|
+
prefix
|
|
23
|
+
delimiter
|
|
24
|
+
opts
|
|
25
|
+
lastAuth
|
|
26
|
+
absoluteMinimumPartSize
|
|
27
|
+
recommendedPartSize
|
|
28
|
+
configuredPartSize
|
|
29
|
+
constructor(keyId, appKey, opts) {
|
|
30
|
+
if (!keyId || !appKey || !opts.bucket) throw new MobilettoError(`b2.StorageClient: key, secret and opts.bucket are required`)
|
|
31
|
+
this.b2 = new B2({
|
|
32
|
+
applicationKeyId: keyId,
|
|
33
|
+
applicationKey: appKey
|
|
34
|
+
})
|
|
35
|
+
this.opts = opts
|
|
36
|
+
this.configuredPartSize = opts.partSize ? opts.partSize : -1
|
|
37
|
+
this.bucket = opts.bucket
|
|
38
|
+
this.delimiter = opts.delimiter || '/'
|
|
39
|
+
this.prefix = opts.prefix
|
|
40
|
+
? opts.prefix.endsWith(this.delimiter)
|
|
41
|
+
? opts.prefix
|
|
42
|
+
: opts.prefix + this.delimiter
|
|
43
|
+
: ''
|
|
44
|
+
this.lastAuth = 0
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
auth = async () => {
|
|
48
|
+
if (Date.now() - this.lastAuth > AUTH_REFRESH) {
|
|
49
|
+
try {
|
|
50
|
+
const response = await this.b2.authorize({})
|
|
51
|
+
this.lastAuth = Date.now()
|
|
52
|
+
this.absoluteMinimumPartSize = response.data.absoluteMinimumPartSize
|
|
53
|
+
this.recommendedPartSize = response.data.recommendedPartSize
|
|
54
|
+
if (this.configuredPartSize === -1) {
|
|
55
|
+
logger.info(`partSize was not configured, using recommendedPartSize=${this.recommendedPartSize}`)
|
|
56
|
+
this.configuredPartSize = this.recommendedPartSize
|
|
57
|
+
} else if (this.configuredPartSize < this.absoluteMinimumPartSize) {
|
|
58
|
+
logger.warn(`partSize was too small (${this.configuredPartSize}), using absoluteMinimumPartSize=${this.absoluteMinimumPartSize}`)
|
|
59
|
+
this.configuredPartSize = this.absoluteMinimumPartSize
|
|
60
|
+
}
|
|
61
|
+
} catch (e) {
|
|
62
|
+
logger.error(`auth error: ${e}`)
|
|
63
|
+
throw e
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
testConfig = async () => await this.b2_list('', false, null, {max: 1})
|
|
69
|
+
|
|
70
|
+
normalizePath = (path) => path.length === 0 ? this.prefix :
|
|
71
|
+
path.startsWith(this.prefix)
|
|
72
|
+
? path
|
|
73
|
+
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path)
|
|
74
|
+
denormalizePath = (path) => {
|
|
75
|
+
const norm = path.startsWith(this.prefix) ? path.substring(this.prefix.length) : path
|
|
76
|
+
return norm.startsWith(this.delimiter) ? norm.substring(1) : norm
|
|
77
|
+
}
|
|
78
|
+
bbError = (err, path, method) => {
|
|
79
|
+
if (err.response && err.response.status && err.response.status === 404) {
|
|
80
|
+
return new MobilettoNotFoundError(path)
|
|
81
|
+
} else if (err.response && err.response.data && err.response.data.message) {
|
|
82
|
+
return new MobilettoError(`${method} error: ${err.response.data.message}`)
|
|
83
|
+
}
|
|
84
|
+
return err
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
b2type = f => f.fileName.endsWith(this.delimiter) && f.contentType === null ? M_DIR : M_FILE
|
|
88
|
+
|
|
89
|
+
file2object(f) {
|
|
90
|
+
return {
|
|
91
|
+
name: this.denormalizePath(f.fileName),
|
|
92
|
+
size: f.contentLength,
|
|
93
|
+
type: this.b2type(f),
|
|
94
|
+
bb2id: f.fileId
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
processList = async (path, response, visitor) => {
|
|
99
|
+
const files = response && response.data && response.data.files ? response.data.files : null
|
|
100
|
+
if (!files) {
|
|
101
|
+
return []
|
|
102
|
+
}
|
|
103
|
+
const objects = files.map((f) => {
|
|
104
|
+
logger.debug(`addListedFiles(${path}): adding file: ${f.fileName}`)
|
|
105
|
+
return this.file2object(f)
|
|
106
|
+
})
|
|
107
|
+
if (visitor) {
|
|
108
|
+
for (const obj of objects) { await visitor(obj) }
|
|
109
|
+
}
|
|
110
|
+
return objects
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
list = (pth = '', recursive = false, visitor = null) =>
|
|
114
|
+
this.b2_list(pth, recursive, visitor)
|
|
115
|
+
|
|
116
|
+
async b2_list (pth = '', recursive = false, visitor = null, opts = {}) {
|
|
117
|
+
try {
|
|
118
|
+
let optsDir = opts.dir || false
|
|
119
|
+
let dirDiscovered = null
|
|
120
|
+
let firstResult = null
|
|
121
|
+
const files = []
|
|
122
|
+
await this.auth()
|
|
123
|
+
const hasMax = opts && opts.max
|
|
124
|
+
const maxFileCount = hasMax ? opts.max : 100
|
|
125
|
+
const pfx = (this.prefix.length === 0 || this.prefix.endsWith(this.delimiter) ? this.prefix : this.prefix + this.delimiter) +
|
|
126
|
+
(pth.startsWith(this.delimiter) ? pth.substring(0) : pth)
|
|
127
|
+
|
|
128
|
+
let response = await this.b2.listFileNames({
|
|
129
|
+
bucketId: this.bucket,
|
|
130
|
+
maxFileCount,
|
|
131
|
+
delimiter: recursive ? '' : this.delimiter,
|
|
132
|
+
prefix: pfx
|
|
133
|
+
})
|
|
134
|
+
const handleResponse = async (files, pth, response, visitor) => {
|
|
135
|
+
const list = await this.processList(pth, response, visitor)
|
|
136
|
+
if (!optsDir && !firstResult && list.length > 0) {
|
|
137
|
+
firstResult = list[0]
|
|
138
|
+
if (list[0].name.endsWith('/')) {
|
|
139
|
+
dirDiscovered = true
|
|
140
|
+
} else if (!dirDiscovered && list.length > 1 && list[1].name.startsWith(list[0].name)) {
|
|
141
|
+
files.push(list[0])
|
|
142
|
+
return true
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (hasMax && files.length + list.length >= maxFileCount) {
|
|
146
|
+
files.push(...list.slice(0, (list.length - maxFileCount) + 1))
|
|
147
|
+
return true
|
|
148
|
+
} else {
|
|
149
|
+
files.push(...list)
|
|
150
|
+
return false
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
while (response.data.nextFileName) {
|
|
154
|
+
if (await handleResponse(files, pth, response, visitor)) {
|
|
155
|
+
return files
|
|
156
|
+
}
|
|
157
|
+
response = await this.b2.listFileNames({
|
|
158
|
+
bucketId: this.bucket,
|
|
159
|
+
startFileName: response.data.nextFileName,
|
|
160
|
+
maxFileCount,
|
|
161
|
+
delimiter: recursive ? '' : this.delimiter,
|
|
162
|
+
prefix: this.prefix
|
|
163
|
+
})
|
|
164
|
+
}
|
|
165
|
+
await handleResponse(files, pth, response, visitor)
|
|
166
|
+
if (dirDiscovered && !pth.endsWith(this.delimiter)) {
|
|
167
|
+
const subdirFiles = await this.b2_list(pth + this.delimiter, recursive, null, { max: 10000, dir: true })
|
|
168
|
+
files.shift()
|
|
169
|
+
files.push(...subdirFiles)
|
|
170
|
+
}
|
|
171
|
+
return files
|
|
172
|
+
} catch (err) {
|
|
173
|
+
throw this.bbError(err, pth, 'list')
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
b2_meta = (path) => (this.driver_metadata ? this.driver_metadata(path) : this.metadata(path))
|
|
178
|
+
|
|
179
|
+
async metadata (pth) {
|
|
180
|
+
await this.auth()
|
|
181
|
+
let files = null
|
|
182
|
+
const normPath = this.normalizePath(pth);
|
|
183
|
+
try {
|
|
184
|
+
files = (await this.b2.listFileNames({
|
|
185
|
+
bucketId: this.bucket,
|
|
186
|
+
startFileName: normPath,
|
|
187
|
+
maxFileCount: 1,
|
|
188
|
+
delimiter: '',
|
|
189
|
+
prefix: this.prefix
|
|
190
|
+
})).data.files
|
|
191
|
+
} catch (err) {
|
|
192
|
+
throw this.bbError(err, pth, 'metadata')
|
|
193
|
+
}
|
|
194
|
+
if (files.length === 0) {
|
|
195
|
+
throw new MobilettoNotFoundError(pth)
|
|
196
|
+
}
|
|
197
|
+
const found = files.filter(f => f.fileName === normPath)
|
|
198
|
+
if (!found || found.length === 0) {
|
|
199
|
+
// if this is a directory, we may have returned the first item in it, that's OK
|
|
200
|
+
if (files[0].fileName.startsWith(normPath.endsWith(this.delimiter) ? normPath : normPath + this.delimiter)) {
|
|
201
|
+
let obj = this.file2object(files[0]);
|
|
202
|
+
obj.name = pth
|
|
203
|
+
obj.type = M_DIR
|
|
204
|
+
return obj
|
|
205
|
+
}
|
|
206
|
+
throw new MobilettoNotFoundError(pth)
|
|
207
|
+
}
|
|
208
|
+
if (found.length > 1) {
|
|
209
|
+
throw new MobilettoError(`metadata(${pth}) multiple files matched: ${JSON.stringify(found)}`)
|
|
210
|
+
}
|
|
211
|
+
return this.file2object(files[0])
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async write (path, generatorOrReadable) {
|
|
215
|
+
await this.auth()
|
|
216
|
+
logger.debug(`write: writing path ${path}`)
|
|
217
|
+
let stream = generatorOrReadable
|
|
218
|
+
if (!isReadable(generatorOrReadable)) {
|
|
219
|
+
stream = new Readable({
|
|
220
|
+
async read (size) {
|
|
221
|
+
let chunk = isAsyncGenerator(generatorOrReadable)
|
|
222
|
+
? (await generatorOrReadable.next()).value
|
|
223
|
+
: generatorOrReadable.next().value
|
|
224
|
+
this.push(typeof chunk !== 'undefined' ? chunk : null)
|
|
225
|
+
}
|
|
226
|
+
})
|
|
227
|
+
}
|
|
228
|
+
let response = null
|
|
229
|
+
try {
|
|
230
|
+
response = await this.b2.uploadAny({
|
|
231
|
+
bucketId: this.bucket,
|
|
232
|
+
fileName: this.normalizePath(path),
|
|
233
|
+
partSize: this.configuredPartSize,
|
|
234
|
+
data: stream
|
|
235
|
+
});
|
|
236
|
+
// response is from either b2_upload_file or b2_finish_large_file
|
|
237
|
+
// It depends on how large the file was, see: https://github.com/gideo-llc/backblaze-b2-upload-any/blob/master/README.md#uploadanyoptions
|
|
238
|
+
if (response.contentLength) {
|
|
239
|
+
return response.contentLength
|
|
240
|
+
}
|
|
241
|
+
} catch (e) {
|
|
242
|
+
logger.error(`write error: ${e}`)
|
|
243
|
+
throw this.bbError(e, path, 'write')
|
|
244
|
+
}
|
|
245
|
+
if (response) {
|
|
246
|
+
throw new MobilettoError(`write error: response status ${response.status} with data ${JSON.stringify(response.data)}`)
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async read (path, callback, endCallback = null) {
|
|
251
|
+
await this.auth()
|
|
252
|
+
const file = this.normalizePath(path)
|
|
253
|
+
logger.debug(`read: reading path: ${path}`)
|
|
254
|
+
let meta = null
|
|
255
|
+
try {
|
|
256
|
+
meta = await (this.b2_meta(file))
|
|
257
|
+
} catch (e) {
|
|
258
|
+
if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
|
|
259
|
+
throw e
|
|
260
|
+
}
|
|
261
|
+
throw new MobilettoError(`read: error reading metadata for path ${path}`, e)
|
|
262
|
+
}
|
|
263
|
+
if (meta.type === M_DIR) {
|
|
264
|
+
throw new MobilettoError(`read: cannot read directory: ${path}`)
|
|
265
|
+
}
|
|
266
|
+
try {
|
|
267
|
+
const stream = await this.b2.downloadFileById({
|
|
268
|
+
bucketId: this.bucket,
|
|
269
|
+
fileId: meta.bb2id,
|
|
270
|
+
responseType: 'stream'
|
|
271
|
+
})
|
|
272
|
+
return await readStream(stream.data, callback, endCallback)
|
|
273
|
+
} catch (err) {
|
|
274
|
+
throw this.bbError(err, path, 'read')
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
async deleteVersions (path, normPath, versions, quiet, errors = {}) {
|
|
279
|
+
const removed = []
|
|
280
|
+
for (const version of versions) {
|
|
281
|
+
logger.debug(`deleteVersions(${path}, ${versions.length} versions) pushing version to delete: fileName=${version.fileName} fileId=${version.fileId}`)
|
|
282
|
+
await this.auth()
|
|
283
|
+
try {
|
|
284
|
+
if (!version.fileName.startsWith(normPath)) {
|
|
285
|
+
logger.debug(`deleteVersions(${path}, ${versions.length} versions) ending iteration on mismatched file: ${version.fileName}`)
|
|
286
|
+
return removed
|
|
287
|
+
}
|
|
288
|
+
const delResponse = await this.b2.deleteFileVersion({
|
|
289
|
+
bucketId: this.bucket,
|
|
290
|
+
fileName: version.fileName,
|
|
291
|
+
fileId: version.fileId
|
|
292
|
+
})
|
|
293
|
+
if (delResponse && delResponse.status !== 200) {
|
|
294
|
+
const message = `deleteVersions(${path}) received HTTP status ${delResponse.status} trying to delete file version: ${JSON.stringify(version)}`
|
|
295
|
+
logger.error(message)
|
|
296
|
+
if (errors) {
|
|
297
|
+
errors[path] = message
|
|
298
|
+
}
|
|
299
|
+
} else if (removed) {
|
|
300
|
+
logger.debug(`deleteVersions(${path}, ${versions.length} versions) REMOVED: fileName=${version.fileName} fileId=${version.fileId}`)
|
|
301
|
+
removed.push(this.denormalizePath(version.fileName))
|
|
302
|
+
}
|
|
303
|
+
} catch (err) {
|
|
304
|
+
const message = `deleteVersions(${path}) error deleting version ${version.fileId}: ${err}`
|
|
305
|
+
logger.error(message)
|
|
306
|
+
if (errors) {
|
|
307
|
+
errors[path] = message
|
|
308
|
+
}
|
|
309
|
+
if (!quiet) throw new MobilettoError(message, err)
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
const errPaths = Object.keys(errors);
|
|
313
|
+
if (errPaths.length > 0) {
|
|
314
|
+
if (quiet) {
|
|
315
|
+
logger.info(`deleteVersions(${path}) had ${errPaths.length} errors but quiet=true, returning ${removed.length} removed paths`)
|
|
316
|
+
return removed
|
|
317
|
+
}
|
|
318
|
+
throw new MobilettoError(`deleteVersions(${path}) had errors: ${JSON.stringify(errors)}`)
|
|
319
|
+
}
|
|
320
|
+
return removed
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async remove (path, recursive, quiet) {
|
|
324
|
+
logger.debug(`b2.remove(${path}, recursive=${recursive}) starting...`)
|
|
325
|
+
await this.auth()
|
|
326
|
+
let meta = null
|
|
327
|
+
const errors = {}
|
|
328
|
+
const normPath = this.normalizePath(path)
|
|
329
|
+
try {
|
|
330
|
+
logger.debug(`b2.remove(${path}) fetching metadata to determine file/directory`)
|
|
331
|
+
meta = await (this.b2_meta(path))
|
|
332
|
+
} catch (e) {
|
|
333
|
+
if (quiet) {
|
|
334
|
+
logger.debug(`b2.remove(${path}) quietly returning, not reporting error fetching metadata for ${path}: ${e}`)
|
|
335
|
+
return path
|
|
336
|
+
}
|
|
337
|
+
if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
|
|
338
|
+
throw e
|
|
339
|
+
}
|
|
340
|
+
throw new MobilettoError(`b2.remove(${path}) error reading metadata for path ${path}`, e)
|
|
341
|
+
}
|
|
342
|
+
if (meta.type === M_FILE) {
|
|
343
|
+
try {
|
|
344
|
+
const response = await this.b2.listFileVersions({
|
|
345
|
+
bucketId: this.bucket,
|
|
346
|
+
startFileName: normPath,
|
|
347
|
+
startFileId: meta.bb2id,
|
|
348
|
+
prefix: this.prefix,
|
|
349
|
+
delimiter: this.delimiter,
|
|
350
|
+
maxFileCount: 1000
|
|
351
|
+
})
|
|
352
|
+
const versions = response.data.files.filter(f => f.fileName === normPath)
|
|
353
|
+
await this.deleteVersions(path, normPath, versions, quiet, errors)
|
|
354
|
+
} catch (err) {
|
|
355
|
+
if (err instanceof MobilettoError) {
|
|
356
|
+
throw err
|
|
357
|
+
}
|
|
358
|
+
throw new MobilettoError(`b2.remove(${path}) error: ${err}`, err)
|
|
359
|
+
}
|
|
360
|
+
return path
|
|
361
|
+
} else if (meta.type === M_DIR) {
|
|
362
|
+
if (!recursive) {
|
|
363
|
+
if (quiet) {
|
|
364
|
+
logger.debug(`b2.remove(${path}) quietly returning, not deleting directory when recursive=false`)
|
|
365
|
+
return path
|
|
366
|
+
}
|
|
367
|
+
throw new MobilettoError(`b2.remove(${path}) cannot remove directory (set recursive=true to remove)`)
|
|
368
|
+
}
|
|
369
|
+
const removed = []
|
|
370
|
+
const errors = {}
|
|
371
|
+
let versionsResponse
|
|
372
|
+
try {
|
|
373
|
+
logger.debug(`b2.remove(${path})`)
|
|
374
|
+
let normPath = this.normalizePath(path);
|
|
375
|
+
versionsResponse = await this.b2.listFileVersions({
|
|
376
|
+
bucketId: this.bucket,
|
|
377
|
+
startFileName: normPath,
|
|
378
|
+
prefix: normPath,
|
|
379
|
+
delimiter: '',
|
|
380
|
+
maxFileCount: 1000
|
|
381
|
+
})
|
|
382
|
+
while (versionsResponse.data.nextFileName &&
|
|
383
|
+
versionsResponse.data.nextFileId &&
|
|
384
|
+
versionsResponse.data.nextFileName.startsWith(normPath)) {
|
|
385
|
+
logger.debug(`b2.remove(${path}) removing a batch of files then asking for more`)
|
|
386
|
+
removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
|
|
387
|
+
logger.debug(`b2.remove(${path})`)
|
|
388
|
+
versionsResponse = await this.b2.listFileVersions({
|
|
389
|
+
bucketId: this.bucket,
|
|
390
|
+
startFileName: versionsResponse.data.nextFileName,
|
|
391
|
+
startFileId: versionsResponse.data.nextFileId,
|
|
392
|
+
prefix: normPath,
|
|
393
|
+
delimiter: ''
|
|
394
|
+
})
|
|
395
|
+
}
|
|
396
|
+
logger.debug(`b2.remove(${path}) removing last batch of files`)
|
|
397
|
+
removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
|
|
398
|
+
} catch (e) {
|
|
399
|
+
const message = `b2.remove(${path}) error listing/deleting: ${e}`
|
|
400
|
+
if (quiet) {
|
|
401
|
+
logger.warn(message)
|
|
402
|
+
} else {
|
|
403
|
+
throw new MobilettoError(message)
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
if (Object.keys(errors).length > 0) {
|
|
407
|
+
if (quiet) {
|
|
408
|
+
logger.warn(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
|
|
409
|
+
} else {
|
|
410
|
+
throw new MobilettoError(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return removed
|
|
414
|
+
|
|
415
|
+
} else {
|
|
416
|
+
throw new MobilettoError(`b2.remove(${path}) cannot remove file of type: ${meta.type}`)
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
function storageClient (key, secret, opts) {
|
|
422
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
423
|
+
throw new MobilettoError('b2.storageClient: key, secret, and opts.bucket are required')
|
|
424
|
+
}
|
|
425
|
+
return new StorageClient(key, secret, opts)
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
module.exports = { storageClient }
|
package/drivers/local/index.js
CHANGED
|
@@ -32,10 +32,13 @@ class StorageClient {
|
|
|
32
32
|
fileMode
|
|
33
33
|
dirMode
|
|
34
34
|
constructor(baseDir, opts) {
|
|
35
|
+
if (!baseDir) {
|
|
36
|
+
throw new MobilettoError('local.StorageClient: key (baseDir) is required')
|
|
37
|
+
}
|
|
35
38
|
const resolved = this.resolveSymlinks(baseDir)
|
|
36
39
|
if (!resolved.stat.isDirectory()) {
|
|
37
40
|
const dest = resolved.path === baseDir ? null : resolved.path
|
|
38
|
-
throw new MobilettoError(`local.
|
|
41
|
+
throw new MobilettoError(`local.StorageClient: not a directory: ${baseDir}${dest ? ` (resolved to ${dest})` : ''}`)
|
|
39
42
|
}
|
|
40
43
|
const dir = resolved.path
|
|
41
44
|
this.baseDir = dir.endsWith('/') ? dir : dir + '/'
|
|
@@ -109,7 +112,7 @@ class StorageClient {
|
|
|
109
112
|
}
|
|
110
113
|
return files
|
|
111
114
|
} catch (e) {
|
|
112
|
-
logger.
|
|
115
|
+
logger.warn(`dirFiles: ${e}`)
|
|
113
116
|
throw this.ioError(e, norm, 'dirFiles')
|
|
114
117
|
}
|
|
115
118
|
}
|
|
@@ -283,4 +286,4 @@ function storageClient (key, secret, opts) {
|
|
|
283
286
|
return new StorageClient(key, opts)
|
|
284
287
|
}
|
|
285
288
|
|
|
286
|
-
module.exports = {storageClient}
|
|
289
|
+
module.exports = { storageClient }
|
package/drivers/s3/index.js
CHANGED
|
@@ -30,7 +30,7 @@ class StorageClient {
|
|
|
30
30
|
delimiter
|
|
31
31
|
constructor(key, secret, opts) {
|
|
32
32
|
if (!key || !secret || !opts || !opts.bucket) {
|
|
33
|
-
throw new MobilettoError('key, secret, and opts.bucket are required')
|
|
33
|
+
throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
|
|
34
34
|
}
|
|
35
35
|
this.bucket = opts.bucket
|
|
36
36
|
const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
|
|
@@ -305,9 +305,9 @@ class StorageClient {
|
|
|
305
305
|
|
|
306
306
|
function storageClient (key, secret, opts) {
|
|
307
307
|
if (!key || !secret || !opts || !opts.bucket) {
|
|
308
|
-
throw new MobilettoError('key, secret, and opts.bucket are required')
|
|
308
|
+
throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
|
|
309
309
|
}
|
|
310
310
|
return new StorageClient(key, secret, opts)
|
|
311
311
|
}
|
|
312
312
|
|
|
313
|
-
module.exports = {storageClient}
|
|
313
|
+
module.exports = { storageClient }
|
package/index.js
CHANGED
|
@@ -2,7 +2,7 @@ const fs = require('fs')
|
|
|
2
2
|
const path = require('path')
|
|
3
3
|
const { Readable, Transform } = require('stream')
|
|
4
4
|
const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
|
|
5
|
-
const { getRedis } = require('./util/redis')
|
|
5
|
+
const { getRedis, teardown } = require('./util/redis')
|
|
6
6
|
const { basename, dirname } = require('path')
|
|
7
7
|
const shasum = require('shasum')
|
|
8
8
|
const randomstring = require('randomstring')
|
|
@@ -38,6 +38,7 @@ function MobilettoError (message, err) {
|
|
|
38
38
|
this.stack = (new Error(this.message)).stack
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
|
+
MobilettoError.prototype.toString = () => JSON.stringify(this)
|
|
41
42
|
|
|
42
43
|
// adapted from https://stackoverflow.com/a/27724419
|
|
43
44
|
function MobilettoNotFoundError (message) {
|
|
@@ -50,6 +51,7 @@ function MobilettoNotFoundError (message) {
|
|
|
50
51
|
this.stack = (new Error(this.message)).stack
|
|
51
52
|
}
|
|
52
53
|
}
|
|
54
|
+
MobilettoNotFoundError.prototype.toString = () => JSON.stringify(this)
|
|
53
55
|
|
|
54
56
|
const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
55
57
|
|
|
@@ -173,6 +175,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
173
175
|
},
|
|
174
176
|
remove: (client) => async (path, opts) => {
|
|
175
177
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
178
|
+
// noinspection JSUnresolvedVariable
|
|
176
179
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
177
180
|
// noinspection JSUnresolvedFunction
|
|
178
181
|
const result = await client.driver_remove(path, recursive, quiet)
|
|
@@ -197,9 +200,11 @@ const UTILITY_FUNCTIONS = {
|
|
|
197
200
|
return data
|
|
198
201
|
},
|
|
199
202
|
write: (client) => async (path, data) => {
|
|
203
|
+
logger.debug(`util.write(${path}) starting ...`)
|
|
200
204
|
// noinspection JSUnresolvedFunction
|
|
201
205
|
const bytesWritten = await client.driver_write(path, data)
|
|
202
206
|
await client.flush()
|
|
207
|
+
logger.debug(`util.write(${path}) wrote ${bytesWritten} bytes`)
|
|
203
208
|
return bytesWritten
|
|
204
209
|
},
|
|
205
210
|
writeFile: (client) => async (path, data) => {
|
|
@@ -220,7 +225,8 @@ const UTILITY_FUNCTIONS = {
|
|
|
220
225
|
const destName = obj.name.startsWith(sourcePath)
|
|
221
226
|
? obj.name.substring(sourcePath.length)
|
|
222
227
|
: obj.name
|
|
223
|
-
const destFullPath = clientPath +
|
|
228
|
+
const destFullPath = (clientPath.endsWith('/') ? clientPath : clientPath + '/') +
|
|
229
|
+
(destName.startsWith('/') ? destName.substring(1) : destName)
|
|
224
230
|
try {
|
|
225
231
|
// if dest already exists and is the same size, don't copy it again
|
|
226
232
|
let srcSize = null
|
|
@@ -338,8 +344,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
338
344
|
logger.info(`mobiletto: connecting with driver ${driverPath}`)
|
|
339
345
|
const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
|
|
340
346
|
const client = driver.storageClient(key, secret, opts)
|
|
341
|
-
|
|
342
|
-
|
|
347
|
+
const configValue = await client.testConfig();
|
|
348
|
+
if (!configValue) {
|
|
349
|
+
const message = `mobiletto(${driverPath}) error: test API call failed: ${configValue}`
|
|
343
350
|
logger.error(message)
|
|
344
351
|
throw new MobilettoError(message)
|
|
345
352
|
}
|
|
@@ -457,8 +464,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
457
464
|
const df = direntFile(direntDir(dirname(path)), path);
|
|
458
465
|
const recursive = false
|
|
459
466
|
const quiet = true
|
|
467
|
+
|
|
468
|
+
logger.debug(`removeDirentFile(${path}) removing df=${df}`)
|
|
460
469
|
await client.remove(df, recursive, quiet)
|
|
470
|
+
|
|
471
|
+
logger.debug(`removeDirentFile(${path}) removing encryptPath(path)=${encryptPath(path)}`)
|
|
461
472
|
await client.remove(encryptPath(path), recursive, quiet)
|
|
473
|
+
|
|
474
|
+
logger.debug(`removeDirentFile(${path}) removing metaPath(path)=${metaPath(path)}`)
|
|
462
475
|
await client.remove(metaPath(path), recursive, quiet)
|
|
463
476
|
}
|
|
464
477
|
|
|
@@ -468,6 +481,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
468
481
|
}
|
|
469
482
|
}
|
|
470
483
|
|
|
484
|
+
// noinspection JSUnusedGlobalSymbols
|
|
471
485
|
const encClient = {
|
|
472
486
|
id: internalIdForDriver(),
|
|
473
487
|
redisConfig: client.redisConfig,
|
|
@@ -490,6 +504,23 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
490
504
|
return thing
|
|
491
505
|
}
|
|
492
506
|
|
|
507
|
+
async function tryParentDirForSingleFile (path, visitor, e) {
|
|
508
|
+
// it might be a single file, try listing the parent dir
|
|
509
|
+
const parentDirent = direntDir(path.dirname(p));
|
|
510
|
+
entries = await client.list(parentDirent, false)
|
|
511
|
+
const objects = await _loadMeta(parentDirent, entries)
|
|
512
|
+
const found = objects.find(o => o.name === p)
|
|
513
|
+
if (found) {
|
|
514
|
+
if (visitor) {
|
|
515
|
+
await visitor(found)
|
|
516
|
+
}
|
|
517
|
+
logger.debug(`tryParentDirForSingleFile(${path}) found ${found.name}`)
|
|
518
|
+
return cacheAndReturn([found])
|
|
519
|
+
}
|
|
520
|
+
logger.debug(`tryParentDirForSingleFile(${path}) nothing found! e=${e}`)
|
|
521
|
+
throw e === null ? new MobilettoNotFoundError(path) : e
|
|
522
|
+
}
|
|
523
|
+
|
|
493
524
|
if (cached) {
|
|
494
525
|
entries = cached
|
|
495
526
|
} else {
|
|
@@ -497,18 +528,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
497
528
|
entries = await client.list(dirent, recursive)
|
|
498
529
|
} catch (e) {
|
|
499
530
|
if (e instanceof MobilettoNotFoundError && p.includes('/')) {
|
|
500
|
-
|
|
501
|
-
const parentDirent = direntDir(path.dirname(p));
|
|
502
|
-
entries = await client.list(parentDirent, false)
|
|
503
|
-
const objects = await _loadMeta(parentDirent, entries)
|
|
504
|
-
const found = objects.find(o => o.name === p)
|
|
505
|
-
if (found) {
|
|
506
|
-
return cacheAndReturn([found])
|
|
507
|
-
}
|
|
508
|
-
throw e
|
|
531
|
+
return await tryParentDirForSingleFile(path, visitor, e)
|
|
509
532
|
}
|
|
510
533
|
}
|
|
511
534
|
}
|
|
535
|
+
if (entries.length === 0 && p.includes('/')) {
|
|
536
|
+
return await tryParentDirForSingleFile(path, visitor, null)
|
|
537
|
+
}
|
|
512
538
|
if (cache) {
|
|
513
539
|
cache.set(cacheKey, entries).then(
|
|
514
540
|
() => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
|
|
@@ -592,12 +618,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
592
618
|
await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
|
|
593
619
|
return generatorBytes
|
|
594
620
|
},
|
|
595
|
-
|
|
621
|
+
|
|
596
622
|
remove: async (path, options) => {
|
|
623
|
+
logger.debug(`enc.remove(${path}) starting`)
|
|
597
624
|
const recursive = options === true || (options && options.recursive) || false
|
|
598
625
|
// const quiet = (options && options.quiet) || false
|
|
599
626
|
if (recursive) {
|
|
600
627
|
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
628
|
+
const removed = []
|
|
601
629
|
async function recRm (path) {
|
|
602
630
|
const dirent = direntDir(path)
|
|
603
631
|
let entries
|
|
@@ -616,13 +644,15 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
616
644
|
await recRm(f.name)
|
|
617
645
|
}
|
|
618
646
|
}
|
|
619
|
-
await removeDirentFile(path)
|
|
647
|
+
await removeDirentFile(path)
|
|
648
|
+
removed.push(path)
|
|
620
649
|
}
|
|
621
650
|
await recRm(path)
|
|
651
|
+
return removed
|
|
622
652
|
}
|
|
623
653
|
|
|
624
654
|
// remove the single file/dir
|
|
625
|
-
await removeDirentFile(path)
|
|
655
|
+
await removeDirentFile(path)
|
|
626
656
|
|
|
627
657
|
// if we were the last dirent file, then also remove dirent directory, and recursively upwards
|
|
628
658
|
let parent = path
|
|
@@ -647,7 +677,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
647
677
|
parent = dirname(parent)
|
|
648
678
|
dirent = direntDir(parent)
|
|
649
679
|
}
|
|
650
|
-
return
|
|
680
|
+
return path
|
|
651
681
|
}
|
|
652
682
|
}
|
|
653
683
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
@@ -703,11 +733,14 @@ const M_SPECIAL = 'special'
|
|
|
703
733
|
|
|
704
734
|
const setLogLevel = level => setLoggerLevel(level)
|
|
705
735
|
|
|
736
|
+
const closeRedis = async () => { await teardown() }
|
|
737
|
+
|
|
706
738
|
module.exports = {
|
|
707
739
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
708
740
|
isAsyncGenerator, isReadable,
|
|
709
741
|
mobiletto, connect,
|
|
710
742
|
MobilettoError, MobilettoNotFoundError,
|
|
711
743
|
setLogLevel,
|
|
712
|
-
readStream, writeStream, closeStream
|
|
744
|
+
readStream, writeStream, closeStream,
|
|
745
|
+
closeRedis
|
|
713
746
|
}
|
package/package.json
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "A storage
|
|
3
|
+
"version": "1.1.3",
|
|
4
|
+
"description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"storage",
|
|
8
8
|
"s3",
|
|
9
|
+
"b2",
|
|
9
10
|
"simple",
|
|
10
11
|
"filesystem"
|
|
11
12
|
],
|
|
@@ -21,7 +22,7 @@
|
|
|
21
22
|
},
|
|
22
23
|
"license": "Apache-2.0",
|
|
23
24
|
"scripts": {
|
|
24
|
-
"test": "mocha --timeout
|
|
25
|
+
"test": "mocha --timeout 60000 ./test/*.spec.js"
|
|
25
26
|
},
|
|
26
27
|
"devDependencies": {
|
|
27
28
|
"@babel/core": "^7.18.10",
|
|
@@ -35,6 +36,8 @@
|
|
|
35
36
|
"dependencies": {
|
|
36
37
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
37
38
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
39
|
+
"@gideo-llc/backblaze-b2-upload-any": "^0.1.4",
|
|
40
|
+
"backblaze-b2": "^1.7.0",
|
|
38
41
|
"ioredis": "^5.2.3",
|
|
39
42
|
"lru-cache": "^7.14.0",
|
|
40
43
|
"node-fetch-commonjs": "^3.1.1",
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// To run the tests, you need a .env file one level above this directory
|
|
2
|
-
|
|
2
|
+
require('dotenv').config()
|
|
3
|
+
|
|
3
4
|
const fs = require('fs')
|
|
4
5
|
const { basename, dirname } = require('path')
|
|
5
|
-
require('dotenv').config()
|
|
6
6
|
|
|
7
7
|
const randomstring = require('randomstring')
|
|
8
8
|
const crypto = require('crypto')
|
|
@@ -12,28 +12,16 @@ const { expect, should, assert } = require('chai')
|
|
|
12
12
|
|
|
13
13
|
const {
|
|
14
14
|
M_DIR, M_FILE,
|
|
15
|
-
mobiletto, connect, MobilettoNotFoundError,
|
|
15
|
+
mobiletto, connect, MobilettoNotFoundError, closeRedis
|
|
16
16
|
} = require("../index")
|
|
17
17
|
|
|
18
|
-
const {
|
|
19
|
-
DEFAULT_CRYPT_ALGO,
|
|
20
|
-
normalizeKey, normalizeIV,
|
|
21
|
-
encrypt, decrypt,
|
|
22
|
-
getCipher, getDecipher
|
|
23
|
-
} = require("../util/crypt")
|
|
24
|
-
|
|
25
|
-
const { logger } = require("../util/logger");
|
|
18
|
+
const { logger } = require("../util/logger")
|
|
26
19
|
|
|
27
20
|
// chunk size used by generator function, used by driver's 'write' function
|
|
28
21
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
29
22
|
const READ_SZ = 8 * 1024 // xfer data in 8k chunks
|
|
30
23
|
const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
|
|
31
24
|
|
|
32
|
-
// encoded bytes written will differ from actual bytes provided
|
|
33
|
-
// likewise, plaintext bytes read will differ from encoded bytes read
|
|
34
|
-
// our tests give some leeway when filesize differences
|
|
35
|
-
const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
|
|
36
|
-
|
|
37
25
|
DRIVER_CONFIG = {
|
|
38
26
|
local: {
|
|
39
27
|
key: process.env.LOCAL_STORAGE || '/tmp'
|
|
@@ -43,9 +31,17 @@ DRIVER_CONFIG = {
|
|
|
43
31
|
secret: process.env.S3_SECRET,
|
|
44
32
|
opts: {
|
|
45
33
|
bucket: process.env.S3_BUCKET,
|
|
46
|
-
prefix: process.env.S3_PREFIX,
|
|
34
|
+
prefix: process.env.S3_PREFIX || 'mobiletto_test',
|
|
47
35
|
region: process.env.S3_REGION
|
|
48
36
|
}
|
|
37
|
+
},
|
|
38
|
+
b2: {
|
|
39
|
+
key: process.env.B2_KEY_ID,
|
|
40
|
+
secret: process.env.B2_KEY_SECRET,
|
|
41
|
+
opts: {
|
|
42
|
+
bucket: process.env.B2_BUCKET,
|
|
43
|
+
prefix: process.env.B2_PREFIX || 'mobiletto_test'
|
|
44
|
+
}
|
|
49
45
|
}
|
|
50
46
|
}
|
|
51
47
|
DRIVER_NAMES = Object.keys(DRIVER_CONFIG)
|
|
@@ -103,62 +99,9 @@ async function writeRandomFile(fixture, size) {
|
|
|
103
99
|
return await fixture.api.write(fixture.name, dataGenerator())
|
|
104
100
|
}
|
|
105
101
|
|
|
106
|
-
describe('crypto test', () => {
|
|
107
|
-
const enc = {
|
|
108
|
-
key: normalizeKey(randomstring.generate(32)),
|
|
109
|
-
iv: normalizeIV(randomstring.generate(16)),
|
|
110
|
-
algo: DEFAULT_CRYPT_ALGO
|
|
111
|
-
}
|
|
112
|
-
it("should encrypt and decrypt a data buffer successfully", async () => {
|
|
113
|
-
const plaintext = randomstring.generate((64 * 1024) + Math.floor(1024*Math.random()))
|
|
114
|
-
const ciphertext = encrypt(plaintext, enc)
|
|
115
|
-
const decrypted = decrypt(ciphertext, enc)
|
|
116
|
-
expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
|
|
117
|
-
})
|
|
118
|
-
it("should encrypt and decrypt a stream successfully", async () => {
|
|
119
|
-
const plaintext = crypto.randomBytes(81920)
|
|
120
|
-
const cipher = getCipher(enc)
|
|
121
|
-
const updateData = cipher.update(plaintext)
|
|
122
|
-
const finalData = cipher.final()
|
|
123
|
-
const encrypted = Buffer.concat([updateData, finalData])
|
|
124
|
-
|
|
125
|
-
const decipher = getDecipher(enc)
|
|
126
|
-
const decryptUpdate = decipher.update(encrypted)
|
|
127
|
-
const decipherFinal = decipher.final()
|
|
128
|
-
const decrypted = Buffer.concat([decryptUpdate, decipherFinal])
|
|
129
|
-
expect(decrypted.length).to.equal(plaintext.length)
|
|
130
|
-
for (let i = 0; i < decrypted.length; i++ ) {
|
|
131
|
-
expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
|
|
132
|
-
}
|
|
133
|
-
})
|
|
134
|
-
it("should encrypt and decrypt a stream successfully using readStream", async () => {
|
|
135
|
-
const plaintext = crypto.randomBytes(81920)
|
|
136
|
-
const cipher = getCipher(enc)
|
|
137
|
-
const updateData = cipher.update(plaintext)
|
|
138
|
-
const finalData = cipher.final()
|
|
139
|
-
const encrypted = Buffer.concat([updateData, finalData])
|
|
140
|
-
|
|
141
|
-
const tempFile = `/tmp/stream_foo_${Date.now}.tmp`
|
|
142
|
-
fs.writeFileSync(tempFile, encrypted)
|
|
143
|
-
|
|
144
|
-
const rs = fs.createReadStream(tempFile)
|
|
145
|
-
const decipher = getDecipher(enc)
|
|
146
|
-
const buffers = []
|
|
147
|
-
await readStream(rs, (chunk) => {
|
|
148
|
-
buffers.push(decipher.update(chunk))
|
|
149
|
-
}, () => {
|
|
150
|
-
buffers.push(decipher.final())
|
|
151
|
-
})
|
|
152
|
-
const decrypted = Buffer.concat(buffers)
|
|
153
|
-
expect(decrypted.length).to.equal(plaintext.length)
|
|
154
|
-
for (let i = 0; i < decrypted.length; i++ ) {
|
|
155
|
-
expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
|
|
156
|
-
}
|
|
157
|
-
})
|
|
158
|
-
|
|
159
|
-
})
|
|
160
|
-
|
|
161
102
|
const encryptionTests = () => [null, { key: rand(32) }]
|
|
103
|
+
// const encryptionTests = () => [null]
|
|
104
|
+
// const encryptionTests = () => [{ key: rand(32) }]
|
|
162
105
|
|
|
163
106
|
const REDIS_ENABLED = {
|
|
164
107
|
name: 'redis-enabled',
|
|
@@ -178,26 +121,13 @@ const REDIS_DISABLED = {
|
|
|
178
121
|
}
|
|
179
122
|
const redisTests = () => [REDIS_ENABLED, REDIS_DISABLED]
|
|
180
123
|
|
|
181
|
-
// For testing single drivers
|
|
182
|
-
// Comment out the for loop for driver tests, and hard-code one of the drivers below
|
|
183
|
-
// const driverName = 'local'
|
|
184
|
-
// const driverName = 's3'
|
|
185
|
-
|
|
186
|
-
// For testing single redis setups
|
|
187
|
-
// Comment out the for loop for redis test, and hard-code one of the setups below
|
|
188
|
-
// const redisSetup = REDIS_ENABLED
|
|
189
|
-
// const redisSetup = REDIS_DISABLED
|
|
190
|
-
|
|
191
|
-
// When you're NOT testing a single driver, it's important that *all* tests remain
|
|
192
|
-
// within the various 'for' loops below. This ensures that all drivers pass exactly
|
|
193
|
-
// the same set of tests, with exactly the same inputs/outputs, both with and without
|
|
194
|
-
// redis caching enabled, both with and without encryption enabled.
|
|
195
|
-
// This greatly supports developer sanity
|
|
196
|
-
|
|
197
124
|
for (const redisSetup of redisTests()) {
|
|
198
125
|
if (!redisSetup.enabled) continue
|
|
199
126
|
|
|
200
127
|
for (const driverName of DRIVER_NAMES) {
|
|
128
|
+
// For testing single drivers
|
|
129
|
+
if (driverName !== 'b2') continue
|
|
130
|
+
|
|
201
131
|
const driverTest = `${driverName} [${redisSetup.name}]`
|
|
202
132
|
const config = redisSetup.config(DRIVER_CONFIG[driverName])
|
|
203
133
|
const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
|
|
@@ -219,7 +149,8 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
219
149
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
220
150
|
should().exist(api, 'expected API object to exist')
|
|
221
151
|
} catch (e) {
|
|
222
|
-
logger.error(`
|
|
152
|
+
logger.error(`error creating API: ${e} ${JSON.stringify(e)}`)
|
|
153
|
+
throw e
|
|
223
154
|
}
|
|
224
155
|
})
|
|
225
156
|
})
|
|
@@ -249,12 +180,12 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
249
180
|
expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
|
|
250
181
|
})
|
|
251
182
|
it("should read the file we just wrote", async () => {
|
|
252
|
-
const data = await fixture.api.readFile(fixture.name)
|
|
183
|
+
const data = await fixture.api.readFile(fixture.name)
|
|
253
184
|
expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
|
|
254
185
|
expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
255
186
|
})
|
|
256
187
|
it("should load metadata on the file we just wrote", async () => {
|
|
257
|
-
await assertMeta
|
|
188
|
+
await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
|
|
258
189
|
})
|
|
259
190
|
it("should delete the file we just wrote", async () => {
|
|
260
191
|
const removed = await fixture.api.remove(fixture.name)
|
|
@@ -331,16 +262,16 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
331
262
|
.finally(done)
|
|
332
263
|
})
|
|
333
264
|
it("should write an encrypted file", async () => {
|
|
334
|
-
const bytesWritten = await writeRandomFile(fixture, size)
|
|
265
|
+
const bytesWritten = await writeRandomFile(fixture, size)
|
|
335
266
|
expect(bytesWritten).is.equal(size, 'expected write API to return correct number of bytes written')
|
|
336
267
|
})
|
|
337
268
|
it("should read the encrypted file we just wrote", async () => {
|
|
338
|
-
const data = await fixture.api.readFile(fixture.name)
|
|
269
|
+
const data = await fixture.api.readFile(fixture.name)
|
|
339
270
|
expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
|
|
340
271
|
expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
341
272
|
})
|
|
342
273
|
it("should load metadata on the encrypted file we just wrote", async () => {
|
|
343
|
-
await assertMeta(fixture.api, fixture.name, fixture.randomData.length
|
|
274
|
+
await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
|
|
344
275
|
})
|
|
345
276
|
it("should delete the encrypted file we just wrote", async () => {
|
|
346
277
|
const removed = await fixture.api.remove(fixture.name)
|
|
@@ -353,7 +284,8 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
353
284
|
|
|
354
285
|
for (const encryption of encryptionTests()) {
|
|
355
286
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
356
|
-
const mirrorDriver = pickAnotherDriver(driverName)
|
|
287
|
+
// const mirrorDriver = pickAnotherDriver(driverName)
|
|
288
|
+
const mirrorDriver = 'local' // for hard-coding the mirror driver
|
|
357
289
|
const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
|
|
358
290
|
const mirrorDest = `mirrorDest_${rand(5)}/`
|
|
359
291
|
describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
|
|
@@ -365,7 +297,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
365
297
|
const subdirName = `subdir_` + Date.now()
|
|
366
298
|
const fullSubdirPath = `${randomParent}/${subdirName}`
|
|
367
299
|
const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
|
|
368
|
-
const randomStreamFileSize = READ_SZ * 10
|
|
300
|
+
const randomStreamFileSize = READ_SZ * 10
|
|
369
301
|
const randomStreamFileData = crypto.randomBytes(randomStreamFileSize)
|
|
370
302
|
//const fileCount = 3 + Math.floor(Math.random() * 10)
|
|
371
303
|
const fileCount = 2
|
|
@@ -396,7 +328,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
396
328
|
})
|
|
397
329
|
it('should write a file in a new directory using a stream', async () => {
|
|
398
330
|
// create a random temp file, write it
|
|
399
|
-
const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}
|
|
331
|
+
const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`
|
|
400
332
|
fs.writeFileSync(tempFile, randomStreamFileData)
|
|
401
333
|
const reader = fs.createReadStream(tempFile)
|
|
402
334
|
const streamFile = fixture.name + '_stream'
|
|
@@ -404,13 +336,14 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
404
336
|
expect(bytesWritten).to.equal(randomStreamFileSize, 'expected write API to return correct number of bytes written for STREAM')
|
|
405
337
|
})
|
|
406
338
|
it("should successfully read the file back that was stream-written", async () => {
|
|
407
|
-
const streamFile = fixture.name + '_stream'
|
|
339
|
+
const streamFile = fixture.name + '_stream'
|
|
408
340
|
const singleFile = (await fixture.api.readFile(streamFile))
|
|
409
341
|
expect(singleFile).to.have.lengthOf(randomStreamFileSize)
|
|
410
342
|
for (let i = 0; i < singleFile.length; i++) {
|
|
411
343
|
expect(singleFile[i]).to.equal(randomStreamFileData[i])
|
|
412
344
|
}
|
|
413
|
-
|
|
345
|
+
const removal = await fixture.api.remove(streamFile)
|
|
346
|
+
expect(removal).to.equal(streamFile, `expected stream file to be deleted, but was: ${removal}`)
|
|
414
347
|
})
|
|
415
348
|
it("should successfully list a file individually", async () => {
|
|
416
349
|
const singleFile = await fixture.api.list(fixture.name)
|
|
@@ -420,7 +353,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
420
353
|
it("should successfully list files in the base directory", async () => {
|
|
421
354
|
const baseListing = await fixture.api.list()
|
|
422
355
|
expect(baseListing).to.have.lengthOf.greaterThanOrEqual(1)
|
|
423
|
-
expect(baseListing.find(o => o.name === dirname(randomParent))).to.not.be.null
|
|
356
|
+
expect(baseListing.find(o => o.name === dirname(randomParent) || o.name === dirname(randomParent)+'/')).to.not.be.null
|
|
424
357
|
})
|
|
425
358
|
it("should successfully list files in a subdirectory, with or without a trailing slash", async () => {
|
|
426
359
|
const noSlash = await fixture.api.list(randomParent)
|
|
@@ -435,7 +368,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
435
368
|
}
|
|
436
369
|
})
|
|
437
370
|
it("should load metadata for one of the new files", async () => {
|
|
438
|
-
await assertMeta(fixture.api, fixture.name, READ_SZ
|
|
371
|
+
await assertMeta(fixture.api, fixture.name, READ_SZ)
|
|
439
372
|
})
|
|
440
373
|
it("should see correct types on objects returned from a listing of the new directory", async () => {
|
|
441
374
|
const objects = await fixture.api.list(randomParent)
|
|
@@ -541,11 +474,12 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
541
474
|
}
|
|
542
475
|
})
|
|
543
476
|
})
|
|
477
|
+
|
|
544
478
|
})
|
|
545
479
|
}
|
|
546
480
|
}
|
|
547
481
|
|
|
548
482
|
after ( (done) => {
|
|
549
483
|
logger.info('all tests finished, tearing down redis...')
|
|
550
|
-
|
|
484
|
+
closeRedis().finally(done)
|
|
551
485
|
})
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
2
|
+
const { expect } = require('chai')
|
|
3
|
+
const randomstring = require('randomstring')
|
|
4
|
+
const crypto = require('crypto')
|
|
5
|
+
|
|
6
|
+
const {
|
|
7
|
+
DEFAULT_CRYPT_ALGO,
|
|
8
|
+
normalizeKey, normalizeIV,
|
|
9
|
+
encrypt, decrypt,
|
|
10
|
+
getCipher, getDecipher
|
|
11
|
+
} = require("../util/crypt")
|
|
12
|
+
|
|
13
|
+
const { readStream } = require("../index")
|
|
14
|
+
|
|
15
|
+
describe('crypto test', () => {
|
|
16
|
+
const enc = {
|
|
17
|
+
key: normalizeKey(randomstring.generate(32)),
|
|
18
|
+
iv: normalizeIV(randomstring.generate(16)),
|
|
19
|
+
algo: DEFAULT_CRYPT_ALGO
|
|
20
|
+
}
|
|
21
|
+
it("should encrypt and decrypt a data buffer successfully", async () => {
|
|
22
|
+
const plaintext = randomstring.generate((64 * 1024) + Math.floor(1024*Math.random()))
|
|
23
|
+
const ciphertext = encrypt(plaintext, enc)
|
|
24
|
+
const decrypted = decrypt(ciphertext, enc)
|
|
25
|
+
expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
|
|
26
|
+
})
|
|
27
|
+
it("should encrypt and decrypt a stream successfully", async () => {
|
|
28
|
+
const plaintext = crypto.randomBytes(81920)
|
|
29
|
+
const cipher = getCipher(enc)
|
|
30
|
+
const updateData = cipher.update(plaintext)
|
|
31
|
+
const finalData = cipher.final()
|
|
32
|
+
const encrypted = Buffer.concat([updateData, finalData])
|
|
33
|
+
|
|
34
|
+
const decipher = getDecipher(enc)
|
|
35
|
+
const decryptUpdate = decipher.update(encrypted)
|
|
36
|
+
const decipherFinal = decipher.final()
|
|
37
|
+
const decrypted = Buffer.concat([decryptUpdate, decipherFinal])
|
|
38
|
+
expect(decrypted.length).to.equal(plaintext.length)
|
|
39
|
+
for (let i = 0; i < decrypted.length; i++ ) {
|
|
40
|
+
expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
|
|
41
|
+
}
|
|
42
|
+
})
|
|
43
|
+
it("should encrypt and decrypt a stream successfully using readStream", async () => {
|
|
44
|
+
const plaintext = crypto.randomBytes(81920)
|
|
45
|
+
const cipher = getCipher(enc)
|
|
46
|
+
const updateData = cipher.update(plaintext)
|
|
47
|
+
const finalData = cipher.final()
|
|
48
|
+
const encrypted = Buffer.concat([updateData, finalData])
|
|
49
|
+
|
|
50
|
+
const tempFile = `/tmp/stream_foo_${Date.now}.tmp`
|
|
51
|
+
fs.writeFileSync(tempFile, encrypted)
|
|
52
|
+
|
|
53
|
+
const rs = fs.createReadStream(tempFile)
|
|
54
|
+
const decipher = getDecipher(enc)
|
|
55
|
+
const buffers = []
|
|
56
|
+
await readStream(rs, (chunk) => {
|
|
57
|
+
buffers.push(decipher.update(chunk))
|
|
58
|
+
}, () => {
|
|
59
|
+
buffers.push(decipher.final())
|
|
60
|
+
})
|
|
61
|
+
const decrypted = Buffer.concat(buffers)
|
|
62
|
+
expect(decrypted.length).to.equal(plaintext.length)
|
|
63
|
+
for (let i = 0; i < decrypted.length; i++ ) {
|
|
64
|
+
expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
|
|
65
|
+
}
|
|
66
|
+
})
|
|
67
|
+
})
|
package/util/logger.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const winston = require('winston')
|
|
2
2
|
|
|
3
3
|
const logger = winston.createLogger({
|
|
4
|
-
level: process.env.MOBILETTO_LOG_LEVEL || '
|
|
4
|
+
level: process.env.MOBILETTO_LOG_LEVEL || 'debug',
|
|
5
5
|
format: winston.format.simple(),
|
|
6
6
|
transports: process.env.MOBILETTO_LOG_FILE
|
|
7
7
|
? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
|