mobiletto 1.0.46 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -55
- package/drivers/b2/index.js +428 -0
- package/drivers/local/index.js +6 -3
- package/drivers/s3/index.js +3 -3
- package/index.js +150 -34
- package/package.json +5 -3
- package/test/{index.spec.js → api.spec.js} +81 -121
- package/test/crypt.spec.js +67 -0
- package/util/logger.js +1 -1
- package/util/redis.js +162 -134
package/README.md
CHANGED
|
@@ -3,28 +3,64 @@ Mobiletto
|
|
|
3
3
|
|
|
4
4
|
Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# Contents
|
|
7
|
+
* [Why Mobiletto?](#Why-Mobiletto?)
|
|
8
|
+
* [Quick start](#Quick-start)
|
|
9
|
+
* [Mobiletto CLI](#mobiletto-cli)
|
|
10
|
+
* [Source](#Source)
|
|
11
|
+
* [Basic usage](#Basic-usage)
|
|
12
|
+
* [Metadata](#Metadata)
|
|
13
|
+
* [Alternate import style](#Alternate-import-style)
|
|
14
|
+
* [Caching](#Caching)
|
|
15
|
+
* [Mirroring](#Mirroring)
|
|
16
|
+
* [Transparent encryption](#Transparent-encryption)
|
|
17
|
+
* [Key rotation](#Key-rotation)
|
|
18
|
+
* [Driver interface](#Driver-interface)
|
|
19
|
+
* [Logging](#Logging)
|
|
20
|
+
|
|
21
|
+
## Why Mobiletto?
|
|
22
|
+
|
|
23
|
+
### Goodbye vendor lock-in!
|
|
24
|
+
The various cloud storage providers have incompatible APIs. Even those that strive for "S3 compatibility"
|
|
25
|
+
have idiosyncratic behaviors.
|
|
26
|
+
|
|
27
|
+
When you choose a particular storage vendor for your app, if you code directly to their API, your app
|
|
28
|
+
is now dependent on that service. As time goes by and code accumulates, changing vendors becomes
|
|
29
|
+
increasingly untenable. Welcome to the fun world of vendor lock-in!
|
|
30
|
+
|
|
31
|
+
Mobiletto was designed to solve this problem. By coding your app to mobiletto's API, you can easily
|
|
32
|
+
change storage providers and know that your app's storage layer will behave identically.
|
|
7
33
|
|
|
34
|
+
### Extensive testing
|
|
8
35
|
All drivers are tested for identical behavior with over 55 tests for each driver.
|
|
36
|
+
Each driver test actually runs *four times*, with every (2x2) combination of:
|
|
37
|
+
* Encryption: both enabled and disabled
|
|
38
|
+
* Redis cache: both enabled and disabled
|
|
9
39
|
|
|
10
|
-
|
|
40
|
+
This approach gives us peace-of-mind that mobiletto will behave the same regardless of which driver you use,
|
|
41
|
+
and regardless of whether you enable caching and/or encryption.
|
|
11
42
|
|
|
12
|
-
|
|
13
|
-
|
|
43
|
+
### Driver support
|
|
44
|
+
Current Mobiletto storage drivers:
|
|
45
|
+
* `s3`: Amazon S3
|
|
46
|
+
* `b2`: Backblaze B2
|
|
47
|
+
* `local`: local filesystem
|
|
14
48
|
|
|
15
|
-
|
|
49
|
+
*Contributions to support more cloud storage providers are very welcome!*
|
|
50
|
+
|
|
51
|
+
## mobiletto-cli
|
|
16
52
|
Mobiletto is intended to be used as a library by other JavaScript code.
|
|
17
53
|
|
|
18
54
|
To work with mobiletto at the command-line, use [mobiletto-cli](https://www.npmjs.com/package/mobiletto-cli)
|
|
19
55
|
|
|
20
|
-
|
|
56
|
+
## Source
|
|
21
57
|
* [mobiletto on GitHub](https://github.com/cobbzilla/mobiletto)
|
|
22
58
|
* [mobiletto on npm](https://www.npmjs.com/package/mobiletto)
|
|
23
59
|
|
|
24
|
-
##
|
|
60
|
+
## Quick Start
|
|
25
61
|
A simple example using the mobiletto `s3` driver.
|
|
26
62
|
|
|
27
|
-
This code would run the same if the driver were `local`.
|
|
63
|
+
This code would run the same if the driver were `b2` or `local`.
|
|
28
64
|
|
|
29
65
|
const storage = require('mobiletto')
|
|
30
66
|
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
|
|
@@ -46,20 +82,12 @@ This code would run the same if the driver were `local`.
|
|
|
46
82
|
// read file as a stream using data callback
|
|
47
83
|
const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
|
|
48
84
|
|
|
49
|
-
// remove a file, returns
|
|
50
|
-
|
|
85
|
+
// remove a file, returns the path removed
|
|
86
|
+
let removed = await s3.remove('some/path')
|
|
87
|
+
|
|
88
|
+
// remove a directory, returns array of paths removed
|
|
89
|
+
removed = await s3.remove('some/directory', {recursive: true})
|
|
51
90
|
|
|
52
|
-
----
|
|
53
|
-
# Contents
|
|
54
|
-
* [Basic usage](#Basic-usage)
|
|
55
|
-
* [Metadata](#Metadata)
|
|
56
|
-
* [Alternate import style](#Alternate-import-style)
|
|
57
|
-
* [Caching](#Caching)
|
|
58
|
-
* [Mirroring](#Mirroring)
|
|
59
|
-
* [Transparent encryption](#Transparent-encryption)
|
|
60
|
-
* [Key rotation](#Key-rotation)
|
|
61
|
-
* [Driver interface](#Driver-interface)
|
|
62
|
-
* [Logging](#Logging)
|
|
63
91
|
----
|
|
64
92
|
## Basic usage
|
|
65
93
|
const { mobiletto } = require('mobiletto')
|
|
@@ -87,44 +115,50 @@ This code would run the same if the driver were `local`.
|
|
|
87
115
|
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
88
116
|
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
89
117
|
|
|
118
|
+
// To use 'b2' driver:
|
|
119
|
+
// * key: Backblaze Key ID
|
|
120
|
+
// * secret: Backblaze Application Key
|
|
121
|
+
// * opts object:
|
|
122
|
+
// * readOnly: optional, never change anything on the bucket; default is false
|
|
123
|
+
// * bucket: required, the ID (**not the name**) of the B2 bucket
|
|
124
|
+
// * prefix: optional, all read/writes within the B2 bucket will be under this prefix
|
|
125
|
+
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
126
|
+
// * partSize: optional, large files will be split into chunks of this size when uploading
|
|
127
|
+
const b3 = await mobiletto('b2', b2_key_id, b2_app_key, {bucket: 'bk', partSize: 10000000})
|
|
128
|
+
|
|
90
129
|
// List files
|
|
91
|
-
|
|
92
|
-
s3.list() // --> returns an array of metadata objects
|
|
130
|
+
api.list() // --> returns an array of metadata objects
|
|
93
131
|
|
|
94
132
|
// List files recursively
|
|
95
|
-
|
|
96
|
-
s3.list({ recursive: true })
|
|
133
|
+
api.list({ recursive: true })
|
|
97
134
|
|
|
98
135
|
// List files in a directory
|
|
99
136
|
const path = 'some/path'
|
|
100
|
-
|
|
101
|
-
|
|
137
|
+
api.list(path)
|
|
138
|
+
api.list(path, { recursive: true }) // also supports recursive flag
|
|
102
139
|
|
|
103
140
|
// Visit files in a directory -- visitor function must be async
|
|
104
|
-
|
|
105
|
-
|
|
141
|
+
api.list(path, { visitor: myAsyncFunc })
|
|
142
|
+
api.list(path, { visitor: myAsyncFunc, recursive: true })
|
|
106
143
|
|
|
107
144
|
// The `list` method throws MobilettoNotFoundError if the path does not exist
|
|
108
145
|
// When you call `safeList` on a non-existent path, it returns an empty array
|
|
109
|
-
|
|
146
|
+
api.safeList('/path/that/does/not/exist') // returns []
|
|
110
147
|
|
|
111
148
|
// Read metadata for a file
|
|
112
|
-
|
|
113
|
-
s3.metadata(path) // returns metadata object
|
|
149
|
+
api.metadata(path) // returns metadata object
|
|
114
150
|
|
|
115
151
|
// The `metadata` method throws MobilettoNotFoundError if the path does not exist
|
|
116
152
|
// When you call `safeMetadata` on a non-existent path, it returns null
|
|
117
|
-
|
|
153
|
+
api.safeMetadata('/tmp/does_not_exist') // returns null
|
|
118
154
|
|
|
119
155
|
// Read a file
|
|
120
156
|
// Provide a callback that writes the data someplace
|
|
121
157
|
const callback = (chunk) => { ... write chunk somewhere ... }
|
|
122
|
-
|
|
123
|
-
s3.read(path, callback) // returns count of bytes read
|
|
158
|
+
api.read(path, callback) // returns count of bytes read
|
|
124
159
|
|
|
125
160
|
// Read an entire file at once (convenience method)
|
|
126
|
-
const
|
|
127
|
-
const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
|
|
161
|
+
const data = await api.readFile(path) // returns a byte Buffer of the file contents
|
|
128
162
|
|
|
129
163
|
// Write a file
|
|
130
164
|
// Provide a generator function that yields chunks of data
|
|
@@ -134,26 +168,22 @@ This code would run the same if the driver were `local`.
|
|
|
134
168
|
yield data
|
|
135
169
|
}
|
|
136
170
|
}
|
|
137
|
-
local.
|
|
138
|
-
s3.write(path, generator) // returns count of bytes written
|
|
171
|
+
local.api(path, generator) // returns count of bytes written
|
|
139
172
|
|
|
140
173
|
// Write an entire file at once (convenience method)
|
|
141
|
-
await
|
|
142
|
-
await s3.writeFile(path, bufferOrString) // returns count of bytes written
|
|
174
|
+
await api.writeFile(path, bufferOrString) // returns count of bytes written
|
|
143
175
|
|
|
144
176
|
// Delete a file
|
|
145
177
|
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
146
178
|
// Always returns a value or throws an error.
|
|
147
179
|
// Return value may be a single string of the file removed, or an array of all files removed (driver-dependent)
|
|
148
180
|
const quiet = true
|
|
149
|
-
|
|
150
|
-
s3.remove(path, {quiet}) // returns single path removed
|
|
181
|
+
api.remove(path, {quiet}) // returns single path removed
|
|
151
182
|
|
|
152
183
|
// Recursively delete a directory and do it quietly (do not report errors)
|
|
153
184
|
const recursive = true
|
|
154
185
|
const quiet = true
|
|
155
|
-
|
|
156
|
-
s3.remove(path, {recursive, quiet}) // returns array containing all paths removed
|
|
186
|
+
api.remove(path, {recursive, quiet}) // returns array of paths removed
|
|
157
187
|
|
|
158
188
|
## Metadata
|
|
159
189
|
The `metadata` command returns metadata about a single filesystem entry.
|
|
@@ -194,15 +224,19 @@ You can override either of these:
|
|
|
194
224
|
Mobiletto will store all of its redis keys with the prefix `_mobiletto__`. You can change this
|
|
195
225
|
by setting the `MOBILETTO_REDIS_PREFIX` env var.
|
|
196
226
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
227
|
+
You can also set per-connection caching with the `opts.redisConfig` object:
|
|
228
|
+
|
|
229
|
+
const redisConfig = {
|
|
230
|
+
enabled: true, // optional, default is true. if false other props are ignored
|
|
231
|
+
host: '127.0.0.1',
|
|
232
|
+
port: 6379,
|
|
233
|
+
prefix: '_mobiletto__'
|
|
234
|
+
}
|
|
235
|
+
const opts = { redisConfig, bucket: 'bk', region: 'us-east-1' }
|
|
236
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
|
|
237
|
+
|
|
238
|
+
### Don't want redis caching?
|
|
239
|
+
To disable: pass `enabled: false` in your `opts.redisConfig` object when you establish your connection.
|
|
206
240
|
|
|
207
241
|
As discussed below, disabling caching will have an adverse effect on performance and incur more requests
|
|
208
242
|
to storage that you really need to.
|
|
@@ -347,7 +381,7 @@ or any other connection configuration information.
|
|
|
347
381
|
Use the `MOBILETTO_LOG_LEVEL` environment variable to set the log level, using one
|
|
348
382
|
of the `npm` levels defined in [https://www.npmjs.com/package/winston#logging-levels](https://www.npmjs.com/package/winston#logging-levels)
|
|
349
383
|
|
|
350
|
-
The default level is `
|
|
384
|
+
The default level is `error`. The most verbose level is `silly`, although currently mobiletto
|
|
351
385
|
does not log at levels below `debug`
|
|
352
386
|
|
|
353
387
|
MOBILETTO_LOG_LEVEL=silly # maximum logs!
|
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
const B2 = require('backblaze-b2')
|
|
2
|
+
|
|
3
|
+
// this library is a godsend for B2 uploads
|
|
4
|
+
B2.prototype.uploadAny = require('@gideo-llc/backblaze-b2-upload-any')
|
|
5
|
+
|
|
6
|
+
const { Readable } = require('stream')
|
|
7
|
+
|
|
8
|
+
const {
|
|
9
|
+
M_FILE, M_DIR,
|
|
10
|
+
readStream, isReadable, isAsyncGenerator,
|
|
11
|
+
MobilettoError, MobilettoNotFoundError
|
|
12
|
+
} = require('../../index')
|
|
13
|
+
|
|
14
|
+
const { logger } = require('../../util/logger')
|
|
15
|
+
|
|
16
|
+
// B2 tokens last 24 hours. Let's refresh ours after 23 hours and 50 minutes
|
|
17
|
+
const AUTH_REFRESH = (1000 * 60 * 60 * 24) - (1000 * 60 * 10)
|
|
18
|
+
|
|
19
|
+
class StorageClient {
|
|
20
|
+
b2
|
|
21
|
+
bucket
|
|
22
|
+
prefix
|
|
23
|
+
delimiter
|
|
24
|
+
opts
|
|
25
|
+
lastAuth
|
|
26
|
+
absoluteMinimumPartSize
|
|
27
|
+
recommendedPartSize
|
|
28
|
+
configuredPartSize
|
|
29
|
+
constructor(keyId, appKey, opts) {
|
|
30
|
+
if (!keyId || !appKey || !opts.bucket) throw new MobilettoError(`b2.StorageClient: key, secret and opts.bucket are required`)
|
|
31
|
+
this.b2 = new B2({
|
|
32
|
+
applicationKeyId: keyId,
|
|
33
|
+
applicationKey: appKey
|
|
34
|
+
})
|
|
35
|
+
this.opts = opts
|
|
36
|
+
this.configuredPartSize = opts.partSize ? opts.partSize : -1
|
|
37
|
+
this.bucket = opts.bucket
|
|
38
|
+
this.delimiter = opts.delimiter || '/'
|
|
39
|
+
this.prefix = opts.prefix
|
|
40
|
+
? opts.prefix.endsWith(this.delimiter)
|
|
41
|
+
? opts.prefix
|
|
42
|
+
: opts.prefix + this.delimiter
|
|
43
|
+
: ''
|
|
44
|
+
this.lastAuth = 0
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
auth = async () => {
|
|
48
|
+
if (Date.now() - this.lastAuth > AUTH_REFRESH) {
|
|
49
|
+
try {
|
|
50
|
+
const response = await this.b2.authorize({})
|
|
51
|
+
this.lastAuth = Date.now()
|
|
52
|
+
this.absoluteMinimumPartSize = response.data.absoluteMinimumPartSize
|
|
53
|
+
this.recommendedPartSize = response.data.recommendedPartSize
|
|
54
|
+
if (this.configuredPartSize === -1) {
|
|
55
|
+
logger.info(`partSize was not configured, using recommendedPartSize=${this.recommendedPartSize}`)
|
|
56
|
+
this.configuredPartSize = this.recommendedPartSize
|
|
57
|
+
} else if (this.configuredPartSize < this.absoluteMinimumPartSize) {
|
|
58
|
+
logger.warn(`partSize was too small (${this.configuredPartSize}), using absoluteMinimumPartSize=${this.absoluteMinimumPartSize}`)
|
|
59
|
+
this.configuredPartSize = this.absoluteMinimumPartSize
|
|
60
|
+
}
|
|
61
|
+
} catch (e) {
|
|
62
|
+
logger.error(`auth error: ${e}`)
|
|
63
|
+
throw e
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
testConfig = async () => await this.b2_list('', false, null, {max: 1})
|
|
69
|
+
|
|
70
|
+
normalizePath = (path) => path.length === 0 ? this.prefix :
|
|
71
|
+
path.startsWith(this.prefix)
|
|
72
|
+
? path
|
|
73
|
+
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path)
|
|
74
|
+
denormalizePath = (path) => {
|
|
75
|
+
const norm = path.startsWith(this.prefix) ? path.substring(this.prefix.length) : path
|
|
76
|
+
return norm.startsWith(this.delimiter) ? norm.substring(1) : norm
|
|
77
|
+
}
|
|
78
|
+
bbError = (err, path, method) => {
|
|
79
|
+
if (err.response && err.response.status && err.response.status === 404) {
|
|
80
|
+
return new MobilettoNotFoundError(path)
|
|
81
|
+
} else if (err.response && err.response.data && err.response.data.message) {
|
|
82
|
+
return new MobilettoError(`${method} error: ${err.response.data.message}`)
|
|
83
|
+
}
|
|
84
|
+
return err
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
b2type = f => f.fileName.endsWith(this.delimiter) && f.contentType === null ? M_DIR : M_FILE
|
|
88
|
+
|
|
89
|
+
file2object(f) {
|
|
90
|
+
return {
|
|
91
|
+
name: this.denormalizePath(f.fileName),
|
|
92
|
+
size: f.contentLength,
|
|
93
|
+
type: this.b2type(f),
|
|
94
|
+
bb2id: f.fileId
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
processList = async (path, response, visitor) => {
|
|
99
|
+
const files = response && response.data && response.data.files ? response.data.files : null
|
|
100
|
+
if (!files) {
|
|
101
|
+
return []
|
|
102
|
+
}
|
|
103
|
+
const objects = files.map((f) => {
|
|
104
|
+
logger.debug(`addListedFiles(${path}): adding file: ${f.fileName}`)
|
|
105
|
+
return this.file2object(f)
|
|
106
|
+
})
|
|
107
|
+
if (visitor) {
|
|
108
|
+
for (const obj of objects) { await visitor(obj) }
|
|
109
|
+
}
|
|
110
|
+
return objects
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
list = (pth = '', recursive = false, visitor = null) =>
|
|
114
|
+
this.b2_list(pth, recursive, visitor)
|
|
115
|
+
|
|
116
|
+
async b2_list (pth = '', recursive = false, visitor = null, opts = {}) {
|
|
117
|
+
try {
|
|
118
|
+
let optsDir = opts.dir || false
|
|
119
|
+
let dirDiscovered = null
|
|
120
|
+
let firstResult = null
|
|
121
|
+
const files = []
|
|
122
|
+
await this.auth()
|
|
123
|
+
const hasMax = opts && opts.max
|
|
124
|
+
const maxFileCount = hasMax ? opts.max : 100
|
|
125
|
+
const pfx = (this.prefix.length === 0 || this.prefix.endsWith(this.delimiter) ? this.prefix : this.prefix + this.delimiter) +
|
|
126
|
+
(pth.startsWith(this.delimiter) ? pth.substring(0) : pth)
|
|
127
|
+
|
|
128
|
+
let response = await this.b2.listFileNames({
|
|
129
|
+
bucketId: this.bucket,
|
|
130
|
+
maxFileCount,
|
|
131
|
+
delimiter: recursive ? '' : this.delimiter,
|
|
132
|
+
prefix: pfx
|
|
133
|
+
})
|
|
134
|
+
const handleResponse = async (files, pth, response, visitor) => {
|
|
135
|
+
const list = await this.processList(pth, response, visitor)
|
|
136
|
+
if (!optsDir && !firstResult && list.length > 0) {
|
|
137
|
+
firstResult = list[0]
|
|
138
|
+
if (list[0].name.endsWith('/')) {
|
|
139
|
+
dirDiscovered = true
|
|
140
|
+
} else if (!dirDiscovered && list.length > 1 && list[1].name.startsWith(list[0].name)) {
|
|
141
|
+
files.push(list[0])
|
|
142
|
+
return true
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (hasMax && files.length + list.length >= maxFileCount) {
|
|
146
|
+
files.push(...list.slice(0, (list.length - maxFileCount) + 1))
|
|
147
|
+
return true
|
|
148
|
+
} else {
|
|
149
|
+
files.push(...list)
|
|
150
|
+
return false
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
while (response.data.nextFileName) {
|
|
154
|
+
if (await handleResponse(files, pth, response, visitor)) {
|
|
155
|
+
return files
|
|
156
|
+
}
|
|
157
|
+
response = await this.b2.listFileNames({
|
|
158
|
+
bucketId: this.bucket,
|
|
159
|
+
startFileName: response.data.nextFileName,
|
|
160
|
+
maxFileCount,
|
|
161
|
+
delimiter: recursive ? '' : this.delimiter,
|
|
162
|
+
prefix: this.prefix
|
|
163
|
+
})
|
|
164
|
+
}
|
|
165
|
+
await handleResponse(files, pth, response, visitor)
|
|
166
|
+
if (dirDiscovered && !pth.endsWith(this.delimiter)) {
|
|
167
|
+
const subdirFiles = await this.b2_list(pth + this.delimiter, recursive, null, { max: 10000, dir: true })
|
|
168
|
+
files.shift()
|
|
169
|
+
files.push(...subdirFiles)
|
|
170
|
+
}
|
|
171
|
+
return files
|
|
172
|
+
} catch (err) {
|
|
173
|
+
throw this.bbError(err, pth, 'list')
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
b2_meta = (path) => (this.driver_metadata ? this.driver_metadata(path) : this.metadata(path))
|
|
178
|
+
|
|
179
|
+
async metadata (pth) {
|
|
180
|
+
await this.auth()
|
|
181
|
+
let files = null
|
|
182
|
+
const normPath = this.normalizePath(pth);
|
|
183
|
+
try {
|
|
184
|
+
files = (await this.b2.listFileNames({
|
|
185
|
+
bucketId: this.bucket,
|
|
186
|
+
startFileName: normPath,
|
|
187
|
+
maxFileCount: 1,
|
|
188
|
+
delimiter: '',
|
|
189
|
+
prefix: this.prefix
|
|
190
|
+
})).data.files
|
|
191
|
+
} catch (err) {
|
|
192
|
+
throw this.bbError(err, pth, 'metadata')
|
|
193
|
+
}
|
|
194
|
+
if (files.length === 0) {
|
|
195
|
+
throw new MobilettoNotFoundError(pth)
|
|
196
|
+
}
|
|
197
|
+
const found = files.filter(f => f.fileName === normPath)
|
|
198
|
+
if (!found || found.length === 0) {
|
|
199
|
+
// if this is a directory, we may have returned the first item in it, that's OK
|
|
200
|
+
if (files[0].fileName.startsWith(normPath.endsWith(this.delimiter) ? normPath : normPath + this.delimiter)) {
|
|
201
|
+
let obj = this.file2object(files[0]);
|
|
202
|
+
obj.name = pth
|
|
203
|
+
obj.type = M_DIR
|
|
204
|
+
return obj
|
|
205
|
+
}
|
|
206
|
+
throw new MobilettoNotFoundError(pth)
|
|
207
|
+
}
|
|
208
|
+
if (found.length > 1) {
|
|
209
|
+
throw new MobilettoError(`metadata(${pth}) multiple files matched: ${JSON.stringify(found)}`)
|
|
210
|
+
}
|
|
211
|
+
return this.file2object(files[0])
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async write (path, generatorOrReadable) {
|
|
215
|
+
await this.auth()
|
|
216
|
+
logger.debug(`write: writing path ${path}`)
|
|
217
|
+
let stream = generatorOrReadable
|
|
218
|
+
if (!isReadable(generatorOrReadable)) {
|
|
219
|
+
stream = new Readable({
|
|
220
|
+
async read (size) {
|
|
221
|
+
let chunk = isAsyncGenerator(generatorOrReadable)
|
|
222
|
+
? (await generatorOrReadable.next()).value
|
|
223
|
+
: generatorOrReadable.next().value
|
|
224
|
+
this.push(typeof chunk !== 'undefined' ? chunk : null)
|
|
225
|
+
}
|
|
226
|
+
})
|
|
227
|
+
}
|
|
228
|
+
let response = null
|
|
229
|
+
try {
|
|
230
|
+
response = await this.b2.uploadAny({
|
|
231
|
+
bucketId: this.bucket,
|
|
232
|
+
fileName: this.normalizePath(path),
|
|
233
|
+
partSize: this.configuredPartSize,
|
|
234
|
+
data: stream
|
|
235
|
+
});
|
|
236
|
+
// response is from either b2_upload_file or b2_finish_large_file
|
|
237
|
+
// It depends on how large the file was, see: https://github.com/gideo-llc/backblaze-b2-upload-any/blob/master/README.md#uploadanyoptions
|
|
238
|
+
if (response.contentLength) {
|
|
239
|
+
return response.contentLength
|
|
240
|
+
}
|
|
241
|
+
} catch (e) {
|
|
242
|
+
logger.error(`write error: ${e}`)
|
|
243
|
+
throw this.bbError(e, path, 'write')
|
|
244
|
+
}
|
|
245
|
+
if (response) {
|
|
246
|
+
throw new MobilettoError(`write error: response status ${response.status} with data ${JSON.stringify(response.data)}`)
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async read (path, callback, endCallback = null) {
|
|
251
|
+
await this.auth()
|
|
252
|
+
const file = this.normalizePath(path)
|
|
253
|
+
logger.debug(`read: reading path: ${path}`)
|
|
254
|
+
let meta = null
|
|
255
|
+
try {
|
|
256
|
+
meta = await (this.b2_meta(file))
|
|
257
|
+
} catch (e) {
|
|
258
|
+
if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
|
|
259
|
+
throw e
|
|
260
|
+
}
|
|
261
|
+
throw new MobilettoError(`read: error reading metadata for path ${path}`, e)
|
|
262
|
+
}
|
|
263
|
+
if (meta.type === M_DIR) {
|
|
264
|
+
throw new MobilettoError(`read: cannot read directory: ${path}`)
|
|
265
|
+
}
|
|
266
|
+
try {
|
|
267
|
+
const stream = await this.b2.downloadFileById({
|
|
268
|
+
bucketId: this.bucket,
|
|
269
|
+
fileId: meta.bb2id,
|
|
270
|
+
responseType: 'stream'
|
|
271
|
+
})
|
|
272
|
+
return await readStream(stream.data, callback, endCallback)
|
|
273
|
+
} catch (err) {
|
|
274
|
+
throw this.bbError(err, path, 'read')
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
async deleteVersions (path, normPath, versions, quiet, errors = {}) {
|
|
279
|
+
const removed = []
|
|
280
|
+
for (const version of versions) {
|
|
281
|
+
logger.debug(`deleteVersions(${path}, ${versions.length} versions) pushing version to delete: fileName=${version.fileName} fileId=${version.fileId}`)
|
|
282
|
+
await this.auth()
|
|
283
|
+
try {
|
|
284
|
+
if (!version.fileName.startsWith(normPath)) {
|
|
285
|
+
logger.debug(`deleteVersions(${path}, ${versions.length} versions) ending iteration on mismatched file: ${version.fileName}`)
|
|
286
|
+
return removed
|
|
287
|
+
}
|
|
288
|
+
const delResponse = await this.b2.deleteFileVersion({
|
|
289
|
+
bucketId: this.bucket,
|
|
290
|
+
fileName: version.fileName,
|
|
291
|
+
fileId: version.fileId
|
|
292
|
+
})
|
|
293
|
+
if (delResponse && delResponse.status !== 200) {
|
|
294
|
+
const message = `deleteVersions(${path}) received HTTP status ${delResponse.status} trying to delete file version: ${JSON.stringify(version)}`
|
|
295
|
+
logger.error(message)
|
|
296
|
+
if (errors) {
|
|
297
|
+
errors[path] = message
|
|
298
|
+
}
|
|
299
|
+
} else if (removed) {
|
|
300
|
+
logger.debug(`deleteVersions(${path}, ${versions.length} versions) REMOVED: fileName=${version.fileName} fileId=${version.fileId}`)
|
|
301
|
+
removed.push(this.denormalizePath(version.fileName))
|
|
302
|
+
}
|
|
303
|
+
} catch (err) {
|
|
304
|
+
const message = `deleteVersions(${path}) error deleting version ${version.fileId}: ${err}`
|
|
305
|
+
logger.error(message)
|
|
306
|
+
if (errors) {
|
|
307
|
+
errors[path] = message
|
|
308
|
+
}
|
|
309
|
+
if (!quiet) throw new MobilettoError(message, err)
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
const errPaths = Object.keys(errors);
|
|
313
|
+
if (errPaths.length > 0) {
|
|
314
|
+
if (quiet) {
|
|
315
|
+
logger.info(`deleteVersions(${path}) had ${errPaths.length} errors but quiet=true, returning ${removed.length} removed paths`)
|
|
316
|
+
return removed
|
|
317
|
+
}
|
|
318
|
+
throw new MobilettoError(`deleteVersions(${path}) had errors: ${JSON.stringify(errors)}`)
|
|
319
|
+
}
|
|
320
|
+
return removed
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async remove (path, recursive, quiet) {
|
|
324
|
+
logger.debug(`b2.remove(${path}, recursive=${recursive}) starting...`)
|
|
325
|
+
await this.auth()
|
|
326
|
+
let meta = null
|
|
327
|
+
const errors = {}
|
|
328
|
+
const normPath = this.normalizePath(path)
|
|
329
|
+
try {
|
|
330
|
+
logger.debug(`b2.remove(${path}) fetching metadata to determine file/directory`)
|
|
331
|
+
meta = await (this.b2_meta(path))
|
|
332
|
+
} catch (e) {
|
|
333
|
+
if (quiet) {
|
|
334
|
+
logger.debug(`b2.remove(${path}) quietly returning, not reporting error fetching metadata for ${path}: ${e}`)
|
|
335
|
+
return path
|
|
336
|
+
}
|
|
337
|
+
if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
|
|
338
|
+
throw e
|
|
339
|
+
}
|
|
340
|
+
throw new MobilettoError(`b2.remove(${path}) error reading metadata for path ${path}`, e)
|
|
341
|
+
}
|
|
342
|
+
if (meta.type === M_FILE) {
|
|
343
|
+
try {
|
|
344
|
+
const response = await this.b2.listFileVersions({
|
|
345
|
+
bucketId: this.bucket,
|
|
346
|
+
startFileName: normPath,
|
|
347
|
+
startFileId: meta.bb2id,
|
|
348
|
+
prefix: this.prefix,
|
|
349
|
+
delimiter: this.delimiter,
|
|
350
|
+
maxFileCount: 1000
|
|
351
|
+
})
|
|
352
|
+
const versions = response.data.files.filter(f => f.fileName === normPath)
|
|
353
|
+
await this.deleteVersions(path, normPath, versions, quiet, errors)
|
|
354
|
+
} catch (err) {
|
|
355
|
+
if (err instanceof MobilettoError) {
|
|
356
|
+
throw err
|
|
357
|
+
}
|
|
358
|
+
throw new MobilettoError(`b2.remove(${path}) error: ${err}`, err)
|
|
359
|
+
}
|
|
360
|
+
return path
|
|
361
|
+
} else if (meta.type === M_DIR) {
|
|
362
|
+
if (!recursive) {
|
|
363
|
+
if (quiet) {
|
|
364
|
+
logger.debug(`b2.remove(${path}) quietly returning, not deleting directory when recursive=false`)
|
|
365
|
+
return path
|
|
366
|
+
}
|
|
367
|
+
throw new MobilettoError(`b2.remove(${path}) cannot remove directory (set recursive=true to remove)`)
|
|
368
|
+
}
|
|
369
|
+
const removed = []
|
|
370
|
+
const errors = {}
|
|
371
|
+
let versionsResponse
|
|
372
|
+
try {
|
|
373
|
+
logger.debug(`b2.remove(${path})`)
|
|
374
|
+
let normPath = this.normalizePath(path);
|
|
375
|
+
versionsResponse = await this.b2.listFileVersions({
|
|
376
|
+
bucketId: this.bucket,
|
|
377
|
+
startFileName: normPath,
|
|
378
|
+
prefix: normPath,
|
|
379
|
+
delimiter: '',
|
|
380
|
+
maxFileCount: 1000
|
|
381
|
+
})
|
|
382
|
+
while (versionsResponse.data.nextFileName &&
|
|
383
|
+
versionsResponse.data.nextFileId &&
|
|
384
|
+
versionsResponse.data.nextFileName.startsWith(normPath)) {
|
|
385
|
+
logger.debug(`b2.remove(${path}) removing a batch of files then asking for more`)
|
|
386
|
+
removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
|
|
387
|
+
logger.debug(`b2.remove(${path})`)
|
|
388
|
+
versionsResponse = await this.b2.listFileVersions({
|
|
389
|
+
bucketId: this.bucket,
|
|
390
|
+
startFileName: versionsResponse.data.nextFileName,
|
|
391
|
+
startFileId: versionsResponse.data.nextFileId,
|
|
392
|
+
prefix: normPath,
|
|
393
|
+
delimiter: ''
|
|
394
|
+
})
|
|
395
|
+
}
|
|
396
|
+
logger.debug(`b2.remove(${path}) removing last batch of files`)
|
|
397
|
+
removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
|
|
398
|
+
} catch (e) {
|
|
399
|
+
const message = `b2.remove(${path}) error listing/deleting: ${e}`
|
|
400
|
+
if (quiet) {
|
|
401
|
+
logger.warn(message)
|
|
402
|
+
} else {
|
|
403
|
+
throw new MobilettoError(message)
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
if (Object.keys(errors).length > 0) {
|
|
407
|
+
if (quiet) {
|
|
408
|
+
logger.warn(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
|
|
409
|
+
} else {
|
|
410
|
+
throw new MobilettoError(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return removed
|
|
414
|
+
|
|
415
|
+
} else {
|
|
416
|
+
throw new MobilettoError(`b2.remove(${path}) cannot remove file of type: ${meta.type}`)
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
function storageClient (key, secret, opts) {
|
|
422
|
+
if (!key || !secret || !opts || !opts.bucket) {
|
|
423
|
+
throw new MobilettoError('b2.storageClient: key, secret, and opts.bucket are required')
|
|
424
|
+
}
|
|
425
|
+
return new StorageClient(key, secret, opts)
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
module.exports = { storageClient }
|