mobiletto 1.1.39 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -27
- package/drivers/local/index.js +1 -1
- package/index.js +7 -862
- package/package.json +14 -19
- package/test/api.spec.js +19 -9
- package/test/crypt.spec.js +1 -1
package/README.md
CHANGED
|
@@ -9,6 +9,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
|
|
|
9
9
|
* [Mobiletto CLI](#mobiletto-cli)
|
|
10
10
|
* [Source](#Source)
|
|
11
11
|
* [Installation](#Installation)
|
|
12
|
+
* [Minimal install](#Minimal-install)
|
|
12
13
|
* [Support and Funding](#Support-and-Funding)
|
|
13
14
|
* [Basic usage](#Basic-usage)
|
|
14
15
|
* [Metadata](#Metadata)
|
|
@@ -91,9 +92,42 @@ Current Mobiletto storage drivers:
|
|
|
91
92
|
* `s3`: Amazon S3
|
|
92
93
|
* `b2`: Backblaze B2
|
|
93
94
|
* `local`: local filesystem
|
|
95
|
+
* `indexeddb`: IndexedDB storage
|
|
94
96
|
|
|
95
97
|
*Contributions to support more cloud storage providers are very welcome!*
|
|
96
98
|
|
|
99
|
+
## Quick Start
|
|
100
|
+
A short example using the mobiletto `s3` driver.
|
|
101
|
+
|
|
102
|
+
This code would run the same with any other mobiletto storage driver.
|
|
103
|
+
|
|
104
|
+
const storage = require('mobiletto')
|
|
105
|
+
const bucket = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
|
|
106
|
+
|
|
107
|
+
// list objects: returns array of metadata objects
|
|
108
|
+
const listing = await bucket.list()
|
|
109
|
+
const dirList = await bucket.list('some/dir/')
|
|
110
|
+
const everything = await bucket.list('', {recursive: true})
|
|
111
|
+
|
|
112
|
+
// write an entire file
|
|
113
|
+
let bytesWritten = await bucket.writeFile('some/path', someBufferOfData)
|
|
114
|
+
|
|
115
|
+
// write a file from a stream/generator
|
|
116
|
+
bytesWritten = await bucket.write('some/path', streamOrGenerator)
|
|
117
|
+
|
|
118
|
+
// read an entire file
|
|
119
|
+
// returns null if an exception would otherwise be thrown
|
|
120
|
+
const bufferOrNull = await bucket.safeReadFile('some/path')
|
|
121
|
+
|
|
122
|
+
// stream-read a file, passing data to callback
|
|
123
|
+
const bytesRead = await bucket.read('some/path', (chunk) => { ...do something with chunk... } )
|
|
124
|
+
|
|
125
|
+
// remove a file, returns the path removed
|
|
126
|
+
let removed = await bucket.remove('some/path') // removed is a string
|
|
127
|
+
|
|
128
|
+
// remove a directory, returns array of paths removed
|
|
129
|
+
removed = await bucket.remove('some/directory', {recursive: true}) // removed is now an array!
|
|
130
|
+
|
|
97
131
|
## mobiletto-cli
|
|
98
132
|
Mobiletto is intended to be used as a library by other JavaScript code.
|
|
99
133
|
|
|
@@ -118,44 +152,44 @@ If you really want the README files in every language, install the full version:
|
|
|
118
152
|
npm install mobiletto
|
|
119
153
|
yarn add mobiletto
|
|
120
154
|
|
|
121
|
-
|
|
122
|
-
|
|
155
|
+
### Minimal install
|
|
156
|
+
The `mobiletto` and `mobiletto-lite` packages include support for all mobiletto storage drivers.
|
|
123
157
|
|
|
124
|
-
|
|
158
|
+
If you know the storage driver(s) that you will need ahead of time, you can lighten your app's
|
|
159
|
+
footprint by installing the `mobiletto-base` package, and then one or more of the `mobiletto-driver-` packages:
|
|
125
160
|
|
|
126
|
-
|
|
127
|
-
const bucket = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
|
|
161
|
+
#### Minimal installation with npm
|
|
128
162
|
|
|
129
|
-
|
|
130
|
-
const listing = await bucket.list()
|
|
131
|
-
const dirList = await bucket.list('some/dir/')
|
|
132
|
-
const everything = await bucket.list('', {recursive: true})
|
|
133
|
-
|
|
134
|
-
// write an entire file
|
|
135
|
-
let bytesWritten = await bucket.writeFile('some/path', someBufferOfData)
|
|
136
|
-
|
|
137
|
-
// write a file from a stream/generator
|
|
138
|
-
bytesWritten = await bucket.write('some/path', streamOrGenerator)
|
|
163
|
+
npm install mobiletto-base # install driver-less mobiletto
|
|
139
164
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
165
|
+
npm install mobiletto-driver-s3 # install Amazon S3 driver
|
|
166
|
+
npm install mobiletto-driver-b2 # install Backblaze B2 driver
|
|
167
|
+
npm install mobiletto-driver-local # install local filesystem driver
|
|
168
|
+
npm install mobiletto-driver-indexeddb # install IndexedDB driver
|
|
143
169
|
|
|
144
|
-
|
|
145
|
-
const bytesRead = await bucket.read('some/path', (chunk) => { ...do something with chunk... } )
|
|
170
|
+
#### Minimal installation with yarn
|
|
146
171
|
|
|
147
|
-
|
|
148
|
-
let removed = await bucket.remove('some/path') // removed is a string
|
|
172
|
+
yarn add mobiletto-base # install driver-less mobiletto
|
|
149
173
|
|
|
150
|
-
|
|
151
|
-
|
|
174
|
+
yarn add mobiletto-driver-s3 # install Amazon S3 driver
|
|
175
|
+
yarn add mobiletto-driver-b2 # install Backblaze B2 driver
|
|
176
|
+
yarn add mobiletto-driver-local # install local filesystem driver
|
|
177
|
+
yarn add mobiletto-driver-indexeddb # install IndexedDB driver
|
|
152
178
|
|
|
153
179
|
----
|
|
154
180
|
## Basic usage
|
|
155
181
|
A much more extensive example, showing most of the features offered:
|
|
156
182
|
|
|
183
|
+
// When using mobiletto or mobiletto-lite, all drivers are included
|
|
157
184
|
const { mobiletto } = require('mobiletto')
|
|
158
185
|
|
|
186
|
+
// When using mobiletto-base, use registerDriver to load a storage driver before connecting
|
|
187
|
+
const { registerDriver, mobiletto } = = require('mobiletto-base')
|
|
188
|
+
registerDriver('s3', require('mobiletto-driver-s3'))
|
|
189
|
+
registerDriver('b2', require('mobiletto-driver-b2'))
|
|
190
|
+
registerDriver('local', require('mobiletto-driver-local'))
|
|
191
|
+
registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
|
|
192
|
+
|
|
159
193
|
// General usage
|
|
160
194
|
const api = await mobiletto(driverName, key, secret, opts)
|
|
161
195
|
|
|
@@ -190,6 +224,13 @@ A much more extensive example, showing most of the features offered:
|
|
|
190
224
|
// * partSize: optional, large files will be split into chunks of this size when uploading
|
|
191
225
|
const b3 = await mobiletto('b2', b2_key_id, b2_app_key, {bucket: 'bk', partSize: 10000000})
|
|
192
226
|
|
|
227
|
+
// To use 'indexeddb' driver:
|
|
228
|
+
// * key: IndexedDB database name
|
|
229
|
+
// * secret: ignored, can be null
|
|
230
|
+
// * opts object:
|
|
231
|
+
// * indexedDB: required, the indexedDB object (an instance of IDBFactory)
|
|
232
|
+
const idb = await mobiletto('indexeddb', 'my_db', null, { indexedDB })
|
|
233
|
+
|
|
193
234
|
// List files
|
|
194
235
|
api.list() // --> returns an array of metadata objects
|
|
195
236
|
|
|
@@ -307,7 +348,7 @@ You can also set per-connection caching with the `opts.redisConfig` object:
|
|
|
307
348
|
To disable: pass `enabled: false` in your `opts.redisConfig` object when you establish your connection.
|
|
308
349
|
|
|
309
350
|
As discussed below, disabling caching will have an adverse effect on performance and incur more requests
|
|
310
|
-
to storage
|
|
351
|
+
to storage than you really need to.
|
|
311
352
|
|
|
312
353
|
### Caching guidance
|
|
313
354
|
**Encrypted storage**: reading/writing encrypted storage is only a little slower than normal,
|
|
@@ -431,11 +472,11 @@ The object that the storageClient function returns must define these functions:
|
|
|
431
472
|
// Read metadata for a path
|
|
432
473
|
async metadata (path)
|
|
433
474
|
|
|
434
|
-
// Read a file
|
|
475
|
+
// Read a file, return bytes read
|
|
435
476
|
// callback receives a chunk of data. endCallback is called at end-of-stream
|
|
436
477
|
async read (path, callback, endCallback = null)
|
|
437
478
|
|
|
438
|
-
// Write a file
|
|
479
|
+
// Write a file, return bytes written
|
|
439
480
|
// driver must be able to handle a generator or a stream
|
|
440
481
|
async write (path, generatorOrReadableStream)
|
|
441
482
|
|
package/drivers/local/index.js
CHANGED
|
@@ -134,7 +134,7 @@ class StorageClient {
|
|
|
134
134
|
if (files.length > 0) {
|
|
135
135
|
return files
|
|
136
136
|
} else {
|
|
137
|
-
logger.warn(`dirFiles (try-file) not found: ${
|
|
137
|
+
logger.warn(`dirFiles (try-file) not found: ${norm}`)
|
|
138
138
|
throw this.ioError(e, norm, 'dirFiles')
|
|
139
139
|
}
|
|
140
140
|
}
|
package/index.js
CHANGED
|
@@ -1,865 +1,10 @@
|
|
|
1
|
-
const
|
|
2
|
-
const { Readable, Transform } = require('stream')
|
|
3
|
-
const Queue = require('bull')
|
|
4
|
-
const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
|
|
5
|
-
const { getRedis, teardown } = require('./util/redis')
|
|
6
|
-
const { basename, dirname } = require('path')
|
|
7
|
-
const shasum = require('shasum')
|
|
8
|
-
const randomstring = require('randomstring')
|
|
1
|
+
const { registerDriver } = require('mobiletto-base')
|
|
9
2
|
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
16
|
-
const ENC_PAD_SEP = ' ~ '
|
|
17
|
-
|
|
18
|
-
const initMobilettoTempDir = () => {
|
|
19
|
-
const dir = process.env.MOBILETTO_TMP || process.env.TMPDIR || '/tmp'
|
|
20
|
-
return dir.endsWith('/') ? dir.substring(0, dir.length - 1) : dir
|
|
21
|
-
}
|
|
22
|
-
const MOBILETTO_TMP = initMobilettoTempDir()
|
|
23
|
-
|
|
24
|
-
const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
|
|
25
|
-
const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
|
|
26
|
-
const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
|
|
27
|
-
|
|
28
|
-
// adapted from https://stackoverflow.com/a/27724419
|
|
29
|
-
function MobilettoError (message, err) {
|
|
30
|
-
this.message = `${message}: ${err ? err : ''}`
|
|
31
|
-
// noinspection JSUnusedGlobalSymbols
|
|
32
|
-
this.err = err
|
|
33
|
-
// Use V8's native method if available, otherwise fallback
|
|
34
|
-
if ('captureStackTrace' in Error) {
|
|
35
|
-
Error.captureStackTrace(this, TypeError)
|
|
36
|
-
} else {
|
|
37
|
-
// noinspection JSUnusedGlobalSymbols
|
|
38
|
-
this.stack = (new Error(this.message)).stack
|
|
39
|
-
}
|
|
40
|
-
MobilettoError.prototype.toString = () => JSON.stringify(this)
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
// adapted from https://stackoverflow.com/a/27724419
|
|
44
|
-
function MobilettoNotFoundError (message) {
|
|
45
|
-
this.message = `${message}`
|
|
46
|
-
// Use V8's native method if available, otherwise fallback
|
|
47
|
-
if ('captureStackTrace' in Error) {
|
|
48
|
-
Error.captureStackTrace(this, TypeError)
|
|
49
|
-
} else {
|
|
50
|
-
// noinspection JSUnusedGlobalSymbols
|
|
51
|
-
this.stack = (new Error(this.message)).stack
|
|
52
|
-
}
|
|
53
|
-
MobilettoNotFoundError.prototype.toString = () => JSON.stringify(this)
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
57
|
-
|
|
58
|
-
const isAsyncGenerator = func => func[Symbol.toStringTag] === 'AsyncGenerator'
|
|
59
|
-
const isReadable = thing => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
|
|
60
|
-
|
|
61
|
-
const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
|
|
62
|
-
|
|
63
|
-
class AwaitableLRU {
|
|
64
|
-
lru
|
|
65
|
-
constructor(size) {
|
|
66
|
-
this.lru = new LRU({ max: size })
|
|
67
|
-
}
|
|
68
|
-
get = key => Promise.resolve(this.lru.get(key))
|
|
69
|
-
set = (key, value) => Promise.resolve(this.lru.set(key, value))
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
const CACHE_FUNCTIONS = {
|
|
73
|
-
redis: client => () => {
|
|
74
|
-
if (typeof client.cache !== 'undefined') return client.cache
|
|
75
|
-
const redisConfig = client.redisConfig || {}
|
|
76
|
-
const enabled = redisConfig.enabled !== false
|
|
77
|
-
if (!enabled) {
|
|
78
|
-
logger.info(`redis: client.redisConfig.enabled === false, disabling cache`)
|
|
79
|
-
client.cache = null
|
|
80
|
-
return client.cache
|
|
81
|
-
}
|
|
82
|
-
const host = redisConfig.host || REDIS_HOST
|
|
83
|
-
const port = redisConfig.port || REDIS_PORT
|
|
84
|
-
const prefix = redisConfig.prefix || REDIS_PREFIX
|
|
85
|
-
if (!client.id) {
|
|
86
|
-
logger.warn(`redis: all nameless connections will share one cache`)
|
|
87
|
-
client.cache = getRedis('~nameless~', host, port, prefix)
|
|
88
|
-
} else {
|
|
89
|
-
client.cache = getRedis(client.id, host, port, prefix)
|
|
90
|
-
}
|
|
91
|
-
return client.cache
|
|
92
|
-
},
|
|
93
|
-
scopedCache: client => (cacheName, size = 100) => {
|
|
94
|
-
const redis = client.redis()
|
|
95
|
-
return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
|
|
96
|
-
},
|
|
97
|
-
flush: client => async () => {
|
|
98
|
-
const redis = client.redis()
|
|
99
|
-
return redis ? redis.flush() : true
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
// noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
|
|
104
|
-
const UTILITY_FUNCTIONS = {
|
|
105
|
-
list: client => async (path, opts) => {
|
|
106
|
-
const cache = client.scopedCache('list')
|
|
107
|
-
let cached = cache ? await cache.get(path) : null
|
|
108
|
-
if (cached) {
|
|
109
|
-
if (Array.isArray(cached)) { return cached }
|
|
110
|
-
throw cached
|
|
111
|
-
}
|
|
112
|
-
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
113
|
-
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
114
|
-
if (visitor && typeof visitor !== 'function') {
|
|
115
|
-
throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
|
|
116
|
-
}
|
|
117
|
-
try {
|
|
118
|
-
// noinspection JSUnresolvedFunction
|
|
119
|
-
const results = await client.driver_list(path, recursive, visitor)
|
|
120
|
-
if (cache) {
|
|
121
|
-
cache.set(path, results).then(
|
|
122
|
-
() => { logger.debug(`list(${path}) cached ${results ? results.length : `unknown? ${JSON.stringify(results)}`} results`) },
|
|
123
|
-
(err) => { logger.error(`list(${path}) error: ${err}`) }
|
|
124
|
-
)
|
|
125
|
-
}
|
|
126
|
-
return results
|
|
127
|
-
} catch (e) {
|
|
128
|
-
if (cache && e instanceof MobilettoNotFoundError) {
|
|
129
|
-
cache.set(path, e).then(
|
|
130
|
-
() => { logger.debug(`list(${path}) cached error ${e}`) },
|
|
131
|
-
(err) => { logger.error(`list(${path}) error ${err} caching MobilettoNotFoundError`) }
|
|
132
|
-
)
|
|
133
|
-
}
|
|
134
|
-
throw e
|
|
135
|
-
}
|
|
136
|
-
},
|
|
137
|
-
safeList: client => async (path, opts) => {
|
|
138
|
-
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
139
|
-
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
140
|
-
try {
|
|
141
|
-
// noinspection JSUnresolvedFunction
|
|
142
|
-
return await client.driver_list(path, recursive, visitor)
|
|
143
|
-
} catch (e) {
|
|
144
|
-
if (e instanceof MobilettoNotFoundError) {
|
|
145
|
-
return []
|
|
146
|
-
}
|
|
147
|
-
throw e
|
|
148
|
-
}
|
|
149
|
-
},
|
|
150
|
-
metadata: client => async (path) => {
|
|
151
|
-
const cache = client.scopedCache('metadata')
|
|
152
|
-
const cached = cache ? await cache.get(path) : null
|
|
153
|
-
if (cached) {
|
|
154
|
-
return cached
|
|
155
|
-
}
|
|
156
|
-
// noinspection JSUnresolvedFunction
|
|
157
|
-
const meta = await client.driver_metadata(path);
|
|
158
|
-
if (cache) {
|
|
159
|
-
cache.set(path, meta).then(
|
|
160
|
-
() => { logger.debug(`metadata(${path}) cached meta = ${JSON.stringify(meta)}`) },
|
|
161
|
-
(err) => { logger.error(`metadata(${path}) error: ${err}`) }
|
|
162
|
-
)
|
|
163
|
-
}
|
|
164
|
-
return meta
|
|
165
|
-
},
|
|
166
|
-
safeMetadata: client => async (path) => {
|
|
167
|
-
try {
|
|
168
|
-
return await client.metadata(path)
|
|
169
|
-
} catch (e) {
|
|
170
|
-
if (e instanceof MobilettoNotFoundError) {
|
|
171
|
-
return null
|
|
172
|
-
}
|
|
173
|
-
throw e
|
|
174
|
-
}
|
|
175
|
-
},
|
|
176
|
-
remove: client => async (path, opts) => {
|
|
177
|
-
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
178
|
-
// noinspection JSUnresolvedVariable
|
|
179
|
-
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
180
|
-
// noinspection JSUnresolvedFunction
|
|
181
|
-
const result = await client.driver_remove(path, recursive, quiet)
|
|
182
|
-
await client.flush()
|
|
183
|
-
return result
|
|
184
|
-
},
|
|
185
|
-
readFile: client => async (path) => {
|
|
186
|
-
const cache = client.scopedCache('readFile')
|
|
187
|
-
const cached = cache ? await cache.get(path) : null
|
|
188
|
-
if (cached) {
|
|
189
|
-
return Buffer.from(cached, 'base64')
|
|
190
|
-
}
|
|
191
|
-
const chunks = []
|
|
192
|
-
await client.read(path, reader(chunks))
|
|
193
|
-
const data = Buffer.concat(chunks);
|
|
194
|
-
if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
|
|
195
|
-
const setVal = cache.set(path, data.toString('base64')) // redis returns promise, LRU does not
|
|
196
|
-
if (setVal instanceof Promise) {
|
|
197
|
-
setVal.then(
|
|
198
|
-
() => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
|
|
199
|
-
(err) => { logger.error(`readFile(${path}) error: ${err}`) }
|
|
200
|
-
)
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return data
|
|
204
|
-
},
|
|
205
|
-
safeReadFile: client => async (path) => {
|
|
206
|
-
try {
|
|
207
|
-
return await client.readFile(path)
|
|
208
|
-
} catch (e) {
|
|
209
|
-
logger.info(`safeReadFile(${path}) ${e}`)
|
|
210
|
-
return null
|
|
211
|
-
}
|
|
212
|
-
},
|
|
213
|
-
write: client => async (path, data) => {
|
|
214
|
-
logger.debug(`util.write(${path}) starting ...`)
|
|
215
|
-
const p = path.startsWith('/') ? path.substring(1) : path
|
|
216
|
-
if (p !== path) {
|
|
217
|
-
logger.debug(`util.write(${path}) removed leading ${this.delimiter}`)
|
|
218
|
-
}
|
|
219
|
-
// noinspection JSUnresolvedFunction
|
|
220
|
-
const bytesWritten = await client.driver_write(p, data)
|
|
221
|
-
await client.flush()
|
|
222
|
-
logger.debug(`util.write(${p}) wrote ${bytesWritten} bytes`)
|
|
223
|
-
return bytesWritten
|
|
224
|
-
},
|
|
225
|
-
writeFile: client => async (path, data) => {
|
|
226
|
-
const readFunc = function* () { yield data }
|
|
227
|
-
return await client.write(path, readFunc())
|
|
228
|
-
},
|
|
229
|
-
mirror: client => async (source, clientPath = '', sourcePath = '') => {
|
|
230
|
-
logger.info(`mirror: starting, sourcePath=${sourcePath} -> clientPath=${clientPath}`)
|
|
231
|
-
const results = {
|
|
232
|
-
success: 0,
|
|
233
|
-
errors: 0
|
|
234
|
-
}
|
|
235
|
-
const visitor = async (obj) => {
|
|
236
|
-
if (obj.type && obj.type === M_FILE) {
|
|
237
|
-
logger.verbose(`mirror: mirroring file: ${obj.name}`)
|
|
238
|
-
const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
|
|
239
|
-
logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
|
|
240
|
-
const destName = obj.name.startsWith(sourcePath)
|
|
241
|
-
? obj.name.substring(sourcePath.length)
|
|
242
|
-
: obj.name
|
|
243
|
-
const destFullPath = (clientPath.endsWith('/') ? clientPath : clientPath + '/') +
|
|
244
|
-
(destName.startsWith('/') ? destName.substring(1) : destName)
|
|
245
|
-
try {
|
|
246
|
-
// if dest already exists and is the same size, don't copy it again
|
|
247
|
-
let srcSize = null
|
|
248
|
-
if (obj.size) {
|
|
249
|
-
srcSize = obj.size
|
|
250
|
-
} else {
|
|
251
|
-
const srcMeta = source.safeMetadata(obj.name)
|
|
252
|
-
if (srcMeta && srcMeta.size) {
|
|
253
|
-
srcSize = srcMeta.size
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
// only continue if we could determine the source size
|
|
257
|
-
if (srcSize) {
|
|
258
|
-
const destMeta = client.safeMetadata(destFullPath)
|
|
259
|
-
if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
|
|
260
|
-
logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
|
|
261
|
-
return
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
// write from source -> write to temp file
|
|
266
|
-
const fd = fs.openSync(tempPath, 'wx', 0o0600)
|
|
267
|
-
const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
|
|
268
|
-
await source.read(obj.name, async (chunk) => {
|
|
269
|
-
if (chunk) { writer.write(chunk) }
|
|
270
|
-
}, () => {
|
|
271
|
-
writer.close((err) => {
|
|
272
|
-
if (err) { throw new MobilettoError(`mirror: error closing temp file: ${err}`) }
|
|
273
|
-
logger.debug(`mirror: finished writing ${obj.name} to temp file ${tempPath}`)
|
|
274
|
-
})
|
|
275
|
-
}).then(async () => {
|
|
276
|
-
// read from temp file -> write to mirror
|
|
277
|
-
const fd = fs.openSync(tempPath, 'r')
|
|
278
|
-
const reader = fs.createReadStream(tempPath, {fd})
|
|
279
|
-
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
280
|
-
await client.write(destFullPath, reader)
|
|
281
|
-
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
282
|
-
})
|
|
283
|
-
results.success++
|
|
284
|
-
} catch (e) {
|
|
285
|
-
logger.warn(`mirror: error copying file: ${e}`)
|
|
286
|
-
results.errors++
|
|
287
|
-
} finally {
|
|
288
|
-
logger.verbose(`mirror: file mirrored successfully: ${obj.name}`)
|
|
289
|
-
fs.rmSync(tempPath, {force: true})
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
await mirrorDir(source, sourcePath, visitor)
|
|
294
|
-
return results
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
async function mirrorDir (source, sourcePath, visitor) {
|
|
299
|
-
logger.verbose(`mirrorDir: mirroring dir: ${sourcePath}`)
|
|
300
|
-
const listing = await source.list(sourcePath, {recursive: false, visitor})
|
|
301
|
-
for (const obj of listing) {
|
|
302
|
-
if (obj.type === M_DIR) {
|
|
303
|
-
const dir = obj.name.startsWith(sourcePath) ? obj.name : sourcePath + obj.name
|
|
304
|
-
await mirrorDir(source, dir, visitor)
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
function utilityFunctionConflict (client, func) {
|
|
310
|
-
if (typeof client[func] === 'function') {
|
|
311
|
-
if (typeof client[`driver_${func}`] !== 'undefined') {
|
|
312
|
-
logger.warn(`utilityFunctionConflict: driver_${func} has already been added`)
|
|
313
|
-
return false
|
|
314
|
-
} else {
|
|
315
|
-
client[`driver_${func}`] = client[func] // save original driver function
|
|
316
|
-
return true
|
|
317
|
-
}
|
|
318
|
-
} else if (typeof client[func] !== 'undefined') {
|
|
319
|
-
throw new MobilettoError(`utilityFunctionConflict: client defines a property ${func}, mobiletto function would overwrite`)
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
const addUtilityFunctions = (client, readOnly = false) => {
|
|
324
|
-
addClientFunctions(client, UTILITY_FUNCTIONS, utilityFunctionConflict)
|
|
325
|
-
if (readOnly) {
|
|
326
|
-
for (const writeFunc of ['write', 'remove', 'writeFile']) {
|
|
327
|
-
client[writeFunc] = async () => {
|
|
328
|
-
logger.warn(`${writeFunc} not supported in readOnly mode`)
|
|
329
|
-
return false
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
return client
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
const addCacheFunctions = client => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
|
|
337
|
-
logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
|
|
338
|
-
return false
|
|
339
|
-
})
|
|
340
|
-
|
|
341
|
-
const addClientFunctions = (client, functions, conflictFunc) => {
|
|
342
|
-
for (const func of Object.keys(functions)) {
|
|
343
|
-
let add = true
|
|
344
|
-
if (client[func]) {
|
|
345
|
-
add = conflictFunc ? conflictFunc(client, func) : false
|
|
346
|
-
}
|
|
347
|
-
if (add) {
|
|
348
|
-
client[func] = functions[func](client)
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
return client
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
async function connect (driverPath, key, secret, opts, encryption = null) {
|
|
355
|
-
return await mobiletto(driverPath, key, secret, opts, encryption)
|
|
3
|
+
const registerDrivers = () => {
|
|
4
|
+
registerDriver('local', require('mobiletto-driver-local'))
|
|
5
|
+
registerDriver('s3', require('mobiletto-driver-s3'))
|
|
6
|
+
registerDriver('b2', require('mobiletto-driver-b2'))
|
|
7
|
+
registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
|
|
356
8
|
}
|
|
357
9
|
|
|
358
|
-
|
|
359
|
-
logger.info(`mobiletto: connecting with driver ${driverPath}`)
|
|
360
|
-
const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
|
|
361
|
-
const client = driver.storageClient(key, secret, opts)
|
|
362
|
-
const configValue = await client.testConfig();
|
|
363
|
-
if (!configValue) {
|
|
364
|
-
const message = `mobiletto(${driverPath}) error: test API call failed: ${configValue}`
|
|
365
|
-
logger.error(message)
|
|
366
|
-
throw new MobilettoError(message)
|
|
367
|
-
}
|
|
368
|
-
const readOnly = opts ? !!opts.readOnly : false
|
|
369
|
-
client.redisConfig = opts && opts.redisConfig ? opts.redisConfig : {}
|
|
370
|
-
|
|
371
|
-
const internalIdForDriver = () =>
|
|
372
|
-
driverPath + '_' + shasum(`${key}\n${JSON.stringify(opts)}\n${(encryption ? JSON.stringify(encryption) : '')}`)
|
|
373
|
-
|
|
374
|
-
// If the driver didn't give the client a name, generate a unique internal name
|
|
375
|
-
if (!client.id) {
|
|
376
|
-
client.id = internalIdForDriver()
|
|
377
|
-
}
|
|
378
|
-
if (encryption === null) {
|
|
379
|
-
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
380
|
-
return addUtilityFunctions(addCacheFunctions(client), readOnly)
|
|
381
|
-
}
|
|
382
|
-
const encKey = normalizeKey(encryption.key)
|
|
383
|
-
if (!encKey) {
|
|
384
|
-
const message = `mobiletto(${driverPath}) invalid encryption key`
|
|
385
|
-
logger.error(message)
|
|
386
|
-
throw new MobilettoError(message)
|
|
387
|
-
}
|
|
388
|
-
const iv = normalizeIV(encryption.iv, encKey)
|
|
389
|
-
if (!iv) {
|
|
390
|
-
const message = `mobiletto(${driverPath}) invalid encryption IV`
|
|
391
|
-
logger.error(message)
|
|
392
|
-
throw new MobilettoError(message)
|
|
393
|
-
}
|
|
394
|
-
const enc = {
|
|
395
|
-
key: encKey,
|
|
396
|
-
iv,
|
|
397
|
-
algo: encryption.algo || DEFAULT_CRYPT_ALGO,
|
|
398
|
-
dirLevels: encryption.dirLevels || 4,
|
|
399
|
-
encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
|
|
400
|
-
}
|
|
401
|
-
function encryptPath (path) {
|
|
402
|
-
const encrypted = shasum(enc.key + ' ' + path)
|
|
403
|
-
let newPath = ''
|
|
404
|
-
for (let i = 0; i <= enc.dirLevels; i++) {
|
|
405
|
-
if (newPath.length > 0) newPath += '/'
|
|
406
|
-
newPath += encrypted.substring(i*2, (i*2)+2)
|
|
407
|
-
}
|
|
408
|
-
return newPath + encrypted
|
|
409
|
-
}
|
|
410
|
-
const metaPath = path => encryptPath(path + ' ~ META')
|
|
411
|
-
const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
|
|
412
|
-
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
413
|
-
|
|
414
|
-
const outerClient = addCacheFunctions(client)
|
|
415
|
-
const _metadata = client => async (path) => {
|
|
416
|
-
const cache = outerClient.scopedCache('metadata')
|
|
417
|
-
const cached = cache ? await cache.get(path) : null
|
|
418
|
-
if (cached) {
|
|
419
|
-
return cached
|
|
420
|
-
}
|
|
421
|
-
let metaObj
|
|
422
|
-
try {
|
|
423
|
-
let chunks = []
|
|
424
|
-
await client.read(metaPath(path), reader(chunks))
|
|
425
|
-
metaObj = JSON.parse(decrypt(Buffer.concat(chunks).toString(), enc))
|
|
426
|
-
} catch (e) {
|
|
427
|
-
metaObj = {}
|
|
428
|
-
}
|
|
429
|
-
let meta
|
|
430
|
-
try {
|
|
431
|
-
meta = await client.metadata(encryptPath(path))
|
|
432
|
-
} catch (e) {
|
|
433
|
-
if (e instanceof MobilettoNotFoundError) {
|
|
434
|
-
const dd = direntDir(path);
|
|
435
|
-
try {
|
|
436
|
-
meta = await client.metadata(dd)
|
|
437
|
-
} catch (err) {
|
|
438
|
-
if (err instanceof MobilettoNotFoundError) {
|
|
439
|
-
const contents = await client.list(dd)
|
|
440
|
-
if (Array.isArray(contents) && contents.length > 0) {
|
|
441
|
-
return { name: path, type: M_DIR }
|
|
442
|
-
} else {
|
|
443
|
-
throw err
|
|
444
|
-
}
|
|
445
|
-
} else {
|
|
446
|
-
throw err
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
} else {
|
|
450
|
-
throw e
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
meta.type ||= M_FILE
|
|
454
|
-
meta.name = path // rewrite name back to plaintext name
|
|
455
|
-
const finalMeta = Object.assign({}, meta, metaObj);
|
|
456
|
-
if (cache) {
|
|
457
|
-
cache.set(path, finalMeta).then(
|
|
458
|
-
() => { logger.debug(`_metadata(${path}) cached meta = ${JSON.stringify(finalMeta)}`) },
|
|
459
|
-
(err) => { logger.error(`_metadata(${path}) error: ${err}`) }
|
|
460
|
-
)
|
|
461
|
-
}
|
|
462
|
-
return finalMeta
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
const _singleMeta = async (job) => {
|
|
466
|
-
const dirent = job.data.dirent
|
|
467
|
-
const entry = job.data.entry
|
|
468
|
-
const logPrefix = `_singleMeta(${dirent}/${basename(entry.name)})`
|
|
469
|
-
return new Promise((resolve, reject) => {
|
|
470
|
-
const cipherText = []
|
|
471
|
-
client.read(dirent + '/' + basename(entry.name), reader(cipherText))
|
|
472
|
-
.then((bytesRead) => {
|
|
473
|
-
if (!bytesRead) {
|
|
474
|
-
logger.warn(`${logPrefix} returned no data`)
|
|
475
|
-
resolve({})
|
|
476
|
-
} else {
|
|
477
|
-
const plain = decrypt(cipherText.toString(), enc)
|
|
478
|
-
const realPath = plain.split(ENC_PAD_SEP)[0]
|
|
479
|
-
_metadata(client)(realPath)
|
|
480
|
-
.then(meta => resolve(meta))
|
|
481
|
-
.catch((err) => {
|
|
482
|
-
const message = `${logPrefix} error fetching _metadata: ${err}`
|
|
483
|
-
logger.warn(message)
|
|
484
|
-
reject(message)
|
|
485
|
-
})
|
|
486
|
-
}
|
|
487
|
-
})
|
|
488
|
-
.catch((err) => {
|
|
489
|
-
const message = `${logPrefix} error reading file: ${err}`
|
|
490
|
-
logger.warn(message)
|
|
491
|
-
reject(message)
|
|
492
|
-
})
|
|
493
|
-
})
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
const META_LOAD_QUEUE_NAME = '_loadMetaQueue'
|
|
497
|
-
const META_LOAD_JOB_NAME = '_loadMetaJob'
|
|
498
|
-
const META_LOAD_CONCURRENCY = process.env.MOBILETTO_META_LOAD_CONCURRENCY || 10
|
|
499
|
-
let META_LOAD_QUEUE = null
|
|
500
|
-
const metaLoadQueue = () => {
|
|
501
|
-
if (META_LOAD_QUEUE === null) {
|
|
502
|
-
if (!client.redisConfig) {
|
|
503
|
-
const message = 'metaLoadQueue: redis is required but not enabled'
|
|
504
|
-
logger.error(message)
|
|
505
|
-
throw new MobilettoError(message)
|
|
506
|
-
}
|
|
507
|
-
META_LOAD_QUEUE = new Queue(META_LOAD_QUEUE_NAME, `redis://${client.redisConfig.host || REDIS_HOST}:${client.redisConfig.port || REDIS_PORT}`)
|
|
508
|
-
META_LOAD_QUEUE.mobilettoHandlers = {}
|
|
509
|
-
META_LOAD_QUEUE.mobilettoErrorHandlers = {}
|
|
510
|
-
META_LOAD_QUEUE.process(META_LOAD_JOB_NAME, META_LOAD_CONCURRENCY, _singleMeta)
|
|
511
|
-
META_LOAD_QUEUE.on('completed', function (job, result) {
|
|
512
|
-
logger.info(`${META_LOAD_JOB_NAME} completed with result: ${JSON.stringify(result)}`)
|
|
513
|
-
if (job.data.mobilettoJobID && META_LOAD_QUEUE.mobilettoHandlers[job.data.mobilettoJobID]) {
|
|
514
|
-
META_LOAD_QUEUE.mobilettoHandlers[job.data.mobilettoJobID](result)
|
|
515
|
-
}
|
|
516
|
-
})
|
|
517
|
-
META_LOAD_QUEUE.on('failed', function (job, result) {
|
|
518
|
-
logger.info(`${META_LOAD_JOB_NAME} failed with result: ${JSON.stringify(result)}`)
|
|
519
|
-
if (job.data.mobilettoJobID && META_LOAD_QUEUE.mobilettoErrorHandlers[job.data.mobilettoJobID]) {
|
|
520
|
-
META_LOAD_QUEUE.mobilettoErrorHandlers[job.data.mobilettoJobID](result)
|
|
521
|
-
}
|
|
522
|
-
})
|
|
523
|
-
}
|
|
524
|
-
return META_LOAD_QUEUE
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
const _loadMeta = async (dirent, entries) => {
|
|
528
|
-
const files = []
|
|
529
|
-
|
|
530
|
-
const waitForFiles = (resolve) => {
|
|
531
|
-
if (files.length === entries.length) {
|
|
532
|
-
resolve(files)
|
|
533
|
-
} else {
|
|
534
|
-
setTimeout(() => waitForFiles(resolve), 1000)
|
|
535
|
-
}
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
const mobilettoJobID = randomstring.generate(10)
|
|
539
|
-
const mq = metaLoadQueue()
|
|
540
|
-
mq.mobilettoHandlers[mobilettoJobID] = meta => files.push(meta)
|
|
541
|
-
mq.mobilettoErrorHandlers[mobilettoJobID] = err => files.push(err)
|
|
542
|
-
for (const entry of entries) {
|
|
543
|
-
const job = { mobilettoJobID, dirent, entry }
|
|
544
|
-
mq.add(META_LOAD_JOB_NAME, job)
|
|
545
|
-
}
|
|
546
|
-
await new Promise(resolve => waitForFiles(resolve))
|
|
547
|
-
delete mq.mobilettoHandlers[mobilettoJobID]
|
|
548
|
-
delete mq.mobilettoErrorHandlers[mobilettoJobID]
|
|
549
|
-
return files
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
async function removeDirentFile(path) {
|
|
553
|
-
const df = direntFile(direntDir(dirname(path)), path);
|
|
554
|
-
const recursive = false
|
|
555
|
-
const quiet = true
|
|
556
|
-
|
|
557
|
-
logger.debug(`removeDirentFile(${path}) removing df=${df}`)
|
|
558
|
-
await client.remove(df, recursive, quiet)
|
|
559
|
-
|
|
560
|
-
logger.debug(`removeDirentFile(${path}) removing encryptPath(path)=${encryptPath(path)}`)
|
|
561
|
-
await client.remove(encryptPath(path), recursive, quiet)
|
|
562
|
-
|
|
563
|
-
logger.debug(`removeDirentFile(${path}) removing metaPath(path)=${metaPath(path)}`)
|
|
564
|
-
await client.remove(metaPath(path), recursive, quiet)
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
function stringGenerator (value, enc) {
|
|
568
|
-
return function* () {
|
|
569
|
-
yield encrypt(value, enc)
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
// noinspection JSUnusedGlobalSymbols
|
|
574
|
-
const encClient = {
|
|
575
|
-
id: internalIdForDriver(),
|
|
576
|
-
redisConfig: client.redisConfig,
|
|
577
|
-
list: async (pth = '', recursive, visitor) => {
|
|
578
|
-
const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
|
|
579
|
-
const dirent = direntDir(p)
|
|
580
|
-
let entries
|
|
581
|
-
|
|
582
|
-
const cacheKey = `${p} ~ ${recursive}`
|
|
583
|
-
const cache = visitor ? null : client.scopedCache('enc_list')
|
|
584
|
-
const cached = cache && await cache.get(cacheKey)
|
|
585
|
-
|
|
586
|
-
function cacheAndReturn (thing) {
|
|
587
|
-
if (cache) {
|
|
588
|
-
cache.set(cacheKey, thing).then(
|
|
589
|
-
() => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
|
|
590
|
-
(err) => { logger.error(`enc_list(${p}) error: ${err}`) }
|
|
591
|
-
)
|
|
592
|
-
}
|
|
593
|
-
return thing
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
async function tryParentDirForSingleFile (p, visitor, e) {
|
|
597
|
-
// it might be a single file, try listing the parent dir
|
|
598
|
-
const parentDirent = direntDir(dirname(p))
|
|
599
|
-
entries = await client.list(parentDirent)
|
|
600
|
-
const objects = await _loadMeta(parentDirent, entries)
|
|
601
|
-
const found = objects.find(o => o.name === p)
|
|
602
|
-
if (found) {
|
|
603
|
-
if (visitor) {
|
|
604
|
-
await visitor(found)
|
|
605
|
-
}
|
|
606
|
-
logger.debug(`tryParentDirForSingleFile(${p}) found ${found.name}`)
|
|
607
|
-
return cacheAndReturn([found])
|
|
608
|
-
}
|
|
609
|
-
logger.debug(`tryParentDirForSingleFile(${p}) nothing found! e=${e}`)
|
|
610
|
-
throw e === null ? new MobilettoNotFoundError(p) : e
|
|
611
|
-
}
|
|
612
|
-
|
|
613
|
-
if (cached) {
|
|
614
|
-
entries = cached
|
|
615
|
-
} else {
|
|
616
|
-
try {
|
|
617
|
-
entries = await _loadMeta(dirent, await client.list(dirent))
|
|
618
|
-
} catch (e) {
|
|
619
|
-
if (e instanceof MobilettoNotFoundError) {
|
|
620
|
-
if (p.includes('/')) {
|
|
621
|
-
return await tryParentDirForSingleFile(p, visitor, e)
|
|
622
|
-
}
|
|
623
|
-
throw e
|
|
624
|
-
} else {
|
|
625
|
-
throw new MobilettoError(`encClient.list(${p}) ${e}`, e)
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
if (recursive) {
|
|
629
|
-
const dirs = entries.filter(obj => obj.type === M_DIR)
|
|
630
|
-
while (dirs.length > 0) {
|
|
631
|
-
const dir = dirs.shift()
|
|
632
|
-
const subdir = direntDir(dir.name)
|
|
633
|
-
const subdirListing = await client.list(subdir)
|
|
634
|
-
if (subdirListing && subdirListing.length > 0) {
|
|
635
|
-
const subdirEntries = await _loadMeta(subdir, subdirListing)
|
|
636
|
-
if (visitor) {
|
|
637
|
-
for (const obj of subdirEntries) {
|
|
638
|
-
await visitor(obj)
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
entries.push(...subdirEntries)
|
|
642
|
-
const moreDirs = subdirEntries.filter(obj => obj.type === M_DIR)
|
|
643
|
-
dirs.unshift(...moreDirs)
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
if (cache) {
|
|
647
|
-
cache.set(cacheKey, entries).then(
|
|
648
|
-
() => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
|
|
649
|
-
(err) => { logger.error(`enc_list(${p}) error: ${err}`) }
|
|
650
|
-
)
|
|
651
|
-
}
|
|
652
|
-
if (!entries || entries.length === 0) {
|
|
653
|
-
return []
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
if (entries.length === 0 && p.includes('/')) {
|
|
658
|
-
return await tryParentDirForSingleFile(p, visitor, null)
|
|
659
|
-
}
|
|
660
|
-
if (cache) {
|
|
661
|
-
cache.set(cacheKey, entries).then(
|
|
662
|
-
() => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
|
|
663
|
-
(err) => { logger.error(`enc_list(${p}) error: ${err}`) }
|
|
664
|
-
)
|
|
665
|
-
}
|
|
666
|
-
if (!entries || entries.length === 0) {
|
|
667
|
-
return []
|
|
668
|
-
}
|
|
669
|
-
if (visitor) {
|
|
670
|
-
for (const obj of entries) {
|
|
671
|
-
await visitor(obj)
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
return entries
|
|
675
|
-
},
|
|
676
|
-
metadata: _metadata(client),
|
|
677
|
-
read: async (path, callback) => {
|
|
678
|
-
const realPath = encryptPath(path)
|
|
679
|
-
const cipher = crypt.getDecipher(enc)
|
|
680
|
-
return await client.read(realPath,
|
|
681
|
-
(chunk) => {
|
|
682
|
-
return callback(cipher.update(chunk))
|
|
683
|
-
}, () => {
|
|
684
|
-
callback(cipher.final())
|
|
685
|
-
})
|
|
686
|
-
},
|
|
687
|
-
write: async (path, readFunc) => {
|
|
688
|
-
// if encrypting paths, write dirent file(s) for all parent directories
|
|
689
|
-
let p = path
|
|
690
|
-
while (true) {
|
|
691
|
-
const direntGenerator = stringGenerator(p + enc.encPathPadding(), enc)
|
|
692
|
-
const dir = direntDir(dirname(p))
|
|
693
|
-
const df = direntFile(dir, p);
|
|
694
|
-
if (!(await client.write(df, direntGenerator()))) {
|
|
695
|
-
throw new MobilettoError('write: error writing dirent file')
|
|
696
|
-
}
|
|
697
|
-
p = dirname(p)
|
|
698
|
-
if (p === '.' || p === '') {
|
|
699
|
-
break
|
|
700
|
-
}
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
const cipher = crypt.getCipher(enc)
|
|
704
|
-
const realPath = encryptPath(path)
|
|
705
|
-
let generatorBytes = 0
|
|
706
|
-
const createEncryptStream = (input) =>
|
|
707
|
-
input.pipe(new Transform({
|
|
708
|
-
transform(chunk, encoding, callback) {
|
|
709
|
-
generatorBytes += chunk.length
|
|
710
|
-
this.push(cipher.update(chunk))
|
|
711
|
-
callback()
|
|
712
|
-
},
|
|
713
|
-
flush(callback) {
|
|
714
|
-
this.push(cipher.final())
|
|
715
|
-
callback()
|
|
716
|
-
}
|
|
717
|
-
}))
|
|
718
|
-
|
|
719
|
-
// If we were passed a Readable, call write with a transform method
|
|
720
|
-
if (isReadable(readFunc)) {
|
|
721
|
-
await client.write(realPath, createEncryptStream(readFunc))
|
|
722
|
-
} else {
|
|
723
|
-
// If we were passed a generator, wrap it with another generator that encrypts
|
|
724
|
-
function* cryptGenerator(plaintextGenerator) {
|
|
725
|
-
let chunk = plaintextGenerator.next().value
|
|
726
|
-
while (chunk) {
|
|
727
|
-
generatorBytes += chunk.length
|
|
728
|
-
yield cipher.update(chunk)
|
|
729
|
-
chunk = plaintextGenerator.next().value
|
|
730
|
-
}
|
|
731
|
-
yield cipher.final()
|
|
732
|
-
}
|
|
733
|
-
await client.write(realPath, cryptGenerator(readFunc))
|
|
734
|
-
}
|
|
735
|
-
// write metadata
|
|
736
|
-
const meta = { name: path, size: generatorBytes, type: M_FILE }
|
|
737
|
-
await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
|
|
738
|
-
return generatorBytes
|
|
739
|
-
},
|
|
740
|
-
|
|
741
|
-
remove: async (path, options) => {
|
|
742
|
-
logger.debug(`enc.remove(${path}) starting`)
|
|
743
|
-
const recursive = options === true || (options && options.recursive) || false
|
|
744
|
-
// const quiet = (options && options.quiet) || false
|
|
745
|
-
if (recursive) {
|
|
746
|
-
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
747
|
-
const removed = []
|
|
748
|
-
async function recRm (path) {
|
|
749
|
-
const dirent = direntDir(path)
|
|
750
|
-
let entries
|
|
751
|
-
try {
|
|
752
|
-
entries = await client.list(dirent)
|
|
753
|
-
} catch (e) {
|
|
754
|
-
if (!(e instanceof MobilettoNotFoundError)) {
|
|
755
|
-
throw e instanceof MobilettoError
|
|
756
|
-
? e
|
|
757
|
-
: new MobilettoError('error listing files for recursive deletion', e)
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
if (entries && entries.length > 0) {
|
|
761
|
-
const files = await _loadMeta(dirent, entries)
|
|
762
|
-
for (const f of files) {
|
|
763
|
-
await recRm(f.name)
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
await removeDirentFile(path)
|
|
767
|
-
removed.push(path)
|
|
768
|
-
}
|
|
769
|
-
await recRm(path)
|
|
770
|
-
return removed
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
// remove the single file/dir
|
|
774
|
-
await removeDirentFile(path)
|
|
775
|
-
|
|
776
|
-
// if we were the last dirent file, then also remove dirent directory, and recursively upwards
|
|
777
|
-
let parent = path
|
|
778
|
-
let dirent = direntDir(parent)
|
|
779
|
-
const df = direntFile(dirent, path)
|
|
780
|
-
await client.remove(df, false, true)
|
|
781
|
-
while (true) {
|
|
782
|
-
try {
|
|
783
|
-
const entries = await client.list(dirent)
|
|
784
|
-
if (entries.length === 0) {
|
|
785
|
-
await removeDirentFile(parent)
|
|
786
|
-
}
|
|
787
|
-
} catch (e) {
|
|
788
|
-
if (!(e instanceof MobilettoNotFoundError)) {
|
|
789
|
-
throw e
|
|
790
|
-
}
|
|
791
|
-
}
|
|
792
|
-
if (parent === '.' || parent === '/') {
|
|
793
|
-
// do not remove dirent for root dir
|
|
794
|
-
break
|
|
795
|
-
}
|
|
796
|
-
parent = dirname(parent)
|
|
797
|
-
dirent = direntDir(parent)
|
|
798
|
-
}
|
|
799
|
-
return path
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
803
|
-
return addUtilityFunctions(addCacheFunctions(encClient), readOnly)
|
|
804
|
-
}
|
|
805
|
-
|
|
806
|
-
async function readStream(stream, callback, endCallback) {
|
|
807
|
-
const counter = {count: 0}
|
|
808
|
-
const streamHandler = stream =>
|
|
809
|
-
new Promise((resolve, reject) => {
|
|
810
|
-
stream.on('data', (data) => {
|
|
811
|
-
counter.count += data ? data.length : 0
|
|
812
|
-
callback(data)
|
|
813
|
-
})
|
|
814
|
-
stream.on('error', reject)
|
|
815
|
-
stream.on('end', () => {
|
|
816
|
-
if (endCallback) {
|
|
817
|
-
endCallback()
|
|
818
|
-
}
|
|
819
|
-
resolve()
|
|
820
|
-
})
|
|
821
|
-
})
|
|
822
|
-
await streamHandler(stream)
|
|
823
|
-
return counter.count
|
|
824
|
-
}
|
|
825
|
-
|
|
826
|
-
function writeStream (stream) {
|
|
827
|
-
return (chunk) => {
|
|
828
|
-
if (chunk) {
|
|
829
|
-
stream.write(chunk, (err) => {
|
|
830
|
-
if (err) {
|
|
831
|
-
logger.error(`writeStream: error writing: ${err}`)
|
|
832
|
-
throw err
|
|
833
|
-
}
|
|
834
|
-
})
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
|
|
839
|
-
function closeStream (stream) {
|
|
840
|
-
return () => stream.close((err) => {
|
|
841
|
-
if (err) {
|
|
842
|
-
logger.error(`closeStream: error closing: ${err}`)
|
|
843
|
-
throw err
|
|
844
|
-
}
|
|
845
|
-
})
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
const M_FILE = 'file'
|
|
849
|
-
const M_DIR = 'dir'
|
|
850
|
-
const M_LINK = 'link'
|
|
851
|
-
const M_SPECIAL = 'special'
|
|
852
|
-
|
|
853
|
-
const setLogLevel = level => setLoggerLevel(level)
|
|
854
|
-
|
|
855
|
-
const closeRedis = async () => { await teardown() }
|
|
856
|
-
|
|
857
|
-
module.exports = {
|
|
858
|
-
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
859
|
-
isAsyncGenerator, isReadable,
|
|
860
|
-
mobiletto, connect,
|
|
861
|
-
MobilettoError, MobilettoNotFoundError,
|
|
862
|
-
setLogLevel,
|
|
863
|
-
readStream, writeStream, closeStream,
|
|
864
|
-
closeRedis
|
|
865
|
-
}
|
|
10
|
+
module.exports = { registerDrivers }
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"keywords": [
|
|
@@ -26,25 +26,20 @@
|
|
|
26
26
|
"test": "mocha --timeout 60000 ./test/*.spec.js"
|
|
27
27
|
},
|
|
28
28
|
"devDependencies": {
|
|
29
|
-
"@babel/core": "^7.
|
|
30
|
-
"@babel/node": "^7.
|
|
31
|
-
"@babel/preset-env": "^7.
|
|
32
|
-
"chai": "^4.3.
|
|
33
|
-
"dotenv": "^16.
|
|
34
|
-
"mocha": "^10.
|
|
35
|
-
"randomstring": "^1.
|
|
29
|
+
"@babel/core": "^7.22.5",
|
|
30
|
+
"@babel/node": "^7.22.5",
|
|
31
|
+
"@babel/preset-env": "^7.22.5",
|
|
32
|
+
"chai": "^4.3.7",
|
|
33
|
+
"dotenv": "^16.3.1",
|
|
34
|
+
"mocha": "^10.2.0",
|
|
35
|
+
"randomstring": "^1.3.0"
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"
|
|
39
|
-
"
|
|
40
|
-
"
|
|
41
|
-
"
|
|
42
|
-
"
|
|
43
|
-
"
|
|
44
|
-
"lru-cache": "^7.14.0",
|
|
45
|
-
"node-fetch-commonjs": "^3.1.1",
|
|
46
|
-
"node-redis-scan": "^1.3.5",
|
|
47
|
-
"shasum": "^1.0.2",
|
|
48
|
-
"winston": "^3.8.1"
|
|
38
|
+
"mobiletto-base": "^1.0.13",
|
|
39
|
+
"mobiletto-common": "^1.0.2",
|
|
40
|
+
"mobiletto-driver-b2": "^1.0.6",
|
|
41
|
+
"mobiletto-driver-local": "^1.0.3",
|
|
42
|
+
"mobiletto-driver-s3": "^1.0.5",
|
|
43
|
+
"mobiletto-driver-indexeddb": "^1.0.1"
|
|
49
44
|
}
|
|
50
45
|
}
|
package/test/api.spec.js
CHANGED
|
@@ -3,25 +3,27 @@ require('dotenv').config()
|
|
|
3
3
|
|
|
4
4
|
const fs = require('fs')
|
|
5
5
|
const { basename, dirname } = require('path')
|
|
6
|
-
|
|
7
6
|
const randomstring = require('randomstring')
|
|
8
7
|
const crypto = require('crypto')
|
|
9
|
-
const redis = require('../util/redis')
|
|
10
|
-
|
|
11
|
-
const { expect, should, assert } = require('chai')
|
|
12
8
|
|
|
13
9
|
const {
|
|
14
|
-
M_DIR, M_FILE,
|
|
15
|
-
|
|
16
|
-
} = require("../index")
|
|
10
|
+
M_DIR, M_FILE, MobilettoNotFoundError, logger
|
|
11
|
+
} = require('mobiletto-common')
|
|
17
12
|
|
|
18
|
-
const {
|
|
13
|
+
const { connect, mobiletto, redis, closeRedis } = require('mobiletto-base')
|
|
14
|
+
|
|
15
|
+
const { registerDrivers } = require("../index")
|
|
16
|
+
registerDrivers()
|
|
17
|
+
|
|
18
|
+
const { expect, should, assert } = require('chai')
|
|
19
19
|
|
|
20
20
|
// chunk size used by generator function, used by driver's 'write' function
|
|
21
21
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
22
22
|
const READ_SZ = 8 * 1024 // xfer data in 8k chunks
|
|
23
23
|
const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
|
|
24
24
|
|
|
25
|
+
const SINGLE_DRIVER = process.env.SINGLE_DRIVER || null
|
|
26
|
+
|
|
25
27
|
DRIVER_CONFIG = {
|
|
26
28
|
local: {
|
|
27
29
|
key: process.env.LOCAL_STORAGE || '/tmp'
|
|
@@ -130,7 +132,7 @@ if (!redisSetup.enabled) continue
|
|
|
130
132
|
|
|
131
133
|
for (const driverName of DRIVER_NAMES) {
|
|
132
134
|
// For testing single drivers
|
|
133
|
-
|
|
135
|
+
if (SINGLE_DRIVER != null && SINGLE_DRIVER !== driverName) continue
|
|
134
136
|
|
|
135
137
|
const driverTest = `${driverName} [${redisSetup.name}]`
|
|
136
138
|
const config = redisSetup.config(DRIVER_CONFIG[driverName])
|
|
@@ -205,6 +207,10 @@ describe(`${driverTest} test`, () => {
|
|
|
205
207
|
|
|
206
208
|
for (const encryption of encryptionTests()) {
|
|
207
209
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
210
|
+
if (encryption && redisSetup.name !== 'redis-enabled' && driverName !== 'local') {
|
|
211
|
+
// encryption without redis is simply too slow, except for local filesystems
|
|
212
|
+
continue
|
|
213
|
+
}
|
|
208
214
|
describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
209
215
|
// describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
|
|
210
216
|
// const encryption = {key: rand(32)}
|
|
@@ -288,6 +294,10 @@ describe(`${driverTest} test`, () => {
|
|
|
288
294
|
|
|
289
295
|
for (const encryption of encryptionTests()) {
|
|
290
296
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
297
|
+
if (encryption && redisSetup.name !== 'redis-enabled') {
|
|
298
|
+
// encryption without redis is simply too slow
|
|
299
|
+
continue
|
|
300
|
+
}
|
|
291
301
|
const mirrorDriver = pickAnotherDriver(driverName)
|
|
292
302
|
// const mirrorDriver = 'local' // for hard-coding the mirror driver
|
|
293
303
|
const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
|