mobiletto 1.0.43 → 1.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -12
- package/drivers/local/index.js +3 -1
- package/drivers/s3/index.js +3 -1
- package/index.js +114 -54
- package/package.json +13 -6
- package/test/index.spec.js +15 -0
- package/util/logger.js +22 -0
- package/util/redis.js +170 -0
package/README.md
CHANGED
|
@@ -53,9 +53,9 @@ This code would run the same if the driver were `local`.
|
|
|
53
53
|
# Contents
|
|
54
54
|
* [Basic usage](#Basic-usage)
|
|
55
55
|
* [Metadata](#Metadata)
|
|
56
|
-
* [Caching](#Caching)
|
|
57
|
-
* [Mirroring](#Caching)
|
|
58
56
|
* [Alternate import style](#Alternate-import-style)
|
|
57
|
+
* [Caching](#Caching)
|
|
58
|
+
* [Mirroring](#Mirroring)
|
|
59
59
|
* [Transparent encryption](#Transparent-encryption)
|
|
60
60
|
* [Key rotation](#Key-rotation)
|
|
61
61
|
* [Driver interface](#Driver-interface)
|
|
@@ -74,8 +74,7 @@ This code would run the same if the driver were `local`.
|
|
|
74
74
|
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
75
75
|
// * fileMode: optional, permissions used when creating new files, default is 0600. can be string or integer
|
|
76
76
|
// * dirMode: optional, permissions used when creating new directories, default is 0700. can be string or integer
|
|
77
|
-
|
|
78
|
-
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700', cacheSize: 100})
|
|
77
|
+
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700'})
|
|
79
78
|
|
|
80
79
|
// To use 's3' driver:
|
|
81
80
|
// * key: AWS access key
|
|
@@ -86,8 +85,7 @@ This code would run the same if the driver were `local`.
|
|
|
86
85
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
87
86
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
88
87
|
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
89
|
-
|
|
90
|
-
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
88
|
+
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
91
89
|
|
|
92
90
|
// List files
|
|
93
91
|
local.list() // --> returns an array of metadata objects
|
|
@@ -180,16 +178,47 @@ should always be present. A subsequent `metadata` command will return all availa
|
|
|
180
178
|
Import the fully-scoped module and use the `connect` function:
|
|
181
179
|
|
|
182
180
|
const storage = require('mobiletto')
|
|
183
|
-
const opts = {bucket: 'bk', region: 'us-east-1'
|
|
181
|
+
const opts = {bucket: 'bk', region: 'us-east-1'}
|
|
184
182
|
const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
|
|
185
183
|
const objectData = await s3.readFile('some/path')
|
|
186
184
|
|
|
187
185
|
## Caching
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
186
|
+
Mobiletto works best with a <a href="https://redis.io">redis</a> cache.
|
|
187
|
+
|
|
188
|
+
Mobiletto will attempt to connect to a redis instance on 127.0.0.1:6379
|
|
189
|
+
|
|
190
|
+
You can override either of these:
|
|
191
|
+
* Set the `MOBILETTO_REDIS_HOST` env var, mobiletto connect here instead of localhost
|
|
192
|
+
* Set the `MOBILETTO_REDIS_PORT` env var, this port will be used
|
|
193
|
+
|
|
194
|
+
Mobiletto will store all of its redis keys with the prefix `_mobiletto__`. You can change this
|
|
195
|
+
by setting the `MOBILETTO_REDIS_PREFIX` env var.
|
|
196
|
+
|
|
197
|
+
### Turn off redis caching
|
|
198
|
+
To disable redis, you can do either of:
|
|
199
|
+
* Not run a redis server on 127.0.0.1:6379 that allows unauthenticated local connections
|
|
200
|
+
* Set `MOBILETTO_REDIS_HOST` to an empty string or anything that won't resolve as a host, for example:
|
|
201
|
+
* `MOBILETTO_REDIS_HOST=`
|
|
202
|
+
* `MOBILETTO_REDIS_HOST=/dev/null`
|
|
203
|
+
* `MOBILETTO_REDIS_HOST=~invalid~hostname~`
|
|
204
|
+
* `MOBILETTO_REDIS_HOST=invalid-hostname` *<-- not this!*
|
|
205
|
+
* **^^^ NO! that last one is a valid hostname** and depending on the name resolution scheme, it might actually be valid and you'll connect to it!
|
|
206
|
+
|
|
207
|
+
As discussed below, disabling caching will have an adverse effect on performance and incur more requests
|
|
208
|
+
to storage that you really need to.
|
|
209
|
+
|
|
210
|
+
### Caching guidance
|
|
211
|
+
**Encrypted storage**: reading/writing encrypted storage is only a little slower than normal,
|
|
212
|
+
but navigating around directories (which some things do) is fairly expensive. Using a redis cache
|
|
213
|
+
will give you a significant performance boost.
|
|
214
|
+
|
|
215
|
+
The default cache is safe, but doesn't perform well if you have a lot of write/remove operations.
|
|
216
|
+
Any write or remove operation invalidates the entire cache, ensuring subsequent reads will see the
|
|
217
|
+
latest changes.
|
|
218
|
+
|
|
219
|
+
### CLI tools
|
|
220
|
+
If you're using a CLI tool like [mobiletto-cli](https://www.npmjs.com/package/mobiletto-cli),
|
|
221
|
+
you'll definitely want the redis cache enabled, as it lasts across invocations of the `mo` command.
|
|
193
222
|
|
|
194
223
|
## Mirroring
|
|
195
224
|
|
package/drivers/local/index.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
const { dirname } = require('path')
|
|
2
2
|
|
|
3
3
|
const {
|
|
4
|
-
M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
|
|
4
|
+
M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
|
|
5
5
|
MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
|
|
6
6
|
} = require('../../index')
|
|
7
7
|
|
|
8
|
+
const { logger } = require('../../util/logger')
|
|
9
|
+
|
|
8
10
|
const fs = require('fs')
|
|
9
11
|
|
|
10
12
|
const DEFAULT_FILE_MODE = '0600'
|
package/drivers/s3/index.js
CHANGED
package/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
2
|
const path = require('path')
|
|
3
3
|
const { Readable, Transform } = require('stream')
|
|
4
|
-
const
|
|
5
|
-
const
|
|
4
|
+
const { logger } = require('./util/logger')
|
|
5
|
+
const redis = require('./util/redis')
|
|
6
6
|
const { basename, dirname } = require('path')
|
|
7
7
|
const shasum = require('shasum')
|
|
8
8
|
const randomstring = require('randomstring')
|
|
@@ -10,21 +10,6 @@ const randomstring = require('randomstring')
|
|
|
10
10
|
const crypt = require('./util/crypt')
|
|
11
11
|
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
|
|
12
12
|
|
|
13
|
-
const logger = winston.createLogger({
|
|
14
|
-
level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
|
|
15
|
-
format: winston.format.simple(),
|
|
16
|
-
transports: process.env.MOBILETTO_LOG_FILE
|
|
17
|
-
? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
|
|
18
|
-
: [ new winston.transports.Console() ]
|
|
19
|
-
})
|
|
20
|
-
|
|
21
|
-
const setLogLevel = (level) => { logger.level = level }
|
|
22
|
-
|
|
23
|
-
const setLogTransports = (transports) => {
|
|
24
|
-
logger.transports.splice(0, logger.transports.length)
|
|
25
|
-
logger.transports.push(...transports)
|
|
26
|
-
}
|
|
27
|
-
|
|
28
13
|
const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
29
14
|
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
30
15
|
const ENC_PAD_SEP = ' ~ '
|
|
@@ -37,12 +22,14 @@ const MOBILETTO_TMP = initMobilettoTempDir()
|
|
|
37
22
|
|
|
38
23
|
// adapted from https://stackoverflow.com/a/27724419
|
|
39
24
|
function MobilettoError (message, err) {
|
|
40
|
-
this.message = `${message}`
|
|
25
|
+
this.message = `${message}: ${err ? err : ''}`
|
|
26
|
+
// noinspection JSUnusedGlobalSymbols
|
|
41
27
|
this.err = err
|
|
42
28
|
// Use V8's native method if available, otherwise fallback
|
|
43
29
|
if ('captureStackTrace' in Error) {
|
|
44
30
|
Error.captureStackTrace(this, TypeError)
|
|
45
31
|
} else {
|
|
32
|
+
// noinspection JSUnusedGlobalSymbols
|
|
46
33
|
this.stack = (new Error(this.message)).stack
|
|
47
34
|
}
|
|
48
35
|
}
|
|
@@ -54,6 +41,7 @@ function MobilettoNotFoundError (message) {
|
|
|
54
41
|
if ('captureStackTrace' in Error) {
|
|
55
42
|
Error.captureStackTrace(this, TypeError)
|
|
56
43
|
} else {
|
|
44
|
+
// noinspection JSUnusedGlobalSymbols
|
|
57
45
|
this.stack = (new Error(this.message)).stack
|
|
58
46
|
}
|
|
59
47
|
}
|
|
@@ -63,16 +51,19 @@ const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
|
63
51
|
const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
|
|
64
52
|
const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
|
|
65
53
|
|
|
54
|
+
const getMetaCache = (size = 100) => redis.scopedCache('meta', size)
|
|
55
|
+
const getListingCache = (size = 100) => redis.scopedCache('listingCache', size)
|
|
56
|
+
|
|
57
|
+
const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
|
|
58
|
+
|
|
59
|
+
// noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
|
|
66
60
|
const UTILITY_FUNCTIONS = {
|
|
67
61
|
list: (client) => async (path, opts) => {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
if (Array.isArray(results)) { return results }
|
|
74
|
-
throw results
|
|
75
|
-
}
|
|
62
|
+
const cache = getListingCache()
|
|
63
|
+
let cached = cache ? await cache.get(path) : null
|
|
64
|
+
if (cached) {
|
|
65
|
+
if (Array.isArray(cached)) { return cached }
|
|
66
|
+
throw cached
|
|
76
67
|
}
|
|
77
68
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
78
69
|
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
@@ -80,14 +71,21 @@ const UTILITY_FUNCTIONS = {
|
|
|
80
71
|
throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
|
|
81
72
|
}
|
|
82
73
|
try {
|
|
74
|
+
// noinspection JSUnresolvedFunction
|
|
83
75
|
const results = await client.driver_list(path, recursive, visitor)
|
|
84
|
-
if (
|
|
85
|
-
|
|
76
|
+
if (cache) {
|
|
77
|
+
cache.set(path, results).then(
|
|
78
|
+
() => { logger.debug(`list(${path}) cached ${results ? results.length : `unknown? ${JSON.stringify(results)}`} results`) },
|
|
79
|
+
(err) => { logger.error(`list(${path}) error: ${err}`) }
|
|
80
|
+
)
|
|
86
81
|
}
|
|
87
82
|
return results
|
|
88
83
|
} catch (e) {
|
|
89
|
-
if (
|
|
90
|
-
|
|
84
|
+
if (cache && e instanceof MobilettoNotFoundError) {
|
|
85
|
+
cache.set(path, e).then(
|
|
86
|
+
() => { logger.debug(`list(${path}) cached error ${e}`) },
|
|
87
|
+
(err) => { logger.error(`list(${path}) error ${err} caching MobilettoNotFoundError`) }
|
|
88
|
+
)
|
|
91
89
|
}
|
|
92
90
|
throw e
|
|
93
91
|
}
|
|
@@ -96,6 +94,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
96
94
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
97
95
|
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
98
96
|
try {
|
|
97
|
+
// noinspection JSUnresolvedFunction
|
|
99
98
|
return await client.driver_list(path, recursive, visitor)
|
|
100
99
|
} catch (e) {
|
|
101
100
|
if (e instanceof MobilettoNotFoundError) {
|
|
@@ -104,6 +103,22 @@ const UTILITY_FUNCTIONS = {
|
|
|
104
103
|
throw e
|
|
105
104
|
}
|
|
106
105
|
},
|
|
106
|
+
metadata: (client) => async (path) => {
|
|
107
|
+
const cache = getMetaCache()
|
|
108
|
+
const cached = cache ? await cache.get(path) : null
|
|
109
|
+
if (cached) {
|
|
110
|
+
return cached
|
|
111
|
+
}
|
|
112
|
+
// noinspection JSUnresolvedFunction
|
|
113
|
+
const meta = await client.driver_metadata(path);
|
|
114
|
+
if (cache) {
|
|
115
|
+
cache.set(path, meta).then(
|
|
116
|
+
() => { logger.debug(`metadata(${path}) cached meta = ${JSON.stringify(meta)}`) },
|
|
117
|
+
(err) => { logger.error(`metadata(${path}) error: ${err}`) }
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
return meta
|
|
121
|
+
},
|
|
107
122
|
safeMetadata: (client) => async (path) => {
|
|
108
123
|
try {
|
|
109
124
|
return await client.metadata(path)
|
|
@@ -117,18 +132,29 @@ const UTILITY_FUNCTIONS = {
|
|
|
117
132
|
remove: (client) => async (path, opts) => {
|
|
118
133
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
119
134
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
135
|
+
// noinspection JSUnresolvedFunction
|
|
120
136
|
const result = await client.driver_remove(path, recursive, quiet)
|
|
121
|
-
|
|
137
|
+
await redis.flushMobiletto()
|
|
122
138
|
return result
|
|
123
139
|
},
|
|
124
140
|
readFile: (client) => async (path) => {
|
|
141
|
+
const cache = redis.scopedCache('readFile')
|
|
142
|
+
const cached = cache ? await cache.get(path) : null
|
|
143
|
+
if (cached) {
|
|
144
|
+
return Buffer.from(cached, 'base64')
|
|
145
|
+
}
|
|
125
146
|
const chunks = []
|
|
126
147
|
await client.read(path, reader(chunks))
|
|
127
|
-
|
|
148
|
+
const data = Buffer.concat(chunks);
|
|
149
|
+
if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
|
|
150
|
+
cache.set(path, data.toString('base64'))
|
|
151
|
+
}
|
|
152
|
+
return data
|
|
128
153
|
},
|
|
129
154
|
write: (client) => async (path, data) => {
|
|
155
|
+
// noinspection JSUnresolvedFunction
|
|
130
156
|
const bytesWritten = await client.driver_write(path, data)
|
|
131
|
-
|
|
157
|
+
await redis.flushMobiletto()
|
|
132
158
|
return bytesWritten
|
|
133
159
|
},
|
|
134
160
|
writeFile: (client) => async (path, data) => {
|
|
@@ -214,8 +240,7 @@ async function mirrorDir (source, sourcePath, visitor) {
|
|
|
214
240
|
}
|
|
215
241
|
}
|
|
216
242
|
|
|
217
|
-
function addUtilityFunctions (client, readOnly = false
|
|
218
|
-
client.listingCache = cacheSize && cacheSize > 0 ? new LRU({ max: cacheSize }) : null
|
|
243
|
+
function addUtilityFunctions (client, readOnly = false) {
|
|
219
244
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
220
245
|
client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
|
|
221
246
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
@@ -245,10 +270,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
245
270
|
throw new MobilettoError(message)
|
|
246
271
|
}
|
|
247
272
|
const readOnly = opts ? !!opts.readOnly : false
|
|
248
|
-
const cacheSize = opts && opts.cacheSize ? opts.cacheSize : null
|
|
249
273
|
if (encryption === null) {
|
|
250
274
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
251
|
-
return addUtilityFunctions(client, readOnly
|
|
275
|
+
return addUtilityFunctions(client, readOnly)
|
|
252
276
|
}
|
|
253
277
|
const encKey = normalizeKey(encryption.key)
|
|
254
278
|
if (!encKey) {
|
|
@@ -283,6 +307,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
283
307
|
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
284
308
|
|
|
285
309
|
const _metadata = (client) => async (path) => {
|
|
310
|
+
const cache = redis.scopedCache('_metadata')
|
|
311
|
+
const cached = cache ? await cache.get(path) : null
|
|
312
|
+
if (cached) {
|
|
313
|
+
return cached
|
|
314
|
+
}
|
|
286
315
|
let metaObj
|
|
287
316
|
try {
|
|
288
317
|
let chunks = []
|
|
@@ -317,7 +346,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
317
346
|
}
|
|
318
347
|
meta.type ||= M_FILE
|
|
319
348
|
meta.name = path // rewrite name back to plaintext name
|
|
320
|
-
|
|
349
|
+
const finalMeta = Object.assign({}, meta, metaObj);
|
|
350
|
+
if (cache) {
|
|
351
|
+
cache.set(path, finalMeta).then(
|
|
352
|
+
() => { logger.debug(`_metadata(${path}) cached meta = ${JSON.stringify(finalMeta)}`) },
|
|
353
|
+
(err) => { logger.error(`_metadata(${path}) error: ${err}`) }
|
|
354
|
+
)
|
|
355
|
+
}
|
|
356
|
+
return finalMeta
|
|
321
357
|
}
|
|
322
358
|
|
|
323
359
|
const _loadMeta = async (dirent, entries) => {
|
|
@@ -353,21 +389,46 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
353
389
|
const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
|
|
354
390
|
const dirent = direntDir(p)
|
|
355
391
|
let entries
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
392
|
+
|
|
393
|
+
const cacheKey = `${p} ~ ${recursive}`
|
|
394
|
+
const cache = visitor ? null : redis.scopedCache('enc_list')
|
|
395
|
+
const cached = cache && await cache.get(cacheKey)
|
|
396
|
+
|
|
397
|
+
function cacheAndReturn (thing) {
|
|
398
|
+
if (cache) {
|
|
399
|
+
cache.set(cacheKey, thing).then(
|
|
400
|
+
() => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
|
|
401
|
+
(err) => { logger.error(`enc_list(${p}) error: ${err}`) }
|
|
402
|
+
)
|
|
403
|
+
}
|
|
404
|
+
return thing
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
if (cached) {
|
|
408
|
+
entries = cached
|
|
409
|
+
} else {
|
|
410
|
+
try {
|
|
411
|
+
entries = await client.list(dirent, recursive)
|
|
412
|
+
} catch (e) {
|
|
413
|
+
if (e instanceof MobilettoNotFoundError && p.includes('/')) {
|
|
414
|
+
// it might be a single file, try listing the parent dir
|
|
415
|
+
const parentDirent = direntDir(path.dirname(p));
|
|
416
|
+
entries = await client.list(parentDirent, false)
|
|
417
|
+
const objects = await _loadMeta(parentDirent, entries)
|
|
418
|
+
const found = objects.find(o => o.name === p)
|
|
419
|
+
if (found) {
|
|
420
|
+
return cacheAndReturn([found])
|
|
421
|
+
}
|
|
422
|
+
throw e
|
|
367
423
|
}
|
|
368
|
-
throw e
|
|
369
424
|
}
|
|
370
425
|
}
|
|
426
|
+
if (cache) {
|
|
427
|
+
cache.set(cacheKey, entries).then(
|
|
428
|
+
() => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
|
|
429
|
+
(err) => { logger.error(`enc_list(${p}) error: ${err}`) }
|
|
430
|
+
)
|
|
431
|
+
}
|
|
371
432
|
if (!entries || entries.length === 0) {
|
|
372
433
|
return []
|
|
373
434
|
}
|
|
@@ -448,7 +509,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
448
509
|
// todo: remove should return an array of the paths that were actually removed
|
|
449
510
|
remove: async (path, options) => {
|
|
450
511
|
const recursive = options === true || (options && options.recursive) || false
|
|
451
|
-
const quiet = (options && options.quiet) || false
|
|
512
|
+
// const quiet = (options && options.quiet) || false
|
|
452
513
|
if (recursive) {
|
|
453
514
|
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
454
515
|
async function recRm (path) {
|
|
@@ -504,7 +565,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
504
565
|
}
|
|
505
566
|
}
|
|
506
567
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
507
|
-
return addUtilityFunctions(encClient, readOnly
|
|
568
|
+
return addUtilityFunctions(encClient, readOnly)
|
|
508
569
|
}
|
|
509
570
|
|
|
510
571
|
async function readStream(stream, callback, endCallback) {
|
|
@@ -557,8 +618,7 @@ const M_SPECIAL = 'special'
|
|
|
557
618
|
module.exports = {
|
|
558
619
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
559
620
|
isAsyncGenerator, isReadable,
|
|
560
|
-
mobiletto, connect,
|
|
561
|
-
setLogLevel, setLogTransports,
|
|
621
|
+
mobiletto, connect,
|
|
562
622
|
MobilettoError, MobilettoNotFoundError,
|
|
563
623
|
readStream, writeStream, closeStream
|
|
564
624
|
}
|
package/package.json
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.44",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
|
-
"keywords": [
|
|
6
|
+
"keywords": [
|
|
7
|
+
"storage",
|
|
8
|
+
"s3",
|
|
9
|
+
"simple",
|
|
10
|
+
"filesystem"
|
|
11
|
+
],
|
|
7
12
|
"homepage": "https://github.com/cobbzilla/mobiletto",
|
|
8
13
|
"author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
|
|
9
14
|
"funding": {
|
|
10
|
-
"type"
|
|
11
|
-
"url"
|
|
15
|
+
"type": "patreon",
|
|
16
|
+
"url": "https://www.patreon.com/cobbzilla"
|
|
12
17
|
},
|
|
13
18
|
"repository": {
|
|
14
|
-
"type"
|
|
15
|
-
"url"
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "https://github.com/cobbzilla/mobiletto"
|
|
16
21
|
},
|
|
17
22
|
"license": "Apache-2.0",
|
|
18
23
|
"scripts": {
|
|
@@ -30,8 +35,10 @@
|
|
|
30
35
|
"dependencies": {
|
|
31
36
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
32
37
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
38
|
+
"ioredis": "^5.2.3",
|
|
33
39
|
"lru-cache": "^7.14.0",
|
|
34
40
|
"node-fetch-commonjs": "^3.1.1",
|
|
41
|
+
"redis-async-gen": "^0.1.3",
|
|
35
42
|
"shasum": "^1.0.2",
|
|
36
43
|
"winston": "^3.8.1"
|
|
37
44
|
}
|
package/test/index.spec.js
CHANGED
|
@@ -6,6 +6,7 @@ require('dotenv').config()
|
|
|
6
6
|
|
|
7
7
|
const randomstring = require('randomstring')
|
|
8
8
|
const crypto = require('crypto')
|
|
9
|
+
const redis = require('../util/redis')
|
|
9
10
|
|
|
10
11
|
const { expect, should, assert } = require('chai')
|
|
11
12
|
|
|
@@ -20,6 +21,7 @@ const {
|
|
|
20
21
|
getCipher, getDecipher
|
|
21
22
|
} = require("../util/crypt")
|
|
22
23
|
const crypt = require("../util/crypt");
|
|
24
|
+
const {logger} = require("../util/logger");
|
|
23
25
|
|
|
24
26
|
// chunk size used by generator function, used by driver's 'write' function
|
|
25
27
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
@@ -77,6 +79,15 @@ async function assertMetaFail (api, name) {
|
|
|
77
79
|
}
|
|
78
80
|
}
|
|
79
81
|
|
|
82
|
+
async function assertSafeMetaNull (api, name) {
|
|
83
|
+
try {
|
|
84
|
+
const meta = await api.safeMetadata(name)
|
|
85
|
+
expect(meta).to.be.null
|
|
86
|
+
} catch (err) {
|
|
87
|
+
assert.fail(`loading metadata with safeMetadata should not throw error: ${err}`)
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
80
91
|
const rand = count => randomstring.generate(count)
|
|
81
92
|
|
|
82
93
|
async function writeRandomFile(fixture, size) {
|
|
@@ -211,6 +222,9 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
211
222
|
it("loading metadata on the file we wrote now fails", async () => {
|
|
212
223
|
await assertMetaFail(fixture.api, fixture.name)
|
|
213
224
|
})
|
|
225
|
+
it("loading metadata on the file we wrote using safeMeta does not fail but returns null", async () => {
|
|
226
|
+
await assertSafeMetaNull(fixture.api, fixture.name)
|
|
227
|
+
})
|
|
214
228
|
})
|
|
215
229
|
|
|
216
230
|
for (const encryption of encryptionTests()) {
|
|
@@ -398,6 +412,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
398
412
|
}
|
|
399
413
|
})
|
|
400
414
|
it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
|
|
415
|
+
await redis.flushMobiletto()
|
|
401
416
|
const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
|
|
402
417
|
expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
|
|
403
418
|
expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
|
package/util/logger.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
const winston = require('winston')
|
|
2
|
+
|
|
3
|
+
const logger = winston.createLogger({
|
|
4
|
+
level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
|
|
5
|
+
format: winston.format.simple(),
|
|
6
|
+
transports: process.env.MOBILETTO_LOG_FILE
|
|
7
|
+
? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
|
|
8
|
+
: [ new winston.transports.Console() ]
|
|
9
|
+
})
|
|
10
|
+
|
|
11
|
+
const setLogLevel = (level) => { logger.level = level }
|
|
12
|
+
|
|
13
|
+
const setLogTransports = (transports) => {
|
|
14
|
+
logger.transports.splice(0, logger.transports.length)
|
|
15
|
+
logger.transports.push(...transports)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
module.exports = {
|
|
19
|
+
logger,
|
|
20
|
+
setLogLevel,
|
|
21
|
+
setLogTransports
|
|
22
|
+
}
|
package/util/redis.js
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
const Redis = require('ioredis')
|
|
2
|
+
const generators = require('redis-async-gen')
|
|
3
|
+
const { logger } = require('./logger')
|
|
4
|
+
const LRU = require("lru-cache");
|
|
5
|
+
|
|
6
|
+
const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
|
|
7
|
+
const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
|
|
8
|
+
const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
|
|
9
|
+
const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
|
|
10
|
+
|
|
11
|
+
const prefix = key => REDIS_PREFIX + key
|
|
12
|
+
const unprefix = key => key && key.startsWith(REDIS_PREFIX) ? key.substring(REDIS_PREFIX.length) : key
|
|
13
|
+
|
|
14
|
+
const NO_REDIS = { redis: 'not-found'}
|
|
15
|
+
|
|
16
|
+
let REDIS_CLIENT = null
|
|
17
|
+
|
|
18
|
+
const getRedis = () => {
|
|
19
|
+
if (REDIS_CLIENT === null) {
|
|
20
|
+
try {
|
|
21
|
+
REDIS_CLIENT = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {
|
|
22
|
+
host: REDIS_HOST,
|
|
23
|
+
port: REDIS_PORT
|
|
24
|
+
}))
|
|
25
|
+
flushMobiletto().then(
|
|
26
|
+
() => { logger.info(`getRedis: successfully flushed`) },
|
|
27
|
+
(e) => { logger.error(`getRedis: error flushing: ${e}`) }
|
|
28
|
+
)
|
|
29
|
+
} catch (e) {
|
|
30
|
+
logger.error(`getRedis(host=${REDIS_HOST}, port=${REDIS_PORT}) connection error: ${e}`)
|
|
31
|
+
REDIS_CLIENT = NO_REDIS
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return REDIS_CLIENT
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
|
|
38
|
+
|
|
39
|
+
const isNoRedis = r => !r || r === NO_REDIS
|
|
40
|
+
|
|
41
|
+
const doRedisAsync = async (func, defaultValue = null) => {
|
|
42
|
+
const redis = getRedis()
|
|
43
|
+
return isNoRedis(redis) ? defaultValue : await func(redis)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const doRedis = (func, defaultValue = null) => {
|
|
47
|
+
const redis = getRedis()
|
|
48
|
+
return isNoRedis(redis) ? defaultValue : func(redis)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function get (key) {
|
|
52
|
+
return doRedis(r => r.get(prefix(key)), null)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function set (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) {
|
|
56
|
+
return await doRedisAsync(r => r.set(prefix(key), val, 'EX', expirationMillis / 1000))
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function sadd (key, val) {
|
|
60
|
+
return await doRedisAsync(r => r.sadd(prefix(key), val))
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function expire (key, expirationMillis) {
|
|
64
|
+
return await doRedisAsync(r => r.expire(prefix(key), expirationMillis / 1000))
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function smembers (key) {
|
|
68
|
+
return doRedis(r => r.smembers(prefix(key)), [])
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function del (key) {
|
|
72
|
+
return await doRedisAsync(r => r.del(prefix(key)))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function flushall () {
|
|
76
|
+
return await doRedisAsync(r => r.flushall())
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function findMatchingKeys (pattern) {
|
|
80
|
+
const redis = getRedis();
|
|
81
|
+
if (isNoRedis(redis)) {
|
|
82
|
+
return []
|
|
83
|
+
}
|
|
84
|
+
const { keysMatching } = generators.using(redis)
|
|
85
|
+
const keys = []
|
|
86
|
+
for await (const key of keysMatching(prefix(pattern))) {
|
|
87
|
+
keys.push(unprefix(key))
|
|
88
|
+
}
|
|
89
|
+
return keys
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function removeMatchingKeys (pattern) {
|
|
93
|
+
return await applyToMatchingKeys(pattern, del)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function applyToMatchingKeys (pattern, asyncFunc) {
|
|
97
|
+
const redis = getRedis();
|
|
98
|
+
if (isNoRedis(redis)) {
|
|
99
|
+
return []
|
|
100
|
+
}
|
|
101
|
+
const { keysMatching } = generators.using(redis)
|
|
102
|
+
const results = []
|
|
103
|
+
for await (const key of keysMatching(prefix(pattern))) {
|
|
104
|
+
results.push(await asyncFunc(unprefix(key)))
|
|
105
|
+
}
|
|
106
|
+
return results
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const CACHES = {}
|
|
110
|
+
|
|
111
|
+
const flushMobiletto = async (redis = getRedis()) => {
|
|
112
|
+
return redis ? await removeMatchingKeys('*') : null
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const scopedCache = (name, size = 100) => {
|
|
116
|
+
if (CACHES[name]) {
|
|
117
|
+
return CACHES[name]
|
|
118
|
+
}
|
|
119
|
+
const redis = getRedis()
|
|
120
|
+
if (isNoRedis(redis)) {
|
|
121
|
+
return size && size > 0 ? new LRU({ max: size }) : null
|
|
122
|
+
}
|
|
123
|
+
const realKey = k => k ? `${name}_${k}` : null
|
|
124
|
+
const cache = {
|
|
125
|
+
get: async key => {
|
|
126
|
+
const val = await get(realKey(key))
|
|
127
|
+
if (!val) return null
|
|
128
|
+
try {
|
|
129
|
+
return JSON.parse(val)
|
|
130
|
+
} catch (e) {
|
|
131
|
+
logger.warn(`get(${key}) error: ${e}`)
|
|
132
|
+
return null
|
|
133
|
+
}
|
|
134
|
+
},
|
|
135
|
+
set: async (key, value) => {
|
|
136
|
+
if (key && value) {
|
|
137
|
+
try {
|
|
138
|
+
await set(realKey(key), JSON.stringify(value))
|
|
139
|
+
} catch (e) {
|
|
140
|
+
logger.warn(`set(${key}) error: ${e}`)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
CACHES[name] = cache
|
|
146
|
+
return cache
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
teardown = () => {
|
|
150
|
+
const redis = getRedis()
|
|
151
|
+
if (redis) {
|
|
152
|
+
redis.shutdown()
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
module.exports = {
|
|
157
|
+
get,
|
|
158
|
+
set,
|
|
159
|
+
del,
|
|
160
|
+
expire,
|
|
161
|
+
sadd,
|
|
162
|
+
smembers,
|
|
163
|
+
flushall,
|
|
164
|
+
findMatchingKeys,
|
|
165
|
+
removeMatchingKeys,
|
|
166
|
+
applyToMatchingKeys,
|
|
167
|
+
scopedCache,
|
|
168
|
+
flushMobiletto,
|
|
169
|
+
teardown
|
|
170
|
+
}
|