mobiletto 1.0.44 → 1.0.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/index.js +108 -19
- package/package.json +1 -1
- package/test/index.spec.js +70 -24
- package/util/redis.js +156 -134
package/README.md
CHANGED
|
@@ -194,15 +194,19 @@ You can override either of these:
|
|
|
194
194
|
Mobiletto will store all of its redis keys with the prefix `_mobiletto__`. You can change this
|
|
195
195
|
by setting the `MOBILETTO_REDIS_PREFIX` env var.
|
|
196
196
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
197
|
+
You can also set per-connection caching with the `opts.redisConfig` object:
|
|
198
|
+
|
|
199
|
+
const redisConfig = {
|
|
200
|
+
enabled: true, // optional, default is true. if false other props are ignored
|
|
201
|
+
host: '127.0.0.1',
|
|
202
|
+
port: 6379,
|
|
203
|
+
prefix: '_mobiletto__'
|
|
204
|
+
}
|
|
205
|
+
const opts = { redisConfig, bucket: 'bk', region: 'us-east-1' }
|
|
206
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
|
|
207
|
+
|
|
208
|
+
### Don't want redis caching?
|
|
209
|
+
To disable: pass `enabled: false` in your `opts.redisConfig` object when you establish your connection.
|
|
206
210
|
|
|
207
211
|
As discussed below, disabling caching will have an adverse effect on performance and incur more requests
|
|
208
212
|
to storage that you really need to.
|
package/index.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
2
|
const path = require('path')
|
|
3
3
|
const { Readable, Transform } = require('stream')
|
|
4
|
-
const { logger } = require('./util/logger')
|
|
5
|
-
const
|
|
4
|
+
const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
|
|
5
|
+
const { getRedis } = require('./util/redis')
|
|
6
6
|
const { basename, dirname } = require('path')
|
|
7
7
|
const shasum = require('shasum')
|
|
8
8
|
const randomstring = require('randomstring')
|
|
9
9
|
|
|
10
10
|
const crypt = require('./util/crypt')
|
|
11
11
|
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
|
|
12
|
+
const LRU = require("lru-cache");
|
|
12
13
|
|
|
13
14
|
const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
14
15
|
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
@@ -20,6 +21,10 @@ const initMobilettoTempDir = () => {
|
|
|
20
21
|
}
|
|
21
22
|
const MOBILETTO_TMP = initMobilettoTempDir()
|
|
22
23
|
|
|
24
|
+
const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
|
|
25
|
+
const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
|
|
26
|
+
const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
|
|
27
|
+
|
|
23
28
|
// adapted from https://stackoverflow.com/a/27724419
|
|
24
29
|
function MobilettoError (message, err) {
|
|
25
30
|
this.message = `${message}: ${err ? err : ''}`
|
|
@@ -51,15 +56,52 @@ const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
|
51
56
|
const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
|
|
52
57
|
const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
|
|
53
58
|
|
|
54
|
-
const getMetaCache = (size = 100) => redis.scopedCache('meta', size)
|
|
55
|
-
const getListingCache = (size = 100) => redis.scopedCache('listingCache', size)
|
|
56
|
-
|
|
57
59
|
const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
|
|
58
60
|
|
|
61
|
+
class AwaitableLRU {
|
|
62
|
+
lru
|
|
63
|
+
constructor(size) {
|
|
64
|
+
this.lru = new LRU({ max: size })
|
|
65
|
+
}
|
|
66
|
+
get = (key) => Promise.resolve(this.lru.get(key))
|
|
67
|
+
set = (key, value) => Promise.resolve(this.lru.set(key, value))
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const CACHE_FUNCTIONS = {
|
|
71
|
+
redis: (client) => () => {
|
|
72
|
+
if (typeof client.cache !== 'undefined') return client.cache
|
|
73
|
+
const redisConfig = client.redisConfig || {}
|
|
74
|
+
const enabled = redisConfig.enabled !== false
|
|
75
|
+
if (!enabled) {
|
|
76
|
+
logger.info(`redis: client.redisConfig.enabled === false, disabling cache`)
|
|
77
|
+
client.cache = null
|
|
78
|
+
return client.cache
|
|
79
|
+
}
|
|
80
|
+
const host = redisConfig.host || REDIS_HOST
|
|
81
|
+
const port = redisConfig.port || REDIS_PORT
|
|
82
|
+
const prefix = redisConfig.prefix || REDIS_PREFIX
|
|
83
|
+
if (!client.id) {
|
|
84
|
+
logger.warn(`redis: all nameless connections will share one cache`)
|
|
85
|
+
client.cache = getRedis('~nameless~', host, port, prefix)
|
|
86
|
+
} else {
|
|
87
|
+
client.cache = getRedis(client.id, host, port, prefix)
|
|
88
|
+
}
|
|
89
|
+
return client.cache
|
|
90
|
+
},
|
|
91
|
+
scopedCache: (client) => (cacheName, size = 100) => {
|
|
92
|
+
const redis = client.redis()
|
|
93
|
+
return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
|
|
94
|
+
},
|
|
95
|
+
flush: (client) => async () => {
|
|
96
|
+
const redis = client.redis()
|
|
97
|
+
return redis ? redis.flush() : true
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
59
101
|
// noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
|
|
60
102
|
const UTILITY_FUNCTIONS = {
|
|
61
103
|
list: (client) => async (path, opts) => {
|
|
62
|
-
const cache =
|
|
104
|
+
const cache = client.scopedCache('list')
|
|
63
105
|
let cached = cache ? await cache.get(path) : null
|
|
64
106
|
if (cached) {
|
|
65
107
|
if (Array.isArray(cached)) { return cached }
|
|
@@ -104,7 +146,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
104
146
|
}
|
|
105
147
|
},
|
|
106
148
|
metadata: (client) => async (path) => {
|
|
107
|
-
const cache =
|
|
149
|
+
const cache = client.scopedCache('metadata')
|
|
108
150
|
const cached = cache ? await cache.get(path) : null
|
|
109
151
|
if (cached) {
|
|
110
152
|
return cached
|
|
@@ -134,11 +176,11 @@ const UTILITY_FUNCTIONS = {
|
|
|
134
176
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
135
177
|
// noinspection JSUnresolvedFunction
|
|
136
178
|
const result = await client.driver_remove(path, recursive, quiet)
|
|
137
|
-
await
|
|
179
|
+
await client.flush()
|
|
138
180
|
return result
|
|
139
181
|
},
|
|
140
182
|
readFile: (client) => async (path) => {
|
|
141
|
-
const cache =
|
|
183
|
+
const cache = client.scopedCache('readFile')
|
|
142
184
|
const cached = cache ? await cache.get(path) : null
|
|
143
185
|
if (cached) {
|
|
144
186
|
return Buffer.from(cached, 'base64')
|
|
@@ -147,14 +189,17 @@ const UTILITY_FUNCTIONS = {
|
|
|
147
189
|
await client.read(path, reader(chunks))
|
|
148
190
|
const data = Buffer.concat(chunks);
|
|
149
191
|
if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
|
|
150
|
-
cache.set(path, data.toString('base64'))
|
|
192
|
+
cache.set(path, data.toString('base64')).then(
|
|
193
|
+
() => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
|
|
194
|
+
(err) => { logger.error(`readFile(${path}) error: ${err}`) }
|
|
195
|
+
)
|
|
151
196
|
}
|
|
152
197
|
return data
|
|
153
198
|
},
|
|
154
199
|
write: (client) => async (path, data) => {
|
|
155
200
|
// noinspection JSUnresolvedFunction
|
|
156
201
|
const bytesWritten = await client.driver_write(path, data)
|
|
157
|
-
await
|
|
202
|
+
await client.flush()
|
|
158
203
|
return bytesWritten
|
|
159
204
|
},
|
|
160
205
|
writeFile: (client) => async (path, data) => {
|
|
@@ -240,11 +285,22 @@ async function mirrorDir (source, sourcePath, visitor) {
|
|
|
240
285
|
}
|
|
241
286
|
}
|
|
242
287
|
|
|
243
|
-
function
|
|
244
|
-
|
|
245
|
-
client[`driver_${func}`]
|
|
246
|
-
|
|
288
|
+
function utilityFunctionConflict (client, func) {
|
|
289
|
+
if (typeof client[func] === 'function') {
|
|
290
|
+
if (typeof client[`driver_${func}`] !== 'undefined') {
|
|
291
|
+
logger.warn(`utilityFunctionConflict: driver_${func} has already been added`)
|
|
292
|
+
return false
|
|
293
|
+
} else {
|
|
294
|
+
client[`driver_${func}`] = client[func] // save original driver function
|
|
295
|
+
return true
|
|
296
|
+
}
|
|
297
|
+
} else if (typeof client[func] !== 'undefined') {
|
|
298
|
+
throw new MobilettoError(`utilityFunctionConflict: client defines a property ${func}, mobiletto function would overwrite`)
|
|
247
299
|
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const addUtilityFunctions = (client, readOnly = false) => {
|
|
303
|
+
addClientFunctions(client, UTILITY_FUNCTIONS, utilityFunctionConflict)
|
|
248
304
|
if (readOnly) {
|
|
249
305
|
for (const writeFunc of ['write', 'remove', 'writeFile']) {
|
|
250
306
|
client[writeFunc] = async () => {
|
|
@@ -256,6 +312,24 @@ function addUtilityFunctions (client, readOnly = false) {
|
|
|
256
312
|
return client
|
|
257
313
|
}
|
|
258
314
|
|
|
315
|
+
const addCacheFunctions = (client) => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
|
|
316
|
+
logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
|
|
317
|
+
return false
|
|
318
|
+
})
|
|
319
|
+
|
|
320
|
+
const addClientFunctions = (client, functions, conflictFunc) => {
|
|
321
|
+
for (const func of Object.keys(functions)) {
|
|
322
|
+
let add = true
|
|
323
|
+
if (client[func]) {
|
|
324
|
+
add = conflictFunc ? conflictFunc(client, func) : false
|
|
325
|
+
}
|
|
326
|
+
if (add) {
|
|
327
|
+
client[func] = functions[func](client)
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return client
|
|
331
|
+
}
|
|
332
|
+
|
|
259
333
|
async function connect (driverPath, key, secret, opts, encryption = null) {
|
|
260
334
|
return await mobiletto(driverPath, key, secret, opts, encryption)
|
|
261
335
|
}
|
|
@@ -270,9 +344,18 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
270
344
|
throw new MobilettoError(message)
|
|
271
345
|
}
|
|
272
346
|
const readOnly = opts ? !!opts.readOnly : false
|
|
347
|
+
client.redisConfig = opts && opts.redisConfig ? opts.redisConfig : {}
|
|
348
|
+
|
|
349
|
+
const internalIdForDriver = () =>
|
|
350
|
+
driverPath + '_' + shasum(`${key}\n${JSON.stringify(opts)}\n${(encryption ? JSON.stringify(encryption) : '')}`)
|
|
351
|
+
|
|
352
|
+
// If the driver didn't give the client a name, generate a unique internal name
|
|
353
|
+
if (!client.id) {
|
|
354
|
+
client.id = internalIdForDriver()
|
|
355
|
+
}
|
|
273
356
|
if (encryption === null) {
|
|
274
357
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
275
|
-
return addUtilityFunctions(client, readOnly)
|
|
358
|
+
return addUtilityFunctions(addCacheFunctions(client), readOnly)
|
|
276
359
|
}
|
|
277
360
|
const encKey = normalizeKey(encryption.key)
|
|
278
361
|
if (!encKey) {
|
|
@@ -306,8 +389,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
306
389
|
const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
|
|
307
390
|
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
308
391
|
|
|
392
|
+
const outerClient = addCacheFunctions(client)
|
|
309
393
|
const _metadata = (client) => async (path) => {
|
|
310
|
-
const cache =
|
|
394
|
+
const cache = outerClient.scopedCache('_metadata')
|
|
311
395
|
const cached = cache ? await cache.get(path) : null
|
|
312
396
|
if (cached) {
|
|
313
397
|
return cached
|
|
@@ -385,13 +469,15 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
385
469
|
}
|
|
386
470
|
|
|
387
471
|
const encClient = {
|
|
472
|
+
id: internalIdForDriver(),
|
|
473
|
+
redisConfig: client.redisConfig,
|
|
388
474
|
list: async (pth = '', recursive, visitor) => {
|
|
389
475
|
const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
|
|
390
476
|
const dirent = direntDir(p)
|
|
391
477
|
let entries
|
|
392
478
|
|
|
393
479
|
const cacheKey = `${p} ~ ${recursive}`
|
|
394
|
-
const cache = visitor ? null :
|
|
480
|
+
const cache = visitor ? null : client.scopedCache('enc_list')
|
|
395
481
|
const cached = cache && await cache.get(cacheKey)
|
|
396
482
|
|
|
397
483
|
function cacheAndReturn (thing) {
|
|
@@ -565,7 +651,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
565
651
|
}
|
|
566
652
|
}
|
|
567
653
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
568
|
-
return addUtilityFunctions(encClient, readOnly)
|
|
654
|
+
return addUtilityFunctions(addCacheFunctions(encClient), readOnly)
|
|
569
655
|
}
|
|
570
656
|
|
|
571
657
|
async function readStream(stream, callback, endCallback) {
|
|
@@ -615,10 +701,13 @@ const M_DIR = 'dir'
|
|
|
615
701
|
const M_LINK = 'link'
|
|
616
702
|
const M_SPECIAL = 'special'
|
|
617
703
|
|
|
704
|
+
const setLogLevel = level => setLoggerLevel(level)
|
|
705
|
+
|
|
618
706
|
module.exports = {
|
|
619
707
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
620
708
|
isAsyncGenerator, isReadable,
|
|
621
709
|
mobiletto, connect,
|
|
622
710
|
MobilettoError, MobilettoNotFoundError,
|
|
711
|
+
setLogLevel,
|
|
623
712
|
readStream, writeStream, closeStream
|
|
624
713
|
}
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -14,14 +14,15 @@ const {
|
|
|
14
14
|
M_DIR, M_FILE,
|
|
15
15
|
mobiletto, connect, MobilettoNotFoundError, readStream
|
|
16
16
|
} = require("../index")
|
|
17
|
+
|
|
17
18
|
const {
|
|
18
19
|
DEFAULT_CRYPT_ALGO,
|
|
19
20
|
normalizeKey, normalizeIV,
|
|
20
21
|
encrypt, decrypt,
|
|
21
22
|
getCipher, getDecipher
|
|
22
23
|
} = require("../util/crypt")
|
|
23
|
-
|
|
24
|
-
const {logger} = require("../util/logger");
|
|
24
|
+
|
|
25
|
+
const { logger } = require("../util/logger");
|
|
25
26
|
|
|
26
27
|
// chunk size used by generator function, used by driver's 'write' function
|
|
27
28
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
@@ -159,31 +160,71 @@ describe('crypto test', () => {
|
|
|
159
160
|
|
|
160
161
|
const encryptionTests = () => [null, { key: rand(32) }]
|
|
161
162
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
163
|
+
const REDIS_ENABLED = {
|
|
164
|
+
name: 'redis-enabled',
|
|
165
|
+
enabled: true,
|
|
166
|
+
config: cfg => Object.assign({}, cfg)
|
|
167
|
+
}
|
|
168
|
+
const REDIS_DISABLED = {
|
|
169
|
+
name: 'redis-disabled',
|
|
170
|
+
enabled: true,
|
|
171
|
+
config: (cfg) => {
|
|
172
|
+
const newConfig = Object.assign({}, cfg)
|
|
173
|
+
if (!newConfig.opts) { newConfig.opts = {} }
|
|
174
|
+
if (!newConfig.opts.redisConfig) { newConfig.opts.redisConfig = {} }
|
|
175
|
+
newConfig.opts.redisConfig.enabled = false
|
|
176
|
+
return newConfig
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const redisTests = () => [REDIS_ENABLED, REDIS_DISABLED]
|
|
165
180
|
|
|
181
|
+
// For testing single drivers
|
|
182
|
+
// Comment out the for loop for driver tests, and hard-code one of the drivers below
|
|
166
183
|
// const driverName = 'local'
|
|
167
184
|
// const driverName = 's3'
|
|
168
185
|
|
|
186
|
+
// For testing single redis setups
|
|
187
|
+
// Comment out the for loop for redis test, and hard-code one of the setups below
|
|
188
|
+
// const redisSetup = REDIS_ENABLED
|
|
189
|
+
// const redisSetup = REDIS_DISABLED
|
|
190
|
+
|
|
169
191
|
// When you're NOT testing a single driver, it's important that *all* tests remain
|
|
170
|
-
// within the
|
|
171
|
-
// the same set of tests, with exactly the same inputs/outputs
|
|
192
|
+
// within the various 'for' loops below. This ensures that all drivers pass exactly
|
|
193
|
+
// the same set of tests, with exactly the same inputs/outputs, both with and without
|
|
194
|
+
// redis caching enabled, both with and without encryption enabled.
|
|
172
195
|
// This greatly supports developer sanity
|
|
173
196
|
|
|
197
|
+
for (const redisSetup of redisTests()) {
|
|
198
|
+
if (!redisSetup.enabled) continue
|
|
199
|
+
|
|
174
200
|
for (const driverName of DRIVER_NAMES) {
|
|
175
|
-
const
|
|
201
|
+
const driverTest = `${driverName} [${redisSetup.name}]`
|
|
202
|
+
const config = redisSetup.config(DRIVER_CONFIG[driverName])
|
|
176
203
|
const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
|
|
177
204
|
const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
|
|
178
|
-
describe(`${
|
|
179
|
-
|
|
205
|
+
describe(`${driverTest} test`, () => {
|
|
206
|
+
beforeEach ((done) => {
|
|
207
|
+
redis.flushAll().then(
|
|
208
|
+
() => { logger.info(`${driverTest} - flushed redis`) },
|
|
209
|
+
(e) => {
|
|
210
|
+
logger.error(`${driverTest} - error flushing redis: ${e}`)
|
|
211
|
+
throw e
|
|
212
|
+
}
|
|
213
|
+
).finally(() => done())
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
describe(`${driverTest} - create api client`, () => {
|
|
180
217
|
it("should validate the config and return an API object", async () => {
|
|
181
|
-
|
|
182
|
-
|
|
218
|
+
try {
|
|
219
|
+
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
220
|
+
should().exist(api, 'expected API object to exist')
|
|
221
|
+
} catch (e) {
|
|
222
|
+
logger.error(`wtf: ${e}`)
|
|
223
|
+
}
|
|
183
224
|
})
|
|
184
225
|
})
|
|
185
226
|
|
|
186
|
-
describe(`${
|
|
227
|
+
describe(`${driverTest} - listing with no arguments returns some results`, () => {
|
|
187
228
|
it("should return some results from a default listing", async () => {
|
|
188
229
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
189
230
|
const results = await api.list()
|
|
@@ -191,7 +232,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
191
232
|
})
|
|
192
233
|
})
|
|
193
234
|
|
|
194
|
-
describe(`${
|
|
235
|
+
describe(`${driverTest} - write a file, read file, read metadata, delete file`, () => {
|
|
195
236
|
// some random data, plus a bit extra
|
|
196
237
|
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
197
238
|
const randomData = rand(size)
|
|
@@ -229,8 +270,8 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
229
270
|
|
|
230
271
|
for (const encryption of encryptionTests()) {
|
|
231
272
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
232
|
-
describe(`${
|
|
233
|
-
// describe(`${
|
|
273
|
+
describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
274
|
+
// describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
|
|
234
275
|
// const encryption = {key: rand(32)}
|
|
235
276
|
const size = 16
|
|
236
277
|
const randomData = rand(size)
|
|
@@ -275,7 +316,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
275
316
|
})
|
|
276
317
|
}
|
|
277
318
|
|
|
278
|
-
describe(`${
|
|
319
|
+
describe(`${driverTest} - write an encrypted file, read file, read metadata, delete file`, () => {
|
|
279
320
|
// some random data, plus a bit extra
|
|
280
321
|
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
281
322
|
const randomData = rand(size)
|
|
@@ -315,8 +356,8 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
315
356
|
const mirrorDriver = pickAnotherDriver(driverName)
|
|
316
357
|
const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
|
|
317
358
|
const mirrorDest = `mirrorDest_${rand(5)}/`
|
|
318
|
-
describe(`${
|
|
319
|
-
// describe(`${
|
|
359
|
+
describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
|
|
360
|
+
// describe(`${driverTest} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|
|
320
361
|
// const encryption = {key: rand(32)}
|
|
321
362
|
// const encryption = null
|
|
322
363
|
// a random directory and file within it
|
|
@@ -412,7 +453,6 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
412
453
|
}
|
|
413
454
|
})
|
|
414
455
|
it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
|
|
415
|
-
await redis.flushMobiletto()
|
|
416
456
|
const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
|
|
417
457
|
expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
|
|
418
458
|
expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
|
|
@@ -455,7 +495,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
455
495
|
})
|
|
456
496
|
}
|
|
457
497
|
|
|
458
|
-
describe(`${
|
|
498
|
+
describe(`${driverTest} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
|
|
459
499
|
it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
|
|
460
500
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
461
501
|
try {
|
|
@@ -468,7 +508,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
468
508
|
})
|
|
469
509
|
})
|
|
470
510
|
|
|
471
|
-
describe(`${
|
|
511
|
+
describe(`${driverTest} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
|
|
472
512
|
it("should throw MobilettoNotFoundError when trying to read metadata for a file that does not exist", async () => {
|
|
473
513
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
474
514
|
try {
|
|
@@ -480,7 +520,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
480
520
|
})
|
|
481
521
|
})
|
|
482
522
|
|
|
483
|
-
describe(`${
|
|
523
|
+
describe(`${driverTest} - expect MobilettoNotFoundError when removing non-existent file`, () => {
|
|
484
524
|
it("should throw MobilettoNotFoundError when trying to remove a file that does not exist", async () => {
|
|
485
525
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
486
526
|
try {
|
|
@@ -493,7 +533,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
493
533
|
})
|
|
494
534
|
})
|
|
495
535
|
|
|
496
|
-
describe(`${
|
|
536
|
+
describe(`${driverTest} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
|
|
497
537
|
it("should NOT throw MobilettoNotFoundError when quietly removing a file that does not exist", async () => {
|
|
498
538
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
499
539
|
if (!(await api.remove(nonexistentFile, {quiet: true}))) {
|
|
@@ -503,3 +543,9 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
503
543
|
})
|
|
504
544
|
})
|
|
505
545
|
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
after ( (done) => {
|
|
549
|
+
logger.info('all tests finished, tearing down redis...')
|
|
550
|
+
redis.teardown().finally(done)
|
|
551
|
+
})
|
package/util/redis.js
CHANGED
|
@@ -1,170 +1,192 @@
|
|
|
1
1
|
const Redis = require('ioredis')
|
|
2
2
|
const generators = require('redis-async-gen')
|
|
3
3
|
const { logger } = require('./logger')
|
|
4
|
-
const LRU = require("lru-cache")
|
|
4
|
+
const LRU = require("lru-cache")
|
|
5
5
|
|
|
6
6
|
const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
|
|
7
|
-
const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
|
|
8
|
-
const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
|
|
9
|
-
const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
|
|
10
7
|
|
|
11
|
-
|
|
12
|
-
const unprefix = key => key && key.startsWith(REDIS_PREFIX) ? key.substring(REDIS_PREFIX.length) : key
|
|
8
|
+
let REDIS_CLIENTS = {}
|
|
13
9
|
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const getRedis = () => {
|
|
19
|
-
if (REDIS_CLIENT === null) {
|
|
20
|
-
try {
|
|
21
|
-
REDIS_CLIENT = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {
|
|
22
|
-
host: REDIS_HOST,
|
|
23
|
-
port: REDIS_PORT
|
|
24
|
-
}))
|
|
25
|
-
flushMobiletto().then(
|
|
26
|
-
() => { logger.info(`getRedis: successfully flushed`) },
|
|
27
|
-
(e) => { logger.error(`getRedis: error flushing: ${e}`) }
|
|
28
|
-
)
|
|
29
|
-
} catch (e) {
|
|
30
|
-
logger.error(`getRedis(host=${REDIS_HOST}, port=${REDIS_PORT}) connection error: ${e}`)
|
|
31
|
-
REDIS_CLIENT = NO_REDIS
|
|
32
|
-
}
|
|
10
|
+
const getRedis = (name, host, port, prefix) => {
|
|
11
|
+
if (typeof REDIS_CLIENTS[name] === 'undefined') {
|
|
12
|
+
REDIS_CLIENTS[name] = new MobilettoCache(name, host, port, prefix + name + '_')
|
|
33
13
|
}
|
|
34
|
-
return
|
|
14
|
+
return REDIS_CLIENTS[name]
|
|
35
15
|
}
|
|
36
16
|
|
|
37
17
|
const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
|
|
38
18
|
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
19
|
+
const ZERO_COUNTERS = {
|
|
20
|
+
get: 0,
|
|
21
|
+
set: 0,
|
|
22
|
+
del: 0,
|
|
23
|
+
flush: 0,
|
|
24
|
+
hit: 0,
|
|
25
|
+
miss: 0
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
class MobilettoCache {
|
|
29
|
+
name
|
|
30
|
+
redis
|
|
31
|
+
prefix
|
|
32
|
+
scopedCaches = {}
|
|
33
|
+
counters = Object.assign({}, ZERO_COUNTERS)
|
|
34
|
+
printStatsInterval = 1000
|
|
35
|
+
constructor(name,
|
|
36
|
+
host = '127.0.0.1',
|
|
37
|
+
port = 6379,
|
|
38
|
+
prefix = '_mobiletto__') {
|
|
39
|
+
this.name = name
|
|
40
|
+
if (host && port) {
|
|
41
|
+
try {
|
|
42
|
+
this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
|
|
43
|
+
} catch (e) {
|
|
44
|
+
logger.error(`redis(${name}): error connecting to redis, using fallback LRU for scoped caches: ${e}`)
|
|
45
|
+
this.redis = null
|
|
46
|
+
}
|
|
47
|
+
} else {
|
|
48
|
+
logger.warn(`redis(${name}): no host or port provided, using fallback LRU for scoped caches`)
|
|
49
|
+
this.redis = null
|
|
50
|
+
}
|
|
51
|
+
this.prefix = prefix
|
|
52
|
+
this.flush().then(
|
|
53
|
+
() => { logger.debug(`redis(${name}): successfully flushed`) },
|
|
54
|
+
(e) => { logger.error(`redis(${name}): error flushing: ${e}`) }
|
|
55
|
+
)
|
|
56
|
+
}
|
|
62
57
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
58
|
+
stats = () => this.counters
|
|
59
|
+
resetStats = () => { this.counters = Object.assign({}, ZERO_COUNTERS) }
|
|
60
|
+
hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
|
|
61
|
+
toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
|
|
66
62
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
}
|
|
63
|
+
pfx = key => this.prefix + key
|
|
64
|
+
unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
|
|
70
65
|
|
|
71
|
-
async
|
|
72
|
-
|
|
73
|
-
|
|
66
|
+
doRedisAsync = async (func, defaultValue = null) => {
|
|
67
|
+
try {
|
|
68
|
+
return this.redis ? await func(this.redis) : defaultValue
|
|
69
|
+
} catch (e) {
|
|
70
|
+
logger.warn(`redis(${this.name}): doRedisAsync(${func}): ${e} (returning default value: ${defaultValue})`)
|
|
71
|
+
return defaultValue
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
74
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
doRedis = (func, defaultValue = null) => {
|
|
76
|
+
try {
|
|
77
|
+
return this.redis ? func(this.redis) : defaultValue
|
|
78
|
+
} catch (e) {
|
|
79
|
+
logger.warn(`redis(${this.name}): doRedis(${func}): ${e} (returning default value: ${defaultValue})`)
|
|
80
|
+
return defaultValue
|
|
81
|
+
}
|
|
82
|
+
}
|
|
78
83
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
get = key => {
|
|
85
|
+
this.counters.get++
|
|
86
|
+
logger.debug(`redis(${this.name}): get(${key}) starting`)
|
|
87
|
+
const val = this.doRedis(r => r.get(this.pfx(key)), null)
|
|
88
|
+
logger.debug(`redis(${this.name}): get(${key}) found value: ${val}`)
|
|
89
|
+
if (val) {
|
|
90
|
+
this.counters.hit++
|
|
91
|
+
} else {
|
|
92
|
+
this.counters.miss++
|
|
93
|
+
}
|
|
94
|
+
if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
|
|
95
|
+
const message = `${new Date()}: ${this}`;
|
|
96
|
+
logger.info(message)
|
|
97
|
+
}
|
|
98
|
+
return val
|
|
83
99
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
100
|
+
|
|
101
|
+
set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
|
|
102
|
+
this.counters.set++
|
|
103
|
+
logger.debug(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) starting`)
|
|
104
|
+
await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
|
|
105
|
+
logger.debug(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) finished`)
|
|
88
106
|
}
|
|
89
|
-
return keys
|
|
90
|
-
}
|
|
91
107
|
|
|
92
|
-
async
|
|
93
|
-
|
|
94
|
-
|
|
108
|
+
del = async key => {
|
|
109
|
+
this.counters.del++
|
|
110
|
+
await this.doRedisAsync(r => r.del(this.pfx(key)))
|
|
111
|
+
}
|
|
95
112
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
113
|
+
findMatchingKeys = async (pattern) => {
|
|
114
|
+
if (!this.redis) return []
|
|
115
|
+
const { keysMatching } = generators.using(this.redis)
|
|
116
|
+
const keys = []
|
|
117
|
+
for await (const key of keysMatching(this.pfx(pattern))) {
|
|
118
|
+
keys.push(this.unprefix(key))
|
|
119
|
+
}
|
|
120
|
+
return keys
|
|
100
121
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
results
|
|
122
|
+
applyToMatchingKeys = async (pattern, asyncFunc) => {
|
|
123
|
+
if (!this.redis) return []
|
|
124
|
+
const { keysMatching } = generators.using(this.redis)
|
|
125
|
+
const results = []
|
|
126
|
+
for await (const key of keysMatching(this.pfx(pattern))) {
|
|
127
|
+
results.push(await asyncFunc(this.unprefix(key)))
|
|
128
|
+
}
|
|
129
|
+
return results
|
|
105
130
|
}
|
|
106
|
-
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const CACHES = {}
|
|
110
|
-
|
|
111
|
-
const flushMobiletto = async (redis = getRedis()) => {
|
|
112
|
-
return redis ? await removeMatchingKeys('*') : null
|
|
113
|
-
}
|
|
131
|
+
removeMatchingKeys = async (pattern) => await this.applyToMatchingKeys(pattern, this.del)
|
|
114
132
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
}
|
|
119
|
-
const redis = getRedis()
|
|
120
|
-
if (isNoRedis(redis)) {
|
|
121
|
-
return size && size > 0 ? new LRU({ max: size }) : null
|
|
133
|
+
flush = async () => {
|
|
134
|
+
this.counters.flush++
|
|
135
|
+
await this.removeMatchingKeys('*')
|
|
122
136
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
if (key && value) {
|
|
137
|
+
|
|
138
|
+
scopedCache = (name, size = 100) => {
|
|
139
|
+
if (this.scopedCaches[name]) {
|
|
140
|
+
return this.scopedCaches[name]
|
|
141
|
+
}
|
|
142
|
+
if (!this.redis) {
|
|
143
|
+
return size && size > 0 ? new LRU({ max: size }) : null
|
|
144
|
+
}
|
|
145
|
+
const realKey = k => k ? `${name}_${k}` : null
|
|
146
|
+
const cache = {
|
|
147
|
+
get: async key => {
|
|
148
|
+
const val = await this.get(realKey(key))
|
|
149
|
+
if (!val) return null
|
|
137
150
|
try {
|
|
138
|
-
|
|
151
|
+
return JSON.parse(val)
|
|
139
152
|
} catch (e) {
|
|
140
|
-
logger.warn(`
|
|
153
|
+
logger.warn(`redis(${this.name}): get(${key}) error: ${e}`)
|
|
154
|
+
return null
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
set: async (key, value) => {
|
|
158
|
+
if (key && value) {
|
|
159
|
+
try {
|
|
160
|
+
await this.set(realKey(key), JSON.stringify(value))
|
|
161
|
+
} catch (e) {
|
|
162
|
+
logger.warn(`redis(${this.name}): set(${key}) error: ${e}`)
|
|
163
|
+
}
|
|
141
164
|
}
|
|
142
165
|
}
|
|
143
166
|
}
|
|
167
|
+
this.scopedCaches[name] = cache
|
|
168
|
+
return cache
|
|
144
169
|
}
|
|
145
|
-
CACHES[name] = cache
|
|
146
|
-
return cache
|
|
147
170
|
}
|
|
148
171
|
|
|
149
|
-
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
172
|
+
const forAllCaches = async (func) => {
|
|
173
|
+
const promises = []
|
|
174
|
+
for (const name of Object.keys(REDIS_CLIENTS)) {
|
|
175
|
+
promises.push(func(REDIS_CLIENTS[name]))
|
|
153
176
|
}
|
|
177
|
+
return Promise.all(promises)
|
|
154
178
|
}
|
|
155
179
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
teardown
|
|
170
|
-
}
|
|
180
|
+
const teardown = async () => await forAllCaches((client) => {
|
|
181
|
+
try {
|
|
182
|
+
if (client.redis) {
|
|
183
|
+
client.redis.disconnect()
|
|
184
|
+
}
|
|
185
|
+
} catch (e) {
|
|
186
|
+
logger.warn(`teardown: error disconnecting from redis(${client.name}): ${e}`)
|
|
187
|
+
}
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
const flushAll = async () => await forAllCaches(client => client.flush())
|
|
191
|
+
|
|
192
|
+
module.exports = { getRedis, flushAll, teardown }
|