mobiletto 1.0.46 → 1.0.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/index.js +104 -18
- package/package.json +1 -1
- package/test/index.spec.js +61 -35
- package/util/redis.js +156 -134
package/README.md
CHANGED
|
@@ -194,15 +194,19 @@ You can override either of these:
|
|
|
194
194
|
Mobiletto will store all of its redis keys with the prefix `_mobiletto__`. You can change this
|
|
195
195
|
by setting the `MOBILETTO_REDIS_PREFIX` env var.
|
|
196
196
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
197
|
+
You can also set per-connection caching with the `opts.redisConfig` object:
|
|
198
|
+
|
|
199
|
+
const redisConfig = {
|
|
200
|
+
enabled: true, // optional, default is true. if false other props are ignored
|
|
201
|
+
host: '127.0.0.1',
|
|
202
|
+
port: 6379,
|
|
203
|
+
prefix: '_mobiletto__'
|
|
204
|
+
}
|
|
205
|
+
const opts = { redisConfig, bucket: 'bk', region: 'us-east-1' }
|
|
206
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
|
|
207
|
+
|
|
208
|
+
### Don't want redis caching?
|
|
209
|
+
To disable: pass `enabled: false` in your `opts.redisConfig` object when you establish your connection.
|
|
206
210
|
|
|
207
211
|
As discussed below, disabling caching will have an adverse effect on performance and incur more requests
|
|
208
212
|
to storage that you really need to.
|
package/index.js
CHANGED
|
@@ -2,13 +2,14 @@ const fs = require('fs')
|
|
|
2
2
|
const path = require('path')
|
|
3
3
|
const { Readable, Transform } = require('stream')
|
|
4
4
|
const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
|
|
5
|
-
const
|
|
5
|
+
const { getRedis } = require('./util/redis')
|
|
6
6
|
const { basename, dirname } = require('path')
|
|
7
7
|
const shasum = require('shasum')
|
|
8
8
|
const randomstring = require('randomstring')
|
|
9
9
|
|
|
10
10
|
const crypt = require('./util/crypt')
|
|
11
11
|
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
|
|
12
|
+
const LRU = require("lru-cache");
|
|
12
13
|
|
|
13
14
|
const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
14
15
|
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
@@ -20,6 +21,10 @@ const initMobilettoTempDir = () => {
|
|
|
20
21
|
}
|
|
21
22
|
const MOBILETTO_TMP = initMobilettoTempDir()
|
|
22
23
|
|
|
24
|
+
const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
|
|
25
|
+
const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
|
|
26
|
+
const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
|
|
27
|
+
|
|
23
28
|
// adapted from https://stackoverflow.com/a/27724419
|
|
24
29
|
function MobilettoError (message, err) {
|
|
25
30
|
this.message = `${message}: ${err ? err : ''}`
|
|
@@ -51,15 +56,52 @@ const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
|
51
56
|
const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
|
|
52
57
|
const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
|
|
53
58
|
|
|
54
|
-
const getMetaCache = (size = 100) => redis.scopedCache('meta', size)
|
|
55
|
-
const getListingCache = (size = 100) => redis.scopedCache('listingCache', size)
|
|
56
|
-
|
|
57
59
|
const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
|
|
58
60
|
|
|
61
|
+
class AwaitableLRU {
|
|
62
|
+
lru
|
|
63
|
+
constructor(size) {
|
|
64
|
+
this.lru = new LRU({ max: size })
|
|
65
|
+
}
|
|
66
|
+
get = (key) => Promise.resolve(this.lru.get(key))
|
|
67
|
+
set = (key, value) => Promise.resolve(this.lru.set(key, value))
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const CACHE_FUNCTIONS = {
|
|
71
|
+
redis: (client) => () => {
|
|
72
|
+
if (typeof client.cache !== 'undefined') return client.cache
|
|
73
|
+
const redisConfig = client.redisConfig || {}
|
|
74
|
+
const enabled = redisConfig.enabled !== false
|
|
75
|
+
if (!enabled) {
|
|
76
|
+
logger.info(`redis: client.redisConfig.enabled === false, disabling cache`)
|
|
77
|
+
client.cache = null
|
|
78
|
+
return client.cache
|
|
79
|
+
}
|
|
80
|
+
const host = redisConfig.host || REDIS_HOST
|
|
81
|
+
const port = redisConfig.port || REDIS_PORT
|
|
82
|
+
const prefix = redisConfig.prefix || REDIS_PREFIX
|
|
83
|
+
if (!client.id) {
|
|
84
|
+
logger.warn(`redis: all nameless connections will share one cache`)
|
|
85
|
+
client.cache = getRedis('~nameless~', host, port, prefix)
|
|
86
|
+
} else {
|
|
87
|
+
client.cache = getRedis(client.id, host, port, prefix)
|
|
88
|
+
}
|
|
89
|
+
return client.cache
|
|
90
|
+
},
|
|
91
|
+
scopedCache: (client) => (cacheName, size = 100) => {
|
|
92
|
+
const redis = client.redis()
|
|
93
|
+
return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
|
|
94
|
+
},
|
|
95
|
+
flush: (client) => async () => {
|
|
96
|
+
const redis = client.redis()
|
|
97
|
+
return redis ? redis.flush() : true
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
59
101
|
// noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
|
|
60
102
|
const UTILITY_FUNCTIONS = {
|
|
61
103
|
list: (client) => async (path, opts) => {
|
|
62
|
-
const cache =
|
|
104
|
+
const cache = client.scopedCache('list')
|
|
63
105
|
let cached = cache ? await cache.get(path) : null
|
|
64
106
|
if (cached) {
|
|
65
107
|
if (Array.isArray(cached)) { return cached }
|
|
@@ -104,7 +146,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
104
146
|
}
|
|
105
147
|
},
|
|
106
148
|
metadata: (client) => async (path) => {
|
|
107
|
-
const cache =
|
|
149
|
+
const cache = client.scopedCache('metadata')
|
|
108
150
|
const cached = cache ? await cache.get(path) : null
|
|
109
151
|
if (cached) {
|
|
110
152
|
return cached
|
|
@@ -134,11 +176,11 @@ const UTILITY_FUNCTIONS = {
|
|
|
134
176
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
135
177
|
// noinspection JSUnresolvedFunction
|
|
136
178
|
const result = await client.driver_remove(path, recursive, quiet)
|
|
137
|
-
await
|
|
179
|
+
await client.flush()
|
|
138
180
|
return result
|
|
139
181
|
},
|
|
140
182
|
readFile: (client) => async (path) => {
|
|
141
|
-
const cache =
|
|
183
|
+
const cache = client.scopedCache('readFile')
|
|
142
184
|
const cached = cache ? await cache.get(path) : null
|
|
143
185
|
if (cached) {
|
|
144
186
|
return Buffer.from(cached, 'base64')
|
|
@@ -147,14 +189,17 @@ const UTILITY_FUNCTIONS = {
|
|
|
147
189
|
await client.read(path, reader(chunks))
|
|
148
190
|
const data = Buffer.concat(chunks);
|
|
149
191
|
if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
|
|
150
|
-
cache.set(path, data.toString('base64'))
|
|
192
|
+
cache.set(path, data.toString('base64')).then(
|
|
193
|
+
() => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
|
|
194
|
+
(err) => { logger.error(`readFile(${path}) error: ${err}`) }
|
|
195
|
+
)
|
|
151
196
|
}
|
|
152
197
|
return data
|
|
153
198
|
},
|
|
154
199
|
write: (client) => async (path, data) => {
|
|
155
200
|
// noinspection JSUnresolvedFunction
|
|
156
201
|
const bytesWritten = await client.driver_write(path, data)
|
|
157
|
-
await
|
|
202
|
+
await client.flush()
|
|
158
203
|
return bytesWritten
|
|
159
204
|
},
|
|
160
205
|
writeFile: (client) => async (path, data) => {
|
|
@@ -240,11 +285,22 @@ async function mirrorDir (source, sourcePath, visitor) {
|
|
|
240
285
|
}
|
|
241
286
|
}
|
|
242
287
|
|
|
243
|
-
function
|
|
244
|
-
|
|
245
|
-
client[`driver_${func}`]
|
|
246
|
-
|
|
288
|
+
function utilityFunctionConflict (client, func) {
|
|
289
|
+
if (typeof client[func] === 'function') {
|
|
290
|
+
if (typeof client[`driver_${func}`] !== 'undefined') {
|
|
291
|
+
logger.warn(`utilityFunctionConflict: driver_${func} has already been added`)
|
|
292
|
+
return false
|
|
293
|
+
} else {
|
|
294
|
+
client[`driver_${func}`] = client[func] // save original driver function
|
|
295
|
+
return true
|
|
296
|
+
}
|
|
297
|
+
} else if (typeof client[func] !== 'undefined') {
|
|
298
|
+
throw new MobilettoError(`utilityFunctionConflict: client defines a property ${func}, mobiletto function would overwrite`)
|
|
247
299
|
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const addUtilityFunctions = (client, readOnly = false) => {
|
|
303
|
+
addClientFunctions(client, UTILITY_FUNCTIONS, utilityFunctionConflict)
|
|
248
304
|
if (readOnly) {
|
|
249
305
|
for (const writeFunc of ['write', 'remove', 'writeFile']) {
|
|
250
306
|
client[writeFunc] = async () => {
|
|
@@ -256,6 +312,24 @@ function addUtilityFunctions (client, readOnly = false) {
|
|
|
256
312
|
return client
|
|
257
313
|
}
|
|
258
314
|
|
|
315
|
+
const addCacheFunctions = (client) => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
|
|
316
|
+
logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
|
|
317
|
+
return false
|
|
318
|
+
})
|
|
319
|
+
|
|
320
|
+
const addClientFunctions = (client, functions, conflictFunc) => {
|
|
321
|
+
for (const func of Object.keys(functions)) {
|
|
322
|
+
let add = true
|
|
323
|
+
if (client[func]) {
|
|
324
|
+
add = conflictFunc ? conflictFunc(client, func) : false
|
|
325
|
+
}
|
|
326
|
+
if (add) {
|
|
327
|
+
client[func] = functions[func](client)
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return client
|
|
331
|
+
}
|
|
332
|
+
|
|
259
333
|
async function connect (driverPath, key, secret, opts, encryption = null) {
|
|
260
334
|
return await mobiletto(driverPath, key, secret, opts, encryption)
|
|
261
335
|
}
|
|
@@ -270,9 +344,18 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
270
344
|
throw new MobilettoError(message)
|
|
271
345
|
}
|
|
272
346
|
const readOnly = opts ? !!opts.readOnly : false
|
|
347
|
+
client.redisConfig = opts && opts.redisConfig ? opts.redisConfig : {}
|
|
348
|
+
|
|
349
|
+
const internalIdForDriver = () =>
|
|
350
|
+
driverPath + '_' + shasum(`${key}\n${JSON.stringify(opts)}\n${(encryption ? JSON.stringify(encryption) : '')}`)
|
|
351
|
+
|
|
352
|
+
// If the driver didn't give the client a name, generate a unique internal name
|
|
353
|
+
if (!client.id) {
|
|
354
|
+
client.id = internalIdForDriver()
|
|
355
|
+
}
|
|
273
356
|
if (encryption === null) {
|
|
274
357
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
275
|
-
return addUtilityFunctions(client, readOnly)
|
|
358
|
+
return addUtilityFunctions(addCacheFunctions(client), readOnly)
|
|
276
359
|
}
|
|
277
360
|
const encKey = normalizeKey(encryption.key)
|
|
278
361
|
if (!encKey) {
|
|
@@ -306,8 +389,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
306
389
|
const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
|
|
307
390
|
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
308
391
|
|
|
392
|
+
const outerClient = addCacheFunctions(client)
|
|
309
393
|
const _metadata = (client) => async (path) => {
|
|
310
|
-
const cache =
|
|
394
|
+
const cache = outerClient.scopedCache('_metadata')
|
|
311
395
|
const cached = cache ? await cache.get(path) : null
|
|
312
396
|
if (cached) {
|
|
313
397
|
return cached
|
|
@@ -385,13 +469,15 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
385
469
|
}
|
|
386
470
|
|
|
387
471
|
const encClient = {
|
|
472
|
+
id: internalIdForDriver(),
|
|
473
|
+
redisConfig: client.redisConfig,
|
|
388
474
|
list: async (pth = '', recursive, visitor) => {
|
|
389
475
|
const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
|
|
390
476
|
const dirent = direntDir(p)
|
|
391
477
|
let entries
|
|
392
478
|
|
|
393
479
|
const cacheKey = `${p} ~ ${recursive}`
|
|
394
|
-
const cache = visitor ? null :
|
|
480
|
+
const cache = visitor ? null : client.scopedCache('enc_list')
|
|
395
481
|
const cached = cache && await cache.get(cacheKey)
|
|
396
482
|
|
|
397
483
|
function cacheAndReturn (thing) {
|
|
@@ -565,7 +651,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
565
651
|
}
|
|
566
652
|
}
|
|
567
653
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
568
|
-
return addUtilityFunctions(encClient, readOnly)
|
|
654
|
+
return addUtilityFunctions(addCacheFunctions(encClient), readOnly)
|
|
569
655
|
}
|
|
570
656
|
|
|
571
657
|
async function readStream(stream, callback, endCallback) {
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -14,14 +14,15 @@ const {
|
|
|
14
14
|
M_DIR, M_FILE,
|
|
15
15
|
mobiletto, connect, MobilettoNotFoundError, readStream
|
|
16
16
|
} = require("../index")
|
|
17
|
+
|
|
17
18
|
const {
|
|
18
19
|
DEFAULT_CRYPT_ALGO,
|
|
19
20
|
normalizeKey, normalizeIV,
|
|
20
21
|
encrypt, decrypt,
|
|
21
22
|
getCipher, getDecipher
|
|
22
23
|
} = require("../util/crypt")
|
|
23
|
-
|
|
24
|
-
const {logger} = require("../util/logger");
|
|
24
|
+
|
|
25
|
+
const { logger } = require("../util/logger");
|
|
25
26
|
|
|
26
27
|
// chunk size used by generator function, used by driver's 'write' function
|
|
27
28
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
@@ -159,41 +160,71 @@ describe('crypto test', () => {
|
|
|
159
160
|
|
|
160
161
|
const encryptionTests = () => [null, { key: rand(32) }]
|
|
161
162
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
163
|
+
const REDIS_ENABLED = {
|
|
164
|
+
name: 'redis-enabled',
|
|
165
|
+
enabled: true,
|
|
166
|
+
config: cfg => Object.assign({}, cfg)
|
|
167
|
+
}
|
|
168
|
+
const REDIS_DISABLED = {
|
|
169
|
+
name: 'redis-disabled',
|
|
170
|
+
enabled: true,
|
|
171
|
+
config: (cfg) => {
|
|
172
|
+
const newConfig = Object.assign({}, cfg)
|
|
173
|
+
if (!newConfig.opts) { newConfig.opts = {} }
|
|
174
|
+
if (!newConfig.opts.redisConfig) { newConfig.opts.redisConfig = {} }
|
|
175
|
+
newConfig.opts.redisConfig.enabled = false
|
|
176
|
+
return newConfig
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const redisTests = () => [REDIS_ENABLED, REDIS_DISABLED]
|
|
165
180
|
|
|
181
|
+
// For testing single drivers
|
|
182
|
+
// Comment out the for loop for driver tests, and hard-code one of the drivers below
|
|
166
183
|
// const driverName = 'local'
|
|
167
184
|
// const driverName = 's3'
|
|
168
185
|
|
|
186
|
+
// For testing single redis setups
|
|
187
|
+
// Comment out the for loop for redis test, and hard-code one of the setups below
|
|
188
|
+
// const redisSetup = REDIS_ENABLED
|
|
189
|
+
// const redisSetup = REDIS_DISABLED
|
|
190
|
+
|
|
169
191
|
// When you're NOT testing a single driver, it's important that *all* tests remain
|
|
170
|
-
// within the
|
|
171
|
-
// the same set of tests, with exactly the same inputs/outputs
|
|
192
|
+
// within the various 'for' loops below. This ensures that all drivers pass exactly
|
|
193
|
+
// the same set of tests, with exactly the same inputs/outputs, both with and without
|
|
194
|
+
// redis caching enabled, both with and without encryption enabled.
|
|
172
195
|
// This greatly supports developer sanity
|
|
173
196
|
|
|
197
|
+
for (const redisSetup of redisTests()) {
|
|
198
|
+
if (!redisSetup.enabled) continue
|
|
199
|
+
|
|
174
200
|
for (const driverName of DRIVER_NAMES) {
|
|
175
|
-
const
|
|
201
|
+
const driverTest = `${driverName} [${redisSetup.name}]`
|
|
202
|
+
const config = redisSetup.config(DRIVER_CONFIG[driverName])
|
|
176
203
|
const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
|
|
177
204
|
const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
|
|
178
|
-
describe(`${
|
|
205
|
+
describe(`${driverTest} test`, () => {
|
|
179
206
|
beforeEach ((done) => {
|
|
180
|
-
redis.
|
|
181
|
-
() => { logger.info(`${
|
|
207
|
+
redis.flushAll().then(
|
|
208
|
+
() => { logger.info(`${driverTest} - flushed redis`) },
|
|
182
209
|
(e) => {
|
|
183
|
-
logger.error(`${
|
|
210
|
+
logger.error(`${driverTest} - error flushing redis: ${e}`)
|
|
184
211
|
throw e
|
|
185
212
|
}
|
|
186
213
|
).finally(() => done())
|
|
187
214
|
})
|
|
188
215
|
|
|
189
|
-
describe(`${
|
|
216
|
+
describe(`${driverTest} - create api client`, () => {
|
|
190
217
|
it("should validate the config and return an API object", async () => {
|
|
191
|
-
|
|
192
|
-
|
|
218
|
+
try {
|
|
219
|
+
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
220
|
+
should().exist(api, 'expected API object to exist')
|
|
221
|
+
} catch (e) {
|
|
222
|
+
logger.error(`wtf: ${e}`)
|
|
223
|
+
}
|
|
193
224
|
})
|
|
194
225
|
})
|
|
195
226
|
|
|
196
|
-
describe(`${
|
|
227
|
+
describe(`${driverTest} - listing with no arguments returns some results`, () => {
|
|
197
228
|
it("should return some results from a default listing", async () => {
|
|
198
229
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
199
230
|
const results = await api.list()
|
|
@@ -201,7 +232,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
201
232
|
})
|
|
202
233
|
})
|
|
203
234
|
|
|
204
|
-
describe(`${
|
|
235
|
+
describe(`${driverTest} - write a file, read file, read metadata, delete file`, () => {
|
|
205
236
|
// some random data, plus a bit extra
|
|
206
237
|
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
207
238
|
const randomData = rand(size)
|
|
@@ -239,8 +270,8 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
239
270
|
|
|
240
271
|
for (const encryption of encryptionTests()) {
|
|
241
272
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
242
|
-
describe(`${
|
|
243
|
-
// describe(`${
|
|
273
|
+
describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
274
|
+
// describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
|
|
244
275
|
// const encryption = {key: rand(32)}
|
|
245
276
|
const size = 16
|
|
246
277
|
const randomData = rand(size)
|
|
@@ -285,7 +316,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
285
316
|
})
|
|
286
317
|
}
|
|
287
318
|
|
|
288
|
-
describe(`${
|
|
319
|
+
describe(`${driverTest} - write an encrypted file, read file, read metadata, delete file`, () => {
|
|
289
320
|
// some random data, plus a bit extra
|
|
290
321
|
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
291
322
|
const randomData = rand(size)
|
|
@@ -325,8 +356,8 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
325
356
|
const mirrorDriver = pickAnotherDriver(driverName)
|
|
326
357
|
const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
|
|
327
358
|
const mirrorDest = `mirrorDest_${rand(5)}/`
|
|
328
|
-
describe(`${
|
|
329
|
-
// describe(`${
|
|
359
|
+
describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
|
|
360
|
+
// describe(`${driverTest} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|
|
330
361
|
// const encryption = {key: rand(32)}
|
|
331
362
|
// const encryption = null
|
|
332
363
|
// a random directory and file within it
|
|
@@ -464,7 +495,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
464
495
|
})
|
|
465
496
|
}
|
|
466
497
|
|
|
467
|
-
describe(`${
|
|
498
|
+
describe(`${driverTest} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
|
|
468
499
|
it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
|
|
469
500
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
470
501
|
try {
|
|
@@ -477,7 +508,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
477
508
|
})
|
|
478
509
|
})
|
|
479
510
|
|
|
480
|
-
describe(`${
|
|
511
|
+
describe(`${driverTest} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
|
|
481
512
|
it("should throw MobilettoNotFoundError when trying to read metadata for a file that does not exist", async () => {
|
|
482
513
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
483
514
|
try {
|
|
@@ -489,7 +520,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
489
520
|
})
|
|
490
521
|
})
|
|
491
522
|
|
|
492
|
-
describe(`${
|
|
523
|
+
describe(`${driverTest} - expect MobilettoNotFoundError when removing non-existent file`, () => {
|
|
493
524
|
it("should throw MobilettoNotFoundError when trying to remove a file that does not exist", async () => {
|
|
494
525
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
495
526
|
try {
|
|
@@ -502,7 +533,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
502
533
|
})
|
|
503
534
|
})
|
|
504
535
|
|
|
505
|
-
describe(`${
|
|
536
|
+
describe(`${driverTest} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
|
|
506
537
|
it("should NOT throw MobilettoNotFoundError when quietly removing a file that does not exist", async () => {
|
|
507
538
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
508
539
|
if (!(await api.remove(nonexistentFile, {quiet: true}))) {
|
|
@@ -512,14 +543,9 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
512
543
|
})
|
|
513
544
|
})
|
|
514
545
|
}
|
|
546
|
+
}
|
|
515
547
|
|
|
516
|
-
after ((done) => {
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
redis.teardown()
|
|
520
|
-
} catch (e) {
|
|
521
|
-
logger.warn(`error calling redis.teardown: ${e}`)
|
|
522
|
-
} finally {
|
|
523
|
-
done()
|
|
524
|
-
}
|
|
548
|
+
after ( (done) => {
|
|
549
|
+
logger.info('all tests finished, tearing down redis...')
|
|
550
|
+
redis.teardown().finally(done)
|
|
525
551
|
})
|
package/util/redis.js
CHANGED
|
@@ -1,170 +1,192 @@
|
|
|
1
1
|
const Redis = require('ioredis')
|
|
2
2
|
const generators = require('redis-async-gen')
|
|
3
3
|
const { logger } = require('./logger')
|
|
4
|
-
const LRU = require("lru-cache")
|
|
4
|
+
const LRU = require("lru-cache")
|
|
5
5
|
|
|
6
6
|
const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
|
|
7
|
-
const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
|
|
8
|
-
const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
|
|
9
|
-
const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
|
|
10
7
|
|
|
11
|
-
|
|
12
|
-
const unprefix = key => key && key.startsWith(REDIS_PREFIX) ? key.substring(REDIS_PREFIX.length) : key
|
|
8
|
+
let REDIS_CLIENTS = {}
|
|
13
9
|
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const getRedis = () => {
|
|
19
|
-
if (REDIS_CLIENT === null) {
|
|
20
|
-
try {
|
|
21
|
-
REDIS_CLIENT = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {
|
|
22
|
-
host: REDIS_HOST,
|
|
23
|
-
port: REDIS_PORT
|
|
24
|
-
}))
|
|
25
|
-
flushMobiletto().then(
|
|
26
|
-
() => { logger.info(`getRedis: successfully flushed`) },
|
|
27
|
-
(e) => { logger.error(`getRedis: error flushing: ${e}`) }
|
|
28
|
-
)
|
|
29
|
-
} catch (e) {
|
|
30
|
-
logger.error(`getRedis(host=${REDIS_HOST}, port=${REDIS_PORT}) connection error: ${e}`)
|
|
31
|
-
REDIS_CLIENT = NO_REDIS
|
|
32
|
-
}
|
|
10
|
+
const getRedis = (name, host, port, prefix) => {
|
|
11
|
+
if (typeof REDIS_CLIENTS[name] === 'undefined') {
|
|
12
|
+
REDIS_CLIENTS[name] = new MobilettoCache(name, host, port, prefix + name + '_')
|
|
33
13
|
}
|
|
34
|
-
return
|
|
14
|
+
return REDIS_CLIENTS[name]
|
|
35
15
|
}
|
|
36
16
|
|
|
37
17
|
const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
|
|
38
18
|
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
19
|
+
const ZERO_COUNTERS = {
|
|
20
|
+
get: 0,
|
|
21
|
+
set: 0,
|
|
22
|
+
del: 0,
|
|
23
|
+
flush: 0,
|
|
24
|
+
hit: 0,
|
|
25
|
+
miss: 0
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
class MobilettoCache {
|
|
29
|
+
name
|
|
30
|
+
redis
|
|
31
|
+
prefix
|
|
32
|
+
scopedCaches = {}
|
|
33
|
+
counters = Object.assign({}, ZERO_COUNTERS)
|
|
34
|
+
printStatsInterval = 1000
|
|
35
|
+
constructor(name,
|
|
36
|
+
host = '127.0.0.1',
|
|
37
|
+
port = 6379,
|
|
38
|
+
prefix = '_mobiletto__') {
|
|
39
|
+
this.name = name
|
|
40
|
+
if (host && port) {
|
|
41
|
+
try {
|
|
42
|
+
this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
|
|
43
|
+
} catch (e) {
|
|
44
|
+
logger.error(`redis(${name}): error connecting to redis, using fallback LRU for scoped caches: ${e}`)
|
|
45
|
+
this.redis = null
|
|
46
|
+
}
|
|
47
|
+
} else {
|
|
48
|
+
logger.warn(`redis(${name}): no host or port provided, using fallback LRU for scoped caches`)
|
|
49
|
+
this.redis = null
|
|
50
|
+
}
|
|
51
|
+
this.prefix = prefix
|
|
52
|
+
this.flush().then(
|
|
53
|
+
() => { logger.debug(`redis(${name}): successfully flushed`) },
|
|
54
|
+
(e) => { logger.error(`redis(${name}): error flushing: ${e}`) }
|
|
55
|
+
)
|
|
56
|
+
}
|
|
62
57
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
58
|
+
stats = () => this.counters
|
|
59
|
+
resetStats = () => { this.counters = Object.assign({}, ZERO_COUNTERS) }
|
|
60
|
+
hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
|
|
61
|
+
toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
|
|
66
62
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
}
|
|
63
|
+
pfx = key => this.prefix + key
|
|
64
|
+
unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
|
|
70
65
|
|
|
71
|
-
async
|
|
72
|
-
|
|
73
|
-
|
|
66
|
+
doRedisAsync = async (func, defaultValue = null) => {
|
|
67
|
+
try {
|
|
68
|
+
return this.redis ? await func(this.redis) : defaultValue
|
|
69
|
+
} catch (e) {
|
|
70
|
+
logger.warn(`redis(${this.name}): doRedisAsync(${func}): ${e} (returning default value: ${defaultValue})`)
|
|
71
|
+
return defaultValue
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
74
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
doRedis = (func, defaultValue = null) => {
|
|
76
|
+
try {
|
|
77
|
+
return this.redis ? func(this.redis) : defaultValue
|
|
78
|
+
} catch (e) {
|
|
79
|
+
logger.warn(`redis(${this.name}): doRedis(${func}): ${e} (returning default value: ${defaultValue})`)
|
|
80
|
+
return defaultValue
|
|
81
|
+
}
|
|
82
|
+
}
|
|
78
83
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
get = key => {
|
|
85
|
+
this.counters.get++
|
|
86
|
+
logger.debug(`redis(${this.name}): get(${key}) starting`)
|
|
87
|
+
const val = this.doRedis(r => r.get(this.pfx(key)), null)
|
|
88
|
+
logger.debug(`redis(${this.name}): get(${key}) found value: ${val}`)
|
|
89
|
+
if (val) {
|
|
90
|
+
this.counters.hit++
|
|
91
|
+
} else {
|
|
92
|
+
this.counters.miss++
|
|
93
|
+
}
|
|
94
|
+
if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
|
|
95
|
+
const message = `${new Date()}: ${this}`;
|
|
96
|
+
logger.info(message)
|
|
97
|
+
}
|
|
98
|
+
return val
|
|
83
99
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
100
|
+
|
|
101
|
+
set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
|
|
102
|
+
this.counters.set++
|
|
103
|
+
logger.debug(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) starting`)
|
|
104
|
+
await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
|
|
105
|
+
logger.debug(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) finished`)
|
|
88
106
|
}
|
|
89
|
-
return keys
|
|
90
|
-
}
|
|
91
107
|
|
|
92
|
-
async
|
|
93
|
-
|
|
94
|
-
|
|
108
|
+
del = async key => {
|
|
109
|
+
this.counters.del++
|
|
110
|
+
await this.doRedisAsync(r => r.del(this.pfx(key)))
|
|
111
|
+
}
|
|
95
112
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
113
|
+
findMatchingKeys = async (pattern) => {
|
|
114
|
+
if (!this.redis) return []
|
|
115
|
+
const { keysMatching } = generators.using(this.redis)
|
|
116
|
+
const keys = []
|
|
117
|
+
for await (const key of keysMatching(this.pfx(pattern))) {
|
|
118
|
+
keys.push(this.unprefix(key))
|
|
119
|
+
}
|
|
120
|
+
return keys
|
|
100
121
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
results
|
|
122
|
+
applyToMatchingKeys = async (pattern, asyncFunc) => {
|
|
123
|
+
if (!this.redis) return []
|
|
124
|
+
const { keysMatching } = generators.using(this.redis)
|
|
125
|
+
const results = []
|
|
126
|
+
for await (const key of keysMatching(this.pfx(pattern))) {
|
|
127
|
+
results.push(await asyncFunc(this.unprefix(key)))
|
|
128
|
+
}
|
|
129
|
+
return results
|
|
105
130
|
}
|
|
106
|
-
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const CACHES = {}
|
|
110
|
-
|
|
111
|
-
const flushMobiletto = async (redis = getRedis()) => {
|
|
112
|
-
return redis ? await removeMatchingKeys('*') : null
|
|
113
|
-
}
|
|
131
|
+
removeMatchingKeys = async (pattern) => await this.applyToMatchingKeys(pattern, this.del)
|
|
114
132
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
}
|
|
119
|
-
const redis = getRedis()
|
|
120
|
-
if (isNoRedis(redis)) {
|
|
121
|
-
return size && size > 0 ? new LRU({ max: size }) : null
|
|
133
|
+
flush = async () => {
|
|
134
|
+
this.counters.flush++
|
|
135
|
+
await this.removeMatchingKeys('*')
|
|
122
136
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
if (key && value) {
|
|
137
|
+
|
|
138
|
+
scopedCache = (name, size = 100) => {
|
|
139
|
+
if (this.scopedCaches[name]) {
|
|
140
|
+
return this.scopedCaches[name]
|
|
141
|
+
}
|
|
142
|
+
if (!this.redis) {
|
|
143
|
+
return size && size > 0 ? new LRU({ max: size }) : null
|
|
144
|
+
}
|
|
145
|
+
const realKey = k => k ? `${name}_${k}` : null
|
|
146
|
+
const cache = {
|
|
147
|
+
get: async key => {
|
|
148
|
+
const val = await this.get(realKey(key))
|
|
149
|
+
if (!val) return null
|
|
137
150
|
try {
|
|
138
|
-
|
|
151
|
+
return JSON.parse(val)
|
|
139
152
|
} catch (e) {
|
|
140
|
-
logger.warn(`
|
|
153
|
+
logger.warn(`redis(${this.name}): get(${key}) error: ${e}`)
|
|
154
|
+
return null
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
set: async (key, value) => {
|
|
158
|
+
if (key && value) {
|
|
159
|
+
try {
|
|
160
|
+
await this.set(realKey(key), JSON.stringify(value))
|
|
161
|
+
} catch (e) {
|
|
162
|
+
logger.warn(`redis(${this.name}): set(${key}) error: ${e}`)
|
|
163
|
+
}
|
|
141
164
|
}
|
|
142
165
|
}
|
|
143
166
|
}
|
|
167
|
+
this.scopedCaches[name] = cache
|
|
168
|
+
return cache
|
|
144
169
|
}
|
|
145
|
-
CACHES[name] = cache
|
|
146
|
-
return cache
|
|
147
170
|
}
|
|
148
171
|
|
|
149
|
-
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
172
|
+
const forAllCaches = async (func) => {
|
|
173
|
+
const promises = []
|
|
174
|
+
for (const name of Object.keys(REDIS_CLIENTS)) {
|
|
175
|
+
promises.push(func(REDIS_CLIENTS[name]))
|
|
153
176
|
}
|
|
177
|
+
return Promise.all(promises)
|
|
154
178
|
}
|
|
155
179
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
teardown
|
|
170
|
-
}
|
|
180
|
+
const teardown = async () => await forAllCaches((client) => {
|
|
181
|
+
try {
|
|
182
|
+
if (client.redis) {
|
|
183
|
+
client.redis.disconnect()
|
|
184
|
+
}
|
|
185
|
+
} catch (e) {
|
|
186
|
+
logger.warn(`teardown: error disconnecting from redis(${client.name}): ${e}`)
|
|
187
|
+
}
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
const flushAll = async () => await forAllCaches(client => client.flush())
|
|
191
|
+
|
|
192
|
+
module.exports = { getRedis, flushAll, teardown }
|