mobiletto 1.0.45 → 1.0.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -194,15 +194,19 @@ You can override either of these:
194
194
  Mobiletto will store all of its redis keys with the prefix `_mobiletto__`. You can change this
195
195
  by setting the `MOBILETTO_REDIS_PREFIX` env var.
196
196
 
197
- ### Turn off redis caching
198
- To disable redis, you can do either of:
199
- * Not run a redis server on 127.0.0.1:6379 that allows unauthenticated local connections
200
- * Set `MOBILETTO_REDIS_HOST` to an empty string or anything that won't resolve as a host, for example:
201
- * `MOBILETTO_REDIS_HOST=`
202
- * `MOBILETTO_REDIS_HOST=/dev/null`
203
- * `MOBILETTO_REDIS_HOST=~invalid~hostname~`
204
- * `MOBILETTO_REDIS_HOST=invalid-hostname` *<-- not this!*
205
- * **^^^ NO! that last one is a valid hostname** and depending on the name resolution scheme, it might actually be valid and you'll connect to it!
197
+ You can also set per-connection caching with the `opts.redisConfig` object:
198
+
199
+ const redisConfig = {
200
+ enabled: true, // optional, default is true. if false other props are ignored
201
+ host: '127.0.0.1',
202
+ port: 6379,
203
+ prefix: '_mobiletto__'
204
+ }
205
+ const opts = { redisConfig, bucket: 'bk', region: 'us-east-1' }
206
+ const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
207
+
208
+ ### Don't want redis caching?
209
+ To disable: pass `enabled: false` in your `opts.redisConfig` object when you establish your connection.
206
210
 
207
211
  As discussed below, disabling caching will have an adverse effect on performance and incur more requests
208
212
  to storage that you really need to.
package/index.js CHANGED
@@ -2,13 +2,14 @@ const fs = require('fs')
2
2
  const path = require('path')
3
3
  const { Readable, Transform } = require('stream')
4
4
  const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
5
- const redis = require('./util/redis')
5
+ const { getRedis } = require('./util/redis')
6
6
  const { basename, dirname } = require('path')
7
7
  const shasum = require('shasum')
8
8
  const randomstring = require('randomstring')
9
9
 
10
10
  const crypt = require('./util/crypt')
11
11
  const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
12
+ const LRU = require("lru-cache");
12
13
 
13
14
  const DIR_ENT_DIR_SUFFIX = '__.dirent'
14
15
  const DIR_ENT_FILE_PREFIX = 'dirent__'
@@ -20,6 +21,10 @@ const initMobilettoTempDir = () => {
20
21
  }
21
22
  const MOBILETTO_TMP = initMobilettoTempDir()
22
23
 
24
+ const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
25
+ const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
26
+ const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
27
+
23
28
  // adapted from https://stackoverflow.com/a/27724419
24
29
  function MobilettoError (message, err) {
25
30
  this.message = `${message}: ${err ? err : ''}`
@@ -51,15 +56,52 @@ const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
51
56
  const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
52
57
  const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
53
58
 
54
- const getMetaCache = (size = 100) => redis.scopedCache('meta', size)
55
- const getListingCache = (size = 100) => redis.scopedCache('listingCache', size)
56
-
57
59
  const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
58
60
 
61
+ class AwaitableLRU {
62
+ lru
63
+ constructor(size) {
64
+ this.lru = new LRU({ max: size })
65
+ }
66
+ get = (key) => Promise.resolve(this.lru.get(key))
67
+ set = (key, value) => Promise.resolve(this.lru.set(key, value))
68
+ }
69
+
70
+ const CACHE_FUNCTIONS = {
71
+ redis: (client) => () => {
72
+ if (typeof client.cache !== 'undefined') return client.cache
73
+ const redisConfig = client.redisConfig || {}
74
+ const enabled = redisConfig.enabled !== false
75
+ if (!enabled) {
76
+ logger.info(`redis: client.redisConfig.enabled === false, disabling cache`)
77
+ client.cache = null
78
+ return client.cache
79
+ }
80
+ const host = redisConfig.host || REDIS_HOST
81
+ const port = redisConfig.port || REDIS_PORT
82
+ const prefix = redisConfig.prefix || REDIS_PREFIX
83
+ if (!client.id) {
84
+ logger.warn(`redis: all nameless connections will share one cache`)
85
+ client.cache = getRedis('~nameless~', host, port, prefix)
86
+ } else {
87
+ client.cache = getRedis(client.id, host, port, prefix)
88
+ }
89
+ return client.cache
90
+ },
91
+ scopedCache: (client) => (cacheName, size = 100) => {
92
+ const redis = client.redis()
93
+ return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
94
+ },
95
+ flush: (client) => async () => {
96
+ const redis = client.redis()
97
+ return redis ? redis.flush() : true
98
+ }
99
+ }
100
+
59
101
  // noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
60
102
  const UTILITY_FUNCTIONS = {
61
103
  list: (client) => async (path, opts) => {
62
- const cache = getListingCache()
104
+ const cache = client.scopedCache('list')
63
105
  let cached = cache ? await cache.get(path) : null
64
106
  if (cached) {
65
107
  if (Array.isArray(cached)) { return cached }
@@ -104,7 +146,7 @@ const UTILITY_FUNCTIONS = {
104
146
  }
105
147
  },
106
148
  metadata: (client) => async (path) => {
107
- const cache = getMetaCache()
149
+ const cache = client.scopedCache('metadata')
108
150
  const cached = cache ? await cache.get(path) : null
109
151
  if (cached) {
110
152
  return cached
@@ -134,11 +176,11 @@ const UTILITY_FUNCTIONS = {
134
176
  const quiet = opts && opts.quiet ? opts.quiet : false
135
177
  // noinspection JSUnresolvedFunction
136
178
  const result = await client.driver_remove(path, recursive, quiet)
137
- await redis.flushMobiletto()
179
+ await client.flush()
138
180
  return result
139
181
  },
140
182
  readFile: (client) => async (path) => {
141
- const cache = redis.scopedCache('readFile')
183
+ const cache = client.scopedCache('readFile')
142
184
  const cached = cache ? await cache.get(path) : null
143
185
  if (cached) {
144
186
  return Buffer.from(cached, 'base64')
@@ -147,14 +189,17 @@ const UTILITY_FUNCTIONS = {
147
189
  await client.read(path, reader(chunks))
148
190
  const data = Buffer.concat(chunks);
149
191
  if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
150
- cache.set(path, data.toString('base64'))
192
+ cache.set(path, data.toString('base64')).then(
193
+ () => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
194
+ (err) => { logger.error(`readFile(${path}) error: ${err}`) }
195
+ )
151
196
  }
152
197
  return data
153
198
  },
154
199
  write: (client) => async (path, data) => {
155
200
  // noinspection JSUnresolvedFunction
156
201
  const bytesWritten = await client.driver_write(path, data)
157
- await redis.flushMobiletto()
202
+ await client.flush()
158
203
  return bytesWritten
159
204
  },
160
205
  writeFile: (client) => async (path, data) => {
@@ -240,11 +285,22 @@ async function mirrorDir (source, sourcePath, visitor) {
240
285
  }
241
286
  }
242
287
 
243
- function addUtilityFunctions (client, readOnly = false) {
244
- for (const func of Object.keys(UTILITY_FUNCTIONS)) {
245
- client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
246
- client[func] = UTILITY_FUNCTIONS[func](client)
288
+ function utilityFunctionConflict (client, func) {
289
+ if (typeof client[func] === 'function') {
290
+ if (typeof client[`driver_${func}`] !== 'undefined') {
291
+ logger.warn(`utilityFunctionConflict: driver_${func} has already been added`)
292
+ return false
293
+ } else {
294
+ client[`driver_${func}`] = client[func] // save original driver function
295
+ return true
296
+ }
297
+ } else if (typeof client[func] !== 'undefined') {
298
+ throw new MobilettoError(`utilityFunctionConflict: client defines a property ${func}, mobiletto function would overwrite`)
247
299
  }
300
+ }
301
+
302
+ const addUtilityFunctions = (client, readOnly = false) => {
303
+ addClientFunctions(client, UTILITY_FUNCTIONS, utilityFunctionConflict)
248
304
  if (readOnly) {
249
305
  for (const writeFunc of ['write', 'remove', 'writeFile']) {
250
306
  client[writeFunc] = async () => {
@@ -256,6 +312,24 @@ function addUtilityFunctions (client, readOnly = false) {
256
312
  return client
257
313
  }
258
314
 
315
+ const addCacheFunctions = (client) => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
316
+ logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
317
+ return false
318
+ })
319
+
320
+ const addClientFunctions = (client, functions, conflictFunc) => {
321
+ for (const func of Object.keys(functions)) {
322
+ let add = true
323
+ if (client[func]) {
324
+ add = conflictFunc ? conflictFunc(client, func) : false
325
+ }
326
+ if (add) {
327
+ client[func] = functions[func](client)
328
+ }
329
+ }
330
+ return client
331
+ }
332
+
259
333
  async function connect (driverPath, key, secret, opts, encryption = null) {
260
334
  return await mobiletto(driverPath, key, secret, opts, encryption)
261
335
  }
@@ -270,9 +344,18 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
270
344
  throw new MobilettoError(message)
271
345
  }
272
346
  const readOnly = opts ? !!opts.readOnly : false
347
+ client.redisConfig = opts && opts.redisConfig ? opts.redisConfig : {}
348
+
349
+ const internalIdForDriver = () =>
350
+ driverPath + '_' + shasum(`${key}\n${JSON.stringify(opts)}\n${(encryption ? JSON.stringify(encryption) : '')}`)
351
+
352
+ // If the driver didn't give the client a name, generate a unique internal name
353
+ if (!client.id) {
354
+ client.id = internalIdForDriver()
355
+ }
273
356
  if (encryption === null) {
274
357
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
275
- return addUtilityFunctions(client, readOnly)
358
+ return addUtilityFunctions(addCacheFunctions(client), readOnly)
276
359
  }
277
360
  const encKey = normalizeKey(encryption.key)
278
361
  if (!encKey) {
@@ -306,8 +389,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
306
389
  const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
307
390
  const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
308
391
 
392
+ const outerClient = addCacheFunctions(client)
309
393
  const _metadata = (client) => async (path) => {
310
- const cache = redis.scopedCache('_metadata')
394
+ const cache = outerClient.scopedCache('_metadata')
311
395
  const cached = cache ? await cache.get(path) : null
312
396
  if (cached) {
313
397
  return cached
@@ -385,13 +469,15 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
385
469
  }
386
470
 
387
471
  const encClient = {
472
+ id: internalIdForDriver(),
473
+ redisConfig: client.redisConfig,
388
474
  list: async (pth = '', recursive, visitor) => {
389
475
  const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
390
476
  const dirent = direntDir(p)
391
477
  let entries
392
478
 
393
479
  const cacheKey = `${p} ~ ${recursive}`
394
- const cache = visitor ? null : redis.scopedCache('enc_list')
480
+ const cache = visitor ? null : client.scopedCache('enc_list')
395
481
  const cached = cache && await cache.get(cacheKey)
396
482
 
397
483
  function cacheAndReturn (thing) {
@@ -565,7 +651,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
565
651
  }
566
652
  }
567
653
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
568
- return addUtilityFunctions(encClient, readOnly)
654
+ return addUtilityFunctions(addCacheFunctions(encClient), readOnly)
569
655
  }
570
656
 
571
657
  async function readStream(stream, callback, endCallback) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.45",
3
+ "version": "1.0.48",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "keywords": [
@@ -14,14 +14,15 @@ const {
14
14
  M_DIR, M_FILE,
15
15
  mobiletto, connect, MobilettoNotFoundError, readStream
16
16
  } = require("../index")
17
+
17
18
  const {
18
19
  DEFAULT_CRYPT_ALGO,
19
20
  normalizeKey, normalizeIV,
20
21
  encrypt, decrypt,
21
22
  getCipher, getDecipher
22
23
  } = require("../util/crypt")
23
- const crypt = require("../util/crypt");
24
- const {logger} = require("../util/logger");
24
+
25
+ const { logger } = require("../util/logger");
25
26
 
26
27
  // chunk size used by generator function, used by driver's 'write' function
27
28
  // the temp file is also TEMP_SZ_MULTIPLE of this number
@@ -159,31 +160,71 @@ describe('crypto test', () => {
159
160
 
160
161
  const encryptionTests = () => [null, { key: rand(32) }]
161
162
 
162
- // To test a single driver:
163
- // - Uncomment one of the lines below to set driverName to the one you want to test
164
- // - Comment out the next `for` line, and its closing curly brace (just before EOF)
163
+ const REDIS_ENABLED = {
164
+ name: 'redis-enabled',
165
+ enabled: true,
166
+ config: cfg => Object.assign({}, cfg)
167
+ }
168
+ const REDIS_DISABLED = {
169
+ name: 'redis-disabled',
170
+ enabled: true,
171
+ config: (cfg) => {
172
+ const newConfig = Object.assign({}, cfg)
173
+ if (!newConfig.opts) { newConfig.opts = {} }
174
+ if (!newConfig.opts.redisConfig) { newConfig.opts.redisConfig = {} }
175
+ newConfig.opts.redisConfig.enabled = false
176
+ return newConfig
177
+ }
178
+ }
179
+ const redisTests = () => [REDIS_ENABLED, REDIS_DISABLED]
165
180
 
181
+ // For testing single drivers
182
+ // Comment out the for loop for driver tests, and hard-code one of the drivers below
166
183
  // const driverName = 'local'
167
184
  // const driverName = 's3'
168
185
 
186
+ // For testing single redis setups
187
+ // Comment out the for loop for redis test, and hard-code one of the setups below
188
+ // const redisSetup = REDIS_ENABLED
189
+ // const redisSetup = REDIS_DISABLED
190
+
169
191
  // When you're NOT testing a single driver, it's important that *all* tests remain
170
- // within the singular 'for' loop below. This ensures that all drivers pass exactly
171
- // the same set of tests, with exactly the same inputs/outputs.
192
+ // within the various 'for' loops below. This ensures that all drivers pass exactly
193
+ // the same set of tests, with exactly the same inputs/outputs, both with and without
194
+ // redis caching enabled, both with and without encryption enabled.
172
195
  // This greatly supports developer sanity
173
196
 
197
+ for (const redisSetup of redisTests()) {
198
+ if (!redisSetup.enabled) continue
199
+
174
200
  for (const driverName of DRIVER_NAMES) {
175
- const config = DRIVER_CONFIG[driverName]
201
+ const driverTest = `${driverName} [${redisSetup.name}]`
202
+ const config = redisSetup.config(DRIVER_CONFIG[driverName])
176
203
  const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
177
204
  const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
178
- describe(`${driverName} test`, () => {
179
- describe(`${driverName} - create api client`, () => {
205
+ describe(`${driverTest} test`, () => {
206
+ beforeEach ((done) => {
207
+ redis.flushAll().then(
208
+ () => { logger.info(`${driverTest} - flushed redis`) },
209
+ (e) => {
210
+ logger.error(`${driverTest} - error flushing redis: ${e}`)
211
+ throw e
212
+ }
213
+ ).finally(() => done())
214
+ })
215
+
216
+ describe(`${driverTest} - create api client`, () => {
180
217
  it("should validate the config and return an API object", async () => {
181
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
182
- should().exist(api, 'expected API object to exist')
218
+ try {
219
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
220
+ should().exist(api, 'expected API object to exist')
221
+ } catch (e) {
222
+ logger.error(`wtf: ${e}`)
223
+ }
183
224
  })
184
225
  })
185
226
 
186
- describe(`${driverName} - listing with no arguments returns some results`, () => {
227
+ describe(`${driverTest} - listing with no arguments returns some results`, () => {
187
228
  it("should return some results from a default listing", async () => {
188
229
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
189
230
  const results = await api.list()
@@ -191,7 +232,7 @@ for (const driverName of DRIVER_NAMES) {
191
232
  })
192
233
  })
193
234
 
194
- describe(`${driverName} - write a file, read file, read metadata, delete file`, () => {
235
+ describe(`${driverTest} - write a file, read file, read metadata, delete file`, () => {
195
236
  // some random data, plus a bit extra
196
237
  const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
197
238
  const randomData = rand(size)
@@ -229,8 +270,8 @@ for (const driverName of DRIVER_NAMES) {
229
270
 
230
271
  for (const encryption of encryptionTests()) {
231
272
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
232
- describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
233
- // describe(`${driverName} - ENC fail to write and delete files in readOnly mode`, () => {
273
+ describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
274
+ // describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
234
275
  // const encryption = {key: rand(32)}
235
276
  const size = 16
236
277
  const randomData = rand(size)
@@ -275,7 +316,7 @@ for (const driverName of DRIVER_NAMES) {
275
316
  })
276
317
  }
277
318
 
278
- describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
319
+ describe(`${driverTest} - write an encrypted file, read file, read metadata, delete file`, () => {
279
320
  // some random data, plus a bit extra
280
321
  const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
281
322
  const randomData = rand(size)
@@ -315,8 +356,8 @@ for (const driverName of DRIVER_NAMES) {
315
356
  const mirrorDriver = pickAnotherDriver(driverName)
316
357
  const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
317
358
  const mirrorDest = `mirrorDest_${rand(5)}/`
318
- describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
319
- // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
359
+ describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
360
+ // describe(`${driverTest} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
320
361
  // const encryption = {key: rand(32)}
321
362
  // const encryption = null
322
363
  // a random directory and file within it
@@ -412,7 +453,6 @@ for (const driverName of DRIVER_NAMES) {
412
453
  }
413
454
  })
414
455
  it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
415
- await redis.flushMobiletto()
416
456
  const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
417
457
  expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
418
458
  expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
@@ -455,7 +495,7 @@ for (const driverName of DRIVER_NAMES) {
455
495
  })
456
496
  }
457
497
 
458
- describe(`${driverName} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
498
+ describe(`${driverTest} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
459
499
  it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
460
500
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
461
501
  try {
@@ -468,7 +508,7 @@ for (const driverName of DRIVER_NAMES) {
468
508
  })
469
509
  })
470
510
 
471
- describe(`${driverName} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
511
+ describe(`${driverTest} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
472
512
  it("should throw MobilettoNotFoundError when trying to read metadata for a file that does not exist", async () => {
473
513
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
474
514
  try {
@@ -480,7 +520,7 @@ for (const driverName of DRIVER_NAMES) {
480
520
  })
481
521
  })
482
522
 
483
- describe(`${driverName} - expect MobilettoNotFoundError when removing non-existent file`, () => {
523
+ describe(`${driverTest} - expect MobilettoNotFoundError when removing non-existent file`, () => {
484
524
  it("should throw MobilettoNotFoundError when trying to remove a file that does not exist", async () => {
485
525
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
486
526
  try {
@@ -493,7 +533,7 @@ for (const driverName of DRIVER_NAMES) {
493
533
  })
494
534
  })
495
535
 
496
- describe(`${driverName} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
536
+ describe(`${driverTest} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
497
537
  it("should NOT throw MobilettoNotFoundError when quietly removing a file that does not exist", async () => {
498
538
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
499
539
  if (!(await api.remove(nonexistentFile, {quiet: true}))) {
@@ -503,3 +543,9 @@ for (const driverName of DRIVER_NAMES) {
503
543
  })
504
544
  })
505
545
  }
546
+ }
547
+
548
+ after ( (done) => {
549
+ logger.info('all tests finished, tearing down redis...')
550
+ redis.teardown().finally(done)
551
+ })
package/util/redis.js CHANGED
@@ -1,170 +1,198 @@
1
1
  const Redis = require('ioredis')
2
2
  const generators = require('redis-async-gen')
3
3
  const { logger } = require('./logger')
4
- const LRU = require("lru-cache");
4
+ const LRU = require("lru-cache")
5
5
 
6
6
  const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
7
- const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
8
- const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
9
- const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
10
7
 
11
- const prefix = key => REDIS_PREFIX + key
12
- const unprefix = key => key && key.startsWith(REDIS_PREFIX) ? key.substring(REDIS_PREFIX.length) : key
8
+ let REDIS_CLIENTS = {}
13
9
 
14
- const NO_REDIS = { redis: 'not-found'}
15
-
16
- let REDIS_CLIENT = null
17
-
18
- const getRedis = () => {
19
- if (REDIS_CLIENT === null) {
20
- try {
21
- REDIS_CLIENT = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {
22
- host: REDIS_HOST,
23
- port: REDIS_PORT
24
- }))
25
- flushMobiletto().then(
26
- () => { logger.info(`getRedis: successfully flushed`) },
27
- (e) => { logger.error(`getRedis: error flushing: ${e}`) }
28
- )
29
- } catch (e) {
30
- logger.error(`getRedis(host=${REDIS_HOST}, port=${REDIS_PORT}) connection error: ${e}`)
31
- REDIS_CLIENT = NO_REDIS
32
- }
10
+ const getRedis = (name, host, port, prefix) => {
11
+ if (typeof REDIS_CLIENTS[name] === 'undefined') {
12
+ REDIS_CLIENTS[name] = new MobilettoCache(name, host, port, prefix + name + '_')
33
13
  }
34
- return REDIS_CLIENT
14
+ return REDIS_CLIENTS[name]
35
15
  }
36
16
 
37
17
  const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
38
18
 
39
- const isNoRedis = r => !r || r === NO_REDIS
40
-
41
- const doRedisAsync = async (func, defaultValue = null) => {
42
- const redis = getRedis()
43
- return isNoRedis(redis) ? defaultValue : await func(redis)
44
- }
45
-
46
- const doRedis = (func, defaultValue = null) => {
47
- const redis = getRedis()
48
- return isNoRedis(redis) ? defaultValue : func(redis)
49
- }
50
-
51
- function get (key) {
52
- return doRedis(r => r.get(prefix(key)), null)
53
- }
54
-
55
- async function set (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) {
56
- return await doRedisAsync(r => r.set(prefix(key), val, 'EX', expirationMillis / 1000))
57
- }
58
-
59
- async function sadd (key, val) {
60
- return await doRedisAsync(r => r.sadd(prefix(key), val))
61
- }
19
+ const ZERO_COUNTERS = {
20
+ get: 0,
21
+ set: 0,
22
+ del: 0,
23
+ flush: 0,
24
+ hit: 0,
25
+ miss: 0
26
+ }
27
+
28
+ class MobilettoCache {
29
+ name
30
+ redis
31
+ prefix
32
+ scopedCaches = {}
33
+ counters = Object.assign({}, ZERO_COUNTERS)
34
+ printStatsInterval = 1000
35
+ constructor(name,
36
+ host = '127.0.0.1',
37
+ port = 6379,
38
+ prefix = '_mobiletto__') {
39
+ this.name = name
40
+ if (host && port) {
41
+ try {
42
+ this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
43
+ } catch (e) {
44
+ logger.error(`redis(${name}): error connecting to redis, using fallback LRU for scoped caches: ${e}`)
45
+ this.redis = null
46
+ }
47
+ } else {
48
+ logger.warn(`redis(${name}): no host or port provided, using fallback LRU for scoped caches`)
49
+ this.redis = null
50
+ }
51
+ this.prefix = prefix
52
+ if (this.redis) {
53
+ // test connection by flushing
54
+ this.flush().then(
55
+ () => { logger.debug(`redis(${name}): successfully flushed`) },
56
+ (e) => {
57
+ logger.error(`redis(${name}): error flushing: ${e}, not using redis!`)
58
+ this.redis = null
59
+ }
60
+ )
61
+ }
62
+ }
62
63
 
63
- async function expire (key, expirationMillis) {
64
- return await doRedisAsync(r => r.expire(prefix(key), expirationMillis / 1000))
65
- }
64
+ stats = () => this.counters
65
+ resetStats = () => { this.counters = Object.assign({}, ZERO_COUNTERS) }
66
+ hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
67
+ toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
66
68
 
67
- function smembers (key) {
68
- return doRedis(r => r.smembers(prefix(key)), [])
69
- }
69
+ pfx = key => this.prefix + key
70
+ unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
70
71
 
71
- async function del (key) {
72
- return await doRedisAsync(r => r.del(prefix(key)))
73
- }
72
+ doRedisAsync = async (func, defaultValue = null) => {
73
+ try {
74
+ return this.redis ? await func(this.redis) : defaultValue
75
+ } catch (e) {
76
+ logger.warn(`redis(${this.name}): doRedisAsync(${func}): ${e} (returning default value: ${defaultValue})`)
77
+ return defaultValue
78
+ }
79
+ }
74
80
 
75
- async function flushall () {
76
- return await doRedisAsync(r => r.flushall())
77
- }
81
+ doRedis = (func, defaultValue = null) => {
82
+ try {
83
+ return this.redis ? func(this.redis) : defaultValue
84
+ } catch (e) {
85
+ logger.warn(`redis(${this.name}): doRedis(${func}): ${e} (returning default value: ${defaultValue})`)
86
+ return defaultValue
87
+ }
88
+ }
78
89
 
79
- async function findMatchingKeys (pattern) {
80
- const redis = getRedis();
81
- if (isNoRedis(redis)) {
82
- return []
90
+ get = async key => {
91
+ this.counters.get++
92
+ logger.silly(`redis(${this.name}): get(${key}) starting`)
93
+ const val = await this.doRedis(r => r.get(this.pfx(key)), null)
94
+ logger.silly(`redis(${this.name}): get(${key}) found value: ${val}`)
95
+ if (val) {
96
+ this.counters.hit++
97
+ } else {
98
+ this.counters.miss++
99
+ }
100
+ if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
101
+ const message = `${new Date()}: ${this}`;
102
+ logger.info(message)
103
+ }
104
+ return val
83
105
  }
84
- const { keysMatching } = generators.using(redis)
85
- const keys = []
86
- for await (const key of keysMatching(prefix(pattern))) {
87
- keys.push(unprefix(key))
106
+
107
+ set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
108
+ this.counters.set++
109
+ logger.silly(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) starting`)
110
+ await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
111
+ logger.silly(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) finished`)
88
112
  }
89
- return keys
90
- }
91
113
 
92
- async function removeMatchingKeys (pattern) {
93
- return await applyToMatchingKeys(pattern, del)
94
- }
114
+ del = async key => {
115
+ this.counters.del++
116
+ await this.doRedisAsync(r => r.del(this.pfx(key)))
117
+ }
95
118
 
96
- async function applyToMatchingKeys (pattern, asyncFunc) {
97
- const redis = getRedis();
98
- if (isNoRedis(redis)) {
99
- return []
119
+ findMatchingKeys = async (pattern) => {
120
+ if (!this.redis) return []
121
+ const { keysMatching } = generators.using(this.redis)
122
+ const keys = []
123
+ for await (const key of keysMatching(this.pfx(pattern))) {
124
+ keys.push(this.unprefix(key))
125
+ }
126
+ return keys
100
127
  }
101
- const { keysMatching } = generators.using(redis)
102
- const results = []
103
- for await (const key of keysMatching(prefix(pattern))) {
104
- results.push(await asyncFunc(unprefix(key)))
128
+ applyToMatchingKeys = async (pattern, asyncFunc) => {
129
+ if (!this.redis) return []
130
+ const { keysMatching } = generators.using(this.redis)
131
+ const results = []
132
+ for await (const key of keysMatching(this.pfx(pattern))) {
133
+ results.push(await asyncFunc(this.unprefix(key)))
134
+ }
135
+ return results
105
136
  }
106
- return results
107
- }
108
-
109
- const CACHES = {}
137
+ removeMatchingKeys = async (pattern) => await this.applyToMatchingKeys(pattern, this.del)
110
138
 
111
- const flushMobiletto = async (redis = getRedis()) => {
112
- return redis ? await removeMatchingKeys('*') : null
113
- }
114
-
115
- const scopedCache = (name, size = 100) => {
116
- if (CACHES[name]) {
117
- return CACHES[name]
139
+ flush = async () => {
140
+ this.counters.flush++
141
+ await this.removeMatchingKeys('*')
118
142
  }
119
- const redis = getRedis()
120
- if (isNoRedis(redis)) {
121
- return size && size > 0 ? new LRU({ max: size }) : null
122
- }
123
- const realKey = k => k ? `${name}_${k}` : null
124
- const cache = {
125
- get: async key => {
126
- const val = await get(realKey(key))
127
- if (!val) return null
128
- try {
129
- return JSON.parse(val)
130
- } catch (e) {
131
- logger.warn(`get(${key}) error: ${e}`)
132
- return null
133
- }
134
- },
135
- set: async (key, value) => {
136
- if (key && value) {
143
+
144
+ scopedCache = (name, size = 100) => {
145
+ if (this.scopedCaches[name]) {
146
+ return this.scopedCaches[name]
147
+ }
148
+ if (!this.redis) {
149
+ return size && size > 0 ? new LRU({ max: size }) : null
150
+ }
151
+ const realKey = k => k ? `${name}_${k}` : null
152
+ const cache = {
153
+ get: async key => {
154
+ const val = await this.get(realKey(key))
155
+ if (!val) return null
137
156
  try {
138
- await set(realKey(key), JSON.stringify(value))
157
+ return JSON.parse(val)
139
158
  } catch (e) {
140
- logger.warn(`set(${key}) error: ${e}`)
159
+ logger.warn(`redis(${this.name}): get(${key}) error: ${e}`)
160
+ return null
161
+ }
162
+ },
163
+ set: async (key, value) => {
164
+ if (key && value) {
165
+ try {
166
+ await this.set(realKey(key), JSON.stringify(value))
167
+ } catch (e) {
168
+ logger.warn(`redis(${this.name}): set(${key}) error: ${e}`)
169
+ }
141
170
  }
142
171
  }
143
172
  }
173
+ this.scopedCaches[name] = cache
174
+ return cache
144
175
  }
145
- CACHES[name] = cache
146
- return cache
147
176
  }
148
177
 
149
- teardown = () => {
150
- const redis = getRedis()
151
- if (redis) {
152
- redis.shutdown()
178
+ const forAllCaches = async (func) => {
179
+ const promises = []
180
+ for (const name of Object.keys(REDIS_CLIENTS)) {
181
+ promises.push(func(REDIS_CLIENTS[name]))
153
182
  }
183
+ return Promise.all(promises)
154
184
  }
155
185
 
156
- module.exports = {
157
- get,
158
- set,
159
- del,
160
- expire,
161
- sadd,
162
- smembers,
163
- flushall,
164
- findMatchingKeys,
165
- removeMatchingKeys,
166
- applyToMatchingKeys,
167
- scopedCache,
168
- flushMobiletto,
169
- teardown
170
- }
186
+ const teardown = async () => await forAllCaches((client) => {
187
+ try {
188
+ if (client.redis) {
189
+ client.redis.disconnect()
190
+ }
191
+ } catch (e) {
192
+ logger.warn(`teardown: error disconnecting from redis(${client.name}): ${e}`)
193
+ }
194
+ })
195
+
196
+ const flushAll = async () => await forAllCaches(client => client.flush())
197
+
198
+ module.exports = { getRedis, flushAll, teardown }