mobiletto 1.0.41 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -27,7 +27,7 @@ A simple example using the mobiletto `s3` driver.
27
27
  This code would run the same if the driver were `local`.
28
28
 
29
29
  const storage = require('mobiletto')
30
- const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
30
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
31
31
 
32
32
  // list objects: returns array of metadata objects
33
33
  const listing = await s3.list()
@@ -53,9 +53,9 @@ This code would run the same if the driver were `local`.
53
53
  # Contents
54
54
  * [Basic usage](#Basic-usage)
55
55
  * [Metadata](#Metadata)
56
- * [Caching](#Caching)
57
- * [Mirroring](#Caching)
58
56
  * [Alternate import style](#Alternate-import-style)
57
+ * [Caching](#Caching)
58
+ * [Mirroring](#Mirroring)
59
59
  * [Transparent encryption](#Transparent-encryption)
60
60
  * [Key rotation](#Key-rotation)
61
61
  * [Driver interface](#Driver-interface)
@@ -74,8 +74,7 @@ This code would run the same if the driver were `local`.
74
74
  // * readOnly: optional, never change anything on the filesystem; default is false
75
75
  // * fileMode: optional, permissions used when creating new files, default is 0600. can be string or integer
76
76
  // * dirMode: optional, permissions used when creating new directories, default is 0700. can be string or integer
77
- // * cacheSize: optional, LRU cache for `list` results, default is no cache
78
- const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700', cacheSize: 100})
77
+ const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700'})
79
78
 
80
79
  // To use 's3' driver:
81
80
  // * key: AWS access key
@@ -86,8 +85,7 @@ This code would run the same if the driver were `local`.
86
85
  // * region: optional, the AWS region to communicate with, default is us-east-1
87
86
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
88
87
  // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
89
- // * cacheSize: optional, LRU cache for `list` results, default is no cache
90
- const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
88
+ const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
91
89
 
92
90
  // List files
93
91
  local.list() // --> returns an array of metadata objects
@@ -180,16 +178,47 @@ should always be present. A subsequent `metadata` command will return all availa
180
178
  Import the fully-scoped module and use the `connect` function:
181
179
 
182
180
  const storage = require('mobiletto')
183
- const opts = {bucket: 'bk', region: 'us-east-1', cacheSize: 100}
181
+ const opts = {bucket: 'bk', region: 'us-east-1'}
184
182
  const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
185
183
  const objectData = await s3.readFile('some/path')
186
184
 
187
185
  ## Caching
188
- Many apps, by design, will repeatedly call `list` even though the data the app is listing hasn't changed at all.
189
-
190
- Save bandwidth and increase performance by passing a `cacheSize` option in the `opts` object.
191
-
192
- Any `write` or `remove` operation will clear the entire cache.
186
+ Mobiletto works best with a <a href="https://redis.io">redis</a> cache.
187
+
188
+ Mobiletto will attempt to connect to a redis instance on 127.0.0.1:6379
189
+
190
+ You can override either of these:
191
+ * Set the `MOBILETTO_REDIS_HOST` env var, mobiletto connect here instead of localhost
192
+ * Set the `MOBILETTO_REDIS_PORT` env var, this port will be used
193
+
194
+ Mobiletto will store all of its redis keys with the prefix `_mobiletto__`. You can change this
195
+ by setting the `MOBILETTO_REDIS_PREFIX` env var.
196
+
197
+ ### Turn off redis caching
198
+ To disable redis, you can do either of:
199
+ * Not run a redis server on 127.0.0.1:6379 that allows unauthenticated local connections
200
+ * Set `MOBILETTO_REDIS_HOST` to an empty string or anything that won't resolve as a host, for example:
201
+ * `MOBILETTO_REDIS_HOST=`
202
+ * `MOBILETTO_REDIS_HOST=/dev/null`
203
+ * `MOBILETTO_REDIS_HOST=~invalid~hostname~`
204
+ * `MOBILETTO_REDIS_HOST=invalid-hostname` *<-- not this!*
205
+ * **^^^ NO! that last one is a valid hostname** and depending on the name resolution scheme, it might actually be valid and you'll connect to it!
206
+
207
+ As discussed below, disabling caching will have an adverse effect on performance and incur more requests
208
+ to storage that you really need to.
209
+
210
+ ### Caching guidance
211
+ **Encrypted storage**: reading/writing encrypted storage is only a little slower than normal,
212
+ but navigating around directories (which some things do) is fairly expensive. Using a redis cache
213
+ will give you a significant performance boost.
214
+
215
+ The default cache is safe, but doesn't perform well if you have a lot of write/remove operations.
216
+ Any write or remove operation invalidates the entire cache, ensuring subsequent reads will see the
217
+ latest changes.
218
+
219
+ ### CLI tools
220
+ If you're using a CLI tool like [mobiletto-cli](https://www.npmjs.com/package/mobiletto-cli),
221
+ you'll definitely want the redis cache enabled, as it lasts across invocations of the `mo` command.
193
222
 
194
223
  ## Mirroring
195
224
 
@@ -1,10 +1,12 @@
1
1
  const { dirname } = require('path')
2
2
 
3
3
  const {
4
- M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable, logger,
4
+ M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
5
5
  MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
6
6
  } = require('../../index')
7
7
 
8
+ const { logger } = require('../../util/logger')
9
+
8
10
  const fs = require('fs')
9
11
 
10
12
  const DEFAULT_FILE_MODE = '0600'
@@ -1,8 +1,10 @@
1
1
  const {
2
- M_DIR, M_FILE, logger,
2
+ M_DIR, M_FILE,
3
3
  MobilettoError, MobilettoNotFoundError, readStream
4
4
  } = require('../../index')
5
5
 
6
+ const { logger } = require('../../util/logger')
7
+
6
8
  const { dirname } = require('path')
7
9
 
8
10
  const {
package/index.js CHANGED
@@ -1,8 +1,8 @@
1
1
  const fs = require('fs')
2
2
  const path = require('path')
3
3
  const { Readable, Transform } = require('stream')
4
- const winston = require('winston')
5
- const LRU = require('lru-cache')
4
+ const { logger } = require('./util/logger')
5
+ const redis = require('./util/redis')
6
6
  const { basename, dirname } = require('path')
7
7
  const shasum = require('shasum')
8
8
  const randomstring = require('randomstring')
@@ -10,21 +10,6 @@ const randomstring = require('randomstring')
10
10
  const crypt = require('./util/crypt')
11
11
  const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
12
12
 
13
- const logger = winston.createLogger({
14
- level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
15
- format: winston.format.simple(),
16
- transports: process.env.MOBILETTO_LOG_FILE
17
- ? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
18
- : [ new winston.transports.Console() ]
19
- })
20
-
21
- const setLogLevel = (level) => { logger.level = level }
22
-
23
- const setLogTransports = (transports) => {
24
- logger.transports.splice(0, logger.transports.length)
25
- logger.transports.push(...transports)
26
- }
27
-
28
13
  const DIR_ENT_DIR_SUFFIX = '__.dirent'
29
14
  const DIR_ENT_FILE_PREFIX = 'dirent__'
30
15
  const ENC_PAD_SEP = ' ~ '
@@ -37,12 +22,14 @@ const MOBILETTO_TMP = initMobilettoTempDir()
37
22
 
38
23
  // adapted from https://stackoverflow.com/a/27724419
39
24
  function MobilettoError (message, err) {
40
- this.message = `${message}`
25
+ this.message = `${message}: ${err ? err : ''}`
26
+ // noinspection JSUnusedGlobalSymbols
41
27
  this.err = err
42
28
  // Use V8's native method if available, otherwise fallback
43
29
  if ('captureStackTrace' in Error) {
44
30
  Error.captureStackTrace(this, TypeError)
45
31
  } else {
32
+ // noinspection JSUnusedGlobalSymbols
46
33
  this.stack = (new Error(this.message)).stack
47
34
  }
48
35
  }
@@ -54,6 +41,7 @@ function MobilettoNotFoundError (message) {
54
41
  if ('captureStackTrace' in Error) {
55
42
  Error.captureStackTrace(this, TypeError)
56
43
  } else {
44
+ // noinspection JSUnusedGlobalSymbols
57
45
  this.stack = (new Error(this.message)).stack
58
46
  }
59
47
  }
@@ -63,16 +51,19 @@ const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
63
51
  const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
64
52
  const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
65
53
 
54
+ const getMetaCache = (size = 100) => redis.scopedCache('meta', size)
55
+ const getListingCache = (size = 100) => redis.scopedCache('listingCache', size)
56
+
57
+ const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
58
+
59
+ // noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
66
60
  const UTILITY_FUNCTIONS = {
67
61
  list: (client) => async (path, opts) => {
68
- let cacheKey
69
- if (client.listingCache) {
70
- cacheKey = shasum(path + ' ' + (opts ? JSON.stringify(opts) : '~'))
71
- const results = client.listingCache.get(cacheKey)
72
- if (results) {
73
- if (Array.isArray(results)) { return results }
74
- throw results
75
- }
62
+ const cache = getListingCache()
63
+ let cached = cache ? await cache.get(path) : null
64
+ if (cached) {
65
+ if (Array.isArray(cached)) { return cached }
66
+ throw cached
76
67
  }
77
68
  const recursive = opts && opts.recursive ? opts.recursive : false
78
69
  const visitor = opts && opts.visitor ? opts.visitor : null
@@ -80,14 +71,21 @@ const UTILITY_FUNCTIONS = {
80
71
  throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
81
72
  }
82
73
  try {
74
+ // noinspection JSUnresolvedFunction
83
75
  const results = await client.driver_list(path, recursive, visitor)
84
- if (client.listingCache) {
85
- client.listingCache.set(cacheKey, results)
76
+ if (cache) {
77
+ cache.set(path, results).then(
78
+ () => { logger.debug(`list(${path}) cached ${results ? results.length : `unknown? ${JSON.stringify(results)}`} results`) },
79
+ (err) => { logger.error(`list(${path}) error: ${err}`) }
80
+ )
86
81
  }
87
82
  return results
88
83
  } catch (e) {
89
- if (client.listingCache && e instanceof MobilettoNotFoundError) {
90
- client.listingCache.set(cacheKey, e)
84
+ if (cache && e instanceof MobilettoNotFoundError) {
85
+ cache.set(path, e).then(
86
+ () => { logger.debug(`list(${path}) cached error ${e}`) },
87
+ (err) => { logger.error(`list(${path}) error ${err} caching MobilettoNotFoundError`) }
88
+ )
91
89
  }
92
90
  throw e
93
91
  }
@@ -96,6 +94,7 @@ const UTILITY_FUNCTIONS = {
96
94
  const recursive = opts && opts.recursive ? opts.recursive : false
97
95
  const visitor = opts && opts.visitor ? opts.visitor : null
98
96
  try {
97
+ // noinspection JSUnresolvedFunction
99
98
  return await client.driver_list(path, recursive, visitor)
100
99
  } catch (e) {
101
100
  if (e instanceof MobilettoNotFoundError) {
@@ -104,6 +103,22 @@ const UTILITY_FUNCTIONS = {
104
103
  throw e
105
104
  }
106
105
  },
106
+ metadata: (client) => async (path) => {
107
+ const cache = getMetaCache()
108
+ const cached = cache ? await cache.get(path) : null
109
+ if (cached) {
110
+ return cached
111
+ }
112
+ // noinspection JSUnresolvedFunction
113
+ const meta = await client.driver_metadata(path);
114
+ if (cache) {
115
+ cache.set(path, meta).then(
116
+ () => { logger.debug(`metadata(${path}) cached meta = ${JSON.stringify(meta)}`) },
117
+ (err) => { logger.error(`metadata(${path}) error: ${err}`) }
118
+ )
119
+ }
120
+ return meta
121
+ },
107
122
  safeMetadata: (client) => async (path) => {
108
123
  try {
109
124
  return await client.metadata(path)
@@ -117,18 +132,29 @@ const UTILITY_FUNCTIONS = {
117
132
  remove: (client) => async (path, opts) => {
118
133
  const recursive = opts && opts.recursive ? opts.recursive : false
119
134
  const quiet = opts && opts.quiet ? opts.quiet : false
135
+ // noinspection JSUnresolvedFunction
120
136
  const result = await client.driver_remove(path, recursive, quiet)
121
- if (client.listingCache) { client.listingCache.clear() }
137
+ await redis.flushMobiletto()
122
138
  return result
123
139
  },
124
140
  readFile: (client) => async (path) => {
141
+ const cache = redis.scopedCache('readFile')
142
+ const cached = cache ? await cache.get(path) : null
143
+ if (cached) {
144
+ return Buffer.from(cached, 'base64')
145
+ }
125
146
  const chunks = []
126
147
  await client.read(path, reader(chunks))
127
- return Buffer.concat(chunks)
148
+ const data = Buffer.concat(chunks);
149
+ if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
150
+ cache.set(path, data.toString('base64'))
151
+ }
152
+ return data
128
153
  },
129
154
  write: (client) => async (path, data) => {
155
+ // noinspection JSUnresolvedFunction
130
156
  const bytesWritten = await client.driver_write(path, data)
131
- if (client.listingCache) { client.listingCache.clear() }
157
+ await redis.flushMobiletto()
132
158
  return bytesWritten
133
159
  },
134
160
  writeFile: (client) => async (path, data) => {
@@ -214,8 +240,7 @@ async function mirrorDir (source, sourcePath, visitor) {
214
240
  }
215
241
  }
216
242
 
217
- function addUtilityFunctions (client, readOnly = false, cacheSize = null) {
218
- client.listingCache = cacheSize && cacheSize > 0 ? new LRU({ max: cacheSize }) : null
243
+ function addUtilityFunctions (client, readOnly = false) {
219
244
  for (const func of Object.keys(UTILITY_FUNCTIONS)) {
220
245
  client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
221
246
  client[func] = UTILITY_FUNCTIONS[func](client)
@@ -245,10 +270,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
245
270
  throw new MobilettoError(message)
246
271
  }
247
272
  const readOnly = opts ? !!opts.readOnly : false
248
- const cacheSize = opts && opts.cacheSize ? opts.cacheSize : null
249
273
  if (encryption === null) {
250
274
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
251
- return addUtilityFunctions(client, readOnly, cacheSize)
275
+ return addUtilityFunctions(client, readOnly)
252
276
  }
253
277
  const encKey = normalizeKey(encryption.key)
254
278
  if (!encKey) {
@@ -283,6 +307,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
283
307
  const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
284
308
 
285
309
  const _metadata = (client) => async (path) => {
310
+ const cache = redis.scopedCache('_metadata')
311
+ const cached = cache ? await cache.get(path) : null
312
+ if (cached) {
313
+ return cached
314
+ }
286
315
  let metaObj
287
316
  try {
288
317
  let chunks = []
@@ -317,7 +346,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
317
346
  }
318
347
  meta.type ||= M_FILE
319
348
  meta.name = path // rewrite name back to plaintext name
320
- return Object.assign({}, meta, metaObj)
349
+ const finalMeta = Object.assign({}, meta, metaObj);
350
+ if (cache) {
351
+ cache.set(path, finalMeta).then(
352
+ () => { logger.debug(`_metadata(${path}) cached meta = ${JSON.stringify(finalMeta)}`) },
353
+ (err) => { logger.error(`_metadata(${path}) error: ${err}`) }
354
+ )
355
+ }
356
+ return finalMeta
321
357
  }
322
358
 
323
359
  const _loadMeta = async (dirent, entries) => {
@@ -353,21 +389,46 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
353
389
  const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
354
390
  const dirent = direntDir(p)
355
391
  let entries
356
- try {
357
- entries = await client.list(dirent, recursive)
358
- } catch (e) {
359
- if (e instanceof MobilettoNotFoundError && p.includes('/')) {
360
- // it might be a single file, try listing the parent dir
361
- const parentDirent = direntDir(path.dirname(p));
362
- entries = await client.list(parentDirent, false)
363
- const objects = await _loadMeta(parentDirent, entries)
364
- const found = objects.find(o => o.name === p)
365
- if (found) {
366
- return [ found ]
392
+
393
+ const cacheKey = `${p} ~ ${recursive}`
394
+ const cache = visitor ? null : redis.scopedCache('enc_list')
395
+ const cached = cache && await cache.get(cacheKey)
396
+
397
+ function cacheAndReturn (thing) {
398
+ if (cache) {
399
+ cache.set(cacheKey, thing).then(
400
+ () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
401
+ (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
402
+ )
403
+ }
404
+ return thing
405
+ }
406
+
407
+ if (cached) {
408
+ entries = cached
409
+ } else {
410
+ try {
411
+ entries = await client.list(dirent, recursive)
412
+ } catch (e) {
413
+ if (e instanceof MobilettoNotFoundError && p.includes('/')) {
414
+ // it might be a single file, try listing the parent dir
415
+ const parentDirent = direntDir(path.dirname(p));
416
+ entries = await client.list(parentDirent, false)
417
+ const objects = await _loadMeta(parentDirent, entries)
418
+ const found = objects.find(o => o.name === p)
419
+ if (found) {
420
+ return cacheAndReturn([found])
421
+ }
422
+ throw e
367
423
  }
368
- throw e
369
424
  }
370
425
  }
426
+ if (cache) {
427
+ cache.set(cacheKey, entries).then(
428
+ () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
429
+ (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
430
+ )
431
+ }
371
432
  if (!entries || entries.length === 0) {
372
433
  return []
373
434
  }
@@ -411,9 +472,8 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
411
472
  const cipher = crypt.getCipher(enc)
412
473
  const realPath = encryptPath(path)
413
474
  let generatorBytes = 0
414
-
415
- function createEncryptStream(input) {
416
- return input.pipe(new Transform({
475
+ const createEncryptStream = (input) =>
476
+ input.pipe(new Transform({
417
477
  transform(chunk, encoding, callback) {
418
478
  generatorBytes += chunk.length
419
479
  this.push(cipher.update(chunk))
@@ -424,10 +484,8 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
424
484
  callback()
425
485
  }
426
486
  }))
427
- }
428
487
 
429
488
  // If we were passed a Readable, call write with a transform method
430
- let streamBytes = -1
431
489
  if (isReadable(readFunc)) {
432
490
  await client.write(realPath, createEncryptStream(readFunc))
433
491
  } else {
@@ -451,7 +509,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
451
509
  // todo: remove should return an array of the paths that were actually removed
452
510
  remove: async (path, options) => {
453
511
  const recursive = options === true || (options && options.recursive) || false
454
- const quiet = (options && options.quiet) || false
512
+ // const quiet = (options && options.quiet) || false
455
513
  if (recursive) {
456
514
  // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
457
515
  async function recRm (path) {
@@ -507,7 +565,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
507
565
  }
508
566
  }
509
567
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
510
- return addUtilityFunctions(encClient, readOnly, cacheSize)
568
+ return addUtilityFunctions(encClient, readOnly)
511
569
  }
512
570
 
513
571
  async function readStream(stream, callback, endCallback) {
@@ -560,8 +618,7 @@ const M_SPECIAL = 'special'
560
618
  module.exports = {
561
619
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
562
620
  isAsyncGenerator, isReadable,
563
- mobiletto, connect, logger,
564
- setLogLevel, setLogTransports,
621
+ mobiletto, connect,
565
622
  MobilettoError, MobilettoNotFoundError,
566
623
  readStream, writeStream, closeStream
567
624
  }
package/package.json CHANGED
@@ -1,9 +1,24 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.41",
3
+ "version": "1.0.44",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
- "author": "Jonathan Cobb",
6
+ "keywords": [
7
+ "storage",
8
+ "s3",
9
+ "simple",
10
+ "filesystem"
11
+ ],
12
+ "homepage": "https://github.com/cobbzilla/mobiletto",
13
+ "author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
14
+ "funding": {
15
+ "type": "patreon",
16
+ "url": "https://www.patreon.com/cobbzilla"
17
+ },
18
+ "repository": {
19
+ "type": "git",
20
+ "url": "https://github.com/cobbzilla/mobiletto"
21
+ },
7
22
  "license": "Apache-2.0",
8
23
  "scripts": {
9
24
  "test": "mocha --timeout 10000"
@@ -20,8 +35,10 @@
20
35
  "dependencies": {
21
36
  "@aws-sdk/client-s3": "^3.150.0",
22
37
  "@aws-sdk/lib-storage": "^3.150.0",
38
+ "ioredis": "^5.2.3",
23
39
  "lru-cache": "^7.14.0",
24
40
  "node-fetch-commonjs": "^3.1.1",
41
+ "redis-async-gen": "^0.1.3",
25
42
  "shasum": "^1.0.2",
26
43
  "winston": "^3.8.1"
27
44
  }
@@ -6,6 +6,7 @@ require('dotenv').config()
6
6
 
7
7
  const randomstring = require('randomstring')
8
8
  const crypto = require('crypto')
9
+ const redis = require('../util/redis')
9
10
 
10
11
  const { expect, should, assert } = require('chai')
11
12
 
@@ -20,6 +21,7 @@ const {
20
21
  getCipher, getDecipher
21
22
  } = require("../util/crypt")
22
23
  const crypt = require("../util/crypt");
24
+ const {logger} = require("../util/logger");
23
25
 
24
26
  // chunk size used by generator function, used by driver's 'write' function
25
27
  // the temp file is also TEMP_SZ_MULTIPLE of this number
@@ -77,6 +79,15 @@ async function assertMetaFail (api, name) {
77
79
  }
78
80
  }
79
81
 
82
+ async function assertSafeMetaNull (api, name) {
83
+ try {
84
+ const meta = await api.safeMetadata(name)
85
+ expect(meta).to.be.null
86
+ } catch (err) {
87
+ assert.fail(`loading metadata with safeMetadata should not throw error: ${err}`)
88
+ }
89
+ }
90
+
80
91
  const rand = count => randomstring.generate(count)
81
92
 
82
93
  async function writeRandomFile(fixture, size) {
@@ -211,6 +222,9 @@ for (const driverName of DRIVER_NAMES) {
211
222
  it("loading metadata on the file we wrote now fails", async () => {
212
223
  await assertMetaFail(fixture.api, fixture.name)
213
224
  })
225
+ it("loading metadata on the file we wrote using safeMeta does not fail but returns null", async () => {
226
+ await assertSafeMetaNull(fixture.api, fixture.name)
227
+ })
214
228
  })
215
229
 
216
230
  for (const encryption of encryptionTests()) {
@@ -398,6 +412,7 @@ for (const driverName of DRIVER_NAMES) {
398
412
  }
399
413
  })
400
414
  it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
415
+ await redis.flushMobiletto()
401
416
  const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
402
417
  expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
403
418
  expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
package/util/logger.js ADDED
@@ -0,0 +1,22 @@
1
+ const winston = require('winston')
2
+
3
+ const logger = winston.createLogger({
4
+ level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
5
+ format: winston.format.simple(),
6
+ transports: process.env.MOBILETTO_LOG_FILE
7
+ ? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
8
+ : [ new winston.transports.Console() ]
9
+ })
10
+
11
+ const setLogLevel = (level) => { logger.level = level }
12
+
13
+ const setLogTransports = (transports) => {
14
+ logger.transports.splice(0, logger.transports.length)
15
+ logger.transports.push(...transports)
16
+ }
17
+
18
+ module.exports = {
19
+ logger,
20
+ setLogLevel,
21
+ setLogTransports
22
+ }
package/util/redis.js ADDED
@@ -0,0 +1,170 @@
1
+ const Redis = require('ioredis')
2
+ const generators = require('redis-async-gen')
3
+ const { logger } = require('./logger')
4
+ const LRU = require("lru-cache");
5
+
6
+ const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
7
+ const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
8
+ const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
9
+ const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
10
+
11
+ const prefix = key => REDIS_PREFIX + key
12
+ const unprefix = key => key && key.startsWith(REDIS_PREFIX) ? key.substring(REDIS_PREFIX.length) : key
13
+
14
+ const NO_REDIS = { redis: 'not-found'}
15
+
16
+ let REDIS_CLIENT = null
17
+
18
+ const getRedis = () => {
19
+ if (REDIS_CLIENT === null) {
20
+ try {
21
+ REDIS_CLIENT = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {
22
+ host: REDIS_HOST,
23
+ port: REDIS_PORT
24
+ }))
25
+ flushMobiletto().then(
26
+ () => { logger.info(`getRedis: successfully flushed`) },
27
+ (e) => { logger.error(`getRedis: error flushing: ${e}`) }
28
+ )
29
+ } catch (e) {
30
+ logger.error(`getRedis(host=${REDIS_HOST}, port=${REDIS_PORT}) connection error: ${e}`)
31
+ REDIS_CLIENT = NO_REDIS
32
+ }
33
+ }
34
+ return REDIS_CLIENT
35
+ }
36
+
37
+ const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
38
+
39
+ const isNoRedis = r => !r || r === NO_REDIS
40
+
41
+ const doRedisAsync = async (func, defaultValue = null) => {
42
+ const redis = getRedis()
43
+ return isNoRedis(redis) ? defaultValue : await func(redis)
44
+ }
45
+
46
+ const doRedis = (func, defaultValue = null) => {
47
+ const redis = getRedis()
48
+ return isNoRedis(redis) ? defaultValue : func(redis)
49
+ }
50
+
51
+ function get (key) {
52
+ return doRedis(r => r.get(prefix(key)), null)
53
+ }
54
+
55
+ async function set (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) {
56
+ return await doRedisAsync(r => r.set(prefix(key), val, 'EX', expirationMillis / 1000))
57
+ }
58
+
59
+ async function sadd (key, val) {
60
+ return await doRedisAsync(r => r.sadd(prefix(key), val))
61
+ }
62
+
63
+ async function expire (key, expirationMillis) {
64
+ return await doRedisAsync(r => r.expire(prefix(key), expirationMillis / 1000))
65
+ }
66
+
67
+ function smembers (key) {
68
+ return doRedis(r => r.smembers(prefix(key)), [])
69
+ }
70
+
71
+ async function del (key) {
72
+ return await doRedisAsync(r => r.del(prefix(key)))
73
+ }
74
+
75
+ async function flushall () {
76
+ return await doRedisAsync(r => r.flushall())
77
+ }
78
+
79
+ async function findMatchingKeys (pattern) {
80
+ const redis = getRedis();
81
+ if (isNoRedis(redis)) {
82
+ return []
83
+ }
84
+ const { keysMatching } = generators.using(redis)
85
+ const keys = []
86
+ for await (const key of keysMatching(prefix(pattern))) {
87
+ keys.push(unprefix(key))
88
+ }
89
+ return keys
90
+ }
91
+
92
+ async function removeMatchingKeys (pattern) {
93
+ return await applyToMatchingKeys(pattern, del)
94
+ }
95
+
96
+ async function applyToMatchingKeys (pattern, asyncFunc) {
97
+ const redis = getRedis();
98
+ if (isNoRedis(redis)) {
99
+ return []
100
+ }
101
+ const { keysMatching } = generators.using(redis)
102
+ const results = []
103
+ for await (const key of keysMatching(prefix(pattern))) {
104
+ results.push(await asyncFunc(unprefix(key)))
105
+ }
106
+ return results
107
+ }
108
+
109
+ const CACHES = {}
110
+
111
+ const flushMobiletto = async (redis = getRedis()) => {
112
+ return redis ? await removeMatchingKeys('*') : null
113
+ }
114
+
115
+ const scopedCache = (name, size = 100) => {
116
+ if (CACHES[name]) {
117
+ return CACHES[name]
118
+ }
119
+ const redis = getRedis()
120
+ if (isNoRedis(redis)) {
121
+ return size && size > 0 ? new LRU({ max: size }) : null
122
+ }
123
+ const realKey = k => k ? `${name}_${k}` : null
124
+ const cache = {
125
+ get: async key => {
126
+ const val = await get(realKey(key))
127
+ if (!val) return null
128
+ try {
129
+ return JSON.parse(val)
130
+ } catch (e) {
131
+ logger.warn(`get(${key}) error: ${e}`)
132
+ return null
133
+ }
134
+ },
135
+ set: async (key, value) => {
136
+ if (key && value) {
137
+ try {
138
+ await set(realKey(key), JSON.stringify(value))
139
+ } catch (e) {
140
+ logger.warn(`set(${key}) error: ${e}`)
141
+ }
142
+ }
143
+ }
144
+ }
145
+ CACHES[name] = cache
146
+ return cache
147
+ }
148
+
149
+ teardown = () => {
150
+ const redis = getRedis()
151
+ if (redis) {
152
+ redis.shutdown()
153
+ }
154
+ }
155
+
156
+ module.exports = {
157
+ get,
158
+ set,
159
+ del,
160
+ expire,
161
+ sadd,
162
+ smembers,
163
+ flushall,
164
+ findMatchingKeys,
165
+ removeMatchingKeys,
166
+ applyToMatchingKeys,
167
+ scopedCache,
168
+ flushMobiletto,
169
+ teardown
170
+ }