mobiletto 1.1.29 → 1.1.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/index.js +91 -51
  2. package/package.json +2 -1
  3. package/util/redis.js +7 -7
package/index.js CHANGED
@@ -1,5 +1,6 @@
1
1
  const fs = require('fs')
2
2
  const { Readable, Transform } = require('stream')
3
+ const Queue = require('bull')
3
4
  const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
4
5
  const { getRedis, teardown } = require('./util/redis')
5
6
  const { basename, dirname } = require('path')
@@ -54,8 +55,8 @@ function MobilettoNotFoundError (message) {
54
55
 
55
56
  const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
56
57
 
57
- const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
58
- const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
58
+ const isAsyncGenerator = func => func[Symbol.toStringTag] === 'AsyncGenerator'
59
+ const isReadable = thing => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
59
60
 
60
61
  const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
61
62
 
@@ -64,12 +65,12 @@ class AwaitableLRU {
64
65
  constructor(size) {
65
66
  this.lru = new LRU({ max: size })
66
67
  }
67
- get = (key) => Promise.resolve(this.lru.get(key))
68
+ get = key => Promise.resolve(this.lru.get(key))
68
69
  set = (key, value) => Promise.resolve(this.lru.set(key, value))
69
70
  }
70
71
 
71
72
  const CACHE_FUNCTIONS = {
72
- redis: (client) => () => {
73
+ redis: client => () => {
73
74
  if (typeof client.cache !== 'undefined') return client.cache
74
75
  const redisConfig = client.redisConfig || {}
75
76
  const enabled = redisConfig.enabled !== false
@@ -89,11 +90,11 @@ const CACHE_FUNCTIONS = {
89
90
  }
90
91
  return client.cache
91
92
  },
92
- scopedCache: (client) => (cacheName, size = 100) => {
93
+ scopedCache: client => (cacheName, size = 100) => {
93
94
  const redis = client.redis()
94
95
  return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
95
96
  },
96
- flush: (client) => async () => {
97
+ flush: client => async () => {
97
98
  const redis = client.redis()
98
99
  return redis ? redis.flush() : true
99
100
  }
@@ -101,7 +102,7 @@ const CACHE_FUNCTIONS = {
101
102
 
102
103
  // noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
103
104
  const UTILITY_FUNCTIONS = {
104
- list: (client) => async (path, opts) => {
105
+ list: client => async (path, opts) => {
105
106
  const cache = client.scopedCache('list')
106
107
  let cached = cache ? await cache.get(path) : null
107
108
  if (cached) {
@@ -133,7 +134,7 @@ const UTILITY_FUNCTIONS = {
133
134
  throw e
134
135
  }
135
136
  },
136
- safeList: (client) => async (path, opts) => {
137
+ safeList: client => async (path, opts) => {
137
138
  const recursive = opts && opts.recursive ? opts.recursive : false
138
139
  const visitor = opts && opts.visitor ? opts.visitor : null
139
140
  try {
@@ -146,7 +147,7 @@ const UTILITY_FUNCTIONS = {
146
147
  throw e
147
148
  }
148
149
  },
149
- metadata: (client) => async (path) => {
150
+ metadata: client => async (path) => {
150
151
  const cache = client.scopedCache('metadata')
151
152
  const cached = cache ? await cache.get(path) : null
152
153
  if (cached) {
@@ -162,7 +163,7 @@ const UTILITY_FUNCTIONS = {
162
163
  }
163
164
  return meta
164
165
  },
165
- safeMetadata: (client) => async (path) => {
166
+ safeMetadata: client => async (path) => {
166
167
  try {
167
168
  return await client.metadata(path)
168
169
  } catch (e) {
@@ -172,7 +173,7 @@ const UTILITY_FUNCTIONS = {
172
173
  throw e
173
174
  }
174
175
  },
175
- remove: (client) => async (path, opts) => {
176
+ remove: client => async (path, opts) => {
176
177
  const recursive = opts && opts.recursive ? opts.recursive : false
177
178
  // noinspection JSUnresolvedVariable
178
179
  const quiet = opts && opts.quiet ? opts.quiet : false
@@ -181,7 +182,7 @@ const UTILITY_FUNCTIONS = {
181
182
  await client.flush()
182
183
  return result
183
184
  },
184
- readFile: (client) => async (path) => {
185
+ readFile: client => async (path) => {
185
186
  const cache = client.scopedCache('readFile')
186
187
  const cached = cache ? await cache.get(path) : null
187
188
  if (cached) {
@@ -201,7 +202,7 @@ const UTILITY_FUNCTIONS = {
201
202
  }
202
203
  return data
203
204
  },
204
- safeReadFile: (client) => async (path) => {
205
+ safeReadFile: client => async (path) => {
205
206
  try {
206
207
  return await client.readFile(path)
207
208
  } catch (e) {
@@ -209,7 +210,7 @@ const UTILITY_FUNCTIONS = {
209
210
  return null
210
211
  }
211
212
  },
212
- write: (client) => async (path, data) => {
213
+ write: client => async (path, data) => {
213
214
  logger.debug(`util.write(${path}) starting ...`)
214
215
  const p = path.startsWith('/') ? path.substring(1) : path
215
216
  if (p !== path) {
@@ -221,11 +222,11 @@ const UTILITY_FUNCTIONS = {
221
222
  logger.debug(`util.write(${p}) wrote ${bytesWritten} bytes`)
222
223
  return bytesWritten
223
224
  },
224
- writeFile: (client) => async (path, data) => {
225
+ writeFile: client => async (path, data) => {
225
226
  const readFunc = function* () { yield data }
226
227
  return await client.write(path, readFunc())
227
228
  },
228
- mirror: (client) => async (source, clientPath = '', sourcePath = '') => {
229
+ mirror: client => async (source, clientPath = '', sourcePath = '') => {
229
230
  logger.info(`mirror: starting, sourcePath=${sourcePath} -> clientPath=${clientPath}`)
230
231
  const results = {
231
232
  success: 0,
@@ -332,7 +333,7 @@ const addUtilityFunctions = (client, readOnly = false) => {
332
333
  return client
333
334
  }
334
335
 
335
- const addCacheFunctions = (client) => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
336
+ const addCacheFunctions = client => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
336
337
  logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
337
338
  return false
338
339
  })
@@ -411,7 +412,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
411
412
  const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
412
413
 
413
414
  const outerClient = addCacheFunctions(client)
414
- const _metadata = (client) => async (path) => {
415
+ const _metadata = client => async (path) => {
415
416
  const cache = outerClient.scopedCache('metadata')
416
417
  const cached = cache ? await cache.get(path) : null
417
418
  if (cached) {
@@ -461,43 +462,82 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
461
462
  return finalMeta
462
463
  }
463
464
 
465
+ const _singleMeta = async (job) => {
466
+ const dirent = job.data.dirent
467
+ const entry = job.data.entry
468
+ const files = job.data.files
469
+ const logPrefix = `_singleMeta(${dirent}/${basename(entry.name)})`
470
+ // every outcome always pushes something onto the files array
471
+ // this is because we know we're done when files.length === entries.length, even if
472
+ // some items in the files array are error messages instead of metadata objects
473
+ return new Promise((resolve, reject) => {
474
+ const cipherText = []
475
+ client.read(dirent + '/' + basename(entry.name), reader(cipherText))
476
+ .then((bytesRead) => {
477
+ if (!bytesRead) {
478
+ logger.warn(`${logPrefix} returned no data`)
479
+ files.push({})
480
+ resolve({})
481
+ } else {
482
+ const plain = decrypt(cipherText.toString(), enc)
483
+ const realPath = plain.split(ENC_PAD_SEP)[0]
484
+ _metadata(client)(realPath)
485
+ .then((meta) => {
486
+ files.push(meta)
487
+ resolve(meta)
488
+ })
489
+ .catch((err) => {
490
+ const message = `${logPrefix} error fetching _metadata: ${err}`
491
+ logger.warn(message)
492
+ files.push(message)
493
+ reject(message)
494
+ })
495
+ }
496
+ })
497
+ .catch((err) => {
498
+ const message = `${logPrefix} error reading file: ${err}`
499
+ logger.warn(message)
500
+ files.push(message)
501
+ reject(message)
502
+ })
503
+ })
504
+ }
505
+
506
+ const META_LOAD_QUEUE_NAME = '_loadMetaQueue'
507
+ const META_LOAD_JOB_NAME = '_loadMetaJob'
508
+ const META_LOAD_CONCURRENCY = 5
509
+ let META_LOAD_QUEUE = null
510
+ const metaLoadQueue = () => {
511
+ if (META_LOAD_QUEUE === null) {
512
+ if (!client.redisConfig) {
513
+ const message = 'metaLoadQueue: redis is required but not enabled'
514
+ logger.error(message)
515
+ throw new MobilettoError(message)
516
+ }
517
+ META_LOAD_QUEUE = new Queue(META_LOAD_QUEUE_NAME, `redis://${client.redisConfig.host || '127.0.0.1'}:${client.redisConfig.port || 6379}`)
518
+ META_LOAD_QUEUE.process(META_LOAD_JOB_NAME, META_LOAD_CONCURRENCY, _singleMeta)
519
+ META_LOAD_QUEUE.on('completed', function (job, result) {
520
+ logger.info(`${META_LOAD_JOB_NAME} completed with result: ${JSON.stringify(result)}`)
521
+ })
522
+ }
523
+ return META_LOAD_QUEUE
524
+ }
525
+
526
+ const waitForFiles = (files, entries, resolve) => {
527
+ if (files.length === entries.length) {
528
+ resolve(files)
529
+ } else {
530
+ setTimeout(() => { waitForFiles(files, entries, resolve) }, 1000)
531
+ }
532
+ }
533
+
464
534
  const _loadMeta = async (dirent, entries) => {
465
535
  const files = []
466
- const promises = []
467
- const promiseWaitPer = 5
468
536
  for (const entry of entries) {
469
- const logPrefix = `_loadMeta(${dirent}/${basename(entry.name)})`
470
- promises.push(new Promise((resolve) => {
471
- const cipherText = []
472
- client.read(dirent + '/' + basename(entry.name), reader(cipherText)).then(
473
- (bytesRead) => {
474
- if (!bytesRead) {
475
- logger.warn(`${logPrefix} returned no data`)
476
- resolve()
477
- } else {
478
- const plain = decrypt(cipherText.toString(), enc)
479
- const realPath = plain.split(ENC_PAD_SEP)[0]
480
- _metadata(client)(realPath).then(
481
- (meta) => {
482
- files.push(meta)
483
- resolve()
484
- },
485
- (err) => {
486
- logger.warn(`${logPrefix} error fetching _metadata: ${err}`)
487
- resolve()
488
- })
489
- }
490
- }).catch((err) => {
491
- logger.warn(`${logPrefix} error reading file: ${err}`)
492
- resolve()
493
- })
494
- }))
495
- if (promises.length % promiseWaitPer === 0) {
496
- logger.debug(`${logPrefix} promises.length = ${promises.length} awaiting`)
497
- await Promise.all(promises)
498
- }
537
+ const job = { dirent, entry, files }
538
+ metaLoadQueue().add(META_LOAD_JOB_NAME, job)
499
539
  }
500
- await Promise.all(promises)
540
+ await new Promise(resolve => waitForFiles(files, entries, resolve))
501
541
  return files
502
542
  }
503
543
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.1.29",
3
+ "version": "1.1.31",
4
4
  "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
@@ -39,6 +39,7 @@
39
39
  "@aws-sdk/lib-storage": "^3.150.0",
40
40
  "@gideo-llc/backblaze-b2-upload-any": "^0.1.4",
41
41
  "backblaze-b2": "^1.7.0",
42
+ "bull": "^4.9.0",
42
43
  "ioredis": "^5.2.3",
43
44
  "lru-cache": "^7.14.0",
44
45
  "node-fetch-commonjs": "^3.1.1",
package/util/redis.js CHANGED
@@ -53,9 +53,9 @@ class MobilettoCache {
53
53
  if (this.redis) {
54
54
  // test connection by flushing
55
55
  this.scanner = new redisScan(this.redis)
56
- this.flush().then(
57
- () => { logger.debug(`redis(${name}) successfully flushed`) },
58
- (e) => {
56
+ this.flush()
57
+ .then(() => { logger.debug(`redis(${name}) successfully flushed`) })
58
+ .catch((e) => {
59
59
  logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
60
60
  this.redis = null
61
61
  })
@@ -141,16 +141,16 @@ class MobilettoCache {
141
141
  keyMatchCount += matchingKeys.length
142
142
  }
143
143
  for (const key of matchingKeys) {
144
- asyncFunc(key).then(
145
- (val) => {
144
+ asyncFunc(key)
145
+ .then((val) => {
146
146
  if (typeof val !== 'undefined') {
147
147
  logger.silly(`${logPrefix} found key ${key} = ${val}`)
148
148
  results.push(val)
149
149
  } else {
150
150
  logger.silly(`${logPrefix} found key with undefined value: ${key}`)
151
151
  }
152
- },
153
- (e) => {
152
+ })
153
+ .catch((e) => {
154
154
  const message = `${logPrefix} error with key ${key}: ${e}`
155
155
  logger.error(message)
156
156
  reject(message)