mobiletto 1.1.28 → 1.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +91 -47
- package/package.json +2 -1
- package/util/redis.js +10 -10
package/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
2
|
const { Readable, Transform } = require('stream')
|
|
3
|
+
const Queue = require('bull')
|
|
3
4
|
const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
|
|
4
5
|
const { getRedis, teardown } = require('./util/redis')
|
|
5
6
|
const { basename, dirname } = require('path')
|
|
@@ -54,8 +55,8 @@ function MobilettoNotFoundError (message) {
|
|
|
54
55
|
|
|
55
56
|
const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
56
57
|
|
|
57
|
-
const isAsyncGenerator =
|
|
58
|
-
const isReadable =
|
|
58
|
+
const isAsyncGenerator = func => func[Symbol.toStringTag] === 'AsyncGenerator'
|
|
59
|
+
const isReadable = thing => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
|
|
59
60
|
|
|
60
61
|
const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
|
|
61
62
|
|
|
@@ -64,12 +65,12 @@ class AwaitableLRU {
|
|
|
64
65
|
constructor(size) {
|
|
65
66
|
this.lru = new LRU({ max: size })
|
|
66
67
|
}
|
|
67
|
-
get =
|
|
68
|
+
get = key => Promise.resolve(this.lru.get(key))
|
|
68
69
|
set = (key, value) => Promise.resolve(this.lru.set(key, value))
|
|
69
70
|
}
|
|
70
71
|
|
|
71
72
|
const CACHE_FUNCTIONS = {
|
|
72
|
-
redis:
|
|
73
|
+
redis: client => () => {
|
|
73
74
|
if (typeof client.cache !== 'undefined') return client.cache
|
|
74
75
|
const redisConfig = client.redisConfig || {}
|
|
75
76
|
const enabled = redisConfig.enabled !== false
|
|
@@ -89,11 +90,11 @@ const CACHE_FUNCTIONS = {
|
|
|
89
90
|
}
|
|
90
91
|
return client.cache
|
|
91
92
|
},
|
|
92
|
-
scopedCache:
|
|
93
|
+
scopedCache: client => (cacheName, size = 100) => {
|
|
93
94
|
const redis = client.redis()
|
|
94
95
|
return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
|
|
95
96
|
},
|
|
96
|
-
flush:
|
|
97
|
+
flush: client => async () => {
|
|
97
98
|
const redis = client.redis()
|
|
98
99
|
return redis ? redis.flush() : true
|
|
99
100
|
}
|
|
@@ -101,7 +102,7 @@ const CACHE_FUNCTIONS = {
|
|
|
101
102
|
|
|
102
103
|
// noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
|
|
103
104
|
const UTILITY_FUNCTIONS = {
|
|
104
|
-
list:
|
|
105
|
+
list: client => async (path, opts) => {
|
|
105
106
|
const cache = client.scopedCache('list')
|
|
106
107
|
let cached = cache ? await cache.get(path) : null
|
|
107
108
|
if (cached) {
|
|
@@ -133,7 +134,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
133
134
|
throw e
|
|
134
135
|
}
|
|
135
136
|
},
|
|
136
|
-
safeList:
|
|
137
|
+
safeList: client => async (path, opts) => {
|
|
137
138
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
138
139
|
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
139
140
|
try {
|
|
@@ -146,7 +147,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
146
147
|
throw e
|
|
147
148
|
}
|
|
148
149
|
},
|
|
149
|
-
metadata:
|
|
150
|
+
metadata: client => async (path) => {
|
|
150
151
|
const cache = client.scopedCache('metadata')
|
|
151
152
|
const cached = cache ? await cache.get(path) : null
|
|
152
153
|
if (cached) {
|
|
@@ -162,7 +163,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
162
163
|
}
|
|
163
164
|
return meta
|
|
164
165
|
},
|
|
165
|
-
safeMetadata:
|
|
166
|
+
safeMetadata: client => async (path) => {
|
|
166
167
|
try {
|
|
167
168
|
return await client.metadata(path)
|
|
168
169
|
} catch (e) {
|
|
@@ -172,7 +173,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
172
173
|
throw e
|
|
173
174
|
}
|
|
174
175
|
},
|
|
175
|
-
remove:
|
|
176
|
+
remove: client => async (path, opts) => {
|
|
176
177
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
177
178
|
// noinspection JSUnresolvedVariable
|
|
178
179
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
@@ -181,7 +182,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
181
182
|
await client.flush()
|
|
182
183
|
return result
|
|
183
184
|
},
|
|
184
|
-
readFile:
|
|
185
|
+
readFile: client => async (path) => {
|
|
185
186
|
const cache = client.scopedCache('readFile')
|
|
186
187
|
const cached = cache ? await cache.get(path) : null
|
|
187
188
|
if (cached) {
|
|
@@ -201,7 +202,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
201
202
|
}
|
|
202
203
|
return data
|
|
203
204
|
},
|
|
204
|
-
safeReadFile:
|
|
205
|
+
safeReadFile: client => async (path) => {
|
|
205
206
|
try {
|
|
206
207
|
return await client.readFile(path)
|
|
207
208
|
} catch (e) {
|
|
@@ -209,7 +210,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
209
210
|
return null
|
|
210
211
|
}
|
|
211
212
|
},
|
|
212
|
-
write:
|
|
213
|
+
write: client => async (path, data) => {
|
|
213
214
|
logger.debug(`util.write(${path}) starting ...`)
|
|
214
215
|
const p = path.startsWith('/') ? path.substring(1) : path
|
|
215
216
|
if (p !== path) {
|
|
@@ -221,11 +222,11 @@ const UTILITY_FUNCTIONS = {
|
|
|
221
222
|
logger.debug(`util.write(${p}) wrote ${bytesWritten} bytes`)
|
|
222
223
|
return bytesWritten
|
|
223
224
|
},
|
|
224
|
-
writeFile:
|
|
225
|
+
writeFile: client => async (path, data) => {
|
|
225
226
|
const readFunc = function* () { yield data }
|
|
226
227
|
return await client.write(path, readFunc())
|
|
227
228
|
},
|
|
228
|
-
mirror:
|
|
229
|
+
mirror: client => async (source, clientPath = '', sourcePath = '') => {
|
|
229
230
|
logger.info(`mirror: starting, sourcePath=${sourcePath} -> clientPath=${clientPath}`)
|
|
230
231
|
const results = {
|
|
231
232
|
success: 0,
|
|
@@ -332,7 +333,7 @@ const addUtilityFunctions = (client, readOnly = false) => {
|
|
|
332
333
|
return client
|
|
333
334
|
}
|
|
334
335
|
|
|
335
|
-
const addCacheFunctions =
|
|
336
|
+
const addCacheFunctions = client => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
|
|
336
337
|
logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
|
|
337
338
|
return false
|
|
338
339
|
})
|
|
@@ -411,7 +412,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
411
412
|
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
412
413
|
|
|
413
414
|
const outerClient = addCacheFunctions(client)
|
|
414
|
-
const _metadata =
|
|
415
|
+
const _metadata = client => async (path) => {
|
|
415
416
|
const cache = outerClient.scopedCache('metadata')
|
|
416
417
|
const cached = cache ? await cache.get(path) : null
|
|
417
418
|
if (cached) {
|
|
@@ -461,39 +462,82 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
461
462
|
return finalMeta
|
|
462
463
|
}
|
|
463
464
|
|
|
465
|
+
const _singleMeta = async (job) => {
|
|
466
|
+
const dirent = job.data.dirent
|
|
467
|
+
const entry = job.data.entry
|
|
468
|
+
const files = job.data.files
|
|
469
|
+
const logPrefix = `_singleMeta(${dirent}/${basename(entry.name)})`
|
|
470
|
+
// every outcome always pushes something onto the files array
|
|
471
|
+
// this is because we know we're done when files.length === entries.length, even if
|
|
472
|
+
// some items in the files array are error messages instead of metadata objects
|
|
473
|
+
return new Promise((resolve, reject) => {
|
|
474
|
+
const cipherText = []
|
|
475
|
+
client.read(dirent + '/' + basename(entry.name), reader(cipherText))
|
|
476
|
+
.then((bytesRead) => {
|
|
477
|
+
if (!bytesRead) {
|
|
478
|
+
logger.warn(`${logPrefix} returned no data`)
|
|
479
|
+
files.push({})
|
|
480
|
+
resolve({})
|
|
481
|
+
} else {
|
|
482
|
+
const plain = decrypt(cipherText.toString(), enc)
|
|
483
|
+
const realPath = plain.split(ENC_PAD_SEP)[0]
|
|
484
|
+
_metadata(client)(realPath)
|
|
485
|
+
.then((meta) => {
|
|
486
|
+
files.push(meta)
|
|
487
|
+
resolve(meta)
|
|
488
|
+
})
|
|
489
|
+
.catch((err) => {
|
|
490
|
+
const message = `${logPrefix} error fetching _metadata: ${err}`
|
|
491
|
+
logger.warn(message)
|
|
492
|
+
files.push(message)
|
|
493
|
+
reject(message)
|
|
494
|
+
})
|
|
495
|
+
}
|
|
496
|
+
})
|
|
497
|
+
.catch((err) => {
|
|
498
|
+
const message = `${logPrefix} error reading file: ${err}`
|
|
499
|
+
logger.warn(message)
|
|
500
|
+
files.push(message)
|
|
501
|
+
reject(message)
|
|
502
|
+
})
|
|
503
|
+
})
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
const META_LOAD_QUEUE_NAME = '_loadMetaQueue'
|
|
507
|
+
const META_LOAD_JOB_NAME = '_loadMetaJob'
|
|
508
|
+
const META_LOAD_CONCURRENCY = 5
|
|
509
|
+
let META_LOAD_QUEUE = null
|
|
510
|
+
const metaLoadQueue = () => {
|
|
511
|
+
if (META_LOAD_QUEUE === null) {
|
|
512
|
+
if (!client.redisConfig) {
|
|
513
|
+
const message = 'metaLoadQueue: redis is required but not enabled'
|
|
514
|
+
logger.error(message)
|
|
515
|
+
throw new MobilettoError(message)
|
|
516
|
+
}
|
|
517
|
+
META_LOAD_QUEUE = new Queue(META_LOAD_QUEUE_NAME, `redis://${client.redisConfig.host || '127.0.0.1'}:${client.redisConfig.port || 6379}`)
|
|
518
|
+
META_LOAD_QUEUE.process(META_LOAD_JOB_NAME, META_LOAD_CONCURRENCY, _singleMeta)
|
|
519
|
+
META_LOAD_QUEUE.on('completed', function (job, result) {
|
|
520
|
+
logger.info(`${META_LOAD_JOB_NAME} completed with result: ${JSON.stringify(result)}`)
|
|
521
|
+
})
|
|
522
|
+
}
|
|
523
|
+
return META_LOAD_QUEUE
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
const waitForFiles = (entries, resolve) => {
|
|
527
|
+
if (files.length === entries.length) {
|
|
528
|
+
resolve(files)
|
|
529
|
+
} else {
|
|
530
|
+
setTimeout(() => { waitForFiles(entries, resolve) }, 1000)
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
464
534
|
const _loadMeta = async (dirent, entries) => {
|
|
465
535
|
const files = []
|
|
466
|
-
const promises = []
|
|
467
536
|
for (const entry of entries) {
|
|
468
|
-
const
|
|
469
|
-
|
|
470
|
-
const cipherText = []
|
|
471
|
-
client.read(dirent + '/' + basename(entry.name), reader(cipherText)).then(
|
|
472
|
-
(bytesRead) => {
|
|
473
|
-
if (!bytesRead) {
|
|
474
|
-
logger.warn(`${logPrefix} returned no data`)
|
|
475
|
-
resolve()
|
|
476
|
-
} else {
|
|
477
|
-
const plain = decrypt(cipherText.toString(), enc)
|
|
478
|
-
const realPath = plain.split(ENC_PAD_SEP)[0]
|
|
479
|
-
_metadata(client)(realPath).then(
|
|
480
|
-
(meta) => {
|
|
481
|
-
files.push(meta)
|
|
482
|
-
resolve()
|
|
483
|
-
},
|
|
484
|
-
(err) => {
|
|
485
|
-
logger.warn(`${logPrefix} error fetching _metadata: ${err}`)
|
|
486
|
-
resolve()
|
|
487
|
-
})
|
|
488
|
-
}
|
|
489
|
-
},
|
|
490
|
-
(err) => {
|
|
491
|
-
logger.warn(`${logPrefix} error reading file: ${err}`)
|
|
492
|
-
resolve()
|
|
493
|
-
})
|
|
494
|
-
}))
|
|
537
|
+
const job = { dirent, entry, files }
|
|
538
|
+
metaLoadQueue().add(META_LOAD_JOB_NAME, job)
|
|
495
539
|
}
|
|
496
|
-
await Promise
|
|
540
|
+
await new Promise(resolve => waitForFiles(entries, resolve))
|
|
497
541
|
return files
|
|
498
542
|
}
|
|
499
543
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.30",
|
|
4
4
|
"description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"keywords": [
|
|
@@ -39,6 +39,7 @@
|
|
|
39
39
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
40
40
|
"@gideo-llc/backblaze-b2-upload-any": "^0.1.4",
|
|
41
41
|
"backblaze-b2": "^1.7.0",
|
|
42
|
+
"bull": "^4.9.0",
|
|
42
43
|
"ioredis": "^5.2.3",
|
|
43
44
|
"lru-cache": "^7.14.0",
|
|
44
45
|
"node-fetch-commonjs": "^3.1.1",
|
package/util/redis.js
CHANGED
|
@@ -53,9 +53,9 @@ class MobilettoCache {
|
|
|
53
53
|
if (this.redis) {
|
|
54
54
|
// test connection by flushing
|
|
55
55
|
this.scanner = new redisScan(this.redis)
|
|
56
|
-
this.flush()
|
|
57
|
-
() => { logger.debug(`redis(${name}) successfully flushed`) }
|
|
58
|
-
(e) => {
|
|
56
|
+
this.flush()
|
|
57
|
+
.then(() => { logger.debug(`redis(${name}) successfully flushed`) })
|
|
58
|
+
.catch((e) => {
|
|
59
59
|
logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
|
|
60
60
|
this.redis = null
|
|
61
61
|
})
|
|
@@ -99,7 +99,7 @@ class MobilettoCache {
|
|
|
99
99
|
this.counters.miss++
|
|
100
100
|
}
|
|
101
101
|
if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
|
|
102
|
-
const message = `${new Date()}: ${this}
|
|
102
|
+
const message = `${new Date()}: ${this}`
|
|
103
103
|
logger.info(message)
|
|
104
104
|
}
|
|
105
105
|
return val
|
|
@@ -134,24 +134,24 @@ class MobilettoCache {
|
|
|
134
134
|
return await new Promise((resolve, reject) => {
|
|
135
135
|
this.scanner.eachScan(this.pfx(pattern), {}, async (matchingKeys) => {
|
|
136
136
|
if (!matchingKeys || typeof matchingKeys.length !== 'number') {
|
|
137
|
-
const message = `${logPrefix}: eachScan received invalid matchingKeys = ${typeof matchingKeys !== 'undefined' ? matchingKeys : 'undefined'}
|
|
137
|
+
const message = `${logPrefix}: eachScan received invalid matchingKeys = ${typeof matchingKeys !== 'undefined' ? matchingKeys : 'undefined'}`
|
|
138
138
|
logger.error(message)
|
|
139
139
|
reject(message)
|
|
140
140
|
} else {
|
|
141
141
|
keyMatchCount += matchingKeys.length
|
|
142
142
|
}
|
|
143
143
|
for (const key of matchingKeys) {
|
|
144
|
-
asyncFunc(key)
|
|
145
|
-
(val) => {
|
|
144
|
+
asyncFunc(key)
|
|
145
|
+
.then((val) => {
|
|
146
146
|
if (typeof val !== 'undefined') {
|
|
147
147
|
logger.silly(`${logPrefix} found key ${key} = ${val}`)
|
|
148
148
|
results.push(val)
|
|
149
149
|
} else {
|
|
150
150
|
logger.silly(`${logPrefix} found key with undefined value: ${key}`)
|
|
151
151
|
}
|
|
152
|
-
}
|
|
153
|
-
(e) => {
|
|
154
|
-
const message = `${logPrefix} error with key ${key}: ${e}
|
|
152
|
+
})
|
|
153
|
+
.catch((e) => {
|
|
154
|
+
const message = `${logPrefix} error with key ${key}: ${e}`
|
|
155
155
|
logger.error(message)
|
|
156
156
|
reject(message)
|
|
157
157
|
})
|