mobiletto 1.1.20 → 1.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +8 -5
- package/package.json +2 -2
- package/test/api.spec.js +1 -1
- package/util/redis.js +32 -17
package/index.js
CHANGED
|
@@ -191,10 +191,13 @@ const UTILITY_FUNCTIONS = {
|
|
|
191
191
|
await client.read(path, reader(chunks))
|
|
192
192
|
const data = Buffer.concat(chunks);
|
|
193
193
|
if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
|
|
194
|
-
cache.set(path, data.toString('base64'))
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
194
|
+
const setVal = cache.set(path, data.toString('base64')) // redis returns promise, LRU does not
|
|
195
|
+
if (setVal instanceof Promise) {
|
|
196
|
+
setVal.then(
|
|
197
|
+
() => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
|
|
198
|
+
(err) => { logger.error(`readFile(${path}) error: ${err}`) }
|
|
199
|
+
)
|
|
200
|
+
}
|
|
198
201
|
}
|
|
199
202
|
return data
|
|
200
203
|
},
|
|
@@ -409,7 +412,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
409
412
|
|
|
410
413
|
const outerClient = addCacheFunctions(client)
|
|
411
414
|
const _metadata = (client) => async (path) => {
|
|
412
|
-
const cache = outerClient.scopedCache('
|
|
415
|
+
const cache = outerClient.scopedCache('metadata')
|
|
413
416
|
const cached = cache ? await cache.get(path) : null
|
|
414
417
|
if (cached) {
|
|
415
418
|
return cached
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.22",
|
|
4
4
|
"description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"keywords": [
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"ioredis": "^5.2.3",
|
|
43
43
|
"lru-cache": "^7.14.0",
|
|
44
44
|
"node-fetch-commonjs": "^3.1.1",
|
|
45
|
-
"redis-
|
|
45
|
+
"node-redis-scan": "^1.3.5",
|
|
46
46
|
"shasum": "^1.0.2",
|
|
47
47
|
"winston": "^3.8.1"
|
|
48
48
|
}
|
package/test/api.spec.js
CHANGED
|
@@ -130,7 +130,7 @@ if (!redisSetup.enabled) continue
|
|
|
130
130
|
|
|
131
131
|
for (const driverName of DRIVER_NAMES) {
|
|
132
132
|
// For testing single drivers
|
|
133
|
-
// if (driverName !== '
|
|
133
|
+
// if (driverName !== 'local') continue
|
|
134
134
|
|
|
135
135
|
const driverTest = `${driverName} [${redisSetup.name}]`
|
|
136
136
|
const config = redisSetup.config(DRIVER_CONFIG[driverName])
|
package/util/redis.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const Redis = require('ioredis')
|
|
2
|
-
const
|
|
2
|
+
const redisScan = require('node-redis-scan')
|
|
3
3
|
const { logger } = require('./logger')
|
|
4
4
|
const LRU = require('lru-cache')
|
|
5
5
|
|
|
@@ -28,6 +28,7 @@ const ZERO_COUNTERS = {
|
|
|
28
28
|
class MobilettoCache {
|
|
29
29
|
name
|
|
30
30
|
redis
|
|
31
|
+
scanner
|
|
31
32
|
prefix
|
|
32
33
|
scopedCaches = {}
|
|
33
34
|
counters = Object.assign({}, ZERO_COUNTERS)
|
|
@@ -51,13 +52,13 @@ class MobilettoCache {
|
|
|
51
52
|
this.prefix = prefix
|
|
52
53
|
if (this.redis) {
|
|
53
54
|
// test connection by flushing
|
|
55
|
+
this.scanner = new redisScan(this.redis)
|
|
54
56
|
this.flush().then(
|
|
55
57
|
() => { logger.debug(`redis(${name}) successfully flushed`) },
|
|
56
58
|
(e) => {
|
|
57
59
|
logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
|
|
58
60
|
this.redis = null
|
|
59
|
-
}
|
|
60
|
-
)
|
|
61
|
+
})
|
|
61
62
|
}
|
|
62
63
|
}
|
|
63
64
|
|
|
@@ -66,7 +67,7 @@ class MobilettoCache {
|
|
|
66
67
|
hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
|
|
67
68
|
toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
|
|
68
69
|
|
|
69
|
-
pfx = key => this.prefix + key
|
|
70
|
+
pfx = key => key.startsWith(this.prefix) ? key : this.prefix + key
|
|
70
71
|
unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
|
|
71
72
|
|
|
72
73
|
doRedisAsync = async (func, defaultValue = null) => {
|
|
@@ -117,23 +118,37 @@ class MobilettoCache {
|
|
|
117
118
|
}
|
|
118
119
|
|
|
119
120
|
findMatchingKeys = async (pattern) => {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}
|
|
126
|
-
return keys
|
|
121
|
+
return await new Promise((resolve, reject) => {
|
|
122
|
+
this.scanner.scan(pattern, (err, matchingKeys) => {
|
|
123
|
+
if (err) reject(err)
|
|
124
|
+
resolve(matchingKeys)
|
|
125
|
+
})
|
|
126
|
+
})
|
|
127
127
|
}
|
|
128
|
+
|
|
128
129
|
applyToMatchingKeys = async (pattern, asyncFunc) => {
|
|
129
|
-
if (!this.redis) return []
|
|
130
|
-
const { keysMatching } = generators.using(this.redis)
|
|
131
130
|
const results = []
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
131
|
+
return await new Promise((resolve, reject) => {
|
|
132
|
+
this.scanner.eachScan(pattern, {}, async (matchingKeys) => {
|
|
133
|
+
for (const key of matchingKeys) {
|
|
134
|
+
asyncFunc(key).then(
|
|
135
|
+
(val) => {
|
|
136
|
+
if (typeof val !== 'undefined') {
|
|
137
|
+
results.push(val)
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
(e) => {
|
|
141
|
+
logger.error(`applyToMatchingKeys: error with key ${key}: ${e}`)
|
|
142
|
+
reject(e)
|
|
143
|
+
})
|
|
144
|
+
}
|
|
145
|
+
}, (err, matchCount) => {
|
|
146
|
+
if (err) reject(err)
|
|
147
|
+
resolve(results)
|
|
148
|
+
})
|
|
149
|
+
})
|
|
136
150
|
}
|
|
151
|
+
|
|
137
152
|
removeMatchingKeys = async (pattern) => await this.applyToMatchingKeys(pattern, this.del)
|
|
138
153
|
|
|
139
154
|
flush = async () => {
|