mobiletto 1.1.20 → 1.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -191,10 +191,13 @@ const UTILITY_FUNCTIONS = {
191
191
  await client.read(path, reader(chunks))
192
192
  const data = Buffer.concat(chunks);
193
193
  if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
194
- cache.set(path, data.toString('base64')).then(
195
- () => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
196
- (err) => { logger.error(`readFile(${path}) error: ${err}`) }
197
- )
194
+ const setVal = cache.set(path, data.toString('base64')) // redis returns promise, LRU does not
195
+ if (setVal instanceof Promise) {
196
+ setVal.then(
197
+ () => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
198
+ (err) => { logger.error(`readFile(${path}) error: ${err}`) }
199
+ )
200
+ }
198
201
  }
199
202
  return data
200
203
  },
@@ -409,7 +412,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
409
412
 
410
413
  const outerClient = addCacheFunctions(client)
411
414
  const _metadata = (client) => async (path) => {
412
- const cache = outerClient.scopedCache('_metadata')
415
+ const cache = outerClient.scopedCache('metadata')
413
416
  const cached = cache ? await cache.get(path) : null
414
417
  if (cached) {
415
418
  return cached
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.1.20",
3
+ "version": "1.1.22",
4
4
  "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
@@ -42,7 +42,7 @@
42
42
  "ioredis": "^5.2.3",
43
43
  "lru-cache": "^7.14.0",
44
44
  "node-fetch-commonjs": "^3.1.1",
45
- "redis-async-gen": "^0.1.3",
45
+ "node-redis-scan": "^1.3.5",
46
46
  "shasum": "^1.0.2",
47
47
  "winston": "^3.8.1"
48
48
  }
package/test/api.spec.js CHANGED
@@ -130,7 +130,7 @@ if (!redisSetup.enabled) continue
130
130
 
131
131
  for (const driverName of DRIVER_NAMES) {
132
132
  // For testing single drivers
133
- // if (driverName !== 'b2') continue
133
+ // if (driverName !== 'local') continue
134
134
 
135
135
  const driverTest = `${driverName} [${redisSetup.name}]`
136
136
  const config = redisSetup.config(DRIVER_CONFIG[driverName])
package/util/redis.js CHANGED
@@ -1,5 +1,5 @@
1
1
  const Redis = require('ioredis')
2
- const generators = require('redis-async-gen')
2
+ const redisScan = require('node-redis-scan')
3
3
  const { logger } = require('./logger')
4
4
  const LRU = require('lru-cache')
5
5
 
@@ -28,6 +28,7 @@ const ZERO_COUNTERS = {
28
28
  class MobilettoCache {
29
29
  name
30
30
  redis
31
+ scanner
31
32
  prefix
32
33
  scopedCaches = {}
33
34
  counters = Object.assign({}, ZERO_COUNTERS)
@@ -51,13 +52,13 @@ class MobilettoCache {
51
52
  this.prefix = prefix
52
53
  if (this.redis) {
53
54
  // test connection by flushing
55
+ this.scanner = new redisScan(this.redis)
54
56
  this.flush().then(
55
57
  () => { logger.debug(`redis(${name}) successfully flushed`) },
56
58
  (e) => {
57
59
  logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
58
60
  this.redis = null
59
- }
60
- )
61
+ })
61
62
  }
62
63
  }
63
64
 
@@ -66,7 +67,7 @@ class MobilettoCache {
66
67
  hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
67
68
  toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
68
69
 
69
- pfx = key => this.prefix + key
70
+ pfx = key => key.startsWith(this.prefix) ? key : this.prefix + key
70
71
  unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
71
72
 
72
73
  doRedisAsync = async (func, defaultValue = null) => {
@@ -117,23 +118,37 @@ class MobilettoCache {
117
118
  }
118
119
 
119
120
  findMatchingKeys = async (pattern) => {
120
- if (!this.redis) return []
121
- const { keysMatching } = generators.using(this.redis)
122
- const keys = []
123
- for await (const key of keysMatching(this.pfx(pattern))) {
124
- keys.push(this.unprefix(key))
125
- }
126
- return keys
121
+ return await new Promise((resolve, reject) => {
122
+ this.scanner.scan(pattern, (err, matchingKeys) => {
123
+ if (err) reject(err)
124
+ resolve(matchingKeys)
125
+ })
126
+ })
127
127
  }
128
+
128
129
  applyToMatchingKeys = async (pattern, asyncFunc) => {
129
- if (!this.redis) return []
130
- const { keysMatching } = generators.using(this.redis)
131
130
  const results = []
132
- for await (const key of keysMatching(this.pfx(pattern))) {
133
- results.push(await asyncFunc(this.unprefix(key)))
134
- }
135
- return results
131
+ return await new Promise((resolve, reject) => {
132
+ this.scanner.eachScan(pattern, {}, async (matchingKeys) => {
133
+ for (const key of matchingKeys) {
134
+ asyncFunc(key).then(
135
+ (val) => {
136
+ if (typeof val !== 'undefined') {
137
+ results.push(val)
138
+ }
139
+ },
140
+ (e) => {
141
+ logger.error(`applyToMatchingKeys: error with key ${key}: ${e}`)
142
+ reject(e)
143
+ })
144
+ }
145
+ }, (err, matchCount) => {
146
+ if (err) reject(err)
147
+ resolve(results)
148
+ })
149
+ })
136
150
  }
151
+
137
152
  removeMatchingKeys = async (pattern) => await this.applyToMatchingKeys(pattern, this.del)
138
153
 
139
154
  flush = async () => {