mobiletto 1.0.46 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -55
- package/drivers/b2/index.js +428 -0
- package/drivers/local/index.js +6 -3
- package/drivers/s3/index.js +3 -3
- package/index.js +150 -34
- package/package.json +5 -3
- package/test/{index.spec.js → api.spec.js} +81 -121
- package/test/crypt.spec.js +67 -0
- package/util/logger.js +1 -1
- package/util/redis.js +162 -134
package/util/redis.js
CHANGED
|
@@ -1,170 +1,198 @@
|
|
|
1
1
|
const Redis = require('ioredis')
|
|
2
2
|
const generators = require('redis-async-gen')
|
|
3
3
|
const { logger } = require('./logger')
|
|
4
|
-
const LRU = require("lru-cache")
|
|
4
|
+
const LRU = require("lru-cache")
|
|
5
5
|
|
|
6
6
|
const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
|
|
7
|
-
const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
|
|
8
|
-
const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
|
|
9
|
-
const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
|
|
10
7
|
|
|
11
|
-
|
|
12
|
-
const unprefix = key => key && key.startsWith(REDIS_PREFIX) ? key.substring(REDIS_PREFIX.length) : key
|
|
8
|
+
let REDIS_CLIENTS = {}
|
|
13
9
|
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const getRedis = () => {
|
|
19
|
-
if (REDIS_CLIENT === null) {
|
|
20
|
-
try {
|
|
21
|
-
REDIS_CLIENT = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {
|
|
22
|
-
host: REDIS_HOST,
|
|
23
|
-
port: REDIS_PORT
|
|
24
|
-
}))
|
|
25
|
-
flushMobiletto().then(
|
|
26
|
-
() => { logger.info(`getRedis: successfully flushed`) },
|
|
27
|
-
(e) => { logger.error(`getRedis: error flushing: ${e}`) }
|
|
28
|
-
)
|
|
29
|
-
} catch (e) {
|
|
30
|
-
logger.error(`getRedis(host=${REDIS_HOST}, port=${REDIS_PORT}) connection error: ${e}`)
|
|
31
|
-
REDIS_CLIENT = NO_REDIS
|
|
32
|
-
}
|
|
10
|
+
const getRedis = (name, host, port, prefix) => {
|
|
11
|
+
if (typeof REDIS_CLIENTS[name] === 'undefined') {
|
|
12
|
+
REDIS_CLIENTS[name] = new MobilettoCache(name, host, port, prefix + name + '_')
|
|
33
13
|
}
|
|
34
|
-
return
|
|
14
|
+
return REDIS_CLIENTS[name]
|
|
35
15
|
}
|
|
36
16
|
|
|
37
17
|
const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
|
|
38
18
|
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
19
|
+
const ZERO_COUNTERS = {
|
|
20
|
+
get: 0,
|
|
21
|
+
set: 0,
|
|
22
|
+
del: 0,
|
|
23
|
+
flush: 0,
|
|
24
|
+
hit: 0,
|
|
25
|
+
miss: 0
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
class MobilettoCache {
|
|
29
|
+
name
|
|
30
|
+
redis
|
|
31
|
+
prefix
|
|
32
|
+
scopedCaches = {}
|
|
33
|
+
counters = Object.assign({}, ZERO_COUNTERS)
|
|
34
|
+
printStatsInterval = 1000
|
|
35
|
+
constructor(name,
|
|
36
|
+
host = '127.0.0.1',
|
|
37
|
+
port = 6379,
|
|
38
|
+
prefix = '_mobiletto__') {
|
|
39
|
+
this.name = name
|
|
40
|
+
if (host && port) {
|
|
41
|
+
try {
|
|
42
|
+
this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
|
|
43
|
+
} catch (e) {
|
|
44
|
+
logger.error(`redis(${name}): error connecting to redis, using fallback LRU for scoped caches: ${e}`)
|
|
45
|
+
this.redis = null
|
|
46
|
+
}
|
|
47
|
+
} else {
|
|
48
|
+
logger.warn(`redis(${name}): no host or port provided, using fallback LRU for scoped caches`)
|
|
49
|
+
this.redis = null
|
|
50
|
+
}
|
|
51
|
+
this.prefix = prefix
|
|
52
|
+
if (this.redis) {
|
|
53
|
+
// test connection by flushing
|
|
54
|
+
this.flush().then(
|
|
55
|
+
() => { logger.debug(`redis(${name}): successfully flushed`) },
|
|
56
|
+
(e) => {
|
|
57
|
+
logger.error(`redis(${name}): error flushing: ${e}, not using redis!`)
|
|
58
|
+
this.redis = null
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
62
63
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
64
|
+
stats = () => this.counters
|
|
65
|
+
resetStats = () => { this.counters = Object.assign({}, ZERO_COUNTERS) }
|
|
66
|
+
hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
|
|
67
|
+
toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
|
|
66
68
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
}
|
|
69
|
+
pfx = key => this.prefix + key
|
|
70
|
+
unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
|
|
70
71
|
|
|
71
|
-
async
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
doRedisAsync = async (func, defaultValue = null) => {
|
|
73
|
+
try {
|
|
74
|
+
return this.redis ? await func(this.redis) : defaultValue
|
|
75
|
+
} catch (e) {
|
|
76
|
+
logger.warn(`redis(${this.name}): doRedisAsync(${func}): ${e} (returning default value: ${defaultValue})`)
|
|
77
|
+
return defaultValue
|
|
78
|
+
}
|
|
79
|
+
}
|
|
74
80
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
81
|
+
doRedis = (func, defaultValue = null) => {
|
|
82
|
+
try {
|
|
83
|
+
return this.redis ? func(this.redis) : defaultValue
|
|
84
|
+
} catch (e) {
|
|
85
|
+
logger.warn(`redis(${this.name}): doRedis(${func}): ${e} (returning default value: ${defaultValue})`)
|
|
86
|
+
return defaultValue
|
|
87
|
+
}
|
|
88
|
+
}
|
|
78
89
|
|
|
79
|
-
async
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
90
|
+
get = async key => {
|
|
91
|
+
this.counters.get++
|
|
92
|
+
logger.silly(`redis(${this.name}): get(${key}) starting`)
|
|
93
|
+
const val = await this.doRedis(r => r.get(this.pfx(key)), null)
|
|
94
|
+
logger.silly(`redis(${this.name}): get(${key}) found value: ${val}`)
|
|
95
|
+
if (val) {
|
|
96
|
+
this.counters.hit++
|
|
97
|
+
} else {
|
|
98
|
+
this.counters.miss++
|
|
99
|
+
}
|
|
100
|
+
if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
|
|
101
|
+
const message = `${new Date()}: ${this}`;
|
|
102
|
+
logger.info(message)
|
|
103
|
+
}
|
|
104
|
+
return val
|
|
83
105
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
106
|
+
|
|
107
|
+
set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
|
|
108
|
+
this.counters.set++
|
|
109
|
+
logger.silly(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) starting`)
|
|
110
|
+
await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
|
|
111
|
+
logger.silly(`redis(${this.name}): set(${key}, ${val}, ${expirationMillis}) finished`)
|
|
88
112
|
}
|
|
89
|
-
return keys
|
|
90
|
-
}
|
|
91
113
|
|
|
92
|
-
async
|
|
93
|
-
|
|
94
|
-
|
|
114
|
+
del = async key => {
|
|
115
|
+
this.counters.del++
|
|
116
|
+
await this.doRedisAsync(r => r.del(this.pfx(key)))
|
|
117
|
+
}
|
|
95
118
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
119
|
+
findMatchingKeys = async (pattern) => {
|
|
120
|
+
if (!this.redis) return []
|
|
121
|
+
const { keysMatching } = generators.using(this.redis)
|
|
122
|
+
const keys = []
|
|
123
|
+
for await (const key of keysMatching(this.pfx(pattern))) {
|
|
124
|
+
keys.push(this.unprefix(key))
|
|
125
|
+
}
|
|
126
|
+
return keys
|
|
100
127
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
results
|
|
128
|
+
applyToMatchingKeys = async (pattern, asyncFunc) => {
|
|
129
|
+
if (!this.redis) return []
|
|
130
|
+
const { keysMatching } = generators.using(this.redis)
|
|
131
|
+
const results = []
|
|
132
|
+
for await (const key of keysMatching(this.pfx(pattern))) {
|
|
133
|
+
results.push(await asyncFunc(this.unprefix(key)))
|
|
134
|
+
}
|
|
135
|
+
return results
|
|
105
136
|
}
|
|
106
|
-
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const CACHES = {}
|
|
137
|
+
removeMatchingKeys = async (pattern) => await this.applyToMatchingKeys(pattern, this.del)
|
|
110
138
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
const scopedCache = (name, size = 100) => {
|
|
116
|
-
if (CACHES[name]) {
|
|
117
|
-
return CACHES[name]
|
|
139
|
+
flush = async () => {
|
|
140
|
+
this.counters.flush++
|
|
141
|
+
await this.removeMatchingKeys('*')
|
|
118
142
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
return null
|
|
133
|
-
}
|
|
134
|
-
},
|
|
135
|
-
set: async (key, value) => {
|
|
136
|
-
if (key && value) {
|
|
143
|
+
|
|
144
|
+
scopedCache = (name, size = 100) => {
|
|
145
|
+
if (this.scopedCaches[name]) {
|
|
146
|
+
return this.scopedCaches[name]
|
|
147
|
+
}
|
|
148
|
+
if (!this.redis) {
|
|
149
|
+
return size && size > 0 ? new LRU({ max: size }) : null
|
|
150
|
+
}
|
|
151
|
+
const realKey = k => k ? `${name}_${k}` : null
|
|
152
|
+
const cache = {
|
|
153
|
+
get: async key => {
|
|
154
|
+
const val = await this.get(realKey(key))
|
|
155
|
+
if (!val) return null
|
|
137
156
|
try {
|
|
138
|
-
|
|
157
|
+
return JSON.parse(val)
|
|
139
158
|
} catch (e) {
|
|
140
|
-
logger.warn(`
|
|
159
|
+
logger.warn(`redis(${this.name}): get(${key}) error: ${e}`)
|
|
160
|
+
return null
|
|
161
|
+
}
|
|
162
|
+
},
|
|
163
|
+
set: async (key, value) => {
|
|
164
|
+
if (key && value) {
|
|
165
|
+
try {
|
|
166
|
+
await this.set(realKey(key), JSON.stringify(value))
|
|
167
|
+
} catch (e) {
|
|
168
|
+
logger.warn(`redis(${this.name}): set(${key}) error: ${e}`)
|
|
169
|
+
}
|
|
141
170
|
}
|
|
142
171
|
}
|
|
143
172
|
}
|
|
173
|
+
this.scopedCaches[name] = cache
|
|
174
|
+
return cache
|
|
144
175
|
}
|
|
145
|
-
CACHES[name] = cache
|
|
146
|
-
return cache
|
|
147
176
|
}
|
|
148
177
|
|
|
149
|
-
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
178
|
+
const forAllCaches = async (func) => {
|
|
179
|
+
const promises = []
|
|
180
|
+
for (const name of Object.keys(REDIS_CLIENTS)) {
|
|
181
|
+
promises.push(func(REDIS_CLIENTS[name]))
|
|
153
182
|
}
|
|
183
|
+
return Promise.all(promises)
|
|
154
184
|
}
|
|
155
185
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
teardown
|
|
170
|
-
}
|
|
186
|
+
const teardown = async () => await forAllCaches((client) => {
|
|
187
|
+
try {
|
|
188
|
+
if (client.redis) {
|
|
189
|
+
client.redis.disconnect()
|
|
190
|
+
}
|
|
191
|
+
} catch (e) {
|
|
192
|
+
logger.warn(`teardown: error disconnecting from redis(${client.name}): ${e}`)
|
|
193
|
+
}
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
const flushAll = async () => await forAllCaches(client => client.flush())
|
|
197
|
+
|
|
198
|
+
module.exports = { getRedis, flushAll, teardown }
|