mobiletto 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +21 -2
- package/package.json +1 -1
- package/test/crypt.spec.js +1 -1
- package/util/crypt.js +0 -80
- package/util/logger.js +0 -27
- package/util/redis.js +0 -233
package/index.js
CHANGED
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
const {
|
|
1
|
+
const {
|
|
2
|
+
registerDriver, mobiletto, connect,
|
|
3
|
+
redis, closeRedis
|
|
4
|
+
} = require('mobiletto-base')
|
|
5
|
+
|
|
6
|
+
const {
|
|
7
|
+
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
8
|
+
MobilettoError, MobilettoNotFoundError,
|
|
9
|
+
logger, setLogLevel
|
|
10
|
+
} = require('mobiletto-common')
|
|
2
11
|
|
|
3
12
|
const registerDrivers = () => {
|
|
4
13
|
registerDriver('local', require('mobiletto-driver-local'))
|
|
@@ -7,4 +16,14 @@ const registerDrivers = () => {
|
|
|
7
16
|
registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
|
|
8
17
|
}
|
|
9
18
|
|
|
10
|
-
module.exports = {
|
|
19
|
+
module.exports = {
|
|
20
|
+
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
21
|
+
MobilettoError, MobilettoNotFoundError,
|
|
22
|
+
logger, setLogLevel,
|
|
23
|
+
registerDrivers,
|
|
24
|
+
registerDriver,
|
|
25
|
+
mobiletto,
|
|
26
|
+
connect,
|
|
27
|
+
redis,
|
|
28
|
+
closeRedis
|
|
29
|
+
}
|
package/package.json
CHANGED
package/test/crypt.spec.js
CHANGED
package/util/crypt.js
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
// adapted from https://stackoverflow.com/a/64136185
|
|
2
|
-
const crypto = require('crypto')
|
|
3
|
-
const shasum = require('shasum')
|
|
4
|
-
const winston = require("winston");
|
|
5
|
-
|
|
6
|
-
const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
|
|
7
|
-
const MIN_KEY_LEN = 16
|
|
8
|
-
const DEFAULT_CRYPT_ALGO = 'aes-256-cbc';
|
|
9
|
-
|
|
10
|
-
function normalizeKey (k) {
|
|
11
|
-
return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
|
|
12
|
-
? Buffer.from(shasum(k.trim())).subarray(0, 32)
|
|
13
|
-
: null
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
function normalizeIV (iv, key) {
|
|
17
|
-
return (typeof iv === 'string' && iv.trim().length >= 16)
|
|
18
|
-
? Buffer.from(shasum(iv.trim())).subarray(0, 16)
|
|
19
|
-
: key
|
|
20
|
-
? Buffer.from(shasum(key)).subarray(0, 16)
|
|
21
|
-
: null
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
// ensure key long enough for security, and is 32 bytes for AES-256
|
|
25
|
-
let KEY = null
|
|
26
|
-
function setDefaultKey (key) {
|
|
27
|
-
KEY = normalizeKey(key)
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// ensure IV is 16 bytes for AES-256
|
|
31
|
-
let CRYPTO_IV = null
|
|
32
|
-
function setDefaultIV (iv) {
|
|
33
|
-
CRYPTO_IV = normalizeIV(iv, KEY)
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function getCipher(enc) {
|
|
37
|
-
try {
|
|
38
|
-
let c = crypto.createCipheriv(enc.algo, enc.key, enc.iv);
|
|
39
|
-
c.setAutoPadding(true)
|
|
40
|
-
return c
|
|
41
|
-
} catch (e) {
|
|
42
|
-
winston.error(`getCipher(${enc}) ${e}`)
|
|
43
|
-
throw e
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
function encrypt (plainText, encryption, outputEncoding = 'base64') {
|
|
48
|
-
if (!encryption || !encryption.key) {
|
|
49
|
-
if (WARN_PLAINTEXT) {
|
|
50
|
-
console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
|
|
51
|
-
}
|
|
52
|
-
return plainText
|
|
53
|
-
}
|
|
54
|
-
const cipher = getCipher(encryption)
|
|
55
|
-
const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
|
|
56
|
-
return encoded.toString(outputEncoding)
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
function getDecipher(enc) {
|
|
60
|
-
let c = crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
|
|
61
|
-
c.setAutoPadding(true)
|
|
62
|
-
return c
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
|
|
66
|
-
if (!encryption || !encryption.key) {
|
|
67
|
-
return cipherText
|
|
68
|
-
}
|
|
69
|
-
const cipher = getDecipher(encryption)
|
|
70
|
-
const data = Buffer.from(cipherText, inputEncoding)
|
|
71
|
-
return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
module.exports = {
|
|
75
|
-
DEFAULT_CRYPT_ALGO,
|
|
76
|
-
setDefaultKey, setDefaultIV,
|
|
77
|
-
encrypt, decrypt,
|
|
78
|
-
normalizeKey, normalizeIV,
|
|
79
|
-
getCipher, getDecipher
|
|
80
|
-
}
|
package/util/logger.js
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
const winston = require('winston')
|
|
2
|
-
|
|
3
|
-
const logger = winston.createLogger({
|
|
4
|
-
levels: winston.config.npm.levels,
|
|
5
|
-
level: process.env.MOBILETTO_LOG_LEVEL || 'error',
|
|
6
|
-
format: winston.format.combine(
|
|
7
|
-
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }),
|
|
8
|
-
winston.format.printf((info) => {
|
|
9
|
-
return `${info.timestamp} [${info.level.toUpperCase()}] ${info.message}`
|
|
10
|
-
})),
|
|
11
|
-
transports: process.env.MOBILETTO_LOG_FILE
|
|
12
|
-
? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
|
|
13
|
-
: [ new winston.transports.Console({ stderrLevels: Object.keys(winston.config.npm.levels) }) ]
|
|
14
|
-
})
|
|
15
|
-
|
|
16
|
-
const setLogLevel = (level) => { logger.level = level }
|
|
17
|
-
|
|
18
|
-
const setLogTransports = (transports) => {
|
|
19
|
-
logger.transports.splice(0, logger.transports.length)
|
|
20
|
-
logger.transports.push(...transports)
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
module.exports = {
|
|
24
|
-
logger,
|
|
25
|
-
setLogLevel,
|
|
26
|
-
setLogTransports
|
|
27
|
-
}
|
package/util/redis.js
DELETED
|
@@ -1,233 +0,0 @@
|
|
|
1
|
-
const Redis = require('ioredis')
|
|
2
|
-
const redisScan = require('node-redis-scan')
|
|
3
|
-
const { logger } = require('./logger')
|
|
4
|
-
const LRU = require('lru-cache')
|
|
5
|
-
|
|
6
|
-
const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
|
|
7
|
-
|
|
8
|
-
let REDIS_CLIENTS = {}
|
|
9
|
-
|
|
10
|
-
const getRedis = (name, host, port, prefix) => {
|
|
11
|
-
if (typeof REDIS_CLIENTS[name] === 'undefined') {
|
|
12
|
-
REDIS_CLIENTS[name] = new MobilettoCache(name, host, port, prefix + name + '_')
|
|
13
|
-
}
|
|
14
|
-
return REDIS_CLIENTS[name]
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
|
|
18
|
-
|
|
19
|
-
const ZERO_COUNTERS = {
|
|
20
|
-
get: 0,
|
|
21
|
-
set: 0,
|
|
22
|
-
del: 0,
|
|
23
|
-
flush: 0,
|
|
24
|
-
hit: 0,
|
|
25
|
-
miss: 0
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
class MobilettoCache {
|
|
29
|
-
name
|
|
30
|
-
redis
|
|
31
|
-
scanner
|
|
32
|
-
prefix
|
|
33
|
-
scopedCaches = {}
|
|
34
|
-
counters = Object.assign({}, ZERO_COUNTERS)
|
|
35
|
-
printStatsInterval = 1000
|
|
36
|
-
constructor(name,
|
|
37
|
-
host = '127.0.0.1',
|
|
38
|
-
port = 6379,
|
|
39
|
-
prefix = '_mobiletto__') {
|
|
40
|
-
this.name = name
|
|
41
|
-
if (host && port) {
|
|
42
|
-
try {
|
|
43
|
-
this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
|
|
44
|
-
} catch (e) {
|
|
45
|
-
logger.error(`redis(${name}) error connecting to redis, using fallback LRU for scoped caches: ${e}`)
|
|
46
|
-
this.redis = null
|
|
47
|
-
}
|
|
48
|
-
} else {
|
|
49
|
-
logger.warn(`redis(${name}) no host or port provided, using fallback LRU for scoped caches`)
|
|
50
|
-
this.redis = null
|
|
51
|
-
}
|
|
52
|
-
this.prefix = prefix
|
|
53
|
-
if (this.redis) {
|
|
54
|
-
// test connection by flushing
|
|
55
|
-
this.scanner = new redisScan(this.redis)
|
|
56
|
-
this.flush()
|
|
57
|
-
.then(() => { logger.debug(`redis(${name}) successfully flushed`) })
|
|
58
|
-
.catch((e) => {
|
|
59
|
-
logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
|
|
60
|
-
this.redis = null
|
|
61
|
-
})
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
stats = () => this.counters
|
|
66
|
-
resetStats = () => { this.counters = Object.assign({}, ZERO_COUNTERS) }
|
|
67
|
-
hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
|
|
68
|
-
toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
|
|
69
|
-
|
|
70
|
-
pfx = key => key.startsWith(this.prefix) ? key : this.prefix + key
|
|
71
|
-
unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
|
|
72
|
-
|
|
73
|
-
doRedisAsync = async (func, defaultValue = null) => {
|
|
74
|
-
try {
|
|
75
|
-
return this.redis ? await func(this.redis) : defaultValue
|
|
76
|
-
} catch (e) {
|
|
77
|
-
logger.warn(`redis(${this.name}) doRedisAsync(${func}) ${e} (returning default value: ${defaultValue})`)
|
|
78
|
-
return defaultValue
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
doRedis = (func, defaultValue = null) => {
|
|
83
|
-
try {
|
|
84
|
-
return this.redis ? func(this.redis) : defaultValue
|
|
85
|
-
} catch (e) {
|
|
86
|
-
logger.warn(`redis(${this.name}) doRedis(${func}) ${e} (returning default value: ${defaultValue})`)
|
|
87
|
-
return defaultValue
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
get = async key => {
|
|
92
|
-
this.counters.get++
|
|
93
|
-
logger.silly(`redis(${this.name}) get(${key}) starting`)
|
|
94
|
-
const val = await this.doRedis(r => r.get(this.pfx(key)), null)
|
|
95
|
-
logger.silly(`redis(${this.name}) get(${key}) found value: ${val}`)
|
|
96
|
-
if (val) {
|
|
97
|
-
this.counters.hit++
|
|
98
|
-
} else {
|
|
99
|
-
this.counters.miss++
|
|
100
|
-
}
|
|
101
|
-
if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
|
|
102
|
-
const message = `${new Date()}: ${this}`
|
|
103
|
-
logger.info(message)
|
|
104
|
-
}
|
|
105
|
-
return val
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
|
|
109
|
-
this.counters.set++
|
|
110
|
-
logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) starting`)
|
|
111
|
-
await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
|
|
112
|
-
logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) finished`)
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
del = async key => {
|
|
116
|
-
this.counters.del++
|
|
117
|
-
await this.doRedisAsync(r => r.del(this.pfx(key)))
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
findMatchingKeys = async (pattern) => {
|
|
121
|
-
return await new Promise((resolve, reject) => {
|
|
122
|
-
this.scanner.scan(this.pfx(pattern), (err, matchingKeys) => {
|
|
123
|
-
if (err) reject(err)
|
|
124
|
-
resolve(matchingKeys)
|
|
125
|
-
})
|
|
126
|
-
})
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
applyToMatchingKeys = async (pattern, asyncFunc) => {
|
|
130
|
-
const results = []
|
|
131
|
-
let keyMatchCount = 0
|
|
132
|
-
const logPrefix = `applyToMatchingKeys(${pattern}, ${asyncFunc && asyncFunc.name ? asyncFunc.name : 'nameless function'})`
|
|
133
|
-
const start = Date.now()
|
|
134
|
-
return await new Promise((resolve, reject) => {
|
|
135
|
-
this.scanner.eachScan(this.pfx(pattern), {}, async (matchingKeys) => {
|
|
136
|
-
if (!matchingKeys || typeof matchingKeys.length !== 'number') {
|
|
137
|
-
const message = `${logPrefix}: eachScan received invalid matchingKeys = ${typeof matchingKeys !== 'undefined' ? matchingKeys : 'undefined'}`
|
|
138
|
-
logger.error(message)
|
|
139
|
-
reject(message)
|
|
140
|
-
} else {
|
|
141
|
-
keyMatchCount += matchingKeys.length
|
|
142
|
-
}
|
|
143
|
-
for (const key of matchingKeys) {
|
|
144
|
-
asyncFunc(key)
|
|
145
|
-
.then((val) => {
|
|
146
|
-
if (typeof val !== 'undefined') {
|
|
147
|
-
logger.silly(`${logPrefix} found key ${key} = ${val}`)
|
|
148
|
-
results.push(val)
|
|
149
|
-
} else {
|
|
150
|
-
logger.silly(`${logPrefix} found key with undefined value: ${key}`)
|
|
151
|
-
}
|
|
152
|
-
})
|
|
153
|
-
.catch((e) => {
|
|
154
|
-
const message = `${logPrefix} error with key ${key}: ${e}`
|
|
155
|
-
logger.error(message)
|
|
156
|
-
reject(message)
|
|
157
|
-
})
|
|
158
|
-
}
|
|
159
|
-
}, (err, matchCount) => {
|
|
160
|
-
if (err) {
|
|
161
|
-
logger.error(`${logPrefix} error in final callback: ${err}`)
|
|
162
|
-
reject(err)
|
|
163
|
-
}
|
|
164
|
-
const duration = Date.now() - start
|
|
165
|
-
logger.debug(`${logPrefix} resolving with matchCount=${matchCount}, keyMatchCount=${keyMatchCount}, results.length=${results.length}, duration ${duration/1000} seconds`)
|
|
166
|
-
resolve(results)
|
|
167
|
-
})
|
|
168
|
-
})
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
removeMatchingKeys = async pattern => await this.applyToMatchingKeys(pattern, this.del)
|
|
172
|
-
|
|
173
|
-
flush = async () => {
|
|
174
|
-
this.counters.flush++
|
|
175
|
-
await this.removeMatchingKeys('*')
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
scopedCache = (name, size = 100) => {
|
|
179
|
-
if (this.scopedCaches[name]) {
|
|
180
|
-
return this.scopedCaches[name]
|
|
181
|
-
}
|
|
182
|
-
if (!this.redis) {
|
|
183
|
-
return size && size > 0 ? new LRU({ max: size }) : null
|
|
184
|
-
}
|
|
185
|
-
const realKey = k => k ? `${name}_${k}` : null
|
|
186
|
-
const cache = {
|
|
187
|
-
get: async key => {
|
|
188
|
-
const val = await this.get(realKey(key))
|
|
189
|
-
if (!val) return null
|
|
190
|
-
try {
|
|
191
|
-
return JSON.parse(val)
|
|
192
|
-
} catch (e) {
|
|
193
|
-
logger.warn(`redis(${this.name}) get(${key}) error: ${e}`)
|
|
194
|
-
return null
|
|
195
|
-
}
|
|
196
|
-
},
|
|
197
|
-
set: async (key, value) => {
|
|
198
|
-
if (key && value) {
|
|
199
|
-
try {
|
|
200
|
-
await this.set(realKey(key), JSON.stringify(value))
|
|
201
|
-
} catch (e) {
|
|
202
|
-
logger.warn(`redis(${this.name}) set(${key}) error: ${e}`)
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
this.scopedCaches[name] = cache
|
|
208
|
-
return cache
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
const forAllCaches = async (func) => {
|
|
213
|
-
const promises = []
|
|
214
|
-
for (const name of Object.keys(REDIS_CLIENTS)) {
|
|
215
|
-
promises.push(func(REDIS_CLIENTS[name]))
|
|
216
|
-
}
|
|
217
|
-
return Promise.all(promises)
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
const teardown = async () => await forAllCaches((client) => {
|
|
221
|
-
try {
|
|
222
|
-
if (client.redis) {
|
|
223
|
-
client.redis.disconnect()
|
|
224
|
-
client.redis = null
|
|
225
|
-
}
|
|
226
|
-
} catch (e) {
|
|
227
|
-
logger.warn(`teardown: error disconnecting from redis(${client.name}) ${e}`)
|
|
228
|
-
}
|
|
229
|
-
})
|
|
230
|
-
|
|
231
|
-
const flushAll = async () => await forAllCaches(client => client.flush())
|
|
232
|
-
|
|
233
|
-
module.exports = { getRedis, flushAll, teardown }
|