mobiletto 1.2.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,4 +1,13 @@
1
- const { registerDriver } = require('mobiletto-base')
1
+ const {
2
+ registerDriver, mobiletto, connect,
3
+ redis, closeRedis
4
+ } = require('mobiletto-base')
5
+
6
+ const {
7
+ M_FILE, M_DIR, M_LINK, M_SPECIAL,
8
+ MobilettoError, MobilettoNotFoundError,
9
+ logger, setLogLevel
10
+ } = require('mobiletto-common')
2
11
 
3
12
  const registerDrivers = () => {
4
13
  registerDriver('local', require('mobiletto-driver-local'))
@@ -7,4 +16,14 @@ const registerDrivers = () => {
7
16
  registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
8
17
  }
9
18
 
10
- module.exports = { registerDrivers }
19
+ module.exports = {
20
+ M_FILE, M_DIR, M_LINK, M_SPECIAL,
21
+ MobilettoError, MobilettoNotFoundError,
22
+ logger, setLogLevel,
23
+ registerDrivers,
24
+ registerDriver,
25
+ mobiletto,
26
+ connect,
27
+ redis,
28
+ closeRedis
29
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.2.0",
3
+ "version": "1.2.1",
4
4
  "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
@@ -8,7 +8,7 @@ const {
8
8
  normalizeKey, normalizeIV,
9
9
  encrypt, decrypt,
10
10
  getCipher, getDecipher
11
- } = require("../util/crypt")
11
+ } = require("mobiletto-base/util/crypt")
12
12
 
13
13
  const { readStream } = require('mobiletto-common')
14
14
 
package/util/crypt.js DELETED
@@ -1,80 +0,0 @@
1
- // adapted from https://stackoverflow.com/a/64136185
2
- const crypto = require('crypto')
3
- const shasum = require('shasum')
4
- const winston = require("winston");
5
-
6
- const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
7
- const MIN_KEY_LEN = 16
8
- const DEFAULT_CRYPT_ALGO = 'aes-256-cbc';
9
-
10
- function normalizeKey (k) {
11
- return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
12
- ? Buffer.from(shasum(k.trim())).subarray(0, 32)
13
- : null
14
- }
15
-
16
- function normalizeIV (iv, key) {
17
- return (typeof iv === 'string' && iv.trim().length >= 16)
18
- ? Buffer.from(shasum(iv.trim())).subarray(0, 16)
19
- : key
20
- ? Buffer.from(shasum(key)).subarray(0, 16)
21
- : null
22
- }
23
-
24
- // ensure key long enough for security, and is 32 bytes for AES-256
25
- let KEY = null
26
- function setDefaultKey (key) {
27
- KEY = normalizeKey(key)
28
- }
29
-
30
- // ensure IV is 16 bytes for AES-256
31
- let CRYPTO_IV = null
32
- function setDefaultIV (iv) {
33
- CRYPTO_IV = normalizeIV(iv, KEY)
34
- }
35
-
36
- function getCipher(enc) {
37
- try {
38
- let c = crypto.createCipheriv(enc.algo, enc.key, enc.iv);
39
- c.setAutoPadding(true)
40
- return c
41
- } catch (e) {
42
- winston.error(`getCipher(${enc}) ${e}`)
43
- throw e
44
- }
45
- }
46
-
47
- function encrypt (plainText, encryption, outputEncoding = 'base64') {
48
- if (!encryption || !encryption.key) {
49
- if (WARN_PLAINTEXT) {
50
- console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
51
- }
52
- return plainText
53
- }
54
- const cipher = getCipher(encryption)
55
- const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
56
- return encoded.toString(outputEncoding)
57
- }
58
-
59
- function getDecipher(enc) {
60
- let c = crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
61
- c.setAutoPadding(true)
62
- return c
63
- }
64
-
65
- function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
66
- if (!encryption || !encryption.key) {
67
- return cipherText
68
- }
69
- const cipher = getDecipher(encryption)
70
- const data = Buffer.from(cipherText, inputEncoding)
71
- return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
72
- }
73
-
74
- module.exports = {
75
- DEFAULT_CRYPT_ALGO,
76
- setDefaultKey, setDefaultIV,
77
- encrypt, decrypt,
78
- normalizeKey, normalizeIV,
79
- getCipher, getDecipher
80
- }
package/util/logger.js DELETED
@@ -1,27 +0,0 @@
1
- const winston = require('winston')
2
-
3
- const logger = winston.createLogger({
4
- levels: winston.config.npm.levels,
5
- level: process.env.MOBILETTO_LOG_LEVEL || 'error',
6
- format: winston.format.combine(
7
- winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }),
8
- winston.format.printf((info) => {
9
- return `${info.timestamp} [${info.level.toUpperCase()}] ${info.message}`
10
- })),
11
- transports: process.env.MOBILETTO_LOG_FILE
12
- ? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
13
- : [ new winston.transports.Console({ stderrLevels: Object.keys(winston.config.npm.levels) }) ]
14
- })
15
-
16
- const setLogLevel = (level) => { logger.level = level }
17
-
18
- const setLogTransports = (transports) => {
19
- logger.transports.splice(0, logger.transports.length)
20
- logger.transports.push(...transports)
21
- }
22
-
23
- module.exports = {
24
- logger,
25
- setLogLevel,
26
- setLogTransports
27
- }
package/util/redis.js DELETED
@@ -1,233 +0,0 @@
1
- const Redis = require('ioredis')
2
- const redisScan = require('node-redis-scan')
3
- const { logger } = require('./logger')
4
- const LRU = require('lru-cache')
5
-
6
- const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
7
-
8
- let REDIS_CLIENTS = {}
9
-
10
- const getRedis = (name, host, port, prefix) => {
11
- if (typeof REDIS_CLIENTS[name] === 'undefined') {
12
- REDIS_CLIENTS[name] = new MobilettoCache(name, host, port, prefix + name + '_')
13
- }
14
- return REDIS_CLIENTS[name]
15
- }
16
-
17
- const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
18
-
19
- const ZERO_COUNTERS = {
20
- get: 0,
21
- set: 0,
22
- del: 0,
23
- flush: 0,
24
- hit: 0,
25
- miss: 0
26
- }
27
-
28
- class MobilettoCache {
29
- name
30
- redis
31
- scanner
32
- prefix
33
- scopedCaches = {}
34
- counters = Object.assign({}, ZERO_COUNTERS)
35
- printStatsInterval = 1000
36
- constructor(name,
37
- host = '127.0.0.1',
38
- port = 6379,
39
- prefix = '_mobiletto__') {
40
- this.name = name
41
- if (host && port) {
42
- try {
43
- this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
44
- } catch (e) {
45
- logger.error(`redis(${name}) error connecting to redis, using fallback LRU for scoped caches: ${e}`)
46
- this.redis = null
47
- }
48
- } else {
49
- logger.warn(`redis(${name}) no host or port provided, using fallback LRU for scoped caches`)
50
- this.redis = null
51
- }
52
- this.prefix = prefix
53
- if (this.redis) {
54
- // test connection by flushing
55
- this.scanner = new redisScan(this.redis)
56
- this.flush()
57
- .then(() => { logger.debug(`redis(${name}) successfully flushed`) })
58
- .catch((e) => {
59
- logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
60
- this.redis = null
61
- })
62
- }
63
- }
64
-
65
- stats = () => this.counters
66
- resetStats = () => { this.counters = Object.assign({}, ZERO_COUNTERS) }
67
- hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
68
- toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
69
-
70
- pfx = key => key.startsWith(this.prefix) ? key : this.prefix + key
71
- unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
72
-
73
- doRedisAsync = async (func, defaultValue = null) => {
74
- try {
75
- return this.redis ? await func(this.redis) : defaultValue
76
- } catch (e) {
77
- logger.warn(`redis(${this.name}) doRedisAsync(${func}) ${e} (returning default value: ${defaultValue})`)
78
- return defaultValue
79
- }
80
- }
81
-
82
- doRedis = (func, defaultValue = null) => {
83
- try {
84
- return this.redis ? func(this.redis) : defaultValue
85
- } catch (e) {
86
- logger.warn(`redis(${this.name}) doRedis(${func}) ${e} (returning default value: ${defaultValue})`)
87
- return defaultValue
88
- }
89
- }
90
-
91
- get = async key => {
92
- this.counters.get++
93
- logger.silly(`redis(${this.name}) get(${key}) starting`)
94
- const val = await this.doRedis(r => r.get(this.pfx(key)), null)
95
- logger.silly(`redis(${this.name}) get(${key}) found value: ${val}`)
96
- if (val) {
97
- this.counters.hit++
98
- } else {
99
- this.counters.miss++
100
- }
101
- if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
102
- const message = `${new Date()}: ${this}`
103
- logger.info(message)
104
- }
105
- return val
106
- }
107
-
108
- set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
109
- this.counters.set++
110
- logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) starting`)
111
- await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
112
- logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) finished`)
113
- }
114
-
115
- del = async key => {
116
- this.counters.del++
117
- await this.doRedisAsync(r => r.del(this.pfx(key)))
118
- }
119
-
120
- findMatchingKeys = async (pattern) => {
121
- return await new Promise((resolve, reject) => {
122
- this.scanner.scan(this.pfx(pattern), (err, matchingKeys) => {
123
- if (err) reject(err)
124
- resolve(matchingKeys)
125
- })
126
- })
127
- }
128
-
129
- applyToMatchingKeys = async (pattern, asyncFunc) => {
130
- const results = []
131
- let keyMatchCount = 0
132
- const logPrefix = `applyToMatchingKeys(${pattern}, ${asyncFunc && asyncFunc.name ? asyncFunc.name : 'nameless function'})`
133
- const start = Date.now()
134
- return await new Promise((resolve, reject) => {
135
- this.scanner.eachScan(this.pfx(pattern), {}, async (matchingKeys) => {
136
- if (!matchingKeys || typeof matchingKeys.length !== 'number') {
137
- const message = `${logPrefix}: eachScan received invalid matchingKeys = ${typeof matchingKeys !== 'undefined' ? matchingKeys : 'undefined'}`
138
- logger.error(message)
139
- reject(message)
140
- } else {
141
- keyMatchCount += matchingKeys.length
142
- }
143
- for (const key of matchingKeys) {
144
- asyncFunc(key)
145
- .then((val) => {
146
- if (typeof val !== 'undefined') {
147
- logger.silly(`${logPrefix} found key ${key} = ${val}`)
148
- results.push(val)
149
- } else {
150
- logger.silly(`${logPrefix} found key with undefined value: ${key}`)
151
- }
152
- })
153
- .catch((e) => {
154
- const message = `${logPrefix} error with key ${key}: ${e}`
155
- logger.error(message)
156
- reject(message)
157
- })
158
- }
159
- }, (err, matchCount) => {
160
- if (err) {
161
- logger.error(`${logPrefix} error in final callback: ${err}`)
162
- reject(err)
163
- }
164
- const duration = Date.now() - start
165
- logger.debug(`${logPrefix} resolving with matchCount=${matchCount}, keyMatchCount=${keyMatchCount}, results.length=${results.length}, duration ${duration/1000} seconds`)
166
- resolve(results)
167
- })
168
- })
169
- }
170
-
171
- removeMatchingKeys = async pattern => await this.applyToMatchingKeys(pattern, this.del)
172
-
173
- flush = async () => {
174
- this.counters.flush++
175
- await this.removeMatchingKeys('*')
176
- }
177
-
178
- scopedCache = (name, size = 100) => {
179
- if (this.scopedCaches[name]) {
180
- return this.scopedCaches[name]
181
- }
182
- if (!this.redis) {
183
- return size && size > 0 ? new LRU({ max: size }) : null
184
- }
185
- const realKey = k => k ? `${name}_${k}` : null
186
- const cache = {
187
- get: async key => {
188
- const val = await this.get(realKey(key))
189
- if (!val) return null
190
- try {
191
- return JSON.parse(val)
192
- } catch (e) {
193
- logger.warn(`redis(${this.name}) get(${key}) error: ${e}`)
194
- return null
195
- }
196
- },
197
- set: async (key, value) => {
198
- if (key && value) {
199
- try {
200
- await this.set(realKey(key), JSON.stringify(value))
201
- } catch (e) {
202
- logger.warn(`redis(${this.name}) set(${key}) error: ${e}`)
203
- }
204
- }
205
- }
206
- }
207
- this.scopedCaches[name] = cache
208
- return cache
209
- }
210
- }
211
-
212
- const forAllCaches = async (func) => {
213
- const promises = []
214
- for (const name of Object.keys(REDIS_CLIENTS)) {
215
- promises.push(func(REDIS_CLIENTS[name]))
216
- }
217
- return Promise.all(promises)
218
- }
219
-
220
- const teardown = async () => await forAllCaches((client) => {
221
- try {
222
- if (client.redis) {
223
- client.redis.disconnect()
224
- client.redis = null
225
- }
226
- } catch (e) {
227
- logger.warn(`teardown: error disconnecting from redis(${client.name}) ${e}`)
228
- }
229
- })
230
-
231
- const flushAll = async () => await forAllCaches(client => client.flush())
232
-
233
- module.exports = { getRedis, flushAll, teardown }