mobiletto 1.1.40 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.1.40",
3
+ "version": "1.2.1",
4
4
  "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
@@ -26,25 +26,20 @@
26
26
  "test": "mocha --timeout 60000 ./test/*.spec.js"
27
27
  },
28
28
  "devDependencies": {
29
- "@babel/core": "^7.18.10",
30
- "@babel/node": "^7.18.10",
31
- "@babel/preset-env": "^7.18.10",
32
- "chai": "^4.3.6",
33
- "dotenv": "^16.0.1",
34
- "mocha": "^10.0.0",
35
- "randomstring": "^1.2.2"
29
+ "@babel/core": "^7.22.5",
30
+ "@babel/node": "^7.22.5",
31
+ "@babel/preset-env": "^7.22.5",
32
+ "chai": "^4.3.7",
33
+ "dotenv": "^16.3.1",
34
+ "mocha": "^10.2.0",
35
+ "randomstring": "^1.3.0"
36
36
  },
37
37
  "dependencies": {
38
- "@aws-sdk/client-s3": "^3.150.0",
39
- "@aws-sdk/lib-storage": "^3.150.0",
40
- "@gideo-llc/backblaze-b2-upload-any": "^0.1.4",
41
- "backblaze-b2": "^1.7.0",
42
- "bull": "^4.9.0",
43
- "ioredis": "^5.2.3",
44
- "lru-cache": "^7.14.0",
45
- "node-fetch-commonjs": "^3.1.1",
46
- "node-redis-scan": "^1.3.5",
47
- "shasum": "^1.0.2",
48
- "winston": "^3.8.1"
38
+ "mobiletto-base": "^1.0.13",
39
+ "mobiletto-common": "^1.0.2",
40
+ "mobiletto-driver-b2": "^1.0.6",
41
+ "mobiletto-driver-local": "^1.0.3",
42
+ "mobiletto-driver-s3": "^1.0.5",
43
+ "mobiletto-driver-indexeddb": "^1.0.1"
49
44
  }
50
45
  }
package/test/api.spec.js CHANGED
@@ -3,25 +3,27 @@ require('dotenv').config()
3
3
 
4
4
  const fs = require('fs')
5
5
  const { basename, dirname } = require('path')
6
-
7
6
  const randomstring = require('randomstring')
8
7
  const crypto = require('crypto')
9
- const redis = require('../util/redis')
10
-
11
- const { expect, should, assert } = require('chai')
12
8
 
13
9
  const {
14
- M_DIR, M_FILE,
15
- mobiletto, connect, MobilettoNotFoundError, closeRedis
16
- } = require("../index")
10
+ M_DIR, M_FILE, MobilettoNotFoundError, logger
11
+ } = require('mobiletto-common')
17
12
 
18
- const { logger } = require("../util/logger")
13
+ const { connect, mobiletto, redis, closeRedis } = require('mobiletto-base')
14
+
15
+ const { registerDrivers } = require("../index")
16
+ registerDrivers()
17
+
18
+ const { expect, should, assert } = require('chai')
19
19
 
20
20
  // chunk size used by generator function, used by driver's 'write' function
21
21
  // the temp file is also TEMP_SZ_MULTIPLE of this number
22
22
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
23
23
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
24
24
 
25
+ const SINGLE_DRIVER = process.env.SINGLE_DRIVER || null
26
+
25
27
  DRIVER_CONFIG = {
26
28
  local: {
27
29
  key: process.env.LOCAL_STORAGE || '/tmp'
@@ -130,7 +132,7 @@ if (!redisSetup.enabled) continue
130
132
 
131
133
  for (const driverName of DRIVER_NAMES) {
132
134
  // For testing single drivers
133
- // if (driverName !== 'local') continue
135
+ if (SINGLE_DRIVER != null && SINGLE_DRIVER !== driverName) continue
134
136
 
135
137
  const driverTest = `${driverName} [${redisSetup.name}]`
136
138
  const config = redisSetup.config(DRIVER_CONFIG[driverName])
@@ -205,6 +207,10 @@ describe(`${driverTest} test`, () => {
205
207
 
206
208
  for (const encryption of encryptionTests()) {
207
209
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
210
+ if (encryption && redisSetup.name !== 'redis-enabled' && driverName !== 'local') {
211
+ // encryption without redis is simply too slow, except for local filesystems
212
+ continue
213
+ }
208
214
  describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
209
215
  // describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
210
216
  // const encryption = {key: rand(32)}
@@ -288,6 +294,10 @@ describe(`${driverTest} test`, () => {
288
294
 
289
295
  for (const encryption of encryptionTests()) {
290
296
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
297
+ if (encryption && redisSetup.name !== 'redis-enabled') {
298
+ // encryption without redis is simply too slow
299
+ continue
300
+ }
291
301
  const mirrorDriver = pickAnotherDriver(driverName)
292
302
  // const mirrorDriver = 'local' // for hard-coding the mirror driver
293
303
  const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
@@ -8,9 +8,9 @@ const {
8
8
  normalizeKey, normalizeIV,
9
9
  encrypt, decrypt,
10
10
  getCipher, getDecipher
11
- } = require("../util/crypt")
11
+ } = require("mobiletto-base/util/crypt")
12
12
 
13
- const { readStream } = require("../index")
13
+ const { readStream } = require('mobiletto-common')
14
14
 
15
15
  describe('crypto test', () => {
16
16
  const enc = {
package/util/crypt.js DELETED
@@ -1,80 +0,0 @@
1
- // adapted from https://stackoverflow.com/a/64136185
2
- const crypto = require('crypto')
3
- const shasum = require('shasum')
4
- const winston = require("winston");
5
-
6
- const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
7
- const MIN_KEY_LEN = 16
8
- const DEFAULT_CRYPT_ALGO = 'aes-256-cbc';
9
-
10
- function normalizeKey (k) {
11
- return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
12
- ? Buffer.from(shasum(k.trim())).subarray(0, 32)
13
- : null
14
- }
15
-
16
- function normalizeIV (iv, key) {
17
- return (typeof iv === 'string' && iv.trim().length >= 16)
18
- ? Buffer.from(shasum(iv.trim())).subarray(0, 16)
19
- : key
20
- ? Buffer.from(shasum(key)).subarray(0, 16)
21
- : null
22
- }
23
-
24
- // ensure key long enough for security, and is 32 bytes for AES-256
25
- let KEY = null
26
- function setDefaultKey (key) {
27
- KEY = normalizeKey(key)
28
- }
29
-
30
- // ensure IV is 16 bytes for AES-256
31
- let CRYPTO_IV = null
32
- function setDefaultIV (iv) {
33
- CRYPTO_IV = normalizeIV(iv, KEY)
34
- }
35
-
36
- function getCipher(enc) {
37
- try {
38
- let c = crypto.createCipheriv(enc.algo, enc.key, enc.iv);
39
- c.setAutoPadding(true)
40
- return c
41
- } catch (e) {
42
- winston.error(`getCipher(${enc}) ${e}`)
43
- throw e
44
- }
45
- }
46
-
47
- function encrypt (plainText, encryption, outputEncoding = 'base64') {
48
- if (!encryption || !encryption.key) {
49
- if (WARN_PLAINTEXT) {
50
- console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
51
- }
52
- return plainText
53
- }
54
- const cipher = getCipher(encryption)
55
- const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
56
- return encoded.toString(outputEncoding)
57
- }
58
-
59
- function getDecipher(enc) {
60
- let c = crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
61
- c.setAutoPadding(true)
62
- return c
63
- }
64
-
65
- function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
66
- if (!encryption || !encryption.key) {
67
- return cipherText
68
- }
69
- const cipher = getDecipher(encryption)
70
- const data = Buffer.from(cipherText, inputEncoding)
71
- return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
72
- }
73
-
74
- module.exports = {
75
- DEFAULT_CRYPT_ALGO,
76
- setDefaultKey, setDefaultIV,
77
- encrypt, decrypt,
78
- normalizeKey, normalizeIV,
79
- getCipher, getDecipher
80
- }
package/util/logger.js DELETED
@@ -1,27 +0,0 @@
1
- const winston = require('winston')
2
-
3
- const logger = winston.createLogger({
4
- levels: winston.config.npm.levels,
5
- level: process.env.MOBILETTO_LOG_LEVEL || 'error',
6
- format: winston.format.combine(
7
- winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }),
8
- winston.format.printf((info) => {
9
- return `${info.timestamp} [${info.level.toUpperCase()}] ${info.message}`
10
- })),
11
- transports: process.env.MOBILETTO_LOG_FILE
12
- ? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
13
- : [ new winston.transports.Console({ stderrLevels: Object.keys(winston.config.npm.levels) }) ]
14
- })
15
-
16
- const setLogLevel = (level) => { logger.level = level }
17
-
18
- const setLogTransports = (transports) => {
19
- logger.transports.splice(0, logger.transports.length)
20
- logger.transports.push(...transports)
21
- }
22
-
23
- module.exports = {
24
- logger,
25
- setLogLevel,
26
- setLogTransports
27
- }
package/util/redis.js DELETED
@@ -1,233 +0,0 @@
1
- const Redis = require('ioredis')
2
- const redisScan = require('node-redis-scan')
3
- const { logger } = require('./logger')
4
- const LRU = require('lru-cache')
5
-
6
- const { DEFAULT_REDIS_OPTIONS } = require('ioredis/built/redis/RedisOptions')
7
-
8
- let REDIS_CLIENTS = {}
9
-
10
- const getRedis = (name, host, port, prefix) => {
11
- if (typeof REDIS_CLIENTS[name] === 'undefined') {
12
- REDIS_CLIENTS[name] = new MobilettoCache(name, host, port, prefix + name + '_')
13
- }
14
- return REDIS_CLIENTS[name]
15
- }
16
-
17
- const DEFAULT_EXPIRATION_MILLIS = 1000 * 60 * 60 * 24 // 1 day
18
-
19
- const ZERO_COUNTERS = {
20
- get: 0,
21
- set: 0,
22
- del: 0,
23
- flush: 0,
24
- hit: 0,
25
- miss: 0
26
- }
27
-
28
- class MobilettoCache {
29
- name
30
- redis
31
- scanner
32
- prefix
33
- scopedCaches = {}
34
- counters = Object.assign({}, ZERO_COUNTERS)
35
- printStatsInterval = 1000
36
- constructor(name,
37
- host = '127.0.0.1',
38
- port = 6379,
39
- prefix = '_mobiletto__') {
40
- this.name = name
41
- if (host && port) {
42
- try {
43
- this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
44
- } catch (e) {
45
- logger.error(`redis(${name}) error connecting to redis, using fallback LRU for scoped caches: ${e}`)
46
- this.redis = null
47
- }
48
- } else {
49
- logger.warn(`redis(${name}) no host or port provided, using fallback LRU for scoped caches`)
50
- this.redis = null
51
- }
52
- this.prefix = prefix
53
- if (this.redis) {
54
- // test connection by flushing
55
- this.scanner = new redisScan(this.redis)
56
- this.flush()
57
- .then(() => { logger.debug(`redis(${name}) successfully flushed`) })
58
- .catch((e) => {
59
- logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
60
- this.redis = null
61
- })
62
- }
63
- }
64
-
65
- stats = () => this.counters
66
- resetStats = () => { this.counters = Object.assign({}, ZERO_COUNTERS) }
67
- hitRate = () => this.counters.get === 0 ? 0 : (100 * this.counters.hit) / this.counters.get
68
- toString = () => `MobilettoCache(${this.name}) [${this.counters.hit}/${this.counters.get} = ${this.hitRate()}% hit] stats=${JSON.stringify(this.counters)}`
69
-
70
- pfx = key => key.startsWith(this.prefix) ? key : this.prefix + key
71
- unprefix = key => key && key.startsWith(this.prefix) ? key.substring(this.prefix.length) : key
72
-
73
- doRedisAsync = async (func, defaultValue = null) => {
74
- try {
75
- return this.redis ? await func(this.redis) : defaultValue
76
- } catch (e) {
77
- logger.warn(`redis(${this.name}) doRedisAsync(${func}) ${e} (returning default value: ${defaultValue})`)
78
- return defaultValue
79
- }
80
- }
81
-
82
- doRedis = (func, defaultValue = null) => {
83
- try {
84
- return this.redis ? func(this.redis) : defaultValue
85
- } catch (e) {
86
- logger.warn(`redis(${this.name}) doRedis(${func}) ${e} (returning default value: ${defaultValue})`)
87
- return defaultValue
88
- }
89
- }
90
-
91
- get = async key => {
92
- this.counters.get++
93
- logger.silly(`redis(${this.name}) get(${key}) starting`)
94
- const val = await this.doRedis(r => r.get(this.pfx(key)), null)
95
- logger.silly(`redis(${this.name}) get(${key}) found value: ${val}`)
96
- if (val) {
97
- this.counters.hit++
98
- } else {
99
- this.counters.miss++
100
- }
101
- if (this.printStatsInterval && this.printStatsInterval > 0 && this.counters.get % this.printStatsInterval === 0) {
102
- const message = `${new Date()}: ${this}`
103
- logger.info(message)
104
- }
105
- return val
106
- }
107
-
108
- set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
109
- this.counters.set++
110
- logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) starting`)
111
- await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
112
- logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) finished`)
113
- }
114
-
115
- del = async key => {
116
- this.counters.del++
117
- await this.doRedisAsync(r => r.del(this.pfx(key)))
118
- }
119
-
120
- findMatchingKeys = async (pattern) => {
121
- return await new Promise((resolve, reject) => {
122
- this.scanner.scan(this.pfx(pattern), (err, matchingKeys) => {
123
- if (err) reject(err)
124
- resolve(matchingKeys)
125
- })
126
- })
127
- }
128
-
129
- applyToMatchingKeys = async (pattern, asyncFunc) => {
130
- const results = []
131
- let keyMatchCount = 0
132
- const logPrefix = `applyToMatchingKeys(${pattern}, ${asyncFunc && asyncFunc.name ? asyncFunc.name : 'nameless function'})`
133
- const start = Date.now()
134
- return await new Promise((resolve, reject) => {
135
- this.scanner.eachScan(this.pfx(pattern), {}, async (matchingKeys) => {
136
- if (!matchingKeys || typeof matchingKeys.length !== 'number') {
137
- const message = `${logPrefix}: eachScan received invalid matchingKeys = ${typeof matchingKeys !== 'undefined' ? matchingKeys : 'undefined'}`
138
- logger.error(message)
139
- reject(message)
140
- } else {
141
- keyMatchCount += matchingKeys.length
142
- }
143
- for (const key of matchingKeys) {
144
- asyncFunc(key)
145
- .then((val) => {
146
- if (typeof val !== 'undefined') {
147
- logger.silly(`${logPrefix} found key ${key} = ${val}`)
148
- results.push(val)
149
- } else {
150
- logger.silly(`${logPrefix} found key with undefined value: ${key}`)
151
- }
152
- })
153
- .catch((e) => {
154
- const message = `${logPrefix} error with key ${key}: ${e}`
155
- logger.error(message)
156
- reject(message)
157
- })
158
- }
159
- }, (err, matchCount) => {
160
- if (err) {
161
- logger.error(`${logPrefix} error in final callback: ${err}`)
162
- reject(err)
163
- }
164
- const duration = Date.now() - start
165
- logger.debug(`${logPrefix} resolving with matchCount=${matchCount}, keyMatchCount=${keyMatchCount}, results.length=${results.length}, duration ${duration/1000} seconds`)
166
- resolve(results)
167
- })
168
- })
169
- }
170
-
171
- removeMatchingKeys = async pattern => await this.applyToMatchingKeys(pattern, this.del)
172
-
173
- flush = async () => {
174
- this.counters.flush++
175
- await this.removeMatchingKeys('*')
176
- }
177
-
178
- scopedCache = (name, size = 100) => {
179
- if (this.scopedCaches[name]) {
180
- return this.scopedCaches[name]
181
- }
182
- if (!this.redis) {
183
- return size && size > 0 ? new LRU({ max: size }) : null
184
- }
185
- const realKey = k => k ? `${name}_${k}` : null
186
- const cache = {
187
- get: async key => {
188
- const val = await this.get(realKey(key))
189
- if (!val) return null
190
- try {
191
- return JSON.parse(val)
192
- } catch (e) {
193
- logger.warn(`redis(${this.name}) get(${key}) error: ${e}`)
194
- return null
195
- }
196
- },
197
- set: async (key, value) => {
198
- if (key && value) {
199
- try {
200
- await this.set(realKey(key), JSON.stringify(value))
201
- } catch (e) {
202
- logger.warn(`redis(${this.name}) set(${key}) error: ${e}`)
203
- }
204
- }
205
- }
206
- }
207
- this.scopedCaches[name] = cache
208
- return cache
209
- }
210
- }
211
-
212
- const forAllCaches = async (func) => {
213
- const promises = []
214
- for (const name of Object.keys(REDIS_CLIENTS)) {
215
- promises.push(func(REDIS_CLIENTS[name]))
216
- }
217
- return Promise.all(promises)
218
- }
219
-
220
- const teardown = async () => await forAllCaches((client) => {
221
- try {
222
- if (client.redis) {
223
- client.redis.disconnect()
224
- client.redis = null
225
- }
226
- } catch (e) {
227
- logger.warn(`teardown: error disconnecting from redis(${client.name}) ${e}`)
228
- }
229
- })
230
-
231
- const flushAll = async () => await forAllCaches(client => client.flush())
232
-
233
- module.exports = { getRedis, flushAll, teardown }