mobiletto 1.1.16 → 1.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/drivers/b2/index.js +1 -1
- package/drivers/s3/index.js +6 -6
- package/index.js +19 -11
- package/package.json +1 -1
- package/test/api.spec.js +3 -3
- package/util/crypt.js +1 -1
- package/util/logger.js +7 -2
- package/util/redis.js +13 -13
package/drivers/b2/index.js
CHANGED
|
@@ -106,7 +106,7 @@ class StorageClient {
|
|
|
106
106
|
return []
|
|
107
107
|
}
|
|
108
108
|
const objects = files.map((f) => {
|
|
109
|
-
logger.debug(`addListedFiles(${path})
|
|
109
|
+
logger.debug(`addListedFiles(${path}) adding file: ${f.fileName}`)
|
|
110
110
|
return this.file2object(f)
|
|
111
111
|
})
|
|
112
112
|
if (visitor) {
|
package/drivers/s3/index.js
CHANGED
|
@@ -79,7 +79,7 @@ class StorageClient {
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
82
|
-
const logPrefix = `_list(path=${path})
|
|
82
|
+
const logPrefix = `_list(path=${path})`
|
|
83
83
|
|
|
84
84
|
// Declare truncated as a flag that the while loop is based on.
|
|
85
85
|
let truncated = true
|
|
@@ -212,7 +212,7 @@ class StorageClient {
|
|
|
212
212
|
})
|
|
213
213
|
let total = 0
|
|
214
214
|
uploader.on('httpUploadProgress', (progress) => {
|
|
215
|
-
logger.debug(`write(${bucketParams.Key})
|
|
215
|
+
logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`)
|
|
216
216
|
total += progress.loaded
|
|
217
217
|
})
|
|
218
218
|
const response = await uploader.done()
|
|
@@ -255,15 +255,15 @@ class StorageClient {
|
|
|
255
255
|
Bucket: this.bucket,
|
|
256
256
|
Delete
|
|
257
257
|
}
|
|
258
|
-
logger.debug(`remove(${path})
|
|
258
|
+
logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`)
|
|
259
259
|
const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
|
|
260
260
|
let statusCode = response.$metadata.httpStatusCode
|
|
261
261
|
let statusClass = Math.floor(statusCode / 100)
|
|
262
262
|
if (statusClass !== 2) {
|
|
263
|
-
throw new MobilettoError(`remove(${path})
|
|
263
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`)
|
|
264
264
|
}
|
|
265
265
|
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
266
|
-
throw new MobilettoError(`remove(${path})
|
|
266
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
267
267
|
}
|
|
268
268
|
if (response.Deleted) {
|
|
269
269
|
removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
|
|
@@ -272,7 +272,7 @@ class StorageClient {
|
|
|
272
272
|
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
273
273
|
} catch (e) {
|
|
274
274
|
if (!(e instanceof MobilettoNotFoundError)) {
|
|
275
|
-
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path})
|
|
275
|
+
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`)
|
|
276
276
|
}
|
|
277
277
|
objects = null
|
|
278
278
|
}
|
package/index.js
CHANGED
|
@@ -202,7 +202,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
202
202
|
try {
|
|
203
203
|
return await client.readFile(path)
|
|
204
204
|
} catch (e) {
|
|
205
|
-
logger.info(`safeReadFile(${path})
|
|
205
|
+
logger.info(`safeReadFile(${path}) ${e}`)
|
|
206
206
|
return null
|
|
207
207
|
}
|
|
208
208
|
},
|
|
@@ -377,13 +377,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
377
377
|
}
|
|
378
378
|
const encKey = normalizeKey(encryption.key)
|
|
379
379
|
if (!encKey) {
|
|
380
|
-
const message = `mobiletto(${driverPath})
|
|
380
|
+
const message = `mobiletto(${driverPath}) invalid encryption key`
|
|
381
381
|
logger.error(message)
|
|
382
382
|
throw new MobilettoError(message)
|
|
383
383
|
}
|
|
384
384
|
const iv = normalizeIV(encryption.iv, encKey)
|
|
385
385
|
if (!iv) {
|
|
386
|
-
const message = `mobiletto(${driverPath})
|
|
386
|
+
const message = `mobiletto(${driverPath}) invalid encryption IV`
|
|
387
387
|
logger.error(message)
|
|
388
388
|
throw new MobilettoError(message)
|
|
389
389
|
}
|
|
@@ -462,24 +462,32 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
462
462
|
const files = []
|
|
463
463
|
const promises = []
|
|
464
464
|
for (const entry of entries) {
|
|
465
|
+
const logPrefix = `_loadMeta(${dirent}/${basename(entry.name)})`
|
|
465
466
|
promises.push(new Promise((resolve) => {
|
|
466
467
|
const cipherText = []
|
|
467
|
-
client.read(dirent + '/' + basename(entry.name), reader(cipherText))
|
|
468
|
-
|
|
468
|
+
client.read(dirent + '/' + basename(entry.name), reader(cipherText)).then(
|
|
469
|
+
(bytesRead) => {
|
|
469
470
|
if (!bytesRead) {
|
|
470
|
-
logger.warn(
|
|
471
|
+
logger.warn(`${logPrefix} returned no data`)
|
|
471
472
|
resolve()
|
|
472
473
|
} else {
|
|
473
474
|
const plain = decrypt(cipherText.toString(), enc)
|
|
474
475
|
const realPath = plain.split(ENC_PAD_SEP)[0]
|
|
475
|
-
_metadata(client)(realPath)
|
|
476
|
-
|
|
476
|
+
_metadata(client)(realPath).then(
|
|
477
|
+
(meta) => {
|
|
477
478
|
files.push(meta)
|
|
478
479
|
resolve()
|
|
480
|
+
},
|
|
481
|
+
(err) => {
|
|
482
|
+
logger.warn(`${logPrefix} error fetching _metadata: ${err}`)
|
|
483
|
+
resolve()
|
|
479
484
|
})
|
|
480
485
|
}
|
|
481
|
-
|
|
482
|
-
)
|
|
486
|
+
},
|
|
487
|
+
(err) => {
|
|
488
|
+
logger.warn(`${logPrefix} error reading file: ${err}`)
|
|
489
|
+
resolve()
|
|
490
|
+
})
|
|
483
491
|
}))
|
|
484
492
|
}
|
|
485
493
|
await Promise.all(promises)
|
|
@@ -559,7 +567,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
559
567
|
}
|
|
560
568
|
throw e
|
|
561
569
|
} else {
|
|
562
|
-
throw new MobilettoError(`encClient.list(${p})
|
|
570
|
+
throw new MobilettoError(`encClient.list(${p}) ${e}`, e)
|
|
563
571
|
}
|
|
564
572
|
}
|
|
565
573
|
if (recursive) {
|
package/package.json
CHANGED
package/test/api.spec.js
CHANGED
|
@@ -100,9 +100,9 @@ async function writeRandomFile(fixture, size) {
|
|
|
100
100
|
return await fixture.api.write(fixture.name, dataGenerator())
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
-
|
|
103
|
+
const encryptionTests = () => [null, { key: rand(32) }]
|
|
104
104
|
// const encryptionTests = () => [null]
|
|
105
|
-
const encryptionTests = () => [{ key: rand(32) }]
|
|
105
|
+
// const encryptionTests = () => [{ key: rand(32) }]
|
|
106
106
|
|
|
107
107
|
const REDIS_ENABLED = {
|
|
108
108
|
name: 'redis-enabled',
|
|
@@ -111,7 +111,7 @@ const REDIS_ENABLED = {
|
|
|
111
111
|
}
|
|
112
112
|
const REDIS_DISABLED = {
|
|
113
113
|
name: 'redis-disabled',
|
|
114
|
-
enabled:
|
|
114
|
+
enabled: true,
|
|
115
115
|
config: (cfg) => {
|
|
116
116
|
const newConfig = Object.assign({}, cfg)
|
|
117
117
|
if (!newConfig.opts) { newConfig.opts = {} }
|
package/util/crypt.js
CHANGED
package/util/logger.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
const winston = require('winston')
|
|
2
2
|
|
|
3
3
|
const logger = winston.createLogger({
|
|
4
|
+
levels: winston.config.npm.levels,
|
|
4
5
|
level: process.env.MOBILETTO_LOG_LEVEL || 'error',
|
|
5
|
-
format: winston.format.
|
|
6
|
+
format: winston.format.combine(
|
|
7
|
+
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }),
|
|
8
|
+
winston.format.printf((info) => {
|
|
9
|
+
return `${info.timestamp} [${info.level.toUpperCase()}] ${info.message}`
|
|
10
|
+
})),
|
|
6
11
|
transports: process.env.MOBILETTO_LOG_FILE
|
|
7
12
|
? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
|
|
8
|
-
: [ new winston.transports.Console() ]
|
|
13
|
+
: [ new winston.transports.Console({ stderrLevels: Object.keys(winston.config.npm.levels) }) ]
|
|
9
14
|
})
|
|
10
15
|
|
|
11
16
|
const setLogLevel = (level) => { logger.level = level }
|
package/util/redis.js
CHANGED
|
@@ -41,20 +41,20 @@ class MobilettoCache {
|
|
|
41
41
|
try {
|
|
42
42
|
this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
|
|
43
43
|
} catch (e) {
|
|
44
|
-
logger.error(`redis(${name})
|
|
44
|
+
logger.error(`redis(${name}) error connecting to redis, using fallback LRU for scoped caches: ${e}`)
|
|
45
45
|
this.redis = null
|
|
46
46
|
}
|
|
47
47
|
} else {
|
|
48
|
-
logger.warn(`redis(${name})
|
|
48
|
+
logger.warn(`redis(${name}) no host or port provided, using fallback LRU for scoped caches`)
|
|
49
49
|
this.redis = null
|
|
50
50
|
}
|
|
51
51
|
this.prefix = prefix
|
|
52
52
|
if (this.redis) {
|
|
53
53
|
// test connection by flushing
|
|
54
54
|
this.flush().then(
|
|
55
|
-
() => { logger.debug(`redis(${name})
|
|
55
|
+
() => { logger.debug(`redis(${name}) successfully flushed`) },
|
|
56
56
|
(e) => {
|
|
57
|
-
logger.warn(`redis(${name})
|
|
57
|
+
logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
|
|
58
58
|
this.redis = null
|
|
59
59
|
}
|
|
60
60
|
)
|
|
@@ -73,7 +73,7 @@ class MobilettoCache {
|
|
|
73
73
|
try {
|
|
74
74
|
return this.redis ? await func(this.redis) : defaultValue
|
|
75
75
|
} catch (e) {
|
|
76
|
-
logger.warn(`redis(${this.name})
|
|
76
|
+
logger.warn(`redis(${this.name}) doRedisAsync(${func}) ${e} (returning default value: ${defaultValue})`)
|
|
77
77
|
return defaultValue
|
|
78
78
|
}
|
|
79
79
|
}
|
|
@@ -82,16 +82,16 @@ class MobilettoCache {
|
|
|
82
82
|
try {
|
|
83
83
|
return this.redis ? func(this.redis) : defaultValue
|
|
84
84
|
} catch (e) {
|
|
85
|
-
logger.warn(`redis(${this.name})
|
|
85
|
+
logger.warn(`redis(${this.name}) doRedis(${func}) ${e} (returning default value: ${defaultValue})`)
|
|
86
86
|
return defaultValue
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
get = async key => {
|
|
91
91
|
this.counters.get++
|
|
92
|
-
logger.silly(`redis(${this.name})
|
|
92
|
+
logger.silly(`redis(${this.name}) get(${key}) starting`)
|
|
93
93
|
const val = await this.doRedis(r => r.get(this.pfx(key)), null)
|
|
94
|
-
logger.silly(`redis(${this.name})
|
|
94
|
+
logger.silly(`redis(${this.name}) get(${key}) found value: ${val}`)
|
|
95
95
|
if (val) {
|
|
96
96
|
this.counters.hit++
|
|
97
97
|
} else {
|
|
@@ -106,9 +106,9 @@ class MobilettoCache {
|
|
|
106
106
|
|
|
107
107
|
set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
|
|
108
108
|
this.counters.set++
|
|
109
|
-
logger.silly(`redis(${this.name})
|
|
109
|
+
logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) starting`)
|
|
110
110
|
await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
|
|
111
|
-
logger.silly(`redis(${this.name})
|
|
111
|
+
logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) finished`)
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
del = async key => {
|
|
@@ -156,7 +156,7 @@ class MobilettoCache {
|
|
|
156
156
|
try {
|
|
157
157
|
return JSON.parse(val)
|
|
158
158
|
} catch (e) {
|
|
159
|
-
logger.warn(`redis(${this.name})
|
|
159
|
+
logger.warn(`redis(${this.name}) get(${key}) error: ${e}`)
|
|
160
160
|
return null
|
|
161
161
|
}
|
|
162
162
|
},
|
|
@@ -165,7 +165,7 @@ class MobilettoCache {
|
|
|
165
165
|
try {
|
|
166
166
|
await this.set(realKey(key), JSON.stringify(value))
|
|
167
167
|
} catch (e) {
|
|
168
|
-
logger.warn(`redis(${this.name})
|
|
168
|
+
logger.warn(`redis(${this.name}) set(${key}) error: ${e}`)
|
|
169
169
|
}
|
|
170
170
|
}
|
|
171
171
|
}
|
|
@@ -190,7 +190,7 @@ const teardown = async () => await forAllCaches((client) => {
|
|
|
190
190
|
client.redis = null
|
|
191
191
|
}
|
|
192
192
|
} catch (e) {
|
|
193
|
-
logger.warn(`teardown: error disconnecting from redis(${client.name})
|
|
193
|
+
logger.warn(`teardown: error disconnecting from redis(${client.name}) ${e}`)
|
|
194
194
|
}
|
|
195
195
|
})
|
|
196
196
|
|