mobiletto 1.1.15 → 1.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/drivers/b2/index.js +1 -1
- package/drivers/s3/index.js +6 -6
- package/index.js +25 -11
- package/package.json +1 -1
- package/util/crypt.js +1 -1
- package/util/logger.js +7 -2
- package/util/redis.js +13 -13
package/drivers/b2/index.js
CHANGED
|
@@ -106,7 +106,7 @@ class StorageClient {
|
|
|
106
106
|
return []
|
|
107
107
|
}
|
|
108
108
|
const objects = files.map((f) => {
|
|
109
|
-
logger.debug(`addListedFiles(${path})
|
|
109
|
+
logger.debug(`addListedFiles(${path}) adding file: ${f.fileName}`)
|
|
110
110
|
return this.file2object(f)
|
|
111
111
|
})
|
|
112
112
|
if (visitor) {
|
package/drivers/s3/index.js
CHANGED
|
@@ -79,7 +79,7 @@ class StorageClient {
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
82
|
-
const logPrefix = `_list(path=${path})
|
|
82
|
+
const logPrefix = `_list(path=${path})`
|
|
83
83
|
|
|
84
84
|
// Declare truncated as a flag that the while loop is based on.
|
|
85
85
|
let truncated = true
|
|
@@ -212,7 +212,7 @@ class StorageClient {
|
|
|
212
212
|
})
|
|
213
213
|
let total = 0
|
|
214
214
|
uploader.on('httpUploadProgress', (progress) => {
|
|
215
|
-
logger.debug(`write(${bucketParams.Key})
|
|
215
|
+
logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`)
|
|
216
216
|
total += progress.loaded
|
|
217
217
|
})
|
|
218
218
|
const response = await uploader.done()
|
|
@@ -255,15 +255,15 @@ class StorageClient {
|
|
|
255
255
|
Bucket: this.bucket,
|
|
256
256
|
Delete
|
|
257
257
|
}
|
|
258
|
-
logger.debug(`remove(${path})
|
|
258
|
+
logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`)
|
|
259
259
|
const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
|
|
260
260
|
let statusCode = response.$metadata.httpStatusCode
|
|
261
261
|
let statusClass = Math.floor(statusCode / 100)
|
|
262
262
|
if (statusClass !== 2) {
|
|
263
|
-
throw new MobilettoError(`remove(${path})
|
|
263
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`)
|
|
264
264
|
}
|
|
265
265
|
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
266
|
-
throw new MobilettoError(`remove(${path})
|
|
266
|
+
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
267
267
|
}
|
|
268
268
|
if (response.Deleted) {
|
|
269
269
|
removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
|
|
@@ -272,7 +272,7 @@ class StorageClient {
|
|
|
272
272
|
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
273
273
|
} catch (e) {
|
|
274
274
|
if (!(e instanceof MobilettoNotFoundError)) {
|
|
275
|
-
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path})
|
|
275
|
+
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`)
|
|
276
276
|
}
|
|
277
277
|
objects = null
|
|
278
278
|
}
|
package/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
|
-
const path = require('path')
|
|
3
2
|
const { Readable, Transform } = require('stream')
|
|
4
3
|
const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
|
|
5
4
|
const { getRedis, teardown } = require('./util/redis')
|
|
@@ -203,7 +202,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
203
202
|
try {
|
|
204
203
|
return await client.readFile(path)
|
|
205
204
|
} catch (e) {
|
|
206
|
-
logger.info(`safeReadFile(${path})
|
|
205
|
+
logger.info(`safeReadFile(${path}) ${e}`)
|
|
207
206
|
return null
|
|
208
207
|
}
|
|
209
208
|
},
|
|
@@ -378,13 +377,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
378
377
|
}
|
|
379
378
|
const encKey = normalizeKey(encryption.key)
|
|
380
379
|
if (!encKey) {
|
|
381
|
-
const message = `mobiletto(${driverPath})
|
|
380
|
+
const message = `mobiletto(${driverPath}) invalid encryption key`
|
|
382
381
|
logger.error(message)
|
|
383
382
|
throw new MobilettoError(message)
|
|
384
383
|
}
|
|
385
384
|
const iv = normalizeIV(encryption.iv, encKey)
|
|
386
385
|
if (!iv) {
|
|
387
|
-
const message = `mobiletto(${driverPath})
|
|
386
|
+
const message = `mobiletto(${driverPath}) invalid encryption IV`
|
|
388
387
|
logger.error(message)
|
|
389
388
|
throw new MobilettoError(message)
|
|
390
389
|
}
|
|
@@ -461,14 +460,29 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
461
460
|
|
|
462
461
|
const _loadMeta = async (dirent, entries) => {
|
|
463
462
|
const files = []
|
|
463
|
+
const promises = []
|
|
464
464
|
for (const entry of entries) {
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
465
|
+
promises.push(new Promise((resolve) => {
|
|
466
|
+
const cipherText = []
|
|
467
|
+
client.read(dirent + '/' + basename(entry.name), reader(cipherText))
|
|
468
|
+
.then((bytesRead) => {
|
|
469
|
+
if (!bytesRead) {
|
|
470
|
+
logger.warn(`_loadMeta(${dirent}/${basename(entry.name)}) returned no data`)
|
|
471
|
+
resolve()
|
|
472
|
+
} else {
|
|
473
|
+
const plain = decrypt(cipherText.toString(), enc)
|
|
474
|
+
const realPath = plain.split(ENC_PAD_SEP)[0]
|
|
475
|
+
_metadata(client)(realPath)
|
|
476
|
+
.then((meta) => {
|
|
477
|
+
files.push(meta)
|
|
478
|
+
resolve()
|
|
479
|
+
})
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
)
|
|
483
|
+
}))
|
|
471
484
|
}
|
|
485
|
+
await Promise.all(promises)
|
|
472
486
|
return files
|
|
473
487
|
}
|
|
474
488
|
|
|
@@ -545,7 +559,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
545
559
|
}
|
|
546
560
|
throw e
|
|
547
561
|
} else {
|
|
548
|
-
throw new MobilettoError(`encClient.list(${p})
|
|
562
|
+
throw new MobilettoError(`encClient.list(${p}) ${e}`, e)
|
|
549
563
|
}
|
|
550
564
|
}
|
|
551
565
|
if (recursive) {
|
package/package.json
CHANGED
package/util/crypt.js
CHANGED
package/util/logger.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
const winston = require('winston')
|
|
2
2
|
|
|
3
3
|
const logger = winston.createLogger({
|
|
4
|
+
levels: winston.config.npm.levels,
|
|
4
5
|
level: process.env.MOBILETTO_LOG_LEVEL || 'error',
|
|
5
|
-
format: winston.format.
|
|
6
|
+
format: winston.format.combine(
|
|
7
|
+
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }),
|
|
8
|
+
winston.format.printf((info) => {
|
|
9
|
+
return `${info.timestamp} [${info.level.toUpperCase()}] ${info.message}`
|
|
10
|
+
})),
|
|
6
11
|
transports: process.env.MOBILETTO_LOG_FILE
|
|
7
12
|
? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
|
|
8
|
-
: [ new winston.transports.Console() ]
|
|
13
|
+
: [ new winston.transports.Console({ stderrLevels: Object.keys(winston.config.npm.levels) }) ]
|
|
9
14
|
})
|
|
10
15
|
|
|
11
16
|
const setLogLevel = (level) => { logger.level = level }
|
package/util/redis.js
CHANGED
|
@@ -41,20 +41,20 @@ class MobilettoCache {
|
|
|
41
41
|
try {
|
|
42
42
|
this.redis = new Redis(Object.assign({}, DEFAULT_REDIS_OPTIONS, {host, port}))
|
|
43
43
|
} catch (e) {
|
|
44
|
-
logger.error(`redis(${name})
|
|
44
|
+
logger.error(`redis(${name}) error connecting to redis, using fallback LRU for scoped caches: ${e}`)
|
|
45
45
|
this.redis = null
|
|
46
46
|
}
|
|
47
47
|
} else {
|
|
48
|
-
logger.warn(`redis(${name})
|
|
48
|
+
logger.warn(`redis(${name}) no host or port provided, using fallback LRU for scoped caches`)
|
|
49
49
|
this.redis = null
|
|
50
50
|
}
|
|
51
51
|
this.prefix = prefix
|
|
52
52
|
if (this.redis) {
|
|
53
53
|
// test connection by flushing
|
|
54
54
|
this.flush().then(
|
|
55
|
-
() => { logger.debug(`redis(${name})
|
|
55
|
+
() => { logger.debug(`redis(${name}) successfully flushed`) },
|
|
56
56
|
(e) => {
|
|
57
|
-
logger.warn(`redis(${name})
|
|
57
|
+
logger.warn(`redis(${name}) error flushing: ${e}, disabling redis`)
|
|
58
58
|
this.redis = null
|
|
59
59
|
}
|
|
60
60
|
)
|
|
@@ -73,7 +73,7 @@ class MobilettoCache {
|
|
|
73
73
|
try {
|
|
74
74
|
return this.redis ? await func(this.redis) : defaultValue
|
|
75
75
|
} catch (e) {
|
|
76
|
-
logger.warn(`redis(${this.name})
|
|
76
|
+
logger.warn(`redis(${this.name}) doRedisAsync(${func}) ${e} (returning default value: ${defaultValue})`)
|
|
77
77
|
return defaultValue
|
|
78
78
|
}
|
|
79
79
|
}
|
|
@@ -82,16 +82,16 @@ class MobilettoCache {
|
|
|
82
82
|
try {
|
|
83
83
|
return this.redis ? func(this.redis) : defaultValue
|
|
84
84
|
} catch (e) {
|
|
85
|
-
logger.warn(`redis(${this.name})
|
|
85
|
+
logger.warn(`redis(${this.name}) doRedis(${func}) ${e} (returning default value: ${defaultValue})`)
|
|
86
86
|
return defaultValue
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
get = async key => {
|
|
91
91
|
this.counters.get++
|
|
92
|
-
logger.silly(`redis(${this.name})
|
|
92
|
+
logger.silly(`redis(${this.name}) get(${key}) starting`)
|
|
93
93
|
const val = await this.doRedis(r => r.get(this.pfx(key)), null)
|
|
94
|
-
logger.silly(`redis(${this.name})
|
|
94
|
+
logger.silly(`redis(${this.name}) get(${key}) found value: ${val}`)
|
|
95
95
|
if (val) {
|
|
96
96
|
this.counters.hit++
|
|
97
97
|
} else {
|
|
@@ -106,9 +106,9 @@ class MobilettoCache {
|
|
|
106
106
|
|
|
107
107
|
set = async (key, val, expirationMillis = DEFAULT_EXPIRATION_MILLIS) => {
|
|
108
108
|
this.counters.set++
|
|
109
|
-
logger.silly(`redis(${this.name})
|
|
109
|
+
logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) starting`)
|
|
110
110
|
await this.doRedisAsync(r => r.set(this.pfx(key), val, 'EX', expirationMillis / 1000))
|
|
111
|
-
logger.silly(`redis(${this.name})
|
|
111
|
+
logger.silly(`redis(${this.name}) set(${key}, ${val}, ${expirationMillis}) finished`)
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
del = async key => {
|
|
@@ -156,7 +156,7 @@ class MobilettoCache {
|
|
|
156
156
|
try {
|
|
157
157
|
return JSON.parse(val)
|
|
158
158
|
} catch (e) {
|
|
159
|
-
logger.warn(`redis(${this.name})
|
|
159
|
+
logger.warn(`redis(${this.name}) get(${key}) error: ${e}`)
|
|
160
160
|
return null
|
|
161
161
|
}
|
|
162
162
|
},
|
|
@@ -165,7 +165,7 @@ class MobilettoCache {
|
|
|
165
165
|
try {
|
|
166
166
|
await this.set(realKey(key), JSON.stringify(value))
|
|
167
167
|
} catch (e) {
|
|
168
|
-
logger.warn(`redis(${this.name})
|
|
168
|
+
logger.warn(`redis(${this.name}) set(${key}) error: ${e}`)
|
|
169
169
|
}
|
|
170
170
|
}
|
|
171
171
|
}
|
|
@@ -190,7 +190,7 @@ const teardown = async () => await forAllCaches((client) => {
|
|
|
190
190
|
client.redis = null
|
|
191
191
|
}
|
|
192
192
|
} catch (e) {
|
|
193
|
-
logger.warn(`teardown: error disconnecting from redis(${client.name})
|
|
193
|
+
logger.warn(`teardown: error disconnecting from redis(${client.name}) ${e}`)
|
|
194
194
|
}
|
|
195
195
|
})
|
|
196
196
|
|