mobiletto 1.0.13 → 1.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/drivers/s3/index.js +2 -1
- package/index.js +52 -8
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -122,9 +122,15 @@ Enable transparent client-side encryption:
|
|
|
122
122
|
|
|
123
123
|
// Pass encryption parameters
|
|
124
124
|
const encryption = {
|
|
125
|
-
key
|
|
126
|
-
|
|
127
|
-
|
|
125
|
+
// key is required, must be >= 16 chars
|
|
126
|
+
key: randomstring.generate(128),
|
|
127
|
+
|
|
128
|
+
// optional, the default is to derive IV from key
|
|
129
|
+
// when set, IV must be >= 16 chars
|
|
130
|
+
iv: randomstring.generate(128),
|
|
131
|
+
|
|
132
|
+
// optional, the default is aes-256-cbc
|
|
133
|
+
algo: 'aes-256-cbc'
|
|
128
134
|
}
|
|
129
135
|
const api = await mobiletto(driverName, key, secret, opts, encryption)
|
|
130
136
|
|
package/drivers/s3/index.js
CHANGED
|
@@ -127,7 +127,8 @@ class StorageClient {
|
|
|
127
127
|
if (recursive && objectCount === 0 && path !== '') {
|
|
128
128
|
throw new MobilettoNotFoundError(path)
|
|
129
129
|
}
|
|
130
|
-
|
|
130
|
+
const filtered = objects.filter(o => o.name !== path)
|
|
131
|
+
return filtered
|
|
131
132
|
}
|
|
132
133
|
|
|
133
134
|
normalizeKey = (path) => {
|
package/index.js
CHANGED
|
@@ -1,10 +1,24 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
|
+
const winston = require('winston')
|
|
2
3
|
const { basename, dirname } = require('path')
|
|
3
4
|
const shasum = require('shasum')
|
|
4
5
|
const randomstring = require('randomstring')
|
|
5
6
|
|
|
6
7
|
const crypt = require('./util/crypt')
|
|
7
|
-
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
|
|
8
|
+
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
|
|
9
|
+
|
|
10
|
+
const logger = winston.createLogger({
|
|
11
|
+
level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
|
|
12
|
+
format: winston.format.simple(),
|
|
13
|
+
transports: [ new winston.transports.Console() ]
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
const setLogLevel = (level) => { logger.level = level }
|
|
17
|
+
|
|
18
|
+
const setLogTransports = (transports) => {
|
|
19
|
+
logger.transports.splice(0, logger.transports.length)
|
|
20
|
+
logger.transports.push(...transports)
|
|
21
|
+
}
|
|
8
22
|
|
|
9
23
|
const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
10
24
|
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
@@ -53,6 +67,18 @@ const UTILITY_FUNCTIONS = {
|
|
|
53
67
|
}
|
|
54
68
|
return await client.driver_list(path, recursive, visitor)
|
|
55
69
|
},
|
|
70
|
+
safeList: (client) => async (path, opts) => {
|
|
71
|
+
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
72
|
+
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
73
|
+
try {
|
|
74
|
+
return await client.driver_list(path, recursive, visitor)
|
|
75
|
+
} catch (e) {
|
|
76
|
+
if (e instanceof MobilettoNotFoundError) {
|
|
77
|
+
return []
|
|
78
|
+
}
|
|
79
|
+
throw e
|
|
80
|
+
}
|
|
81
|
+
},
|
|
56
82
|
remove: (client) => async (path, opts) => {
|
|
57
83
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
58
84
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
@@ -68,13 +94,16 @@ const UTILITY_FUNCTIONS = {
|
|
|
68
94
|
return await client.write(path, readFunc())
|
|
69
95
|
},
|
|
70
96
|
mirror: (client) => async (source, clientPath = '', sourcePath = '') => {
|
|
97
|
+
logger.info(`mirror: starting, sourcePath=${sourcePath} -> clientPath=${clientPath}`)
|
|
71
98
|
const results = {
|
|
72
99
|
success: 0,
|
|
73
100
|
errors: 0
|
|
74
101
|
}
|
|
75
102
|
const visitor = async (obj) => {
|
|
76
103
|
if (obj.type && obj.type === M_FILE) {
|
|
104
|
+
logger.verbose(`mirror: mirroring file: ${obj.name}`)
|
|
77
105
|
const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
|
|
106
|
+
logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
|
|
78
107
|
try {
|
|
79
108
|
// write from source -> write to temp file
|
|
80
109
|
const fd = fs.openSync(tempPath, 'wx', 0o0600)
|
|
@@ -84,6 +113,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
84
113
|
}, () => {
|
|
85
114
|
writer.close((err) => {
|
|
86
115
|
if (err) { throw new MobilettoError(`mirror: error closing temp file: ${err}`) }
|
|
116
|
+
logger.debug(`mirror: finished writing ${obj.name} to temp file ${tempPath}`)
|
|
87
117
|
})
|
|
88
118
|
}).then(async () => {
|
|
89
119
|
// read from temp file -> write to mirror
|
|
@@ -92,13 +122,16 @@ const UTILITY_FUNCTIONS = {
|
|
|
92
122
|
const destName = obj.name.startsWith(sourcePath)
|
|
93
123
|
? obj.name.substring(sourcePath.length)
|
|
94
124
|
: obj.name
|
|
125
|
+
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
95
126
|
await client.write(clientPath + destName, reader)
|
|
127
|
+
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
96
128
|
})
|
|
97
129
|
results.success++
|
|
98
130
|
} catch (e) {
|
|
99
|
-
|
|
131
|
+
logger.warn(`mirror: error copying file: ${e}`)
|
|
100
132
|
results.errors++
|
|
101
133
|
} finally {
|
|
134
|
+
logger.verbose(`mirror: file mirrored successfully: ${obj.name}`)
|
|
102
135
|
fs.rmSync(tempPath, {force: true})
|
|
103
136
|
}
|
|
104
137
|
}
|
|
@@ -109,6 +142,7 @@ const UTILITY_FUNCTIONS = {
|
|
|
109
142
|
}
|
|
110
143
|
|
|
111
144
|
async function mirrorDir (source, sourcePath, visitor) {
|
|
145
|
+
logger.verbose(`mirrorDir: mirroring dir: ${sourcePath}`)
|
|
112
146
|
const listing = await source.list(sourcePath, {recursive: false, visitor})
|
|
113
147
|
for (const obj of listing) {
|
|
114
148
|
if (obj.type === M_DIR) {
|
|
@@ -126,7 +160,7 @@ function addUtilityFunctions (client, readOnly = false) {
|
|
|
126
160
|
if (readOnly) {
|
|
127
161
|
for (const writeFunc of ['write', 'remove', 'writeFile']) {
|
|
128
162
|
client[writeFunc] = async () => {
|
|
129
|
-
|
|
163
|
+
logger.warn(`${writeFunc} not supported in readOnly mode`)
|
|
130
164
|
return false
|
|
131
165
|
}
|
|
132
166
|
}
|
|
@@ -139,22 +173,30 @@ async function connect (driverPath, key, secret, opts, encryption = null) {
|
|
|
139
173
|
}
|
|
140
174
|
|
|
141
175
|
async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
176
|
+
logger.info(`mobiletto: connecting with driver ${driverPath}`)
|
|
142
177
|
const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
|
|
143
178
|
const client = driver.storageClient(key, secret, opts)
|
|
144
179
|
if (!(await client.testConfig())) {
|
|
145
|
-
|
|
180
|
+
const message = `mobiletto(${driverPath}) error: test API call failed`
|
|
181
|
+
logger.error(message)
|
|
182
|
+
throw new MobilettoError(message)
|
|
146
183
|
}
|
|
147
184
|
const readOnly = opts ? !!opts.readOnly : false
|
|
148
185
|
if (encryption === null) {
|
|
186
|
+
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
149
187
|
return addUtilityFunctions(client, readOnly)
|
|
150
188
|
}
|
|
151
189
|
const encKey = normalizeKey(encryption.key)
|
|
152
190
|
if (!encKey) {
|
|
153
|
-
|
|
191
|
+
const message = `mobiletto(${driverPath}): invalid encryption key`
|
|
192
|
+
logger.error(message)
|
|
193
|
+
throw new MobilettoError(message)
|
|
154
194
|
}
|
|
155
195
|
const iv = normalizeIV(encryption.iv, encKey)
|
|
156
196
|
if (!iv) {
|
|
157
|
-
|
|
197
|
+
const message = `mobiletto(${driverPath}): invalid encryption IV`
|
|
198
|
+
logger.error(message)
|
|
199
|
+
throw new MobilettoError(message)
|
|
158
200
|
}
|
|
159
201
|
const enc = {
|
|
160
202
|
key: encKey,
|
|
@@ -360,6 +402,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
360
402
|
return true
|
|
361
403
|
}
|
|
362
404
|
}
|
|
405
|
+
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
363
406
|
return addUtilityFunctions(encClient, readOnly)
|
|
364
407
|
}
|
|
365
408
|
|
|
@@ -388,7 +431,7 @@ function writeStream (stream) {
|
|
|
388
431
|
if (chunk) {
|
|
389
432
|
stream.write(chunk, (err) => {
|
|
390
433
|
if (err) {
|
|
391
|
-
|
|
434
|
+
logger.error(`writeStream: error writing: ${err}`)
|
|
392
435
|
throw err
|
|
393
436
|
}
|
|
394
437
|
})
|
|
@@ -399,7 +442,7 @@ function writeStream (stream) {
|
|
|
399
442
|
function closeStream (stream) {
|
|
400
443
|
return () => stream.close((err) => {
|
|
401
444
|
if (err) {
|
|
402
|
-
|
|
445
|
+
logger.error(`closeStream: error closing: ${err}`)
|
|
403
446
|
throw err
|
|
404
447
|
}
|
|
405
448
|
})
|
|
@@ -414,6 +457,7 @@ module.exports = {
|
|
|
414
457
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
415
458
|
isAsyncGenerator, isReadable,
|
|
416
459
|
mobiletto, connect,
|
|
460
|
+
setLogLevel, setLogTransports,
|
|
417
461
|
MobilettoError, MobilettoNotFoundError,
|
|
418
462
|
readStream, writeStream, closeStream
|
|
419
463
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.16",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Jonathan Cobb",
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
22
22
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
23
23
|
"node-fetch-commonjs": "^3.1.1",
|
|
24
|
-
"shasum": "^1.0.2"
|
|
24
|
+
"shasum": "^1.0.2",
|
|
25
|
+
"winston": "^3.8.1"
|
|
25
26
|
}
|
|
26
27
|
}
|