mobiletto 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/drivers/local/index.js +3 -14
- package/drivers/s3/index.js +6 -14
- package/index.js +84 -10
- package/package.json +3 -2
- package/test/index.spec.js +71 -25
- package/util/crypt.js +71 -0
package/README.md
CHANGED
|
@@ -85,6 +85,9 @@ A driver is any JS file that exports a 'storageClient' function with this signat
|
|
|
85
85
|
|
|
86
86
|
The object that the storageClient function returns must define these functions:
|
|
87
87
|
|
|
88
|
+
// Test the driver before using, ensure proper configuration
|
|
89
|
+
async testConfig ()
|
|
90
|
+
|
|
88
91
|
// Find DNS records that match the given domain, type (if provided) and name (if provided)
|
|
89
92
|
async list (path) // path may be omitted
|
|
90
93
|
|
package/drivers/local/index.js
CHANGED
|
@@ -2,7 +2,7 @@ const { dirname } = require('path')
|
|
|
2
2
|
|
|
3
3
|
const {
|
|
4
4
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
5
|
-
MobilettoError, writeStream, closeStream
|
|
5
|
+
MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
|
|
6
6
|
} = require('../../index')
|
|
7
7
|
|
|
8
8
|
const fs = require('fs')
|
|
@@ -143,23 +143,12 @@ class StorageClient {
|
|
|
143
143
|
return Promise.resolve(count)
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
async read (path, callback) {
|
|
146
|
+
async read (path, callback, endCallback = null) {
|
|
147
147
|
const file = this.normalizePath(path)
|
|
148
148
|
// console.log(`read: reading path: ${path} - ${file}`)
|
|
149
149
|
try {
|
|
150
150
|
const stream = fs.createReadStream(file)
|
|
151
|
-
|
|
152
|
-
const streamHandler = stream =>
|
|
153
|
-
new Promise((resolve, reject) => {
|
|
154
|
-
stream.on('data', (data) => {
|
|
155
|
-
count += data ? data.length : 0
|
|
156
|
-
callback(data)
|
|
157
|
-
})
|
|
158
|
-
stream.on('error', reject)
|
|
159
|
-
stream.on('end', resolve)
|
|
160
|
-
})
|
|
161
|
-
await streamHandler(stream)
|
|
162
|
-
return count
|
|
151
|
+
return await readStream(stream, callback, endCallback)
|
|
163
152
|
} catch (err) {
|
|
164
153
|
if (err.code && err.code === 'ENOENT') {
|
|
165
154
|
throw new MobilettoNotFoundError(path)
|
package/drivers/s3/index.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
const {
|
|
1
|
+
const {
|
|
2
|
+
M_DIR, M_FILE,
|
|
3
|
+
MobilettoError, MobilettoNotFoundError, readStream
|
|
4
|
+
} = require('../../index')
|
|
2
5
|
|
|
3
6
|
const {
|
|
4
7
|
S3Client,
|
|
@@ -166,7 +169,7 @@ class StorageClient {
|
|
|
166
169
|
return total
|
|
167
170
|
}
|
|
168
171
|
|
|
169
|
-
async read (path, callback) {
|
|
172
|
+
async read (path, callback, endCallback = null) {
|
|
170
173
|
const Key = this.normalizeKey(path)
|
|
171
174
|
// console.log(`read: reading Key: ${path} - ${Key}`)
|
|
172
175
|
const bucketParams = {
|
|
@@ -177,18 +180,7 @@ class StorageClient {
|
|
|
177
180
|
}
|
|
178
181
|
try {
|
|
179
182
|
const data = await this.client.send(new GetObjectCommand(bucketParams))
|
|
180
|
-
|
|
181
|
-
const streamHandler = stream =>
|
|
182
|
-
new Promise((resolve, reject) => {
|
|
183
|
-
stream.on('data', (data) => {
|
|
184
|
-
count += data ? data.length : 0
|
|
185
|
-
callback(data)
|
|
186
|
-
})
|
|
187
|
-
stream.on('error', reject)
|
|
188
|
-
stream.on('end', resolve)
|
|
189
|
-
})
|
|
190
|
-
await streamHandler(data.Body)
|
|
191
|
-
return count
|
|
183
|
+
return await readStream(data.Body, callback, endCallback)
|
|
192
184
|
} catch (err) {
|
|
193
185
|
throw this.s3error(err, Key, path, 'read')
|
|
194
186
|
}
|
package/index.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
const crypt = require('./util/crypt')
|
|
2
|
+
const {normalizeKey, normalizeIV} = require("./util/crypt");
|
|
3
|
+
|
|
1
4
|
// adapted from https://stackoverflow.com/a/27724419
|
|
2
5
|
function MobilettoError (message, err) {
|
|
3
6
|
this.message = `${message}`
|
|
@@ -21,13 +24,67 @@ function MobilettoNotFoundError (message) {
|
|
|
21
24
|
}
|
|
22
25
|
}
|
|
23
26
|
|
|
24
|
-
async function mobiletto (driverPath, key, secret, opts) {
|
|
27
|
+
async function mobiletto (driverPath, key, secret, opts, encryptionKey = null, encryptionIV = null) {
|
|
25
28
|
const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
|
|
26
29
|
const client = driver.storageClient(key, secret, opts)
|
|
27
30
|
if (!(await client.testConfig())) {
|
|
28
31
|
throw new MobilettoError(`mobiletti(${driverPath}) error: test API call failed`)
|
|
29
32
|
}
|
|
30
|
-
|
|
33
|
+
if (encryptionKey === null) {
|
|
34
|
+
return client
|
|
35
|
+
}
|
|
36
|
+
const encKey = normalizeKey(encryptionKey)
|
|
37
|
+
const iv = normalizeIV(encryptionIV, key)
|
|
38
|
+
return {
|
|
39
|
+
list: async (path) => client.list(path),
|
|
40
|
+
metadata: async (path) => client.metadata(path),
|
|
41
|
+
read: async (path, callback) => {
|
|
42
|
+
const cipher = crypt.startDecryptStream(encKey, iv)
|
|
43
|
+
return client.read(path,
|
|
44
|
+
(chunk) => {
|
|
45
|
+
return callback(crypt.updateCryptStream(cipher, chunk))
|
|
46
|
+
}, () => {
|
|
47
|
+
return callback(crypt.closeCryptStream(cipher))
|
|
48
|
+
})
|
|
49
|
+
},
|
|
50
|
+
write: async (path, readFunc) => {
|
|
51
|
+
function* cryptGenerator(plaintextGenerator) {
|
|
52
|
+
let chunk = plaintextGenerator.next().value
|
|
53
|
+
while (chunk) {
|
|
54
|
+
yield cipher.update(chunk)
|
|
55
|
+
chunk = plaintextGenerator.next().value
|
|
56
|
+
}
|
|
57
|
+
yield cipher.final()
|
|
58
|
+
}
|
|
59
|
+
const cipher = crypt.startEncryptStream(encKey, iv)
|
|
60
|
+
return client.write(path, cryptGenerator(readFunc))
|
|
61
|
+
},
|
|
62
|
+
remove: async (path, options) => {
|
|
63
|
+
const recursive = (options && options.recursive) || false
|
|
64
|
+
const quiet = (options && options.quiet) || false
|
|
65
|
+
return client.remove(path, {recursive, quiet})
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function readStream(stream, callback, endCallback) {
|
|
71
|
+
const counter = {count: 0}
|
|
72
|
+
const streamHandler = stream =>
|
|
73
|
+
new Promise((resolve, reject) => {
|
|
74
|
+
stream.on('data', (data) => {
|
|
75
|
+
counter.count += data ? data.length : 0
|
|
76
|
+
callback(data)
|
|
77
|
+
})
|
|
78
|
+
stream.on('error', reject)
|
|
79
|
+
stream.on('end', () => {
|
|
80
|
+
if (endCallback) {
|
|
81
|
+
endCallback()
|
|
82
|
+
}
|
|
83
|
+
resolve()
|
|
84
|
+
})
|
|
85
|
+
})
|
|
86
|
+
await streamHandler(stream)
|
|
87
|
+
return counter.count
|
|
31
88
|
}
|
|
32
89
|
|
|
33
90
|
function writeStream (stream) {
|
|
@@ -52,13 +109,31 @@ function closeStream (stream) {
|
|
|
52
109
|
})
|
|
53
110
|
}
|
|
54
111
|
|
|
55
|
-
function
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
112
|
+
async function streamReader (stream, callback, endCallback) {
|
|
113
|
+
let count = 0
|
|
114
|
+
const streamHandler = async (stream) => {
|
|
115
|
+
new Promise((resolve, reject) => {
|
|
116
|
+
stream.on('data', (data) => {
|
|
117
|
+
count += data ? data.length : 0
|
|
118
|
+
callback(data)
|
|
119
|
+
})
|
|
120
|
+
stream.on('error', reject)
|
|
121
|
+
stream.on('end', (resolve) => {
|
|
122
|
+
if (typeof endCallback === 'function') {
|
|
123
|
+
const endData = endCallback()
|
|
124
|
+
if (endData) {
|
|
125
|
+
count += endData.length
|
|
126
|
+
callback(endData)
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
resolve()
|
|
130
|
+
})
|
|
131
|
+
})
|
|
61
132
|
}
|
|
133
|
+
await streamHandler(stream).then(() => {
|
|
134
|
+
console.log('streamhandler ended')
|
|
135
|
+
})
|
|
136
|
+
return count
|
|
62
137
|
}
|
|
63
138
|
|
|
64
139
|
const M_FILE = 'file'
|
|
@@ -70,6 +145,5 @@ module.exports = {
|
|
|
70
145
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
71
146
|
mobiletto,
|
|
72
147
|
MobilettoError, MobilettoNotFoundError,
|
|
73
|
-
writeStream, closeStream,
|
|
148
|
+
readStream, writeStream, closeStream, streamReader
|
|
74
149
|
}
|
|
75
|
-
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Jonathan Cobb",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
22
22
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
23
|
-
"node-fetch-commonjs": "^3.1.1"
|
|
23
|
+
"node-fetch-commonjs": "^3.1.1",
|
|
24
|
+
"shasum": "^1.0.2"
|
|
24
25
|
}
|
|
25
26
|
}
|
package/test/index.spec.js
CHANGED
|
@@ -13,6 +13,7 @@ const {MobilettoNotFoundError, M_FILE, M_DIR} = require("../index")
|
|
|
13
13
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
14
14
|
const READ_SZ = 8 * 1024 // xfer data in 8k chunks
|
|
15
15
|
const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
|
|
16
|
+
const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.05;
|
|
16
17
|
|
|
17
18
|
DRIVER_CONFIG = {
|
|
18
19
|
local: {
|
|
@@ -29,11 +30,16 @@ DRIVER_CONFIG = {
|
|
|
29
30
|
}
|
|
30
31
|
}
|
|
31
32
|
|
|
32
|
-
async function assertMeta (api, name, expectedSize) {
|
|
33
|
+
async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
|
|
33
34
|
const meta = await api.metadata(name)
|
|
34
35
|
should().exist(meta, 'expected return value from metadata call')
|
|
35
|
-
expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
|
|
36
36
|
expect(meta.name).equals(name, 'expected name of written file to be correct')
|
|
37
|
+
if (closeEnoughPercent) {
|
|
38
|
+
expect(Math.abs(expectedSize - meta.size)).to.be.lessThan(Math.floor(meta.size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
|
|
39
|
+
'expected write API to return within 5% of bytes written (due to encryption)')
|
|
40
|
+
} else {
|
|
41
|
+
expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
|
|
42
|
+
}
|
|
37
43
|
}
|
|
38
44
|
|
|
39
45
|
async function assertMetaFail (api, name) {
|
|
@@ -48,8 +54,30 @@ async function assertMetaFail (api, name) {
|
|
|
48
54
|
}
|
|
49
55
|
}
|
|
50
56
|
|
|
51
|
-
|
|
52
|
-
|
|
57
|
+
const rand = count => randomstring.generate(count)
|
|
58
|
+
|
|
59
|
+
async function writeRandomFile(fixture, size) {
|
|
60
|
+
const data = fixture.randomData
|
|
61
|
+
function* dataGenerator() {
|
|
62
|
+
// read in 8k chunks
|
|
63
|
+
for (let i = 0; i < data.length; i += size) {
|
|
64
|
+
const end = (i + size) > data.length ? data.length : (i + size)
|
|
65
|
+
yield data.slice(i, end)
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return await fixture.api.write(fixture.name, dataGenerator())
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function readFile(fixture) {
|
|
72
|
+
const chunks = []
|
|
73
|
+
function reader(chunk) {
|
|
74
|
+
if (chunk) {
|
|
75
|
+
chunks.push(chunk)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const response = await fixture.api.read(fixture.name, reader)
|
|
79
|
+
const data = Buffer.concat(chunks)
|
|
80
|
+
return { response, data }
|
|
53
81
|
}
|
|
54
82
|
|
|
55
83
|
// To test a single driver:
|
|
@@ -97,28 +125,11 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
97
125
|
.finally(done)
|
|
98
126
|
})
|
|
99
127
|
it("should write a file", async () => {
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
function* dataGenerator() {
|
|
103
|
-
// read in 8k chunks
|
|
104
|
-
for (let i = 0; i < data.length; i += READ_SZ) {
|
|
105
|
-
const end = (i + READ_SZ) > data.length ? data.length : (i + READ_SZ)
|
|
106
|
-
yield data.slice(i, end)
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
const response = await fixture.api.write(fixture.name, dataGenerator())
|
|
111
|
-
expect(response).to.equal(data.length, 'expected write API to return correct number of bytes written')
|
|
128
|
+
const response = await writeRandomFile(fixture, size)
|
|
129
|
+
expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
|
|
112
130
|
})
|
|
113
131
|
it("should read the file we just wrote", async () => {
|
|
114
|
-
const
|
|
115
|
-
function reader(chunk) {
|
|
116
|
-
if (chunk) {
|
|
117
|
-
chunks.push(chunk)
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
const response = await fixture.api.read(fixture.name, reader)
|
|
121
|
-
const data = Buffer.concat(chunks)
|
|
132
|
+
const { response, data } = await readFile(fixture);
|
|
122
133
|
expect(response).is.equal(data.length, 'expected read API to return correct number of bytes read')
|
|
123
134
|
expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
124
135
|
})
|
|
@@ -134,6 +145,42 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
134
145
|
})
|
|
135
146
|
})
|
|
136
147
|
|
|
148
|
+
describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
|
|
149
|
+
// some random data, plus a bit extra
|
|
150
|
+
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
151
|
+
const randomData = rand(size)
|
|
152
|
+
const fileSuffix = '' + Date.now()
|
|
153
|
+
const encryptionKey = rand(32)
|
|
154
|
+
let encryptedByteCount = null
|
|
155
|
+
let fixture
|
|
156
|
+
beforeEach((done) => {
|
|
157
|
+
const name = `test_file_${fileSuffix}`
|
|
158
|
+
mobiletto(driverName, config.key, config.secret, config.opts, encryptionKey)
|
|
159
|
+
.then(api => { fixture = {api, name, randomData} })
|
|
160
|
+
.finally(done)
|
|
161
|
+
})
|
|
162
|
+
it("should write an encrypted file", async () => {
|
|
163
|
+
encryptedByteCount = await writeRandomFile(fixture, size);
|
|
164
|
+
expect(Math.abs(encryptedByteCount - size)).to.be.lessThan(Math.floor(size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
|
|
165
|
+
'expected write API to return within 5% of bytes written (due to encryption)')
|
|
166
|
+
})
|
|
167
|
+
it("should read the encrypted file we just wrote", async () => {
|
|
168
|
+
const { response, data } = await readFile(fixture);
|
|
169
|
+
expect(response).is.equal(encryptedByteCount, 'expected read API to return correct number of bytes read')
|
|
170
|
+
expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
171
|
+
})
|
|
172
|
+
it("should load metadata on the encrypted file we just wrote", async () => {
|
|
173
|
+
await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
|
|
174
|
+
})
|
|
175
|
+
it("should delete the encrypted file we just wrote", async () => {
|
|
176
|
+
const removed = await fixture.api.remove(fixture.name)
|
|
177
|
+
expect(removed).to.be.true
|
|
178
|
+
})
|
|
179
|
+
it("loading metadata on the encrypted file we wrote now fails", async () => {
|
|
180
|
+
await assertMetaFail(fixture.api, fixture.name)
|
|
181
|
+
})
|
|
182
|
+
})
|
|
183
|
+
|
|
137
184
|
describe(`${driverName} - write files in a new dir, read metadata, recursively delete`, () => {
|
|
138
185
|
// a random directory and file within it
|
|
139
186
|
const randomParent = `random_dir_${rand(10)}`
|
|
@@ -233,6 +280,5 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
233
280
|
}
|
|
234
281
|
})
|
|
235
282
|
})
|
|
236
|
-
|
|
237
283
|
})
|
|
238
284
|
}
|
package/util/crypt.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// adapted from https://stackoverflow.com/a/64136185
|
|
2
|
+
const crypto = require('crypto')
|
|
3
|
+
const shasum = require('shasum')
|
|
4
|
+
|
|
5
|
+
const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
|
|
6
|
+
const MIN_KEY_LEN = 16
|
|
7
|
+
|
|
8
|
+
function normalizeKey (k) {
|
|
9
|
+
return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
|
|
10
|
+
? Buffer.from(shasum(k)).subarray(0, 32)
|
|
11
|
+
: null
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function normalizeIV (iv, key) {
|
|
15
|
+
return (typeof iv === 'string')
|
|
16
|
+
? Buffer.from(shasum(iv)).subarray(0, 16)
|
|
17
|
+
: key
|
|
18
|
+
? Buffer.from(shasum(key)).subarray(0, 16)
|
|
19
|
+
: null
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// ensure key long enough for security, and is 32 bytes for AES-256
|
|
23
|
+
let KEY = null
|
|
24
|
+
function setDefaultKey (key) {
|
|
25
|
+
KEY = normalizeKey(key)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// ensure IV is 16 bytes for AES-256
|
|
29
|
+
let CRYPTO_IV = null
|
|
30
|
+
function setDefaultIV (iv) {
|
|
31
|
+
CRYPTO_IV = normalizeIV(iv, KEY)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function getCipher(key, iv) {
|
|
35
|
+
return crypto.createCipheriv('aes-256-cbc', key, iv);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function encrypt (plainText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'base64') {
|
|
39
|
+
if (!key) {
|
|
40
|
+
if (WARN_PLAINTEXT) {
|
|
41
|
+
console.warn(` ****** key is undefined, encryption of user-data is DISABLED`)
|
|
42
|
+
}
|
|
43
|
+
return plainText
|
|
44
|
+
}
|
|
45
|
+
const cipher = getCipher(key, iv)
|
|
46
|
+
return Buffer.concat([cipher.update(plainText), cipher.final()]).toString(outputEncoding)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function getDecipher(key, iv) {
|
|
50
|
+
return crypto.createDecipheriv('aes-256-cbc', key, iv);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function decrypt (cipherText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'utf8') {
|
|
54
|
+
if (!key) {
|
|
55
|
+
return cipherText
|
|
56
|
+
}
|
|
57
|
+
const cipher = getDecipher(key, iv)
|
|
58
|
+
const data = Buffer.from(cipherText, 'base64')
|
|
59
|
+
return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const startEncryptStream = (key = KEY, iv = CRYPTO_IV) => getCipher(key, iv)
|
|
63
|
+
const startDecryptStream = (key = KEY, iv = CRYPTO_IV) => getDecipher(key, iv)
|
|
64
|
+
const updateCryptStream = (cipher, data) => cipher.update(data)
|
|
65
|
+
const closeCryptStream = (cipher) => cipher.final()
|
|
66
|
+
|
|
67
|
+
module.exports = {
|
|
68
|
+
encrypt, decrypt,
|
|
69
|
+
normalizeKey, normalizeIV,
|
|
70
|
+
startEncryptStream, startDecryptStream, updateCryptStream, closeCryptStream
|
|
71
|
+
}
|