mobiletto 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -85,6 +85,9 @@ A driver is any JS file that exports a 'storageClient' function with this signat
85
85
 
86
86
  The object that the storageClient function returns must define these functions:
87
87
 
88
+ // Test the driver before using, ensure proper configuration
89
+ async testConfig ()
90
+
88
91
  // Find DNS records that match the given domain, type (if provided) and name (if provided)
89
92
  async list (path) // path may be omitted
90
93
 
@@ -2,7 +2,7 @@ const { dirname } = require('path')
2
2
 
3
3
  const {
4
4
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
5
- MobilettoError, writeStream, closeStream, MobilettoNotFoundError
5
+ MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
6
6
  } = require('../../index')
7
7
 
8
8
  const fs = require('fs')
@@ -143,23 +143,12 @@ class StorageClient {
143
143
  return Promise.resolve(count)
144
144
  }
145
145
 
146
- async read (path, callback) {
146
+ async read (path, callback, endCallback = null) {
147
147
  const file = this.normalizePath(path)
148
148
  // console.log(`read: reading path: ${path} - ${file}`)
149
149
  try {
150
150
  const stream = fs.createReadStream(file)
151
- let count = 0
152
- const streamHandler = stream =>
153
- new Promise((resolve, reject) => {
154
- stream.on('data', (data) => {
155
- count += data ? data.length : 0
156
- callback(data)
157
- })
158
- stream.on('error', reject)
159
- stream.on('end', resolve)
160
- })
161
- await streamHandler(stream)
162
- return count
151
+ return await readStream(stream, callback, endCallback)
163
152
  } catch (err) {
164
153
  if (err.code && err.code === 'ENOENT') {
165
154
  throw new MobilettoNotFoundError(path)
@@ -1,4 +1,7 @@
1
- const { MobilettoError, M_DIR, M_FILE, MobilettoNotFoundError} = require('../../index')
1
+ const {
2
+ M_DIR, M_FILE,
3
+ MobilettoError, MobilettoNotFoundError, readStream
4
+ } = require('../../index')
2
5
 
3
6
  const {
4
7
  S3Client,
@@ -166,7 +169,7 @@ class StorageClient {
166
169
  return total
167
170
  }
168
171
 
169
- async read (path, callback) {
172
+ async read (path, callback, endCallback = null) {
170
173
  const Key = this.normalizeKey(path)
171
174
  // console.log(`read: reading Key: ${path} - ${Key}`)
172
175
  const bucketParams = {
@@ -177,18 +180,7 @@ class StorageClient {
177
180
  }
178
181
  try {
179
182
  const data = await this.client.send(new GetObjectCommand(bucketParams))
180
- let count = 0
181
- const streamHandler = stream =>
182
- new Promise((resolve, reject) => {
183
- stream.on('data', (data) => {
184
- count += data ? data.length : 0
185
- callback(data)
186
- })
187
- stream.on('error', reject)
188
- stream.on('end', resolve)
189
- })
190
- await streamHandler(data.Body)
191
- return count
183
+ return await readStream(data.Body, callback, endCallback)
192
184
  } catch (err) {
193
185
  throw this.s3error(err, Key, path, 'read')
194
186
  }
package/index.js CHANGED
@@ -1,3 +1,6 @@
1
+ const crypt = require('./util/crypt')
2
+ const {normalizeKey, normalizeIV} = require("./util/crypt");
3
+
1
4
  // adapted from https://stackoverflow.com/a/27724419
2
5
  function MobilettoError (message, err) {
3
6
  this.message = `${message}`
@@ -21,13 +24,67 @@ function MobilettoNotFoundError (message) {
21
24
  }
22
25
  }
23
26
 
24
- async function mobiletto (driverPath, key, secret, opts) {
27
+ async function mobiletto (driverPath, key, secret, opts, encryptionKey = null, encryptionIV = null) {
25
28
  const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
26
29
  const client = driver.storageClient(key, secret, opts)
27
30
  if (!(await client.testConfig())) {
28
31
  throw new MobilettoError(`mobiletti(${driverPath}) error: test API call failed`)
29
32
  }
30
- return client
33
+ if (encryptionKey === null) {
34
+ return client
35
+ }
36
+ const encKey = normalizeKey(encryptionKey)
37
+ const iv = normalizeIV(encryptionIV, key)
38
+ return {
39
+ list: async (path) => client.list(path),
40
+ metadata: async (path) => client.metadata(path),
41
+ read: async (path, callback) => {
42
+ const cipher = crypt.startDecryptStream(encKey, iv)
43
+ return client.read(path,
44
+ (chunk) => {
45
+ return callback(crypt.updateCryptStream(cipher, chunk))
46
+ }, () => {
47
+ return callback(crypt.closeCryptStream(cipher))
48
+ })
49
+ },
50
+ write: async (path, readFunc) => {
51
+ function* cryptGenerator(plaintextGenerator) {
52
+ let chunk = plaintextGenerator.next().value
53
+ while (chunk) {
54
+ yield cipher.update(chunk)
55
+ chunk = plaintextGenerator.next().value
56
+ }
57
+ yield cipher.final()
58
+ }
59
+ const cipher = crypt.startEncryptStream(encKey, iv)
60
+ return client.write(path, cryptGenerator(readFunc))
61
+ },
62
+ remove: async (path, options) => {
63
+ const recursive = (options && options.recursive) || false
64
+ const quiet = (options && options.quiet) || false
65
+ return client.remove(path, {recursive, quiet})
66
+ }
67
+ }
68
+ }
69
+
70
+ async function readStream(stream, callback, endCallback) {
71
+ const counter = {count: 0}
72
+ const streamHandler = stream =>
73
+ new Promise((resolve, reject) => {
74
+ stream.on('data', (data) => {
75
+ counter.count += data ? data.length : 0
76
+ callback(data)
77
+ })
78
+ stream.on('error', reject)
79
+ stream.on('end', () => {
80
+ if (endCallback) {
81
+ endCallback()
82
+ }
83
+ resolve()
84
+ })
85
+ })
86
+ await streamHandler(stream)
87
+ return counter.count
31
88
  }
32
89
 
33
90
  function writeStream (stream) {
@@ -52,13 +109,31 @@ function closeStream (stream) {
52
109
  })
53
110
  }
54
111
 
55
- function readCloseHandler (promise, closeHandler) {
56
- return () => {
57
- if (closeHandler) {
58
- closeHandler()
59
- }
60
- promise.resolve()
112
+ async function streamReader (stream, callback, endCallback) {
113
+ let count = 0
114
+ const streamHandler = async (stream) => {
115
+ new Promise((resolve, reject) => {
116
+ stream.on('data', (data) => {
117
+ count += data ? data.length : 0
118
+ callback(data)
119
+ })
120
+ stream.on('error', reject)
121
+ stream.on('end', (resolve) => {
122
+ if (typeof endCallback === 'function') {
123
+ const endData = endCallback()
124
+ if (endData) {
125
+ count += endData.length
126
+ callback(endData)
127
+ }
128
+ }
129
+ resolve()
130
+ })
131
+ })
61
132
  }
133
+ await streamHandler(stream).then(() => {
134
+ console.log('streamhandler ended')
135
+ })
136
+ return count
62
137
  }
63
138
 
64
139
  const M_FILE = 'file'
@@ -70,6 +145,5 @@ module.exports = {
70
145
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
71
146
  mobiletto,
72
147
  MobilettoError, MobilettoNotFoundError,
73
- writeStream, closeStream, readCloseHandler
148
+ readStream, writeStream, closeStream, streamReader
74
149
  }
75
-
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.2",
3
+ "version": "1.0.3",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -20,6 +20,7 @@
20
20
  "dependencies": {
21
21
  "@aws-sdk/client-s3": "^3.150.0",
22
22
  "@aws-sdk/lib-storage": "^3.150.0",
23
- "node-fetch-commonjs": "^3.1.1"
23
+ "node-fetch-commonjs": "^3.1.1",
24
+ "shasum": "^1.0.2"
24
25
  }
25
26
  }
@@ -13,6 +13,7 @@ const {MobilettoNotFoundError, M_FILE, M_DIR} = require("../index")
13
13
  // the temp file is also TEMP_SZ_MULTIPLE of this number
14
14
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
15
15
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
16
+ const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.05;
16
17
 
17
18
  DRIVER_CONFIG = {
18
19
  local: {
@@ -29,11 +30,16 @@ DRIVER_CONFIG = {
29
30
  }
30
31
  }
31
32
 
32
- async function assertMeta (api, name, expectedSize) {
33
+ async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
33
34
  const meta = await api.metadata(name)
34
35
  should().exist(meta, 'expected return value from metadata call')
35
- expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
36
36
  expect(meta.name).equals(name, 'expected name of written file to be correct')
37
+ if (closeEnoughPercent) {
38
+ expect(Math.abs(expectedSize - meta.size)).to.be.lessThan(Math.floor(meta.size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
39
+ 'expected write API to return within 5% of bytes written (due to encryption)')
40
+ } else {
41
+ expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
42
+ }
37
43
  }
38
44
 
39
45
  async function assertMetaFail (api, name) {
@@ -48,8 +54,30 @@ async function assertMetaFail (api, name) {
48
54
  }
49
55
  }
50
56
 
51
- function rand (count) {
52
- return randomstring.generate(count)
57
+ const rand = count => randomstring.generate(count)
58
+
59
+ async function writeRandomFile(fixture, size) {
60
+ const data = fixture.randomData
61
+ function* dataGenerator() {
62
+ // read in 8k chunks
63
+ for (let i = 0; i < data.length; i += size) {
64
+ const end = (i + size) > data.length ? data.length : (i + size)
65
+ yield data.slice(i, end)
66
+ }
67
+ }
68
+ return await fixture.api.write(fixture.name, dataGenerator())
69
+ }
70
+
71
+ async function readFile(fixture) {
72
+ const chunks = []
73
+ function reader(chunk) {
74
+ if (chunk) {
75
+ chunks.push(chunk)
76
+ }
77
+ }
78
+ const response = await fixture.api.read(fixture.name, reader)
79
+ const data = Buffer.concat(chunks)
80
+ return { response, data }
53
81
  }
54
82
 
55
83
  // To test a single driver:
@@ -97,28 +125,11 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
97
125
  .finally(done)
98
126
  })
99
127
  it("should write a file", async () => {
100
- const data = fixture.randomData
101
-
102
- function* dataGenerator() {
103
- // read in 8k chunks
104
- for (let i = 0; i < data.length; i += READ_SZ) {
105
- const end = (i + READ_SZ) > data.length ? data.length : (i + READ_SZ)
106
- yield data.slice(i, end)
107
- }
108
- }
109
-
110
- const response = await fixture.api.write(fixture.name, dataGenerator())
111
- expect(response).to.equal(data.length, 'expected write API to return correct number of bytes written')
128
+ const response = await writeRandomFile(fixture, size)
129
+ expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
112
130
  })
113
131
  it("should read the file we just wrote", async () => {
114
- const chunks = []
115
- function reader(chunk) {
116
- if (chunk) {
117
- chunks.push(chunk)
118
- }
119
- }
120
- const response = await fixture.api.read(fixture.name, reader)
121
- const data = Buffer.concat(chunks)
132
+ const { response, data } = await readFile(fixture);
122
133
  expect(response).is.equal(data.length, 'expected read API to return correct number of bytes read')
123
134
  expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
124
135
  })
@@ -134,6 +145,42 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
134
145
  })
135
146
  })
136
147
 
148
+ describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
149
+ // some random data, plus a bit extra
150
+ const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
151
+ const randomData = rand(size)
152
+ const fileSuffix = '' + Date.now()
153
+ const encryptionKey = rand(32)
154
+ let encryptedByteCount = null
155
+ let fixture
156
+ beforeEach((done) => {
157
+ const name = `test_file_${fileSuffix}`
158
+ mobiletto(driverName, config.key, config.secret, config.opts, encryptionKey)
159
+ .then(api => { fixture = {api, name, randomData} })
160
+ .finally(done)
161
+ })
162
+ it("should write an encrypted file", async () => {
163
+ encryptedByteCount = await writeRandomFile(fixture, size);
164
+ expect(Math.abs(encryptedByteCount - size)).to.be.lessThan(Math.floor(size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
165
+ 'expected write API to return within 5% of bytes written (due to encryption)')
166
+ })
167
+ it("should read the encrypted file we just wrote", async () => {
168
+ const { response, data } = await readFile(fixture);
169
+ expect(response).is.equal(encryptedByteCount, 'expected read API to return correct number of bytes read')
170
+ expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
171
+ })
172
+ it("should load metadata on the encrypted file we just wrote", async () => {
173
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
174
+ })
175
+ it("should delete the encrypted file we just wrote", async () => {
176
+ const removed = await fixture.api.remove(fixture.name)
177
+ expect(removed).to.be.true
178
+ })
179
+ it("loading metadata on the encrypted file we wrote now fails", async () => {
180
+ await assertMetaFail(fixture.api, fixture.name)
181
+ })
182
+ })
183
+
137
184
  describe(`${driverName} - write files in a new dir, read metadata, recursively delete`, () => {
138
185
  // a random directory and file within it
139
186
  const randomParent = `random_dir_${rand(10)}`
@@ -233,6 +280,5 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
233
280
  }
234
281
  })
235
282
  })
236
-
237
283
  })
238
284
  }
package/util/crypt.js ADDED
@@ -0,0 +1,71 @@
1
+ // adapted from https://stackoverflow.com/a/64136185
2
+ const crypto = require('crypto')
3
+ const shasum = require('shasum')
4
+
5
+ const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
6
+ const MIN_KEY_LEN = 16
7
+
8
+ function normalizeKey (k) {
9
+ return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
10
+ ? Buffer.from(shasum(k)).subarray(0, 32)
11
+ : null
12
+ }
13
+
14
+ function normalizeIV (iv, key) {
15
+ return (typeof iv === 'string')
16
+ ? Buffer.from(shasum(iv)).subarray(0, 16)
17
+ : key
18
+ ? Buffer.from(shasum(key)).subarray(0, 16)
19
+ : null
20
+ }
21
+
22
+ // ensure key long enough for security, and is 32 bytes for AES-256
23
+ let KEY = null
24
+ function setDefaultKey (key) {
25
+ KEY = normalizeKey(key)
26
+ }
27
+
28
+ // ensure IV is 16 bytes for AES-256
29
+ let CRYPTO_IV = null
30
+ function setDefaultIV (iv) {
31
+ CRYPTO_IV = normalizeIV(iv, KEY)
32
+ }
33
+
34
+ function getCipher(key, iv) {
35
+ return crypto.createCipheriv('aes-256-cbc', key, iv);
36
+ }
37
+
38
+ function encrypt (plainText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'base64') {
39
+ if (!key) {
40
+ if (WARN_PLAINTEXT) {
41
+ console.warn(` ****** key is undefined, encryption of user-data is DISABLED`)
42
+ }
43
+ return plainText
44
+ }
45
+ const cipher = getCipher(key, iv)
46
+ return Buffer.concat([cipher.update(plainText), cipher.final()]).toString(outputEncoding)
47
+ }
48
+
49
+ function getDecipher(key, iv) {
50
+ return crypto.createDecipheriv('aes-256-cbc', key, iv);
51
+ }
52
+
53
+ function decrypt (cipherText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'utf8') {
54
+ if (!key) {
55
+ return cipherText
56
+ }
57
+ const cipher = getDecipher(key, iv)
58
+ const data = Buffer.from(cipherText, 'base64')
59
+ return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
60
+ }
61
+
62
+ const startEncryptStream = (key = KEY, iv = CRYPTO_IV) => getCipher(key, iv)
63
+ const startDecryptStream = (key = KEY, iv = CRYPTO_IV) => getDecipher(key, iv)
64
+ const updateCryptStream = (cipher, data) => cipher.update(data)
65
+ const closeCryptStream = (cipher) => cipher.final()
66
+
67
+ module.exports = {
68
+ encrypt, decrypt,
69
+ normalizeKey, normalizeIV,
70
+ startEncryptStream, startDecryptStream, updateCryptStream, closeCryptStream
71
+ }