mobiletto 1.0.2 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,53 +1,59 @@
1
1
  Mobiletto
2
2
  =========
3
3
 
4
- Mobiletto is a JavaScript storage abstraction layer.
4
+ Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
5
5
 
6
6
  It supports apps that are agnostic to where files are stored.
7
7
 
8
- # Using storage services
9
- const { mobiletto } = require('mobiletto')
8
+ All drivers are tested for identical behavior with 30 tests for each driver.
10
9
 
11
- // general usage
10
+ # Basic usage
11
+ const { mobiletto, readFile } = require('mobiletto')
12
+
13
+ // General usage
12
14
  const api = await mobiletto(driverName, key, secret, opts)
13
15
 
14
- // to use 'local' driver:
16
+ // To use 'local' driver:
15
17
  // * key: base directory
16
18
  // * secret: ignored, can be null
17
19
  // * opts object:
18
20
  // * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
19
21
  const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
20
22
 
21
- // to use 's3' driver:
23
+ // To use 's3' driver:
22
24
  // * key: AWS access key
23
25
  // * secret: AWS secret key
24
26
  // * opts object:
25
27
  // * bucket: required, name of the S3 bucket
26
28
  // * region: optional, the AWS region to communicate with, default is us-east-1
27
29
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
28
- // * delimiter: optional, directory delimiter, default is '/'
30
+ // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
29
31
  const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
30
32
 
31
- // list files
33
+ // List files
32
34
  local.list() // --> returns an array of file objects
33
35
  s3.list() // --> returns an array of file objects
34
36
 
35
- // list files in a directory
37
+ // List files in a directory
36
38
  const path = 'some/path'
37
39
  local.list(path) // --> returns an array of file objects
38
40
  s3.list(path) // --> returns an array of file objects
39
41
 
40
- // read metadata for a file
42
+ // Read metadata for a file
41
43
  local.metadata(path)
42
44
  s3.metadata(path)
43
45
 
44
- // read a file
46
+ // Read a file
45
47
  // Provide a callback that writes the data someplace
46
48
  const callback = (chunk) => { ... write chunk somewhere ... }
47
49
  local.read(path, callback)
48
50
  s3.read(path, callback)
51
+
52
+ // Read an entire file at once (convenience method)
53
+ const localData = await local.readFile(path) // returns a byte Buffer of the file contents
54
+ const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
49
55
 
50
- // write a file
56
+ // Write a file
51
57
  // Provide a generator function that yields chunks of data
52
58
  const generator = function* () {
53
59
  while ( ... more-data-to-return ... ) {
@@ -58,17 +64,55 @@ It supports apps that are agnostic to where files are stored.
58
64
  local.write(path, generator)
59
65
  s3.write(path, generator)
60
66
 
61
- // delete a file
67
+ // Delete a file
62
68
  // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
63
69
  const quiet = true
64
70
  local.remove(path, {quiet})
65
71
  s3.remove(path, {quiet})
66
72
 
67
- // recursively delete a directory
73
+ // Recursively delete a directory
68
74
  const recursive = true
69
75
  local.remove(path, {recursive, quiet})
70
76
  s3.remove(path, {recursive, quiet})
71
77
 
78
+ # Transparent Encryption
79
+ Enable transparent client-side encryption:
80
+
81
+ // Pass encryption parameters
82
+ const encryption = {
83
+ key: randomstring.generate(128), // required, must be >= 32 chars
84
+ iv: randomstring.generate(128), // optional, default is to derive IV from key
85
+ algo: 'aes-256-cbc', // optional, aes-256-cbc is the default
86
+ encryptPaths: true // optional, default is true
87
+ }
88
+ const api = await mobiletto(driverName, key, secret, opts, encryption)
89
+
90
+ // Subsequent write operations will encrypt data (client side) when writing
91
+ // Subsequent read operations will decrypt data (client side) when reading
92
+ // If `encryptPaths` is true, then path names will be hashed using the encrpytion key
93
+
94
+ Note that when `encryptPaths` is enabled, `list` commands are considerably more inefficient,
95
+ especially for directories with a large number of files.
96
+
97
+ What's happening? A separate "directory entry" directory (encrypted) tracks what files are in that
98
+ directory (aka the dirent directory).
99
+ * The `list` command reads the directory entry files, decrypts each path listed; then returns metadata for each file
100
+ * The `write` command writes dirent files in each parent's dirent directory, recursively; then writes the file
101
+ * The `remove` command removes the corresponding dirent file, and its parent if empty, recursively; then removes the file
102
+ * Note: Recursive removal on large and deep filesystems can be an expensive operation
103
+
104
+ Note that even with `encryptPaths` enabled, an adversary with full visibility into your encrypted storage,
105
+ even without the key, can still see the total number of directories and how many files are in each, and with some
106
+ effort, discover some or all of the overall hierarchical structure. They would not know the names of the
107
+ directories/files unless they also knew your encryption key or had otherwise successfully cracked the encryption.
108
+ All bets are off then!
109
+
110
+ # Alternate usage: import fully scoped module and use `connect`
111
+
112
+ const storage = require('mobiletto')
113
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
114
+ const objectData = await s3.readFile('some/path')
115
+
72
116
  # Driver Interface
73
117
  A driver is any JS file that exports a 'storageClient' function with this signature:
74
118
 
@@ -85,6 +129,9 @@ A driver is any JS file that exports a 'storageClient' function with this signat
85
129
 
86
130
  The object that the storageClient function returns must define these functions:
87
131
 
132
+ // Test the driver before using, ensure proper configuration
133
+ async testConfig ()
134
+
88
135
  // Find DNS records that match the given domain, type (if provided) and name (if provided)
89
136
  async list (path) // path may be omitted
90
137
 
@@ -92,10 +139,12 @@ The object that the storageClient function returns must define these functions:
92
139
  async metadata (path)
93
140
 
94
141
  // Read a file
95
- async read (path, callback) // callback receives a chunk of data; a null chunk signals end of data
142
+ // callback receives a chunk of data. endCallback is called at end-of-stream
143
+ async read (path, callback, endCallback = null)
96
144
 
97
145
  // Write a file
98
- async write (path, readFunc) // readFunc returns the data you want to write, or null to finish
146
+ // readFunc returns the data you want to write
147
+ async write (path, readFunc)
99
148
 
100
149
  // Remove a file, or recursively delete a directory
101
150
  async remove (path, {recursive = false, quiet = false})
@@ -2,13 +2,23 @@ const { dirname } = require('path')
2
2
 
3
3
  const {
4
4
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
5
- MobilettoError, writeStream, closeStream, MobilettoNotFoundError
5
+ MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
6
6
  } = require('../../index')
7
7
 
8
8
  const fs = require('fs')
9
9
 
10
10
  const DEFAULT_MODE = '0700'
11
11
 
12
+ const isENOENT = (err) => err.code && err.code === 'ENOENT'
13
+
14
+ const ioError = (err, path, method) => isENOENT(err)
15
+ ? err instanceof MobilettoNotFoundError
16
+ ? err
17
+ : new MobilettoNotFoundError(path)
18
+ : err instanceof MobilettoError || err instanceof MobilettoNotFoundError
19
+ ? err
20
+ : new MobilettoError(`${method}(${path}) error: ${err}`, err)
21
+
12
22
  class StorageClient {
13
23
  baseDir
14
24
  mode
@@ -74,7 +84,7 @@ class StorageClient {
74
84
  return entry
75
85
  })
76
86
  } catch (err) {
77
- throw new MobilettoError(`list(${path}) error: ${err}`, err)
87
+ throw ioError(err, path, 'list')
78
88
  }
79
89
  }
80
90
  async metadata (path) {
@@ -82,19 +92,23 @@ class StorageClient {
82
92
  try {
83
93
  const lstat = fs.lstatSync(file)
84
94
  if (!lstat) {
85
- throw new MobilettoError(`metadata: lstat error`)
95
+ throw new MobilettoError('metadata: lstat error')
96
+ }
97
+ const type = this.fileType(lstat)
98
+ if (type === M_DIR && path !== '') {
99
+ const contents = await this.list(path)
100
+ if (contents.length === 0) {
101
+ throw new MobilettoNotFoundError(path)
102
+ }
86
103
  }
87
104
  return {
88
105
  name: file.startsWith(this.baseDir) ? file.substring(this.baseDir.length) : file,
89
- type: this.fileType(lstat),
106
+ type,
90
107
  size: lstat.size,
91
108
  mtime: lstat.mtimeMs
92
109
  }
93
110
  } catch (err) {
94
- if (err.code && err.code === 'ENOENT') {
95
- throw new MobilettoNotFoundError(path)
96
- }
97
- throw new MobilettoError(`metadata(${path}) error: ${err}`, err)
111
+ throw ioError(err, path, 'metadata')
98
112
  }
99
113
  }
100
114
 
@@ -143,28 +157,14 @@ class StorageClient {
143
157
  return Promise.resolve(count)
144
158
  }
145
159
 
146
- async read (path, callback) {
160
+ async read (path, callback, endCallback = null) {
147
161
  const file = this.normalizePath(path)
148
162
  // console.log(`read: reading path: ${path} - ${file}`)
149
163
  try {
150
164
  const stream = fs.createReadStream(file)
151
- let count = 0
152
- const streamHandler = stream =>
153
- new Promise((resolve, reject) => {
154
- stream.on('data', (data) => {
155
- count += data ? data.length : 0
156
- callback(data)
157
- })
158
- stream.on('error', reject)
159
- stream.on('end', resolve)
160
- })
161
- await streamHandler(stream)
162
- return count
165
+ return await readStream(stream, callback, endCallback)
163
166
  } catch (err) {
164
- if (err.code && err.code === 'ENOENT') {
165
- throw new MobilettoNotFoundError(path)
166
- }
167
- throw new MobilettoError(`read(${path}) error: ${err}`, err)
167
+ throw ioError(err, path, 'read')
168
168
  }
169
169
  }
170
170
 
@@ -176,7 +176,7 @@ class StorageClient {
176
176
  try {
177
177
  fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
178
178
  } catch (err) {
179
- if (err.code && err.code === 'ENOENT') {
179
+ if (isENOENT(err)) {
180
180
  if (!quiet) {
181
181
  throw new MobilettoNotFoundError(path)
182
182
  }
@@ -1,4 +1,7 @@
1
- const { MobilettoError, M_DIR, M_FILE, MobilettoNotFoundError} = require('../../index')
1
+ const {
2
+ M_DIR, M_FILE,
3
+ MobilettoError, MobilettoNotFoundError, readStream
4
+ } = require('../../index')
2
5
 
3
6
  const {
4
7
  S3Client,
@@ -77,12 +80,14 @@ class StorageClient {
77
80
  while (truncated) {
78
81
  try {
79
82
  const response = await this.client.send(new ListObjectsCommand(bucketParams))
80
- if (typeof response.Contents !== 'undefined') {
83
+ const hasContents = typeof response.Contents !== 'undefined'
84
+ if (hasContents) {
81
85
  response.Contents.forEach((item) => {
82
86
  objects.push(item.Key) // todo: transform into standard object
83
87
  })
84
88
  }
85
- if (typeof response.CommonPrefixes !== 'undefined') {
89
+ const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
90
+ if (hasCommonPrefixes) {
86
91
  response.CommonPrefixes.forEach((item) => {
87
92
  objects.push(item.Prefix) // todo: transform into standard object
88
93
  })
@@ -91,13 +96,22 @@ class StorageClient {
91
96
  // If truncated is true, advance the marker
92
97
  if (truncated) {
93
98
  bucketParams.Marker = response.NextMarker
99
+ } else if (!hasContents && !hasCommonPrefixes) {
100
+ if (path === '') {
101
+ break
102
+ }
103
+ throw new MobilettoNotFoundError(path)
94
104
  }
95
- // At end of the list, response.truncated is false, and the function exits the while loop.
96
105
  } catch (err) {
97
- console.log(`${logPrefix} Error: ${err}`)
98
- truncated = false
106
+ if (err instanceof MobilettoNotFoundError) {
107
+ throw err
108
+ }
109
+ throw new MobilettoError(`${logPrefix} Error: ${err}`)
99
110
  }
100
111
  }
112
+ if (recursive && objects.length === 0) {
113
+ throw new MobilettoNotFoundError(path)
114
+ }
101
115
  return objects
102
116
  }
103
117
 
@@ -166,7 +180,7 @@ class StorageClient {
166
180
  return total
167
181
  }
168
182
 
169
- async read (path, callback) {
183
+ async read (path, callback, endCallback = null) {
170
184
  const Key = this.normalizeKey(path)
171
185
  // console.log(`read: reading Key: ${path} - ${Key}`)
172
186
  const bucketParams = {
@@ -177,18 +191,7 @@ class StorageClient {
177
191
  }
178
192
  try {
179
193
  const data = await this.client.send(new GetObjectCommand(bucketParams))
180
- let count = 0
181
- const streamHandler = stream =>
182
- new Promise((resolve, reject) => {
183
- stream.on('data', (data) => {
184
- count += data ? data.length : 0
185
- callback(data)
186
- })
187
- stream.on('error', reject)
188
- stream.on('end', resolve)
189
- })
190
- await streamHandler(data.Body)
191
- return count
194
+ return await readStream(data.Body, callback, endCallback)
192
195
  } catch (err) {
193
196
  throw this.s3error(err, Key, path, 'read')
194
197
  }
@@ -221,7 +224,14 @@ class StorageClient {
221
224
  if (!quiet && response.Errors && response.Errors.length > 0) {
222
225
  throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
223
226
  }
224
- objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
227
+ try {
228
+ objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
229
+ } catch (e) {
230
+ if (!(e instanceof MobilettoNotFoundError)) {
231
+ throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
232
+ }
233
+ objects = null
234
+ }
225
235
  }
226
236
  } else {
227
237
  const Key = this.normalizeKey(path)
package/index.js CHANGED
@@ -1,3 +1,14 @@
1
+ const { basename, dirname } = require('path')
2
+ const shasum = require('shasum')
3
+ const randomstring = require('randomstring')
4
+
5
+ const crypt = require('./util/crypt')
6
+ const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt");
7
+
8
+ const DIR_ENT_DIR_SUFFIX = '__.dirent'
9
+ const DIR_ENT_FILE_PREFIX = 'dirent__'
10
+ const ENC_PAD_SEP = ' ~ '
11
+
1
12
  // adapted from https://stackoverflow.com/a/27724419
2
13
  function MobilettoError (message, err) {
3
14
  this.message = `${message}`
@@ -21,13 +32,257 @@ function MobilettoNotFoundError (message) {
21
32
  }
22
33
  }
23
34
 
24
- async function mobiletto (driverPath, key, secret, opts) {
35
+ const reader = (chunks) => (chunk) => {
36
+ if (chunk) {
37
+ chunks.push(chunk)
38
+ }
39
+ }
40
+
41
+ const UTILITY_FUNCTIONS = {
42
+ readFile: (client) => async (path) => {
43
+ const chunks = []
44
+ await client.read(path, reader(chunks))
45
+ return Buffer.concat(chunks)
46
+ }
47
+ }
48
+
49
+ function addUtilityFunctions (client) {
50
+ for (const func of Object.keys(UTILITY_FUNCTIONS)) {
51
+ client[func] = UTILITY_FUNCTIONS[func](client)
52
+ }
53
+ return client
54
+ }
55
+
56
+ async function connect (driverPath, key, secret, opts, encryption = null) {
57
+ return await mobiletto(driverPath, key, secret, opts, encryption)
58
+ }
59
+
60
+ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
25
61
  const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
26
62
  const client = driver.storageClient(key, secret, opts)
27
63
  if (!(await client.testConfig())) {
28
- throw new MobilettoError(`mobiletti(${driverPath}) error: test API call failed`)
64
+ throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
29
65
  }
30
- return client
66
+ if (encryption === null) {
67
+ return addUtilityFunctions(client)
68
+ }
69
+ const encKey = normalizeKey(encryption.key)
70
+ if (!encKey) {
71
+ throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption key`)
72
+ }
73
+ const iv = normalizeIV(encryption.iv, encKey)
74
+ if (!iv) {
75
+ throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption IV`)
76
+ }
77
+ const encryptPaths = encryption.encryptPaths || true;
78
+ const enc = {
79
+ key: encKey,
80
+ iv,
81
+ algo: encryption.algo || DEFAULT_CRYPT_ALGO,
82
+ encryptPaths,
83
+ dirLevels: encryption.dirLevels || 4,
84
+ encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
85
+ }
86
+ function encryptPath (path) {
87
+ const encrypted = shasum(enc.key + ' ' + path)
88
+ let newPath = ''
89
+ for (let i = 0; i <= enc.dirLevels; i++) {
90
+ if (newPath.length > 0) newPath += '/'
91
+ newPath += encrypted.substring(i*2, (i*2)+2)
92
+ }
93
+ return newPath + encrypted
94
+ }
95
+ const direntDir = dir => encryptPaths ? encryptPath(dir + DIR_ENT_DIR_SUFFIX) : null
96
+ const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
97
+
98
+ const _metadata = (client) => async (path) => {
99
+ if (!encryptPaths) {
100
+ return client.metadata(path)
101
+ }
102
+ let meta
103
+ try {
104
+ meta = await client.metadata(encryptPath(path))
105
+ } catch (e) {
106
+ if (e instanceof MobilettoNotFoundError) {
107
+ const dd = direntDir(path);
108
+ try {
109
+ meta = await client.metadata(dd)
110
+ } catch (err) {
111
+ if (err instanceof MobilettoNotFoundError) {
112
+ const contents = await client.list(dd)
113
+ if (Array.isArray(contents) && contents.length > 0) {
114
+ return { name: path, type: M_DIR }
115
+ } else {
116
+ throw err
117
+ }
118
+ } else {
119
+ throw err
120
+ }
121
+ }
122
+ } else {
123
+ throw e
124
+ }
125
+ }
126
+ meta.name = path // rewrite name back to plaintext name
127
+ return meta
128
+ }
129
+
130
+ const _loadMeta = async (dirent, entries) => {
131
+ const files = []
132
+ for (const entry of entries) {
133
+ const cipherText = []
134
+ await client.read(dirent + '/' + basename(entry.name), reader(cipherText))
135
+ const plain = decrypt(cipherText.toString(), enc)
136
+ const realPath = plain.split(ENC_PAD_SEP)[0]
137
+ const meta = await _metadata(client)(realPath)
138
+ files.push(meta)
139
+ }
140
+ return files
141
+ }
142
+
143
+ async function removeDirentFile(path) {
144
+ const df = direntFile(direntDir(dirname(path)), path);
145
+ await client.remove(df, {recursive: false, quiet: true})
146
+ await client.remove(encryptPath(path), {recursive: false, quiet: true})
147
+ }
148
+
149
+ const encClient = {
150
+ list: async (path) => {
151
+ if (!encryptPaths) {
152
+ return await client.list(path)
153
+ }
154
+ const dirent = direntDir(path)
155
+ const entries = await client.list(dirent)
156
+ if (!entries || entries.length === 0) {
157
+ return []
158
+ }
159
+ return _loadMeta(dirent, entries)
160
+ },
161
+ metadata: _metadata(client),
162
+ read: async (path, callback) => {
163
+ const realPath = encryptPaths ? encryptPath(path) : path
164
+ const cipher = crypt.startDecryptStream(enc)
165
+ return client.read(realPath,
166
+ (chunk) => {
167
+ return callback(crypt.updateCryptStream(cipher, chunk))
168
+ }, () => {
169
+ return callback(crypt.closeCryptStream(cipher))
170
+ })
171
+ },
172
+ write: async (path, readFunc) => {
173
+ // if encrypting paths, write dirent file(s) for all parent directories
174
+ if (encryptPaths) {
175
+ let p = path
176
+ while (true) {
177
+ const direntGenerator = function* () {
178
+ yield encrypt(p + enc.encPathPadding(), enc)
179
+ }
180
+ const dir = direntDir(dirname(p))
181
+ const df = direntFile(dir, p);
182
+ if (!(await client.write(df, direntGenerator()))) {
183
+ throw new MobilettoError('write: error writing dirent file')
184
+ }
185
+ p = dirname(p)
186
+ if (p === '.' || p === '') {
187
+ break
188
+ }
189
+ }
190
+ }
191
+
192
+ function* cryptGenerator(plaintextGenerator) {
193
+ let chunk = plaintextGenerator.next().value
194
+ while (chunk) {
195
+ yield cipher.update(chunk)
196
+ chunk = plaintextGenerator.next().value
197
+ }
198
+ yield cipher.final()
199
+ }
200
+ const cipher = crypt.startEncryptStream(enc)
201
+ const realPath = encryptPath(path)
202
+ return client.write(realPath, cryptGenerator(readFunc))
203
+ },
204
+ remove: async (path, options) => {
205
+ const recursive = (options && options.recursive) || false
206
+ const quiet = (options && options.quiet) || false
207
+ if (!encryptPaths) {
208
+ return client.remove(path, {recursive, quiet})
209
+ }
210
+
211
+ if (recursive) {
212
+ // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
213
+ async function recRm (path) {
214
+ const dirent = direntDir(path)
215
+ let entries
216
+ try {
217
+ entries = await client.list(dirent)
218
+ } catch (e) {
219
+ if (!(e instanceof MobilettoNotFoundError)) {
220
+ throw e instanceof MobilettoError
221
+ ? e
222
+ : new MobilettoError('error listing files for recursive deletion', e)
223
+ }
224
+ }
225
+ if (entries && entries.length > 0) {
226
+ const files = await _loadMeta(dirent, entries)
227
+ for (const f of files) {
228
+ await recRm(f.name)
229
+ }
230
+ }
231
+ await removeDirentFile(path);
232
+ }
233
+ await recRm(path)
234
+ }
235
+
236
+ // remove the single file/dir
237
+ await removeDirentFile(path);
238
+
239
+ // if we were the last dirent file, then also remove dirent directory, and recursively upwards
240
+ let parent = path
241
+ let dirent = direntDir(parent)
242
+ const df = direntFile(dirent, path)
243
+ await client.remove(df, {recursive: false, quiet: true})
244
+ while (true) {
245
+ try {
246
+ const entries = await client.list(dirent)
247
+ if (entries.length === 0) {
248
+ await removeDirentFile(parent)
249
+ }
250
+ } catch (e) {
251
+ if (!(e instanceof MobilettoNotFoundError)) {
252
+ throw e
253
+ }
254
+ }
255
+ if (parent === '.' || parent === '/') {
256
+ // do not remove dirent for root dir
257
+ break
258
+ }
259
+ parent = dirname(parent)
260
+ dirent = direntDir(parent)
261
+ }
262
+ return true
263
+ }
264
+ }
265
+ return addUtilityFunctions(encClient)
266
+ }
267
+
268
+ async function readStream(stream, callback, endCallback) {
269
+ const counter = {count: 0}
270
+ const streamHandler = stream =>
271
+ new Promise((resolve, reject) => {
272
+ stream.on('data', (data) => {
273
+ counter.count += data ? data.length : 0
274
+ callback(data)
275
+ })
276
+ stream.on('error', reject)
277
+ stream.on('end', () => {
278
+ if (endCallback) {
279
+ endCallback()
280
+ }
281
+ resolve()
282
+ })
283
+ })
284
+ await streamHandler(stream)
285
+ return counter.count
31
286
  }
32
287
 
33
288
  function writeStream (stream) {
@@ -35,7 +290,7 @@ function writeStream (stream) {
35
290
  if (chunk) {
36
291
  stream.write(chunk, (err) => {
37
292
  if (err) {
38
- console.log(`writeStream: error writing: ${err}`)
293
+ console.error(`writeStream: error writing: ${err}`)
39
294
  throw err
40
295
  }
41
296
  })
@@ -46,21 +301,12 @@ function writeStream (stream) {
46
301
  function closeStream (stream) {
47
302
  return () => stream.close((err) => {
48
303
  if (err) {
49
- console.log(`closeStream: error closing: ${err}`)
304
+ console.error(`closeStream: error closing: ${err}`)
50
305
  throw err
51
306
  }
52
307
  })
53
308
  }
54
309
 
55
- function readCloseHandler (promise, closeHandler) {
56
- return () => {
57
- if (closeHandler) {
58
- closeHandler()
59
- }
60
- promise.resolve()
61
- }
62
- }
63
-
64
310
  const M_FILE = 'file'
65
311
  const M_DIR = 'dir'
66
312
  const M_LINK = 'link'
@@ -68,8 +314,7 @@ const M_SPECIAL = 'special'
68
314
 
69
315
  module.exports = {
70
316
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
71
- mobiletto,
317
+ mobiletto, connect,
72
318
  MobilettoError, MobilettoNotFoundError,
73
- writeStream, closeStream, readCloseHandler
319
+ readStream, writeStream, closeStream
74
320
  }
75
-
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.2",
3
+ "version": "1.0.6",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -20,6 +20,7 @@
20
20
  "dependencies": {
21
21
  "@aws-sdk/client-s3": "^3.150.0",
22
22
  "@aws-sdk/lib-storage": "^3.150.0",
23
- "node-fetch-commonjs": "^3.1.1"
23
+ "node-fetch-commonjs": "^3.1.1",
24
+ "shasum": "^1.0.2"
24
25
  }
25
26
  }
@@ -6,17 +6,22 @@ const randomstring = require('randomstring')
6
6
 
7
7
  const { expect, should, assert } = require('chai')
8
8
 
9
- const { mobiletto } = require('../index.js')
10
- const {MobilettoNotFoundError, M_FILE, M_DIR} = require("../index")
9
+ const { mobiletto, connect, MobilettoNotFoundError, M_FILE, M_DIR } = require("../index")
10
+ const { encrypt, decrypt, normalizeKey, normalizeIV, DEFAULT_CRYPT_ALGO } = require("../util/crypt")
11
11
 
12
12
  // chunk size used by generator function, used by driver's 'write' function
13
13
  // the temp file is also TEMP_SZ_MULTIPLE of this number
14
14
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
15
15
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
16
16
 
17
+ // encoded bytes written will differ from actual bytes provided
18
+ // likewise, plaintext bytes read will differ from encoded bytes read
19
+ // our tests give some leeway when filesize differences
20
+ const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
21
+
17
22
  DRIVER_CONFIG = {
18
23
  local: {
19
- key: '/tmp'
24
+ key: process.env.LOCAL_STORAGE || '/tmp'
20
25
  },
21
26
  s3: {
22
27
  key: process.env.S3_ACCESS,
@@ -29,11 +34,20 @@ DRIVER_CONFIG = {
29
34
  }
30
35
  }
31
36
 
32
- async function assertMeta (api, name, expectedSize) {
37
+ function closeEnough (expected, actual, percent = ENC_SIZE_CLOSE_ENOUGH_PERCENT) {
38
+ expect(Math.abs(expected - actual)).to.be.lessThan(Math.floor(expected * percent),
39
+ 'expected size within 5% of actual (due to encryption)')
40
+ }
41
+
42
+ async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
33
43
  const meta = await api.metadata(name)
34
44
  should().exist(meta, 'expected return value from metadata call')
35
- expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
36
45
  expect(meta.name).equals(name, 'expected name of written file to be correct')
46
+ if (closeEnoughPercent) {
47
+ closeEnough(expectedSize, meta.size, closeEnoughPercent)
48
+ } else {
49
+ expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
50
+ }
37
51
  }
38
52
 
39
53
  async function assertMetaFail (api, name) {
@@ -48,10 +62,35 @@ async function assertMetaFail (api, name) {
48
62
  }
49
63
  }
50
64
 
51
- function rand (count) {
52
- return randomstring.generate(count)
65
+ const rand = count => randomstring.generate(count)
66
+
67
+ async function writeRandomFile(fixture, size) {
68
+ const data = fixture.randomData
69
+ function* dataGenerator() {
70
+ // read in 8k chunks
71
+ for (let i = 0; i < data.length; i += size) {
72
+ const end = (i + size) > data.length ? data.length : (i + size)
73
+ yield data.slice(i, end)
74
+ }
75
+ }
76
+ return await fixture.api.write(fixture.name, dataGenerator())
53
77
  }
54
78
 
79
+ describe('crypto test', () => {
80
+ it("should encrypt and decrypt successfully", async () => {
81
+ const enc = {
82
+ key: normalizeKey(randomstring.generate(32)),
83
+ iv: normalizeIV(randomstring.generate(16)),
84
+ algo: DEFAULT_CRYPT_ALGO
85
+ }
86
+ const plaintext = randomstring.generate(1024 + Math.floor(1024*Math.random()))
87
+ const ciphertext = encrypt(plaintext, enc)
88
+ const decrypted = decrypt(ciphertext, enc)
89
+ expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
90
+ })
91
+
92
+ })
93
+
55
94
  // To test a single driver:
56
95
  // - Uncomment one of the lines below to set driverName to the one you want to test
57
96
  // - Comment out the next `for` line, and its closing curly brace (just before EOF)
@@ -92,34 +131,17 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
92
131
  let fixture
93
132
  beforeEach((done) => {
94
133
  const name = `test_file_${fileSuffix}`
95
- mobiletto(driverName, config.key, config.secret, config.opts)
134
+ connect(driverName, config.key, config.secret, config.opts)
96
135
  .then(api => { fixture = {api, name, randomData} })
97
136
  .finally(done)
98
137
  })
99
138
  it("should write a file", async () => {
100
- const data = fixture.randomData
101
-
102
- function* dataGenerator() {
103
- // read in 8k chunks
104
- for (let i = 0; i < data.length; i += READ_SZ) {
105
- const end = (i + READ_SZ) > data.length ? data.length : (i + READ_SZ)
106
- yield data.slice(i, end)
107
- }
108
- }
109
-
110
- const response = await fixture.api.write(fixture.name, dataGenerator())
111
- expect(response).to.equal(data.length, 'expected write API to return correct number of bytes written')
139
+ const response = await writeRandomFile(fixture, size)
140
+ expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
112
141
  })
113
142
  it("should read the file we just wrote", async () => {
114
- const chunks = []
115
- function reader(chunk) {
116
- if (chunk) {
117
- chunks.push(chunk)
118
- }
119
- }
120
- const response = await fixture.api.read(fixture.name, reader)
121
- const data = Buffer.concat(chunks)
122
- expect(response).is.equal(data.length, 'expected read API to return correct number of bytes read')
143
+ const data = await fixture.api.readFile(fixture.name);
144
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
123
145
  expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
124
146
  })
125
147
  it("should load metadata on the file we just wrote", async () => {
@@ -134,59 +156,107 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
134
156
  })
135
157
  })
136
158
 
137
- describe(`${driverName} - write files in a new dir, read metadata, recursively delete`, () => {
138
- // a random directory and file within it
139
- const randomParent = `random_dir_${rand(10)}`
140
- const subdirName = `subdir_` + Date.now()
141
- const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
142
- const fileCount = 3 + Math.floor(Math.random() * 10)
159
+ describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
160
+ // some random data, plus a bit extra
161
+ const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
162
+ const randomData = rand(size)
163
+ const fileSuffix = '' + Date.now()
164
+ const encryptionKey = rand(32)
165
+ let encryptedByteCount = null
143
166
  let fixture
144
167
  beforeEach((done) => {
145
- mobiletto(driverName, config.key, config.secret, config.opts)
146
- .then(api => { fixture = {api, name: randomPath} })
168
+ const name = `test_file_${fileSuffix}`
169
+ mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
170
+ .then(api => { fixture = {api, name, randomData} })
147
171
  .finally(done)
148
172
  })
149
- it(`should write ${fileCount} files in a new directory`, async () => {
150
- function* dataGenerator() {
151
- // return one chunk of random data
152
- yield rand(READ_SZ)
153
- }
154
- for (let i = 0; i < fileCount; i++) {
155
- const response = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
156
- expect(response).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
157
- }
173
+ it("should write an encrypted file", async () => {
174
+ encryptedByteCount = await writeRandomFile(fixture, size);
175
+ closeEnough(size, encryptedByteCount)
158
176
  })
159
- it("should load metadata for one of the new files", async () => {
160
- await assertMeta (fixture.api, fixture.name, READ_SZ)
161
- })
162
- it("should see correct types on objects returned from a listing of the new directory", async () => {
163
- const objects = await fixture.api.list(randomParent)
164
- expect(objects).to.have.lengthOf(1)
165
- expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
166
- })
167
- it("should see correct types on objects returned from a listing of the subdirectory", async () => {
168
- const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
169
- expect(objects).to.have.lengthOf(fileCount)
170
- for (let i = 0; i < fileCount; i++) {
171
- // we should find all the files, and they should all have the correct type
172
- expect(objects
173
- .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
174
- ).to.not.be.null
175
- }
177
+ it("should read the encrypted file we just wrote", async () => {
178
+ const data = await fixture.api.readFile(fixture.name);
179
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
180
+ expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
181
+ })
182
+ it("should load metadata on the encrypted file we just wrote", async () => {
183
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
176
184
  })
177
- it("should recursively delete the directory and file we just created", async () => {
178
- const recursive = true
179
- const removed = await fixture.api.remove(randomParent, {recursive})
185
+ it("should delete the encrypted file we just wrote", async () => {
186
+ const removed = await fixture.api.remove(fixture.name)
180
187
  expect(removed).to.be.true
181
188
  })
182
- it("loading metadata on the file we wrote now fails", async () => {
189
+ it("loading metadata on the encrypted file we wrote now fails", async () => {
183
190
  await assertMetaFail(fixture.api, fixture.name)
184
191
  })
185
- it("loading metadata on the parent dir we wrote now fails", async () => {
186
- await assertMetaFail(fixture.api, randomParent)
187
- })
188
192
  })
189
193
 
194
+ for (const encryption of [null, {key: rand(32)}]) {
195
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
196
+ describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
197
+ // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
198
+ // const encryption = {key: rand(32)}
199
+ // const encryption = null
200
+ // a random directory and file within it
201
+ const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
202
+ const subdirName = `subdir_` + Date.now()
203
+ const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
204
+ //const fileCount = 3 + Math.floor(Math.random() * 10)
205
+ const fileCount = 2
206
+ let fixture
207
+ beforeEach((done) => {
208
+ mobiletto(driverName, config.key, config.secret, config.opts, encryption)
209
+ .then(api => { fixture = {api, name: randomPath} })
210
+ .catch((err) => { throw err })
211
+ .finally(done)
212
+ })
213
+ it(`should write ${fileCount} files in a new directory`, async () => {
214
+ function* dataGenerator() {
215
+ // return one chunk of random data
216
+ yield rand(READ_SZ)
217
+ }
218
+
219
+ for (let i = 0; i < fileCount; i++) {
220
+ const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
221
+ if (encryption) {
222
+ closeEnough(READ_SZ, bytesWritten)
223
+ } else {
224
+ expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
225
+ }
226
+ }
227
+ })
228
+ it("should load metadata for one of the new files", async () => {
229
+ await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
230
+ })
231
+ it("should see correct types on objects returned from a listing of the new directory", async () => {
232
+ const objects = await fixture.api.list(randomParent)
233
+ expect(objects).to.have.lengthOf(1)
234
+ expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
235
+ })
236
+ it("should see correct types on objects returned from a listing of the subdirectory", async () => {
237
+ const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
238
+ expect(objects).to.have.lengthOf(fileCount)
239
+ for (let i = 0; i < fileCount; i++) {
240
+ // we should find all the files, and they should all have the correct type
241
+ expect(objects
242
+ .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
243
+ ).to.not.be.null
244
+ }
245
+ })
246
+ it("should recursively delete the directory and file we just created", async () => {
247
+ const recursive = true
248
+ const removed = await fixture.api.remove(randomParent, {recursive})
249
+ expect(removed).to.be.true
250
+ })
251
+ it("loading metadata on the file we wrote now fails", async () => {
252
+ await assertMetaFail(fixture.api, fixture.name)
253
+ })
254
+ it("loading metadata on the parent dir we wrote now fails", async () => {
255
+ await assertMetaFail(fixture.api, randomParent)
256
+ })
257
+ })
258
+ }
259
+
190
260
  describe(`${driverName} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
191
261
  it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
192
262
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
@@ -233,6 +303,5 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
233
303
  }
234
304
  })
235
305
  })
236
-
237
306
  })
238
307
  }
package/util/crypt.js ADDED
@@ -0,0 +1,75 @@
1
+ // adapted from https://stackoverflow.com/a/64136185
2
+ const crypto = require('crypto')
3
+ const shasum = require('shasum')
4
+
5
+ const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
6
+ const MIN_KEY_LEN = 16
7
+ const DEFAULT_CRYPT_ALGO = 'aes-256-cbc';
8
+
9
+ function normalizeKey (k) {
10
+ return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
11
+ ? Buffer.from(shasum(k)).subarray(0, 32)
12
+ : null
13
+ }
14
+
15
+ function normalizeIV (iv, key) {
16
+ return (typeof iv === 'string')
17
+ ? Buffer.from(shasum(iv)).subarray(0, 16)
18
+ : key
19
+ ? Buffer.from(shasum(key)).subarray(0, 16)
20
+ : null
21
+ }
22
+
23
+ // ensure key long enough for security, and is 32 bytes for AES-256
24
+ let KEY = null
25
+ function setDefaultKey (key) {
26
+ KEY = normalizeKey(key)
27
+ }
28
+
29
+ // ensure IV is 16 bytes for AES-256
30
+ let CRYPTO_IV = null
31
+ function setDefaultIV (iv) {
32
+ CRYPTO_IV = normalizeIV(iv, KEY)
33
+ }
34
+
35
+ function getCipher(enc) {
36
+ return crypto.createCipheriv(enc.algo, enc.key, enc.iv);
37
+ }
38
+
39
+ function encrypt (plainText, encryption, outputEncoding = 'base64') {
40
+ if (!encryption || !encryption.key) {
41
+ if (WARN_PLAINTEXT) {
42
+ console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
43
+ }
44
+ return plainText
45
+ }
46
+ const cipher = getCipher(encryption)
47
+ const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
48
+ return encoded.toString(outputEncoding)
49
+ }
50
+
51
+ function getDecipher(enc) {
52
+ return crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
53
+ }
54
+
55
+ function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
56
+ if (!encryption || !encryption.key) {
57
+ return cipherText
58
+ }
59
+ const cipher = getDecipher(encryption)
60
+ const data = Buffer.from(cipherText, inputEncoding)
61
+ return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
62
+ }
63
+
64
+ const startEncryptStream = (enc) => getCipher(enc)
65
+ const startDecryptStream = (enc) => getDecipher(enc)
66
+ const updateCryptStream = (cipher, data) => cipher.update(data)
67
+ const closeCryptStream = (cipher) => cipher.final()
68
+
69
+ module.exports = {
70
+ DEFAULT_CRYPT_ALGO,
71
+ setDefaultKey, setDefaultIV,
72
+ encrypt, decrypt,
73
+ normalizeKey, normalizeIV,
74
+ startEncryptStream, startDecryptStream, updateCryptStream, closeCryptStream
75
+ }