mobiletto 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,12 +1,14 @@
1
1
  Mobiletto
2
2
  =========
3
3
 
4
- Mobiletto is a JavaScript storage abstraction layer.
4
+ Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
5
5
 
6
6
  It supports apps that are agnostic to where files are stored.
7
7
 
8
+ All drivers are tested for identical behavior with 30 tests for each driver.
9
+
8
10
  # Basic usage
9
- const { mobiletto } = require('mobiletto')
11
+ const { mobiletto, readFile } = require('mobiletto')
10
12
 
11
13
  // General usage
12
14
  const api = await mobiletto(driverName, key, secret, opts)
@@ -25,7 +27,7 @@ It supports apps that are agnostic to where files are stored.
25
27
  // * bucket: required, name of the S3 bucket
26
28
  // * region: optional, the AWS region to communicate with, default is us-east-1
27
29
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
28
- // * delimiter: optional, directory delimiter, default is '/'
30
+ // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
29
31
  const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
30
32
 
31
33
  // List files
@@ -46,6 +48,10 @@ It supports apps that are agnostic to where files are stored.
46
48
  const callback = (chunk) => { ... write chunk somewhere ... }
47
49
  local.read(path, callback)
48
50
  s3.read(path, callback)
51
+
52
+ // Read an entire file at once (convenience method)
53
+ const localData = await local.readFile(path) // returns a byte Buffer of the file contents
54
+ const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
49
55
 
50
56
  // Write a file
51
57
  // Provide a generator function that yields chunks of data
@@ -74,15 +80,38 @@ Enable transparent client-side encryption:
74
80
 
75
81
  // Pass encryption parameters
76
82
  const encryption = {
77
- key: randomstring.generate(128), // required, must be >= 16 chars
78
- iv: randomstring.generate(128), // optional
79
- algo: 'aes-256-cbc' // optional, aes-256-cbc is the default
83
+ key: randomstring.generate(128), // required, must be >= 32 chars
84
+ iv: randomstring.generate(128), // optional, default is to derive IV from key
85
+ algo: 'aes-256-cbc', // optional, aes-256-cbc is the default
86
+ encryptPaths: true // optional, default is true
80
87
  }
81
88
  const api = await mobiletto(driverName, key, secret, opts, encryption)
82
89
 
83
90
  // Subsequent write operations will encrypt data (client side) when writing
84
91
  // Subsequent read operations will decrypt data (client side) when reading
85
- // Path names will also be encrypted
92
+ // If `encryptPaths` is true, then path names will be hashed using the encrpytion key
93
+
94
+ Note that when `encryptPaths` is enabled, `list` commands are considerably more inefficient,
95
+ especially for directories with a large number of files.
96
+
97
+ What's happening? A separate "directory entry" directory (encrypted) tracks what files are in that
98
+ directory (aka the dirent directory).
99
+ * The `list` command reads the directory entry files, decrypts each path listed; then returns metadata for each file
100
+ * The `write` command writes dirent files in each parent's dirent directory, recursively; then writes the file
101
+ * The `remove` command removes the corresponding dirent file, and its parent if empty, recursively; then removes the file
102
+ * Note: Recursive removal on large and deep filesystems can be an expensive operation
103
+
104
+ Note that even with `encryptPaths` enabled, an adversary with full visibility into your encrypted storage,
105
+ even without the key, can still see the total number of directories and how many files are in each, and with some
106
+ effort, discover some or all of the overall hierarchical structure. They would not know the names of the
107
+ directories/files unless they also knew your encryption key or had otherwise successfully cracked the encryption.
108
+ All bets are off then!
109
+
110
+ # Alternate usage: import fully scoped module and use `connect`
111
+
112
+ const storage = require('mobiletto')
113
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
114
+ const objectData = await s3.readFile('some/path')
86
115
 
87
116
  # Driver Interface
88
117
  A driver is any JS file that exports a 'storageClient' function with this signature:
@@ -110,10 +139,12 @@ The object that the storageClient function returns must define these functions:
110
139
  async metadata (path)
111
140
 
112
141
  // Read a file
113
- async read (path, callback) // callback receives a chunk of data; a null chunk signals end of data
142
+ // callback receives a chunk of data. endCallback is called at end-of-stream
143
+ async read (path, callback, endCallback = null)
114
144
 
115
145
  // Write a file
116
- async write (path, readFunc) // readFunc returns the data you want to write, or null to finish
146
+ // readFunc returns the data you want to write
147
+ async write (path, readFunc)
117
148
 
118
149
  // Remove a file, or recursively delete a directory
119
150
  async remove (path, {recursive = false, quiet = false})
@@ -9,6 +9,16 @@ const fs = require('fs')
9
9
 
10
10
  const DEFAULT_MODE = '0700'
11
11
 
12
+ const isENOENT = (err) => err.code && err.code === 'ENOENT'
13
+
14
+ const ioError = (err, path, method) => isENOENT(err)
15
+ ? err instanceof MobilettoNotFoundError
16
+ ? err
17
+ : new MobilettoNotFoundError(path)
18
+ : err instanceof MobilettoError || err instanceof MobilettoNotFoundError
19
+ ? err
20
+ : new MobilettoError(`${method}(${path}) error: ${err}`, err)
21
+
12
22
  class StorageClient {
13
23
  baseDir
14
24
  mode
@@ -74,7 +84,7 @@ class StorageClient {
74
84
  return entry
75
85
  })
76
86
  } catch (err) {
77
- throw new MobilettoError(`list(${path}) error: ${err}`, err)
87
+ throw ioError(err, path, 'list')
78
88
  }
79
89
  }
80
90
  async metadata (path) {
@@ -82,19 +92,23 @@ class StorageClient {
82
92
  try {
83
93
  const lstat = fs.lstatSync(file)
84
94
  if (!lstat) {
85
- throw new MobilettoError(`metadata: lstat error`)
95
+ throw new MobilettoError('metadata: lstat error')
96
+ }
97
+ const type = this.fileType(lstat)
98
+ if (type === M_DIR && path !== '') {
99
+ const contents = await this.list(path)
100
+ if (contents.length === 0) {
101
+ throw new MobilettoNotFoundError(path)
102
+ }
86
103
  }
87
104
  return {
88
105
  name: file.startsWith(this.baseDir) ? file.substring(this.baseDir.length) : file,
89
- type: this.fileType(lstat),
106
+ type,
90
107
  size: lstat.size,
91
108
  mtime: lstat.mtimeMs
92
109
  }
93
110
  } catch (err) {
94
- if (err.code && err.code === 'ENOENT') {
95
- throw new MobilettoNotFoundError(path)
96
- }
97
- throw new MobilettoError(`metadata(${path}) error: ${err}`, err)
111
+ throw ioError(err, path, 'metadata')
98
112
  }
99
113
  }
100
114
 
@@ -150,10 +164,7 @@ class StorageClient {
150
164
  const stream = fs.createReadStream(file)
151
165
  return await readStream(stream, callback, endCallback)
152
166
  } catch (err) {
153
- if (err.code && err.code === 'ENOENT') {
154
- throw new MobilettoNotFoundError(path)
155
- }
156
- throw new MobilettoError(`read(${path}) error: ${err}`, err)
167
+ throw ioError(err, path, 'read')
157
168
  }
158
169
  }
159
170
 
@@ -165,7 +176,7 @@ class StorageClient {
165
176
  try {
166
177
  fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
167
178
  } catch (err) {
168
- if (err.code && err.code === 'ENOENT') {
179
+ if (isENOENT(err)) {
169
180
  if (!quiet) {
170
181
  throw new MobilettoNotFoundError(path)
171
182
  }
@@ -80,12 +80,14 @@ class StorageClient {
80
80
  while (truncated) {
81
81
  try {
82
82
  const response = await this.client.send(new ListObjectsCommand(bucketParams))
83
- if (typeof response.Contents !== 'undefined') {
83
+ const hasContents = typeof response.Contents !== 'undefined'
84
+ if (hasContents) {
84
85
  response.Contents.forEach((item) => {
85
86
  objects.push(item.Key) // todo: transform into standard object
86
87
  })
87
88
  }
88
- if (typeof response.CommonPrefixes !== 'undefined') {
89
+ const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
90
+ if (hasCommonPrefixes) {
89
91
  response.CommonPrefixes.forEach((item) => {
90
92
  objects.push(item.Prefix) // todo: transform into standard object
91
93
  })
@@ -94,13 +96,22 @@ class StorageClient {
94
96
  // If truncated is true, advance the marker
95
97
  if (truncated) {
96
98
  bucketParams.Marker = response.NextMarker
99
+ } else if (!hasContents && !hasCommonPrefixes) {
100
+ if (path === '') {
101
+ break
102
+ }
103
+ throw new MobilettoNotFoundError(path)
97
104
  }
98
- // At end of the list, response.truncated is false, and the function exits the while loop.
99
105
  } catch (err) {
100
- console.log(`${logPrefix} Error: ${err}`)
101
- truncated = false
106
+ if (err instanceof MobilettoNotFoundError) {
107
+ throw err
108
+ }
109
+ throw new MobilettoError(`${logPrefix} Error: ${err}`)
102
110
  }
103
111
  }
112
+ if (recursive && objects.length === 0) {
113
+ throw new MobilettoNotFoundError(path)
114
+ }
104
115
  return objects
105
116
  }
106
117
 
@@ -213,7 +224,14 @@ class StorageClient {
213
224
  if (!quiet && response.Errors && response.Errors.length > 0) {
214
225
  throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
215
226
  }
216
- objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
227
+ try {
228
+ objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
229
+ } catch (e) {
230
+ if (!(e instanceof MobilettoNotFoundError)) {
231
+ throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
232
+ }
233
+ objects = null
234
+ }
217
235
  }
218
236
  } else {
219
237
  const Key = this.normalizeKey(path)
package/index.js CHANGED
@@ -1,5 +1,13 @@
1
+ const { basename, dirname } = require('path')
2
+ const shasum = require('shasum')
3
+ const randomstring = require('randomstring')
4
+
1
5
  const crypt = require('./util/crypt')
2
- const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV} = require("./util/crypt");
6
+ const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt");
7
+
8
+ const DIR_ENT_DIR_SUFFIX = '__.dirent'
9
+ const DIR_ENT_FILE_PREFIX = 'dirent__'
10
+ const ENC_PAD_SEP = ' ~ '
3
11
 
4
12
  // adapted from https://stackoverflow.com/a/27724419
5
13
  function MobilettoError (message, err) {
@@ -24,6 +32,31 @@ function MobilettoNotFoundError (message) {
24
32
  }
25
33
  }
26
34
 
35
+ const reader = (chunks) => (chunk) => {
36
+ if (chunk) {
37
+ chunks.push(chunk)
38
+ }
39
+ }
40
+
41
+ const UTILITY_FUNCTIONS = {
42
+ readFile: (client) => async (path) => {
43
+ const chunks = []
44
+ await client.read(path, reader(chunks))
45
+ return Buffer.concat(chunks)
46
+ }
47
+ }
48
+
49
+ function addUtilityFunctions (client) {
50
+ for (const func of Object.keys(UTILITY_FUNCTIONS)) {
51
+ client[func] = UTILITY_FUNCTIONS[func](client)
52
+ }
53
+ return client
54
+ }
55
+
56
+ async function connect (driverPath, key, secret, opts, encryption = null) {
57
+ return await mobiletto(driverPath, key, secret, opts, encryption)
58
+ }
59
+
27
60
  async function mobiletto (driverPath, key, secret, opts, encryption = null) {
28
61
  const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
29
62
  const client = driver.storageClient(key, secret, opts)
@@ -31,7 +64,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
31
64
  throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
32
65
  }
33
66
  if (encryption === null) {
34
- return client
67
+ return addUtilityFunctions(client)
35
68
  }
36
69
  const encKey = normalizeKey(encryption.key)
37
70
  if (!encKey) {
@@ -41,17 +74,95 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
41
74
  if (!iv) {
42
75
  throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption IV`)
43
76
  }
77
+ const encryptPaths = encryption.encryptPaths || true;
44
78
  const enc = {
45
79
  key: encKey,
46
80
  iv,
47
- algo: encryption.algo || DEFAULT_CRYPT_ALGO
81
+ algo: encryption.algo || DEFAULT_CRYPT_ALGO,
82
+ encryptPaths,
83
+ dirLevels: encryption.dirLevels || 4,
84
+ encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
85
+ }
86
+ function encryptPath (path) {
87
+ const encrypted = shasum(enc.key + ' ' + path)
88
+ let newPath = ''
89
+ for (let i = 0; i <= enc.dirLevels; i++) {
90
+ if (newPath.length > 0) newPath += '/'
91
+ newPath += encrypted.substring(i*2, (i*2)+2)
92
+ }
93
+ return newPath + encrypted
94
+ }
95
+ const direntDir = dir => encryptPaths ? encryptPath(dir + DIR_ENT_DIR_SUFFIX) : null
96
+ const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
97
+
98
+ const _metadata = (client) => async (path) => {
99
+ if (!encryptPaths) {
100
+ return client.metadata(path)
101
+ }
102
+ let meta
103
+ try {
104
+ meta = await client.metadata(encryptPath(path))
105
+ } catch (e) {
106
+ if (e instanceof MobilettoNotFoundError) {
107
+ const dd = direntDir(path);
108
+ try {
109
+ meta = await client.metadata(dd)
110
+ } catch (err) {
111
+ if (err instanceof MobilettoNotFoundError) {
112
+ const contents = await client.list(dd)
113
+ if (Array.isArray(contents) && contents.length > 0) {
114
+ return { name: path, type: M_DIR }
115
+ } else {
116
+ throw err
117
+ }
118
+ } else {
119
+ throw err
120
+ }
121
+ }
122
+ } else {
123
+ throw e
124
+ }
125
+ }
126
+ meta.name = path // rewrite name back to plaintext name
127
+ return meta
128
+ }
129
+
130
+ const _loadMeta = async (dirent, entries) => {
131
+ const files = []
132
+ for (const entry of entries) {
133
+ const cipherText = []
134
+ await client.read(dirent + '/' + basename(entry.name), reader(cipherText))
135
+ const plain = decrypt(cipherText.toString(), enc)
136
+ const realPath = plain.split(ENC_PAD_SEP)[0]
137
+ const meta = await _metadata(client)(realPath)
138
+ files.push(meta)
139
+ }
140
+ return files
141
+ }
142
+
143
+ async function removeDirentFile(path) {
144
+ const df = direntFile(direntDir(dirname(path)), path);
145
+ await client.remove(df, {recursive: false, quiet: true})
146
+ await client.remove(encryptPath(path), {recursive: false, quiet: true})
48
147
  }
49
- return {
50
- list: async (path) => client.list(path),
51
- metadata: async (path) => client.metadata(path),
148
+
149
+ const encClient = {
150
+ list: async (path) => {
151
+ if (!encryptPaths) {
152
+ return await client.list(path)
153
+ }
154
+ const dirent = direntDir(path)
155
+ const entries = await client.list(dirent)
156
+ if (!entries || entries.length === 0) {
157
+ return []
158
+ }
159
+ return _loadMeta(dirent, entries)
160
+ },
161
+ metadata: _metadata(client),
52
162
  read: async (path, callback) => {
163
+ const realPath = encryptPaths ? encryptPath(path) : path
53
164
  const cipher = crypt.startDecryptStream(enc)
54
- return client.read(path,
165
+ return client.read(realPath,
55
166
  (chunk) => {
56
167
  return callback(crypt.updateCryptStream(cipher, chunk))
57
168
  }, () => {
@@ -59,6 +170,25 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
59
170
  })
60
171
  },
61
172
  write: async (path, readFunc) => {
173
+ // if encrypting paths, write dirent file(s) for all parent directories
174
+ if (encryptPaths) {
175
+ let p = path
176
+ while (true) {
177
+ const direntGenerator = function* () {
178
+ yield encrypt(p + enc.encPathPadding(), enc)
179
+ }
180
+ const dir = direntDir(dirname(p))
181
+ const df = direntFile(dir, p);
182
+ if (!(await client.write(df, direntGenerator()))) {
183
+ throw new MobilettoError('write: error writing dirent file')
184
+ }
185
+ p = dirname(p)
186
+ if (p === '.' || p === '') {
187
+ break
188
+ }
189
+ }
190
+ }
191
+
62
192
  function* cryptGenerator(plaintextGenerator) {
63
193
  let chunk = plaintextGenerator.next().value
64
194
  while (chunk) {
@@ -68,14 +198,71 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
68
198
  yield cipher.final()
69
199
  }
70
200
  const cipher = crypt.startEncryptStream(enc)
71
- return client.write(path, cryptGenerator(readFunc))
201
+ const realPath = encryptPath(path)
202
+ return client.write(realPath, cryptGenerator(readFunc))
72
203
  },
73
204
  remove: async (path, options) => {
74
205
  const recursive = (options && options.recursive) || false
75
206
  const quiet = (options && options.quiet) || false
76
- return client.remove(path, {recursive, quiet})
207
+ if (!encryptPaths) {
208
+ return client.remove(path, {recursive, quiet})
209
+ }
210
+
211
+ if (recursive) {
212
+ // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
213
+ async function recRm (path) {
214
+ const dirent = direntDir(path)
215
+ let entries
216
+ try {
217
+ entries = await client.list(dirent)
218
+ } catch (e) {
219
+ if (!(e instanceof MobilettoNotFoundError)) {
220
+ throw e instanceof MobilettoError
221
+ ? e
222
+ : new MobilettoError('error listing files for recursive deletion', e)
223
+ }
224
+ }
225
+ if (entries && entries.length > 0) {
226
+ const files = await _loadMeta(dirent, entries)
227
+ for (const f of files) {
228
+ await recRm(f.name)
229
+ }
230
+ }
231
+ await removeDirentFile(path);
232
+ }
233
+ await recRm(path)
234
+ }
235
+
236
+ // remove the single file/dir
237
+ await removeDirentFile(path);
238
+
239
+ // if we were the last dirent file, then also remove dirent directory, and recursively upwards
240
+ let parent = path
241
+ let dirent = direntDir(parent)
242
+ const df = direntFile(dirent, path)
243
+ await client.remove(df, {recursive: false, quiet: true})
244
+ while (true) {
245
+ try {
246
+ const entries = await client.list(dirent)
247
+ if (entries.length === 0) {
248
+ await removeDirentFile(parent)
249
+ }
250
+ } catch (e) {
251
+ if (!(e instanceof MobilettoNotFoundError)) {
252
+ throw e
253
+ }
254
+ }
255
+ if (parent === '.' || parent === '/') {
256
+ // do not remove dirent for root dir
257
+ break
258
+ }
259
+ parent = dirname(parent)
260
+ dirent = direntDir(parent)
261
+ }
262
+ return true
77
263
  }
78
264
  }
265
+ return addUtilityFunctions(encClient)
79
266
  }
80
267
 
81
268
  async function readStream(stream, callback, endCallback) {
@@ -103,7 +290,7 @@ function writeStream (stream) {
103
290
  if (chunk) {
104
291
  stream.write(chunk, (err) => {
105
292
  if (err) {
106
- console.log(`writeStream: error writing: ${err}`)
293
+ console.error(`writeStream: error writing: ${err}`)
107
294
  throw err
108
295
  }
109
296
  })
@@ -114,39 +301,12 @@ function writeStream (stream) {
114
301
  function closeStream (stream) {
115
302
  return () => stream.close((err) => {
116
303
  if (err) {
117
- console.log(`closeStream: error closing: ${err}`)
304
+ console.error(`closeStream: error closing: ${err}`)
118
305
  throw err
119
306
  }
120
307
  })
121
308
  }
122
309
 
123
- async function streamReader (stream, callback, endCallback) {
124
- let count = 0
125
- const streamHandler = async (stream) => {
126
- new Promise((resolve, reject) => {
127
- stream.on('data', (data) => {
128
- count += data ? data.length : 0
129
- callback(data)
130
- })
131
- stream.on('error', reject)
132
- stream.on('end', (resolve) => {
133
- if (typeof endCallback === 'function') {
134
- const endData = endCallback()
135
- if (endData) {
136
- count += endData.length
137
- callback(endData)
138
- }
139
- }
140
- resolve()
141
- })
142
- })
143
- }
144
- await streamHandler(stream).then(() => {
145
- // console.log('streamHandler ended')
146
- })
147
- return count
148
- }
149
-
150
310
  const M_FILE = 'file'
151
311
  const M_DIR = 'dir'
152
312
  const M_LINK = 'link'
@@ -154,7 +314,7 @@ const M_SPECIAL = 'special'
154
314
 
155
315
  module.exports = {
156
316
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
157
- mobiletto,
317
+ mobiletto, connect,
158
318
  MobilettoError, MobilettoNotFoundError,
159
- readStream, writeStream, closeStream, streamReader
319
+ readStream, writeStream, closeStream
160
320
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.4",
3
+ "version": "1.0.6",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -6,18 +6,22 @@ const randomstring = require('randomstring')
6
6
 
7
7
  const { expect, should, assert } = require('chai')
8
8
 
9
- const { mobiletto } = require('../index.js')
10
- const {MobilettoNotFoundError, M_FILE, M_DIR} = require("../index")
9
+ const { mobiletto, connect, MobilettoNotFoundError, M_FILE, M_DIR } = require("../index")
10
+ const { encrypt, decrypt, normalizeKey, normalizeIV, DEFAULT_CRYPT_ALGO } = require("../util/crypt")
11
11
 
12
12
  // chunk size used by generator function, used by driver's 'write' function
13
13
  // the temp file is also TEMP_SZ_MULTIPLE of this number
14
14
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
15
15
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
16
- const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.05;
16
+
17
+ // encoded bytes written will differ from actual bytes provided
18
+ // likewise, plaintext bytes read will differ from encoded bytes read
19
+ // our tests give some leeway when filesize differences
20
+ const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
17
21
 
18
22
  DRIVER_CONFIG = {
19
23
  local: {
20
- key: '/tmp'
24
+ key: process.env.LOCAL_STORAGE || '/tmp'
21
25
  },
22
26
  s3: {
23
27
  key: process.env.S3_ACCESS,
@@ -30,13 +34,17 @@ DRIVER_CONFIG = {
30
34
  }
31
35
  }
32
36
 
37
+ function closeEnough (expected, actual, percent = ENC_SIZE_CLOSE_ENOUGH_PERCENT) {
38
+ expect(Math.abs(expected - actual)).to.be.lessThan(Math.floor(expected * percent),
39
+ 'expected size within 5% of actual (due to encryption)')
40
+ }
41
+
33
42
  async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
34
43
  const meta = await api.metadata(name)
35
44
  should().exist(meta, 'expected return value from metadata call')
36
45
  expect(meta.name).equals(name, 'expected name of written file to be correct')
37
46
  if (closeEnoughPercent) {
38
- expect(Math.abs(expectedSize - meta.size)).to.be.lessThan(Math.floor(meta.size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
39
- 'expected write API to return within 5% of bytes written (due to encryption)')
47
+ closeEnough(expectedSize, meta.size, closeEnoughPercent)
40
48
  } else {
41
49
  expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
42
50
  }
@@ -68,17 +76,20 @@ async function writeRandomFile(fixture, size) {
68
76
  return await fixture.api.write(fixture.name, dataGenerator())
69
77
  }
70
78
 
71
- async function readFile(fixture) {
72
- const chunks = []
73
- function reader(chunk) {
74
- if (chunk) {
75
- chunks.push(chunk)
79
+ describe('crypto test', () => {
80
+ it("should encrypt and decrypt successfully", async () => {
81
+ const enc = {
82
+ key: normalizeKey(randomstring.generate(32)),
83
+ iv: normalizeIV(randomstring.generate(16)),
84
+ algo: DEFAULT_CRYPT_ALGO
76
85
  }
77
- }
78
- const response = await fixture.api.read(fixture.name, reader)
79
- const data = Buffer.concat(chunks)
80
- return { response, data }
81
- }
86
+ const plaintext = randomstring.generate(1024 + Math.floor(1024*Math.random()))
87
+ const ciphertext = encrypt(plaintext, enc)
88
+ const decrypted = decrypt(ciphertext, enc)
89
+ expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
90
+ })
91
+
92
+ })
82
93
 
83
94
  // To test a single driver:
84
95
  // - Uncomment one of the lines below to set driverName to the one you want to test
@@ -120,7 +131,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
120
131
  let fixture
121
132
  beforeEach((done) => {
122
133
  const name = `test_file_${fileSuffix}`
123
- mobiletto(driverName, config.key, config.secret, config.opts)
134
+ connect(driverName, config.key, config.secret, config.opts)
124
135
  .then(api => { fixture = {api, name, randomData} })
125
136
  .finally(done)
126
137
  })
@@ -129,8 +140,8 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
129
140
  expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
130
141
  })
131
142
  it("should read the file we just wrote", async () => {
132
- const { response, data } = await readFile(fixture);
133
- expect(response).is.equal(data.length, 'expected read API to return correct number of bytes read')
143
+ const data = await fixture.api.readFile(fixture.name);
144
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
134
145
  expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
135
146
  })
136
147
  it("should load metadata on the file we just wrote", async () => {
@@ -161,13 +172,12 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
161
172
  })
162
173
  it("should write an encrypted file", async () => {
163
174
  encryptedByteCount = await writeRandomFile(fixture, size);
164
- expect(Math.abs(encryptedByteCount - size)).to.be.lessThan(Math.floor(size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
165
- 'expected write API to return within 5% of bytes written (due to encryption)')
175
+ closeEnough(size, encryptedByteCount)
166
176
  })
167
177
  it("should read the encrypted file we just wrote", async () => {
168
- const { response, data } = await readFile(fixture);
169
- expect(response).is.equal(encryptedByteCount, 'expected read API to return correct number of bytes read')
170
- expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
178
+ const data = await fixture.api.readFile(fixture.name);
179
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
180
+ expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
171
181
  })
172
182
  it("should load metadata on the encrypted file we just wrote", async () => {
173
183
  await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
@@ -181,58 +191,71 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
181
191
  })
182
192
  })
183
193
 
184
- describe(`${driverName} - write files in a new dir, read metadata, recursively delete`, () => {
185
- // a random directory and file within it
186
- const randomParent = `random_dir_${rand(10)}`
187
- const subdirName = `subdir_` + Date.now()
188
- const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
189
- const fileCount = 3 + Math.floor(Math.random() * 10)
190
- let fixture
191
- beforeEach((done) => {
192
- mobiletto(driverName, config.key, config.secret, config.opts)
193
- .then(api => { fixture = {api, name: randomPath} })
194
- .finally(done)
195
- })
196
- it(`should write ${fileCount} files in a new directory`, async () => {
197
- function* dataGenerator() {
198
- // return one chunk of random data
199
- yield rand(READ_SZ)
200
- }
201
- for (let i = 0; i < fileCount; i++) {
202
- const response = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
203
- expect(response).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
204
- }
205
- })
206
- it("should load metadata for one of the new files", async () => {
207
- await assertMeta (fixture.api, fixture.name, READ_SZ)
208
- })
209
- it("should see correct types on objects returned from a listing of the new directory", async () => {
210
- const objects = await fixture.api.list(randomParent)
211
- expect(objects).to.have.lengthOf(1)
212
- expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
213
- })
214
- it("should see correct types on objects returned from a listing of the subdirectory", async () => {
215
- const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
216
- expect(objects).to.have.lengthOf(fileCount)
217
- for (let i = 0; i < fileCount; i++) {
218
- // we should find all the files, and they should all have the correct type
219
- expect(objects
220
- .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
221
- ).to.not.be.null
222
- }
223
- })
224
- it("should recursively delete the directory and file we just created", async () => {
225
- const recursive = true
226
- const removed = await fixture.api.remove(randomParent, {recursive})
227
- expect(removed).to.be.true
228
- })
229
- it("loading metadata on the file we wrote now fails", async () => {
230
- await assertMetaFail(fixture.api, fixture.name)
231
- })
232
- it("loading metadata on the parent dir we wrote now fails", async () => {
233
- await assertMetaFail(fixture.api, randomParent)
194
+ for (const encryption of [null, {key: rand(32)}]) {
195
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
196
+ describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
197
+ // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
198
+ // const encryption = {key: rand(32)}
199
+ // const encryption = null
200
+ // a random directory and file within it
201
+ const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
202
+ const subdirName = `subdir_` + Date.now()
203
+ const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
204
+ //const fileCount = 3 + Math.floor(Math.random() * 10)
205
+ const fileCount = 2
206
+ let fixture
207
+ beforeEach((done) => {
208
+ mobiletto(driverName, config.key, config.secret, config.opts, encryption)
209
+ .then(api => { fixture = {api, name: randomPath} })
210
+ .catch((err) => { throw err })
211
+ .finally(done)
212
+ })
213
+ it(`should write ${fileCount} files in a new directory`, async () => {
214
+ function* dataGenerator() {
215
+ // return one chunk of random data
216
+ yield rand(READ_SZ)
217
+ }
218
+
219
+ for (let i = 0; i < fileCount; i++) {
220
+ const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
221
+ if (encryption) {
222
+ closeEnough(READ_SZ, bytesWritten)
223
+ } else {
224
+ expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
225
+ }
226
+ }
227
+ })
228
+ it("should load metadata for one of the new files", async () => {
229
+ await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
230
+ })
231
+ it("should see correct types on objects returned from a listing of the new directory", async () => {
232
+ const objects = await fixture.api.list(randomParent)
233
+ expect(objects).to.have.lengthOf(1)
234
+ expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
235
+ })
236
+ it("should see correct types on objects returned from a listing of the subdirectory", async () => {
237
+ const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
238
+ expect(objects).to.have.lengthOf(fileCount)
239
+ for (let i = 0; i < fileCount; i++) {
240
+ // we should find all the files, and they should all have the correct type
241
+ expect(objects
242
+ .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
243
+ ).to.not.be.null
244
+ }
245
+ })
246
+ it("should recursively delete the directory and file we just created", async () => {
247
+ const recursive = true
248
+ const removed = await fixture.api.remove(randomParent, {recursive})
249
+ expect(removed).to.be.true
250
+ })
251
+ it("loading metadata on the file we wrote now fails", async () => {
252
+ await assertMetaFail(fixture.api, fixture.name)
253
+ })
254
+ it("loading metadata on the parent dir we wrote now fails", async () => {
255
+ await assertMetaFail(fixture.api, randomParent)
256
+ })
234
257
  })
235
- })
258
+ }
236
259
 
237
260
  describe(`${driverName} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
238
261
  it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
package/util/crypt.js CHANGED
@@ -36,27 +36,28 @@ function getCipher(enc) {
36
36
  return crypto.createCipheriv(enc.algo, enc.key, enc.iv);
37
37
  }
38
38
 
39
- function encrypt (plainText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'base64') {
40
- if (!key) {
39
+ function encrypt (plainText, encryption, outputEncoding = 'base64') {
40
+ if (!encryption || !encryption.key) {
41
41
  if (WARN_PLAINTEXT) {
42
- console.warn(` ****** key is undefined, encryption of user-data is DISABLED`)
42
+ console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
43
43
  }
44
44
  return plainText
45
45
  }
46
- const cipher = getCipher(key, iv)
47
- return Buffer.concat([cipher.update(plainText), cipher.final()]).toString(outputEncoding)
46
+ const cipher = getCipher(encryption)
47
+ const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
48
+ return encoded.toString(outputEncoding)
48
49
  }
49
50
 
50
51
  function getDecipher(enc) {
51
52
  return crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
52
53
  }
53
54
 
54
- function decrypt (cipherText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'utf8') {
55
- if (!key) {
55
+ function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
56
+ if (!encryption || !encryption.key) {
56
57
  return cipherText
57
58
  }
58
- const cipher = getDecipher(key, iv)
59
- const data = Buffer.from(cipherText, 'base64')
59
+ const cipher = getDecipher(encryption)
60
+ const data = Buffer.from(cipherText, inputEncoding)
60
61
  return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
61
62
  }
62
63