mobiletto 1.0.4 → 1.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,10 +1,17 @@
1
1
  Mobiletto
2
2
  =========
3
3
 
4
- Mobiletto is a JavaScript storage abstraction layer.
4
+ Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
5
5
 
6
6
  It supports apps that are agnostic to where files are stored.
7
7
 
8
+ All drivers are tested for identical behavior with over 35 tests for each driver.
9
+
10
+ Today the supported drivers are:
11
+
12
+ * `s3`: read/write an Amazon S3 bucket
13
+ * `local`: read/write to local filesystem
14
+
8
15
  # Basic usage
9
16
  const { mobiletto } = require('mobiletto')
10
17
 
@@ -15,6 +22,7 @@ It supports apps that are agnostic to where files are stored.
15
22
  // * key: base directory
16
23
  // * secret: ignored, can be null
17
24
  // * opts object:
25
+ // * readOnly: optional, never change anything on the filesystem; default is false
18
26
  // * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
19
27
  const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
20
28
 
@@ -22,10 +30,11 @@ It supports apps that are agnostic to where files are stored.
22
30
  // * key: AWS access key
23
31
  // * secret: AWS secret key
24
32
  // * opts object:
33
+ // * readOnly: optional, never change anything on the bucket; default is false
25
34
  // * bucket: required, name of the S3 bucket
26
35
  // * region: optional, the AWS region to communicate with, default is us-east-1
27
36
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
28
- // * delimiter: optional, directory delimiter, default is '/'
37
+ // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
29
38
  const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
30
39
 
31
40
  // List files
@@ -46,6 +55,10 @@ It supports apps that are agnostic to where files are stored.
46
55
  const callback = (chunk) => { ... write chunk somewhere ... }
47
56
  local.read(path, callback)
48
57
  s3.read(path, callback)
58
+
59
+ // Read an entire file at once (convenience method)
60
+ const localData = await local.readFile(path) // returns a byte Buffer of the file contents
61
+ const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
49
62
 
50
63
  // Write a file
51
64
  // Provide a generator function that yields chunks of data
@@ -69,20 +82,43 @@ It supports apps that are agnostic to where files are stored.
69
82
  local.remove(path, {recursive, quiet})
70
83
  s3.remove(path, {recursive, quiet})
71
84
 
85
+ # Alternate usage
86
+ Import the fully-scoped module and use the `connect` function:
87
+
88
+ const storage = require('mobiletto')
89
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
90
+ const objectData = await s3.readFile('some/path')
91
+
72
92
  # Transparent Encryption
73
93
  Enable transparent client-side encryption:
74
94
 
75
95
  // Pass encryption parameters
76
96
  const encryption = {
77
- key: randomstring.generate(128), // required, must be >= 16 chars
78
- iv: randomstring.generate(128), // optional
79
- algo: 'aes-256-cbc' // optional, aes-256-cbc is the default
97
+ key: randomstring.generate(128), // required, must be >= 32 chars
98
+ iv: randomstring.generate(128), // optional, default is to derive IV from key
99
+ algo: 'aes-256-cbc' // optional, aes-256-cbc is the default
80
100
  }
81
101
  const api = await mobiletto(driverName, key, secret, opts, encryption)
82
102
 
83
103
  // Subsequent write operations will encrypt data (client side) when writing
84
104
  // Subsequent read operations will decrypt data (client side) when reading
85
- // Path names will also be encrypted
105
+
106
+ What's happening? A separate "directory entry" (dirent) directory (encrypted) tracks what files are in that
107
+ directory (aka the dirent directory).
108
+ * The `list` command reads the directory entry files, decrypts each path listed; then returns metadata for each file
109
+ * `list` commands are more inefficient, especially for directories with a large number of files
110
+ * The `write` command writes dirent files in each parent's dirent directory, recursively; then writes the file
111
+ * `write` commands will incur O(N) writes, with N = depth in the directory hierarchy
112
+ * The `remove` command removes the corresponding dirent file, and its parent if empty, recursively; then removes the file
113
+ * Non-recursive `remove` commands will incur O(N) reads and potentially as many deletes, with N = depth in the directory hierarchy
114
+ * Recursive `remove` commands on large and deep filesystems can be expensive
115
+
116
+ Note that even with client-side encryption enabled, an adversary with full visibility into your encrypted server-side
117
+ storage, even without the key, can still see the total number of directories and how many files are in each, and with
118
+ some effort, discover some or all of the overall structure of the directory hierarchy.
119
+ *Note: Use a relatively flat structure for better security.*
120
+ The adversary would not know the names of the directories/files unless they also knew your encryption
121
+ key or had otherwise successfully cracked the encryption. All bets are off then!
86
122
 
87
123
  # Driver Interface
88
124
  A driver is any JS file that exports a 'storageClient' function with this signature:
@@ -110,10 +146,12 @@ The object that the storageClient function returns must define these functions:
110
146
  async metadata (path)
111
147
 
112
148
  // Read a file
113
- async read (path, callback) // callback receives a chunk of data; a null chunk signals end of data
149
+ // callback receives a chunk of data. endCallback is called at end-of-stream
150
+ async read (path, callback, endCallback = null)
114
151
 
115
152
  // Write a file
116
- async write (path, readFunc) // readFunc returns the data you want to write, or null to finish
153
+ // readFunc returns the data you want to write
154
+ async write (path, readFunc)
117
155
 
118
156
  // Remove a file, or recursively delete a directory
119
157
  async remove (path, {recursive = false, quiet = false})
@@ -9,6 +9,16 @@ const fs = require('fs')
9
9
 
10
10
  const DEFAULT_MODE = '0700'
11
11
 
12
+ const isENOENT = (err) => err.code && err.code === 'ENOENT'
13
+
14
+ const ioError = (err, path, method) => isENOENT(err)
15
+ ? err instanceof MobilettoNotFoundError
16
+ ? err
17
+ : new MobilettoNotFoundError(path)
18
+ : err instanceof MobilettoError || err instanceof MobilettoNotFoundError
19
+ ? err
20
+ : new MobilettoError(`${method}(${path}) error: ${err}`, err)
21
+
12
22
  class StorageClient {
13
23
  baseDir
14
24
  mode
@@ -74,7 +84,7 @@ class StorageClient {
74
84
  return entry
75
85
  })
76
86
  } catch (err) {
77
- throw new MobilettoError(`list(${path}) error: ${err}`, err)
87
+ throw ioError(err, path, 'list')
78
88
  }
79
89
  }
80
90
  async metadata (path) {
@@ -82,19 +92,23 @@ class StorageClient {
82
92
  try {
83
93
  const lstat = fs.lstatSync(file)
84
94
  if (!lstat) {
85
- throw new MobilettoError(`metadata: lstat error`)
95
+ throw new MobilettoError('metadata: lstat error')
96
+ }
97
+ const type = this.fileType(lstat)
98
+ if (type === M_DIR && path !== '') {
99
+ const contents = await this.list(path)
100
+ if (contents.length === 0) {
101
+ throw new MobilettoNotFoundError(path)
102
+ }
86
103
  }
87
104
  return {
88
105
  name: file.startsWith(this.baseDir) ? file.substring(this.baseDir.length) : file,
89
- type: this.fileType(lstat),
106
+ type,
90
107
  size: lstat.size,
91
108
  mtime: lstat.mtimeMs
92
109
  }
93
110
  } catch (err) {
94
- if (err.code && err.code === 'ENOENT') {
95
- throw new MobilettoNotFoundError(path)
96
- }
97
- throw new MobilettoError(`metadata(${path}) error: ${err}`, err)
111
+ throw ioError(err, path, 'metadata')
98
112
  }
99
113
  }
100
114
 
@@ -150,10 +164,7 @@ class StorageClient {
150
164
  const stream = fs.createReadStream(file)
151
165
  return await readStream(stream, callback, endCallback)
152
166
  } catch (err) {
153
- if (err.code && err.code === 'ENOENT') {
154
- throw new MobilettoNotFoundError(path)
155
- }
156
- throw new MobilettoError(`read(${path}) error: ${err}`, err)
167
+ throw ioError(err, path, 'read')
157
168
  }
158
169
  }
159
170
 
@@ -165,7 +176,7 @@ class StorageClient {
165
176
  try {
166
177
  fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
167
178
  } catch (err) {
168
- if (err.code && err.code === 'ENOENT') {
179
+ if (isENOENT(err)) {
169
180
  if (!quiet) {
170
181
  throw new MobilettoNotFoundError(path)
171
182
  }
@@ -80,12 +80,14 @@ class StorageClient {
80
80
  while (truncated) {
81
81
  try {
82
82
  const response = await this.client.send(new ListObjectsCommand(bucketParams))
83
- if (typeof response.Contents !== 'undefined') {
83
+ const hasContents = typeof response.Contents !== 'undefined'
84
+ if (hasContents) {
84
85
  response.Contents.forEach((item) => {
85
86
  objects.push(item.Key) // todo: transform into standard object
86
87
  })
87
88
  }
88
- if (typeof response.CommonPrefixes !== 'undefined') {
89
+ const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
90
+ if (hasCommonPrefixes) {
89
91
  response.CommonPrefixes.forEach((item) => {
90
92
  objects.push(item.Prefix) // todo: transform into standard object
91
93
  })
@@ -94,13 +96,22 @@ class StorageClient {
94
96
  // If truncated is true, advance the marker
95
97
  if (truncated) {
96
98
  bucketParams.Marker = response.NextMarker
99
+ } else if (!hasContents && !hasCommonPrefixes) {
100
+ if (path === '') {
101
+ break
102
+ }
103
+ throw new MobilettoNotFoundError(path)
97
104
  }
98
- // At end of the list, response.truncated is false, and the function exits the while loop.
99
105
  } catch (err) {
100
- console.log(`${logPrefix} Error: ${err}`)
101
- truncated = false
106
+ if (err instanceof MobilettoNotFoundError) {
107
+ throw err
108
+ }
109
+ throw new MobilettoError(`${logPrefix} Error: ${err}`)
102
110
  }
103
111
  }
112
+ if (recursive && objects.length === 0) {
113
+ throw new MobilettoNotFoundError(path)
114
+ }
104
115
  return objects
105
116
  }
106
117
 
@@ -213,7 +224,14 @@ class StorageClient {
213
224
  if (!quiet && response.Errors && response.Errors.length > 0) {
214
225
  throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
215
226
  }
216
- objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
227
+ try {
228
+ objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
229
+ } catch (e) {
230
+ if (!(e instanceof MobilettoNotFoundError)) {
231
+ throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
232
+ }
233
+ objects = null
234
+ }
217
235
  }
218
236
  } else {
219
237
  const Key = this.normalizeKey(path)
package/index.js CHANGED
@@ -1,5 +1,13 @@
1
+ const { basename, dirname } = require('path')
2
+ const shasum = require('shasum')
3
+ const randomstring = require('randomstring')
4
+
1
5
  const crypt = require('./util/crypt')
2
- const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV} = require("./util/crypt");
6
+ const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt");
7
+
8
+ const DIR_ENT_DIR_SUFFIX = '__.dirent'
9
+ const DIR_ENT_FILE_PREFIX = 'dirent__'
10
+ const ENC_PAD_SEP = ' ~ '
3
11
 
4
12
  // adapted from https://stackoverflow.com/a/27724419
5
13
  function MobilettoError (message, err) {
@@ -24,14 +32,48 @@ function MobilettoNotFoundError (message) {
24
32
  }
25
33
  }
26
34
 
35
+ const reader = (chunks) => (chunk) => {
36
+ if (chunk) {
37
+ chunks.push(chunk)
38
+ }
39
+ }
40
+
41
+ const UTILITY_FUNCTIONS = {
42
+ readFile: (client) => async (path) => {
43
+ const chunks = []
44
+ await client.read(path, reader(chunks))
45
+ return Buffer.concat(chunks)
46
+ }
47
+ }
48
+
49
+ function addUtilityFunctions (client, readOnly = false) {
50
+ for (const func of Object.keys(UTILITY_FUNCTIONS)) {
51
+ client[func] = UTILITY_FUNCTIONS[func](client)
52
+ }
53
+ if (readOnly) {
54
+ for (const writeFunc of ['write', 'remove']) {
55
+ client[writeFunc] = async () => {
56
+ // console.warn(`${writeFunc} not supported in readOnly mode`)
57
+ return false
58
+ }
59
+ }
60
+ }
61
+ return client
62
+ }
63
+
64
+ async function connect (driverPath, key, secret, opts, encryption = null) {
65
+ return await mobiletto(driverPath, key, secret, opts, encryption)
66
+ }
67
+
27
68
  async function mobiletto (driverPath, key, secret, opts, encryption = null) {
28
69
  const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
29
70
  const client = driver.storageClient(key, secret, opts)
30
71
  if (!(await client.testConfig())) {
31
72
  throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
32
73
  }
74
+ const readOnly = opts ? !!opts.readOnly : false
33
75
  if (encryption === null) {
34
- return client
76
+ return addUtilityFunctions(client, readOnly)
35
77
  }
36
78
  const encKey = normalizeKey(encryption.key)
37
79
  if (!encKey) {
@@ -44,14 +86,84 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
44
86
  const enc = {
45
87
  key: encKey,
46
88
  iv,
47
- algo: encryption.algo || DEFAULT_CRYPT_ALGO
89
+ algo: encryption.algo || DEFAULT_CRYPT_ALGO,
90
+ dirLevels: encryption.dirLevels || 4,
91
+ encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
48
92
  }
49
- return {
50
- list: async (path) => client.list(path),
51
- metadata: async (path) => client.metadata(path),
93
+ function encryptPath (path) {
94
+ const encrypted = shasum(enc.key + ' ' + path)
95
+ let newPath = ''
96
+ for (let i = 0; i <= enc.dirLevels; i++) {
97
+ if (newPath.length > 0) newPath += '/'
98
+ newPath += encrypted.substring(i*2, (i*2)+2)
99
+ }
100
+ return newPath + encrypted
101
+ }
102
+ const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
103
+ const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
104
+
105
+ const _metadata = (client) => async (path) => {
106
+ let meta
107
+ try {
108
+ meta = await client.metadata(encryptPath(path))
109
+ } catch (e) {
110
+ if (e instanceof MobilettoNotFoundError) {
111
+ const dd = direntDir(path);
112
+ try {
113
+ meta = await client.metadata(dd)
114
+ } catch (err) {
115
+ if (err instanceof MobilettoNotFoundError) {
116
+ const contents = await client.list(dd)
117
+ if (Array.isArray(contents) && contents.length > 0) {
118
+ return { name: path, type: M_DIR }
119
+ } else {
120
+ throw err
121
+ }
122
+ } else {
123
+ throw err
124
+ }
125
+ }
126
+ } else {
127
+ throw e
128
+ }
129
+ }
130
+ meta.name = path // rewrite name back to plaintext name
131
+ return meta
132
+ }
133
+
134
+ const _loadMeta = async (dirent, entries) => {
135
+ const files = []
136
+ for (const entry of entries) {
137
+ const cipherText = []
138
+ await client.read(dirent + '/' + basename(entry.name), reader(cipherText))
139
+ const plain = decrypt(cipherText.toString(), enc)
140
+ const realPath = plain.split(ENC_PAD_SEP)[0]
141
+ const meta = await _metadata(client)(realPath)
142
+ files.push(meta)
143
+ }
144
+ return files
145
+ }
146
+
147
+ async function removeDirentFile(path) {
148
+ const df = direntFile(direntDir(dirname(path)), path);
149
+ await client.remove(df, {recursive: false, quiet: true})
150
+ await client.remove(encryptPath(path), {recursive: false, quiet: true})
151
+ }
152
+
153
+ const encClient = {
154
+ list: async (path) => {
155
+ const dirent = direntDir(path)
156
+ const entries = await client.list(dirent)
157
+ if (!entries || entries.length === 0) {
158
+ return []
159
+ }
160
+ return _loadMeta(dirent, entries)
161
+ },
162
+ metadata: _metadata(client),
52
163
  read: async (path, callback) => {
164
+ const realPath = encryptPath(path)
53
165
  const cipher = crypt.startDecryptStream(enc)
54
- return client.read(path,
166
+ return client.read(realPath,
55
167
  (chunk) => {
56
168
  return callback(crypt.updateCryptStream(cipher, chunk))
57
169
  }, () => {
@@ -59,6 +171,23 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
59
171
  })
60
172
  },
61
173
  write: async (path, readFunc) => {
174
+ // if encrypting paths, write dirent file(s) for all parent directories
175
+ let p = path
176
+ while (true) {
177
+ const direntGenerator = function* () {
178
+ yield encrypt(p + enc.encPathPadding(), enc)
179
+ }
180
+ const dir = direntDir(dirname(p))
181
+ const df = direntFile(dir, p);
182
+ if (!(await client.write(df, direntGenerator()))) {
183
+ throw new MobilettoError('write: error writing dirent file')
184
+ }
185
+ p = dirname(p)
186
+ if (p === '.' || p === '') {
187
+ break
188
+ }
189
+ }
190
+
62
191
  function* cryptGenerator(plaintextGenerator) {
63
192
  let chunk = plaintextGenerator.next().value
64
193
  while (chunk) {
@@ -68,14 +197,66 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
68
197
  yield cipher.final()
69
198
  }
70
199
  const cipher = crypt.startEncryptStream(enc)
71
- return client.write(path, cryptGenerator(readFunc))
200
+ const realPath = encryptPath(path)
201
+ return client.write(realPath, cryptGenerator(readFunc))
72
202
  },
73
203
  remove: async (path, options) => {
74
204
  const recursive = (options && options.recursive) || false
75
- const quiet = (options && options.quiet) || false
76
- return client.remove(path, {recursive, quiet})
205
+ if (recursive) {
206
+ // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
207
+ async function recRm (path) {
208
+ const dirent = direntDir(path)
209
+ let entries
210
+ try {
211
+ entries = await client.list(dirent)
212
+ } catch (e) {
213
+ if (!(e instanceof MobilettoNotFoundError)) {
214
+ throw e instanceof MobilettoError
215
+ ? e
216
+ : new MobilettoError('error listing files for recursive deletion', e)
217
+ }
218
+ }
219
+ if (entries && entries.length > 0) {
220
+ const files = await _loadMeta(dirent, entries)
221
+ for (const f of files) {
222
+ await recRm(f.name)
223
+ }
224
+ }
225
+ await removeDirentFile(path);
226
+ }
227
+ await recRm(path)
228
+ }
229
+
230
+ // remove the single file/dir
231
+ await removeDirentFile(path);
232
+
233
+ // if we were the last dirent file, then also remove dirent directory, and recursively upwards
234
+ let parent = path
235
+ let dirent = direntDir(parent)
236
+ const df = direntFile(dirent, path)
237
+ await client.remove(df, {recursive: false, quiet: true})
238
+ while (true) {
239
+ try {
240
+ const entries = await client.list(dirent)
241
+ if (entries.length === 0) {
242
+ await removeDirentFile(parent)
243
+ }
244
+ } catch (e) {
245
+ if (!(e instanceof MobilettoNotFoundError)) {
246
+ throw e
247
+ }
248
+ }
249
+ if (parent === '.' || parent === '/') {
250
+ // do not remove dirent for root dir
251
+ break
252
+ }
253
+ parent = dirname(parent)
254
+ dirent = direntDir(parent)
255
+ }
256
+ return true
77
257
  }
78
258
  }
259
+ return addUtilityFunctions(encClient, readOnly)
79
260
  }
80
261
 
81
262
  async function readStream(stream, callback, endCallback) {
@@ -103,7 +284,7 @@ function writeStream (stream) {
103
284
  if (chunk) {
104
285
  stream.write(chunk, (err) => {
105
286
  if (err) {
106
- console.log(`writeStream: error writing: ${err}`)
287
+ console.error(`writeStream: error writing: ${err}`)
107
288
  throw err
108
289
  }
109
290
  })
@@ -114,39 +295,12 @@ function writeStream (stream) {
114
295
  function closeStream (stream) {
115
296
  return () => stream.close((err) => {
116
297
  if (err) {
117
- console.log(`closeStream: error closing: ${err}`)
298
+ console.error(`closeStream: error closing: ${err}`)
118
299
  throw err
119
300
  }
120
301
  })
121
302
  }
122
303
 
123
- async function streamReader (stream, callback, endCallback) {
124
- let count = 0
125
- const streamHandler = async (stream) => {
126
- new Promise((resolve, reject) => {
127
- stream.on('data', (data) => {
128
- count += data ? data.length : 0
129
- callback(data)
130
- })
131
- stream.on('error', reject)
132
- stream.on('end', (resolve) => {
133
- if (typeof endCallback === 'function') {
134
- const endData = endCallback()
135
- if (endData) {
136
- count += endData.length
137
- callback(endData)
138
- }
139
- }
140
- resolve()
141
- })
142
- })
143
- }
144
- await streamHandler(stream).then(() => {
145
- // console.log('streamHandler ended')
146
- })
147
- return count
148
- }
149
-
150
304
  const M_FILE = 'file'
151
305
  const M_DIR = 'dir'
152
306
  const M_LINK = 'link'
@@ -154,7 +308,7 @@ const M_SPECIAL = 'special'
154
308
 
155
309
  module.exports = {
156
310
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
157
- mobiletto,
311
+ mobiletto, connect,
158
312
  MobilettoError, MobilettoNotFoundError,
159
- readStream, writeStream, closeStream, streamReader
313
+ readStream, writeStream, closeStream
160
314
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.4",
3
+ "version": "1.0.8",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -6,18 +6,22 @@ const randomstring = require('randomstring')
6
6
 
7
7
  const { expect, should, assert } = require('chai')
8
8
 
9
- const { mobiletto } = require('../index.js')
10
- const {MobilettoNotFoundError, M_FILE, M_DIR} = require("../index")
9
+ const { mobiletto, connect, MobilettoNotFoundError, M_FILE, M_DIR } = require("../index")
10
+ const { encrypt, decrypt, normalizeKey, normalizeIV, DEFAULT_CRYPT_ALGO } = require("../util/crypt")
11
11
 
12
12
  // chunk size used by generator function, used by driver's 'write' function
13
13
  // the temp file is also TEMP_SZ_MULTIPLE of this number
14
14
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
15
15
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
16
- const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.05;
16
+
17
+ // encoded bytes written will differ from actual bytes provided
18
+ // likewise, plaintext bytes read will differ from encoded bytes read
19
+ // our tests give some leeway when filesize differences
20
+ const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
17
21
 
18
22
  DRIVER_CONFIG = {
19
23
  local: {
20
- key: '/tmp'
24
+ key: process.env.LOCAL_STORAGE || '/tmp'
21
25
  },
22
26
  s3: {
23
27
  key: process.env.S3_ACCESS,
@@ -30,13 +34,17 @@ DRIVER_CONFIG = {
30
34
  }
31
35
  }
32
36
 
37
+ function closeEnough (expected, actual, percent = ENC_SIZE_CLOSE_ENOUGH_PERCENT) {
38
+ expect(Math.abs(expected - actual)).to.be.lessThan(Math.floor(expected * percent),
39
+ 'expected size within 5% of actual (due to encryption)')
40
+ }
41
+
33
42
  async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
34
43
  const meta = await api.metadata(name)
35
44
  should().exist(meta, 'expected return value from metadata call')
36
45
  expect(meta.name).equals(name, 'expected name of written file to be correct')
37
46
  if (closeEnoughPercent) {
38
- expect(Math.abs(expectedSize - meta.size)).to.be.lessThan(Math.floor(meta.size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
39
- 'expected write API to return within 5% of bytes written (due to encryption)')
47
+ closeEnough(expectedSize, meta.size, closeEnoughPercent)
40
48
  } else {
41
49
  expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
42
50
  }
@@ -68,17 +76,22 @@ async function writeRandomFile(fixture, size) {
68
76
  return await fixture.api.write(fixture.name, dataGenerator())
69
77
  }
70
78
 
71
- async function readFile(fixture) {
72
- const chunks = []
73
- function reader(chunk) {
74
- if (chunk) {
75
- chunks.push(chunk)
79
+ describe('crypto test', () => {
80
+ it("should encrypt and decrypt successfully", async () => {
81
+ const enc = {
82
+ key: normalizeKey(randomstring.generate(32)),
83
+ iv: normalizeIV(randomstring.generate(16)),
84
+ algo: DEFAULT_CRYPT_ALGO
76
85
  }
77
- }
78
- const response = await fixture.api.read(fixture.name, reader)
79
- const data = Buffer.concat(chunks)
80
- return { response, data }
81
- }
86
+ const plaintext = randomstring.generate(1024 + Math.floor(1024*Math.random()))
87
+ const ciphertext = encrypt(plaintext, enc)
88
+ const decrypted = decrypt(ciphertext, enc)
89
+ expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
90
+ })
91
+
92
+ })
93
+
94
+ const ENC_TESTS = [null, {key: rand(32)}]
82
95
 
83
96
  // To test a single driver:
84
97
  // - Uncomment one of the lines below to set driverName to the one you want to test
@@ -120,7 +133,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
120
133
  let fixture
121
134
  beforeEach((done) => {
122
135
  const name = `test_file_${fileSuffix}`
123
- mobiletto(driverName, config.key, config.secret, config.opts)
136
+ connect(driverName, config.key, config.secret, config.opts)
124
137
  .then(api => { fixture = {api, name, randomData} })
125
138
  .finally(done)
126
139
  })
@@ -129,8 +142,8 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
129
142
  expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
130
143
  })
131
144
  it("should read the file we just wrote", async () => {
132
- const { response, data } = await readFile(fixture);
133
- expect(response).is.equal(data.length, 'expected read API to return correct number of bytes read')
145
+ const data = await fixture.api.readFile(fixture.name);
146
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
134
147
  expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
135
148
  })
136
149
  it("should load metadata on the file we just wrote", async () => {
@@ -145,6 +158,54 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
145
158
  })
146
159
  })
147
160
 
161
+ for (const encryption of ENC_TESTS) {
162
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
163
+ describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
164
+ // some random data, plus a bit extra
165
+ const size = 16
166
+ const randomData = rand(size)
167
+ const fileSuffix = '' + Date.now()
168
+ const encryption = {key: rand(32)}
169
+ let fixture
170
+ beforeEach((done) => {
171
+ const name = `test_file_${fileSuffix}`
172
+ const opts = Object.assign({}, config.opts, {readOnly: true})
173
+ connect(driverName, config.key, config.secret, opts, encryption)
174
+ .then(api => { fixture = {api, name, randomData} })
175
+ .catch(err => { throw err })
176
+ .finally(done)
177
+ })
178
+ it("should try to write a file and fail", async () => {
179
+ const response = await writeRandomFile(fixture, size)
180
+ expect(response).to.be.false
181
+ })
182
+ it("should fail to read the file we did not write", async () => {
183
+ try {
184
+ await fixture.api.readFile(fixture.name)
185
+ assert.fail('should not have been able to read non-existent file that we did not write')
186
+ } catch (e) {
187
+ if (!(e instanceof MobilettoNotFoundError)) {
188
+ assert.fail(`unexpected error: ${e} trying to read non-existent file`)
189
+ }
190
+ }
191
+ })
192
+ it("should fail to load metadata on the file we did not write", async () => {
193
+ try {
194
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
195
+ assert.fail('should not have been able to read metadata for non-existent file that we did not write')
196
+ } catch (e) {
197
+ if (!(e instanceof MobilettoNotFoundError)) {
198
+ assert.fail(`unexpected error: ${e} trying to read metadata on non-existent file`)
199
+ }
200
+ }
201
+ })
202
+ it("should fail to delete the file we did not write", async () => {
203
+ const removed = await fixture.api.remove(fixture.name)
204
+ expect(removed).to.be.false
205
+ })
206
+ })
207
+ }
208
+
148
209
  describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
149
210
  // some random data, plus a bit extra
150
211
  const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
@@ -157,17 +218,17 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
157
218
  const name = `test_file_${fileSuffix}`
158
219
  mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
159
220
  .then(api => { fixture = {api, name, randomData} })
221
+ .catch((err) => { throw err })
160
222
  .finally(done)
161
223
  })
162
224
  it("should write an encrypted file", async () => {
163
225
  encryptedByteCount = await writeRandomFile(fixture, size);
164
- expect(Math.abs(encryptedByteCount - size)).to.be.lessThan(Math.floor(size * ENC_SIZE_CLOSE_ENOUGH_PERCENT),
165
- 'expected write API to return within 5% of bytes written (due to encryption)')
226
+ closeEnough(size, encryptedByteCount)
166
227
  })
167
228
  it("should read the encrypted file we just wrote", async () => {
168
- const { response, data } = await readFile(fixture);
169
- expect(response).is.equal(encryptedByteCount, 'expected read API to return correct number of bytes read')
170
- expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
229
+ const data = await fixture.api.readFile(fixture.name);
230
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
231
+ expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
171
232
  })
172
233
  it("should load metadata on the encrypted file we just wrote", async () => {
173
234
  await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
@@ -181,58 +242,71 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
181
242
  })
182
243
  })
183
244
 
184
- describe(`${driverName} - write files in a new dir, read metadata, recursively delete`, () => {
185
- // a random directory and file within it
186
- const randomParent = `random_dir_${rand(10)}`
187
- const subdirName = `subdir_` + Date.now()
188
- const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
189
- const fileCount = 3 + Math.floor(Math.random() * 10)
190
- let fixture
191
- beforeEach((done) => {
192
- mobiletto(driverName, config.key, config.secret, config.opts)
193
- .then(api => { fixture = {api, name: randomPath} })
194
- .finally(done)
195
- })
196
- it(`should write ${fileCount} files in a new directory`, async () => {
197
- function* dataGenerator() {
198
- // return one chunk of random data
199
- yield rand(READ_SZ)
200
- }
201
- for (let i = 0; i < fileCount; i++) {
202
- const response = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
203
- expect(response).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
204
- }
205
- })
206
- it("should load metadata for one of the new files", async () => {
207
- await assertMeta (fixture.api, fixture.name, READ_SZ)
208
- })
209
- it("should see correct types on objects returned from a listing of the new directory", async () => {
210
- const objects = await fixture.api.list(randomParent)
211
- expect(objects).to.have.lengthOf(1)
212
- expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
213
- })
214
- it("should see correct types on objects returned from a listing of the subdirectory", async () => {
215
- const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
216
- expect(objects).to.have.lengthOf(fileCount)
217
- for (let i = 0; i < fileCount; i++) {
218
- // we should find all the files, and they should all have the correct type
219
- expect(objects
220
- .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
221
- ).to.not.be.null
222
- }
223
- })
224
- it("should recursively delete the directory and file we just created", async () => {
225
- const recursive = true
226
- const removed = await fixture.api.remove(randomParent, {recursive})
227
- expect(removed).to.be.true
228
- })
229
- it("loading metadata on the file we wrote now fails", async () => {
230
- await assertMetaFail(fixture.api, fixture.name)
231
- })
232
- it("loading metadata on the parent dir we wrote now fails", async () => {
233
- await assertMetaFail(fixture.api, randomParent)
245
+ for (const encryption of ENC_TESTS) {
246
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
247
+ describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
248
+ // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
249
+ // const encryption = {key: rand(32)}
250
+ // const encryption = null
251
+ // a random directory and file within it
252
+ const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
253
+ const subdirName = `subdir_` + Date.now()
254
+ const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
255
+ //const fileCount = 3 + Math.floor(Math.random() * 10)
256
+ const fileCount = 2
257
+ let fixture
258
+ beforeEach((done) => {
259
+ mobiletto(driverName, config.key, config.secret, config.opts, encryption)
260
+ .then(api => { fixture = {api, name: randomPath} })
261
+ .catch((err) => { throw err })
262
+ .finally(done)
263
+ })
264
+ it(`should write ${fileCount} files in a new directory`, async () => {
265
+ function* dataGenerator() {
266
+ // return one chunk of random data
267
+ yield rand(READ_SZ)
268
+ }
269
+
270
+ for (let i = 0; i < fileCount; i++) {
271
+ const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
272
+ if (encryption) {
273
+ closeEnough(READ_SZ, bytesWritten)
274
+ } else {
275
+ expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
276
+ }
277
+ }
278
+ })
279
+ it("should load metadata for one of the new files", async () => {
280
+ await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
281
+ })
282
+ it("should see correct types on objects returned from a listing of the new directory", async () => {
283
+ const objects = await fixture.api.list(randomParent)
284
+ expect(objects).to.have.lengthOf(1)
285
+ expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
286
+ })
287
+ it("should see correct types on objects returned from a listing of the subdirectory", async () => {
288
+ const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
289
+ expect(objects).to.have.lengthOf(fileCount)
290
+ for (let i = 0; i < fileCount; i++) {
291
+ // we should find all the files, and they should all have the correct type
292
+ expect(objects
293
+ .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
294
+ ).to.not.be.null
295
+ }
296
+ })
297
+ it("should recursively delete the directory and file we just created", async () => {
298
+ const recursive = true
299
+ const removed = await fixture.api.remove(randomParent, {recursive})
300
+ expect(removed).to.be.true
301
+ })
302
+ it("loading metadata on the file we wrote now fails", async () => {
303
+ await assertMetaFail(fixture.api, fixture.name)
304
+ })
305
+ it("loading metadata on the parent dir we wrote now fails", async () => {
306
+ await assertMetaFail(fixture.api, randomParent)
307
+ })
234
308
  })
235
- })
309
+ }
236
310
 
237
311
  describe(`${driverName} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
238
312
  it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
package/util/crypt.js CHANGED
@@ -36,27 +36,28 @@ function getCipher(enc) {
36
36
  return crypto.createCipheriv(enc.algo, enc.key, enc.iv);
37
37
  }
38
38
 
39
- function encrypt (plainText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'base64') {
40
- if (!key) {
39
+ function encrypt (plainText, encryption, outputEncoding = 'base64') {
40
+ if (!encryption || !encryption.key) {
41
41
  if (WARN_PLAINTEXT) {
42
- console.warn(` ****** key is undefined, encryption of user-data is DISABLED`)
42
+ console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
43
43
  }
44
44
  return plainText
45
45
  }
46
- const cipher = getCipher(key, iv)
47
- return Buffer.concat([cipher.update(plainText), cipher.final()]).toString(outputEncoding)
46
+ const cipher = getCipher(encryption)
47
+ const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
48
+ return encoded.toString(outputEncoding)
48
49
  }
49
50
 
50
51
  function getDecipher(enc) {
51
52
  return crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
52
53
  }
53
54
 
54
- function decrypt (cipherText, key = KEY, iv = CRYPTO_IV, outputEncoding = 'utf8') {
55
- if (!key) {
55
+ function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
56
+ if (!encryption || !encryption.key) {
56
57
  return cipherText
57
58
  }
58
- const cipher = getDecipher(key, iv)
59
- const data = Buffer.from(cipherText, 'base64')
59
+ const cipher = getDecipher(encryption)
60
+ const data = Buffer.from(cipherText, inputEncoding)
60
61
  return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
61
62
  }
62
63