mobiletto 1.0.6 → 1.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -5,10 +5,15 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
5
5
 
6
6
  It supports apps that are agnostic to where files are stored.
7
7
 
8
- All drivers are tested for identical behavior with 30 tests for each driver.
8
+ All drivers are tested for identical behavior with over 35 tests for each driver.
9
+
10
+ Today the supported drivers are:
11
+
12
+ * `s3`: read/write an Amazon S3 bucket
13
+ * `local`: read/write to local filesystem
9
14
 
10
15
  # Basic usage
11
- const { mobiletto, readFile } = require('mobiletto')
16
+ const { mobiletto } = require('mobiletto')
12
17
 
13
18
  // General usage
14
19
  const api = await mobiletto(driverName, key, secret, opts)
@@ -17,6 +22,7 @@ All drivers are tested for identical behavior with 30 tests for each driver.
17
22
  // * key: base directory
18
23
  // * secret: ignored, can be null
19
24
  // * opts object:
25
+ // * readOnly: optional, never change anything on the filesystem; default is false
20
26
  // * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
21
27
  const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
22
28
 
@@ -24,6 +30,7 @@ All drivers are tested for identical behavior with 30 tests for each driver.
24
30
  // * key: AWS access key
25
31
  // * secret: AWS secret key
26
32
  // * opts object:
33
+ // * readOnly: optional, never change anything on the bucket; default is false
27
34
  // * bucket: required, name of the S3 bucket
28
35
  // * region: optional, the AWS region to communicate with, default is us-east-1
29
36
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
@@ -64,6 +71,10 @@ All drivers are tested for identical behavior with 30 tests for each driver.
64
71
  local.write(path, generator)
65
72
  s3.write(path, generator)
66
73
 
74
+ // Write an entire file at once (convenience method)
75
+ await local.writeFile(path, bufferOrString) // returns count of bytes written
76
+ await s3.writeFile(path, bufferOrString) // returns count of bytes written
77
+
67
78
  // Delete a file
68
79
  // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
69
80
  const quiet = true
@@ -75,6 +86,13 @@ All drivers are tested for identical behavior with 30 tests for each driver.
75
86
  local.remove(path, {recursive, quiet})
76
87
  s3.remove(path, {recursive, quiet})
77
88
 
89
+ # Alternate usage
90
+ Import the fully-scoped module and use the `connect` function:
91
+
92
+ const storage = require('mobiletto')
93
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
94
+ const objectData = await s3.readFile('some/path')
95
+
78
96
  # Transparent Encryption
79
97
  Enable transparent client-side encryption:
80
98
 
@@ -82,36 +100,29 @@ Enable transparent client-side encryption:
82
100
  const encryption = {
83
101
  key: randomstring.generate(128), // required, must be >= 32 chars
84
102
  iv: randomstring.generate(128), // optional, default is to derive IV from key
85
- algo: 'aes-256-cbc', // optional, aes-256-cbc is the default
86
- encryptPaths: true // optional, default is true
103
+ algo: 'aes-256-cbc' // optional, aes-256-cbc is the default
87
104
  }
88
105
  const api = await mobiletto(driverName, key, secret, opts, encryption)
89
106
 
90
107
  // Subsequent write operations will encrypt data (client side) when writing
91
108
  // Subsequent read operations will decrypt data (client side) when reading
92
- // If `encryptPaths` is true, then path names will be hashed using the encrpytion key
93
109
 
94
- Note that when `encryptPaths` is enabled, `list` commands are considerably more inefficient,
95
- especially for directories with a large number of files.
96
-
97
- What's happening? A separate "directory entry" directory (encrypted) tracks what files are in that
110
+ What's happening? A separate "directory entry" (dirent) directory (encrypted) tracks what files are in that
98
111
  directory (aka the dirent directory).
99
112
  * The `list` command reads the directory entry files, decrypts each path listed; then returns metadata for each file
113
+ * `list` commands are more inefficient, especially for directories with a large number of files
100
114
  * The `write` command writes dirent files in each parent's dirent directory, recursively; then writes the file
115
+ * `write` commands will incur O(N) writes, with N = depth in the directory hierarchy
101
116
  * The `remove` command removes the corresponding dirent file, and its parent if empty, recursively; then removes the file
102
- * Note: Recursive removal on large and deep filesystems can be an expensive operation
103
-
104
- Note that even with `encryptPaths` enabled, an adversary with full visibility into your encrypted storage,
105
- even without the key, can still see the total number of directories and how many files are in each, and with some
106
- effort, discover some or all of the overall hierarchical structure. They would not know the names of the
107
- directories/files unless they also knew your encryption key or had otherwise successfully cracked the encryption.
108
- All bets are off then!
109
-
110
- # Alternate usage: import fully scoped module and use `connect`
111
-
112
- const storage = require('mobiletto')
113
- const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
114
- const objectData = await s3.readFile('some/path')
117
+ * Non-recursive `remove` commands will incur O(N) reads and potentially as many deletes, with N = depth in the directory hierarchy
118
+ * Recursive `remove` commands on large and deep filesystems can be expensive
119
+
120
+ Note that even with client-side encryption enabled, an adversary with full visibility into your encrypted server-side
121
+ storage, even without the key, can still see the total number of directories and how many files are in each, and with
122
+ some effort, discover some or all of the overall structure of the directory hierarchy.
123
+ *Note: Use a relatively flat structure for better security.*
124
+ The adversary would not know the names of the directories/files unless they also knew your encryption
125
+ key or had otherwise successfully cracked the encryption. All bets are off then!
115
126
 
116
127
  # Driver Interface
117
128
  A driver is any JS file that exports a 'storageClient' function with this signature:
package/index.js CHANGED
@@ -43,13 +43,25 @@ const UTILITY_FUNCTIONS = {
43
43
  const chunks = []
44
44
  await client.read(path, reader(chunks))
45
45
  return Buffer.concat(chunks)
46
+ },
47
+ writeFile: (client) => async (path, data) => {
48
+ const readFunc = function* () { yield data }
49
+ return await client.write(path, readFunc)
46
50
  }
47
51
  }
48
52
 
49
- function addUtilityFunctions (client) {
53
+ function addUtilityFunctions (client, readOnly = false) {
50
54
  for (const func of Object.keys(UTILITY_FUNCTIONS)) {
51
55
  client[func] = UTILITY_FUNCTIONS[func](client)
52
56
  }
57
+ if (readOnly) {
58
+ for (const writeFunc of ['write', 'remove', 'writeFile']) {
59
+ client[writeFunc] = async () => {
60
+ // console.warn(`${writeFunc} not supported in readOnly mode`)
61
+ return false
62
+ }
63
+ }
64
+ }
53
65
  return client
54
66
  }
55
67
 
@@ -63,8 +75,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
63
75
  if (!(await client.testConfig())) {
64
76
  throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
65
77
  }
78
+ const readOnly = opts ? !!opts.readOnly : false
66
79
  if (encryption === null) {
67
- return addUtilityFunctions(client)
80
+ return addUtilityFunctions(client, readOnly)
68
81
  }
69
82
  const encKey = normalizeKey(encryption.key)
70
83
  if (!encKey) {
@@ -74,12 +87,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
74
87
  if (!iv) {
75
88
  throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption IV`)
76
89
  }
77
- const encryptPaths = encryption.encryptPaths || true;
78
90
  const enc = {
79
91
  key: encKey,
80
92
  iv,
81
93
  algo: encryption.algo || DEFAULT_CRYPT_ALGO,
82
- encryptPaths,
83
94
  dirLevels: encryption.dirLevels || 4,
84
95
  encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
85
96
  }
@@ -92,13 +103,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
92
103
  }
93
104
  return newPath + encrypted
94
105
  }
95
- const direntDir = dir => encryptPaths ? encryptPath(dir + DIR_ENT_DIR_SUFFIX) : null
106
+ const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
96
107
  const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
97
108
 
98
109
  const _metadata = (client) => async (path) => {
99
- if (!encryptPaths) {
100
- return client.metadata(path)
101
- }
102
110
  let meta
103
111
  try {
104
112
  meta = await client.metadata(encryptPath(path))
@@ -148,9 +156,6 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
148
156
 
149
157
  const encClient = {
150
158
  list: async (path) => {
151
- if (!encryptPaths) {
152
- return await client.list(path)
153
- }
154
159
  const dirent = direntDir(path)
155
160
  const entries = await client.list(dirent)
156
161
  if (!entries || entries.length === 0) {
@@ -160,7 +165,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
160
165
  },
161
166
  metadata: _metadata(client),
162
167
  read: async (path, callback) => {
163
- const realPath = encryptPaths ? encryptPath(path) : path
168
+ const realPath = encryptPath(path)
164
169
  const cipher = crypt.startDecryptStream(enc)
165
170
  return client.read(realPath,
166
171
  (chunk) => {
@@ -171,21 +176,19 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
171
176
  },
172
177
  write: async (path, readFunc) => {
173
178
  // if encrypting paths, write dirent file(s) for all parent directories
174
- if (encryptPaths) {
175
- let p = path
176
- while (true) {
177
- const direntGenerator = function* () {
178
- yield encrypt(p + enc.encPathPadding(), enc)
179
- }
180
- const dir = direntDir(dirname(p))
181
- const df = direntFile(dir, p);
182
- if (!(await client.write(df, direntGenerator()))) {
183
- throw new MobilettoError('write: error writing dirent file')
184
- }
185
- p = dirname(p)
186
- if (p === '.' || p === '') {
187
- break
188
- }
179
+ let p = path
180
+ while (true) {
181
+ const direntGenerator = function* () {
182
+ yield encrypt(p + enc.encPathPadding(), enc)
183
+ }
184
+ const dir = direntDir(dirname(p))
185
+ const df = direntFile(dir, p);
186
+ if (!(await client.write(df, direntGenerator()))) {
187
+ throw new MobilettoError('write: error writing dirent file')
188
+ }
189
+ p = dirname(p)
190
+ if (p === '.' || p === '') {
191
+ break
189
192
  }
190
193
  }
191
194
 
@@ -203,11 +206,6 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
203
206
  },
204
207
  remove: async (path, options) => {
205
208
  const recursive = (options && options.recursive) || false
206
- const quiet = (options && options.quiet) || false
207
- if (!encryptPaths) {
208
- return client.remove(path, {recursive, quiet})
209
- }
210
-
211
209
  if (recursive) {
212
210
  // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
213
211
  async function recRm (path) {
@@ -262,7 +260,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
262
260
  return true
263
261
  }
264
262
  }
265
- return addUtilityFunctions(encClient)
263
+ return addUtilityFunctions(encClient, readOnly)
266
264
  }
267
265
 
268
266
  async function readStream(stream, callback, endCallback) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.6",
3
+ "version": "1.0.9",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -91,6 +91,8 @@ describe('crypto test', () => {
91
91
 
92
92
  })
93
93
 
94
+ const ENC_TESTS = [null, {key: rand(32)}]
95
+
94
96
  // To test a single driver:
95
97
  // - Uncomment one of the lines below to set driverName to the one you want to test
96
98
  // - Comment out the next `for` line, and its closing curly brace (just before EOF)
@@ -156,6 +158,54 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
156
158
  })
157
159
  })
158
160
 
161
+ for (const encryption of ENC_TESTS) {
162
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
163
+ describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
164
+ // some random data, plus a bit extra
165
+ const size = 16
166
+ const randomData = rand(size)
167
+ const fileSuffix = '' + Date.now()
168
+ const encryption = {key: rand(32)}
169
+ let fixture
170
+ beforeEach((done) => {
171
+ const name = `test_file_${fileSuffix}`
172
+ const opts = Object.assign({}, config.opts, {readOnly: true})
173
+ connect(driverName, config.key, config.secret, opts, encryption)
174
+ .then(api => { fixture = {api, name, randomData} })
175
+ .catch(err => { throw err })
176
+ .finally(done)
177
+ })
178
+ it("should try to write a file and fail", async () => {
179
+ const response = await writeRandomFile(fixture, size)
180
+ expect(response).to.be.false
181
+ })
182
+ it("should fail to read the file we did not write", async () => {
183
+ try {
184
+ await fixture.api.readFile(fixture.name)
185
+ assert.fail('should not have been able to read non-existent file that we did not write')
186
+ } catch (e) {
187
+ if (!(e instanceof MobilettoNotFoundError)) {
188
+ assert.fail(`unexpected error: ${e} trying to read non-existent file`)
189
+ }
190
+ }
191
+ })
192
+ it("should fail to load metadata on the file we did not write", async () => {
193
+ try {
194
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
195
+ assert.fail('should not have been able to read metadata for non-existent file that we did not write')
196
+ } catch (e) {
197
+ if (!(e instanceof MobilettoNotFoundError)) {
198
+ assert.fail(`unexpected error: ${e} trying to read metadata on non-existent file`)
199
+ }
200
+ }
201
+ })
202
+ it("should fail to delete the file we did not write", async () => {
203
+ const removed = await fixture.api.remove(fixture.name)
204
+ expect(removed).to.be.false
205
+ })
206
+ })
207
+ }
208
+
159
209
  describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
160
210
  // some random data, plus a bit extra
161
211
  const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
@@ -168,6 +218,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
168
218
  const name = `test_file_${fileSuffix}`
169
219
  mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
170
220
  .then(api => { fixture = {api, name, randomData} })
221
+ .catch((err) => { throw err })
171
222
  .finally(done)
172
223
  })
173
224
  it("should write an encrypted file", async () => {
@@ -191,7 +242,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
191
242
  })
192
243
  })
193
244
 
194
- for (const encryption of [null, {key: rand(32)}]) {
245
+ for (const encryption of ENC_TESTS) {
195
246
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
196
247
  describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
197
248
  // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {