mobiletto 1.0.6 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -22
- package/index.js +31 -33
- package/package.json +1 -1
- package/test/index.spec.js +52 -1
package/README.md
CHANGED
|
@@ -5,10 +5,15 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
|
|
|
5
5
|
|
|
6
6
|
It supports apps that are agnostic to where files are stored.
|
|
7
7
|
|
|
8
|
-
All drivers are tested for identical behavior with
|
|
8
|
+
All drivers are tested for identical behavior with over 35 tests for each driver.
|
|
9
|
+
|
|
10
|
+
Today the supported drivers are:
|
|
11
|
+
|
|
12
|
+
* `s3`: read/write an Amazon S3 bucket
|
|
13
|
+
* `local`: read/write to local filesystem
|
|
9
14
|
|
|
10
15
|
# Basic usage
|
|
11
|
-
const { mobiletto
|
|
16
|
+
const { mobiletto } = require('mobiletto')
|
|
12
17
|
|
|
13
18
|
// General usage
|
|
14
19
|
const api = await mobiletto(driverName, key, secret, opts)
|
|
@@ -17,6 +22,7 @@ All drivers are tested for identical behavior with 30 tests for each driver.
|
|
|
17
22
|
// * key: base directory
|
|
18
23
|
// * secret: ignored, can be null
|
|
19
24
|
// * opts object:
|
|
25
|
+
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
20
26
|
// * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
|
|
21
27
|
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
|
|
22
28
|
|
|
@@ -24,6 +30,7 @@ All drivers are tested for identical behavior with 30 tests for each driver.
|
|
|
24
30
|
// * key: AWS access key
|
|
25
31
|
// * secret: AWS secret key
|
|
26
32
|
// * opts object:
|
|
33
|
+
// * readOnly: optional, never change anything on the bucket; default is false
|
|
27
34
|
// * bucket: required, name of the S3 bucket
|
|
28
35
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
29
36
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
@@ -64,6 +71,10 @@ All drivers are tested for identical behavior with 30 tests for each driver.
|
|
|
64
71
|
local.write(path, generator)
|
|
65
72
|
s3.write(path, generator)
|
|
66
73
|
|
|
74
|
+
// Write an entire file at once (convenience method)
|
|
75
|
+
await local.writeFile(path, bufferOrString) // returns count of bytes written
|
|
76
|
+
await s3.writeFile(path, bufferOrString) // returns count of bytes written
|
|
77
|
+
|
|
67
78
|
// Delete a file
|
|
68
79
|
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
69
80
|
const quiet = true
|
|
@@ -75,6 +86,13 @@ All drivers are tested for identical behavior with 30 tests for each driver.
|
|
|
75
86
|
local.remove(path, {recursive, quiet})
|
|
76
87
|
s3.remove(path, {recursive, quiet})
|
|
77
88
|
|
|
89
|
+
# Alternate usage
|
|
90
|
+
Import the fully-scoped module and use the `connect` function:
|
|
91
|
+
|
|
92
|
+
const storage = require('mobiletto')
|
|
93
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
94
|
+
const objectData = await s3.readFile('some/path')
|
|
95
|
+
|
|
78
96
|
# Transparent Encryption
|
|
79
97
|
Enable transparent client-side encryption:
|
|
80
98
|
|
|
@@ -82,36 +100,29 @@ Enable transparent client-side encryption:
|
|
|
82
100
|
const encryption = {
|
|
83
101
|
key: randomstring.generate(128), // required, must be >= 32 chars
|
|
84
102
|
iv: randomstring.generate(128), // optional, default is to derive IV from key
|
|
85
|
-
algo: 'aes-256-cbc'
|
|
86
|
-
encryptPaths: true // optional, default is true
|
|
103
|
+
algo: 'aes-256-cbc' // optional, aes-256-cbc is the default
|
|
87
104
|
}
|
|
88
105
|
const api = await mobiletto(driverName, key, secret, opts, encryption)
|
|
89
106
|
|
|
90
107
|
// Subsequent write operations will encrypt data (client side) when writing
|
|
91
108
|
// Subsequent read operations will decrypt data (client side) when reading
|
|
92
|
-
// If `encryptPaths` is true, then path names will be hashed using the encrpytion key
|
|
93
109
|
|
|
94
|
-
|
|
95
|
-
especially for directories with a large number of files.
|
|
96
|
-
|
|
97
|
-
What's happening? A separate "directory entry" directory (encrypted) tracks what files are in that
|
|
110
|
+
What's happening? A separate "directory entry" (dirent) directory (encrypted) tracks what files are in that
|
|
98
111
|
directory (aka the dirent directory).
|
|
99
112
|
* The `list` command reads the directory entry files, decrypts each path listed; then returns metadata for each file
|
|
113
|
+
* `list` commands are more inefficient, especially for directories with a large number of files
|
|
100
114
|
* The `write` command writes dirent files in each parent's dirent directory, recursively; then writes the file
|
|
115
|
+
* `write` commands will incur O(N) writes, with N = depth in the directory hierarchy
|
|
101
116
|
* The `remove` command removes the corresponding dirent file, and its parent if empty, recursively; then removes the file
|
|
102
|
-
*
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
const storage = require('mobiletto')
|
|
113
|
-
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
114
|
-
const objectData = await s3.readFile('some/path')
|
|
117
|
+
* Non-recursive `remove` commands will incur O(N) reads and potentially as many deletes, with N = depth in the directory hierarchy
|
|
118
|
+
* Recursive `remove` commands on large and deep filesystems can be expensive
|
|
119
|
+
|
|
120
|
+
Note that even with client-side encryption enabled, an adversary with full visibility into your encrypted server-side
|
|
121
|
+
storage, even without the key, can still see the total number of directories and how many files are in each, and with
|
|
122
|
+
some effort, discover some or all of the overall structure of the directory hierarchy.
|
|
123
|
+
*Note: Use a relatively flat structure for better security.*
|
|
124
|
+
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
125
|
+
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
115
126
|
|
|
116
127
|
# Driver Interface
|
|
117
128
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
package/index.js
CHANGED
|
@@ -43,13 +43,25 @@ const UTILITY_FUNCTIONS = {
|
|
|
43
43
|
const chunks = []
|
|
44
44
|
await client.read(path, reader(chunks))
|
|
45
45
|
return Buffer.concat(chunks)
|
|
46
|
+
},
|
|
47
|
+
writeFile: (client) => async (path, data) => {
|
|
48
|
+
const readFunc = function* () { yield data }
|
|
49
|
+
return await client.write(path, readFunc)
|
|
46
50
|
}
|
|
47
51
|
}
|
|
48
52
|
|
|
49
|
-
function addUtilityFunctions (client) {
|
|
53
|
+
function addUtilityFunctions (client, readOnly = false) {
|
|
50
54
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
51
55
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
52
56
|
}
|
|
57
|
+
if (readOnly) {
|
|
58
|
+
for (const writeFunc of ['write', 'remove', 'writeFile']) {
|
|
59
|
+
client[writeFunc] = async () => {
|
|
60
|
+
// console.warn(`${writeFunc} not supported in readOnly mode`)
|
|
61
|
+
return false
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
53
65
|
return client
|
|
54
66
|
}
|
|
55
67
|
|
|
@@ -63,8 +75,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
63
75
|
if (!(await client.testConfig())) {
|
|
64
76
|
throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
|
|
65
77
|
}
|
|
78
|
+
const readOnly = opts ? !!opts.readOnly : false
|
|
66
79
|
if (encryption === null) {
|
|
67
|
-
return addUtilityFunctions(client)
|
|
80
|
+
return addUtilityFunctions(client, readOnly)
|
|
68
81
|
}
|
|
69
82
|
const encKey = normalizeKey(encryption.key)
|
|
70
83
|
if (!encKey) {
|
|
@@ -74,12 +87,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
74
87
|
if (!iv) {
|
|
75
88
|
throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption IV`)
|
|
76
89
|
}
|
|
77
|
-
const encryptPaths = encryption.encryptPaths || true;
|
|
78
90
|
const enc = {
|
|
79
91
|
key: encKey,
|
|
80
92
|
iv,
|
|
81
93
|
algo: encryption.algo || DEFAULT_CRYPT_ALGO,
|
|
82
|
-
encryptPaths,
|
|
83
94
|
dirLevels: encryption.dirLevels || 4,
|
|
84
95
|
encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
|
|
85
96
|
}
|
|
@@ -92,13 +103,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
92
103
|
}
|
|
93
104
|
return newPath + encrypted
|
|
94
105
|
}
|
|
95
|
-
const direntDir = dir =>
|
|
106
|
+
const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
|
|
96
107
|
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
97
108
|
|
|
98
109
|
const _metadata = (client) => async (path) => {
|
|
99
|
-
if (!encryptPaths) {
|
|
100
|
-
return client.metadata(path)
|
|
101
|
-
}
|
|
102
110
|
let meta
|
|
103
111
|
try {
|
|
104
112
|
meta = await client.metadata(encryptPath(path))
|
|
@@ -148,9 +156,6 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
148
156
|
|
|
149
157
|
const encClient = {
|
|
150
158
|
list: async (path) => {
|
|
151
|
-
if (!encryptPaths) {
|
|
152
|
-
return await client.list(path)
|
|
153
|
-
}
|
|
154
159
|
const dirent = direntDir(path)
|
|
155
160
|
const entries = await client.list(dirent)
|
|
156
161
|
if (!entries || entries.length === 0) {
|
|
@@ -160,7 +165,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
160
165
|
},
|
|
161
166
|
metadata: _metadata(client),
|
|
162
167
|
read: async (path, callback) => {
|
|
163
|
-
const realPath =
|
|
168
|
+
const realPath = encryptPath(path)
|
|
164
169
|
const cipher = crypt.startDecryptStream(enc)
|
|
165
170
|
return client.read(realPath,
|
|
166
171
|
(chunk) => {
|
|
@@ -171,21 +176,19 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
171
176
|
},
|
|
172
177
|
write: async (path, readFunc) => {
|
|
173
178
|
// if encrypting paths, write dirent file(s) for all parent directories
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
break
|
|
188
|
-
}
|
|
179
|
+
let p = path
|
|
180
|
+
while (true) {
|
|
181
|
+
const direntGenerator = function* () {
|
|
182
|
+
yield encrypt(p + enc.encPathPadding(), enc)
|
|
183
|
+
}
|
|
184
|
+
const dir = direntDir(dirname(p))
|
|
185
|
+
const df = direntFile(dir, p);
|
|
186
|
+
if (!(await client.write(df, direntGenerator()))) {
|
|
187
|
+
throw new MobilettoError('write: error writing dirent file')
|
|
188
|
+
}
|
|
189
|
+
p = dirname(p)
|
|
190
|
+
if (p === '.' || p === '') {
|
|
191
|
+
break
|
|
189
192
|
}
|
|
190
193
|
}
|
|
191
194
|
|
|
@@ -203,11 +206,6 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
203
206
|
},
|
|
204
207
|
remove: async (path, options) => {
|
|
205
208
|
const recursive = (options && options.recursive) || false
|
|
206
|
-
const quiet = (options && options.quiet) || false
|
|
207
|
-
if (!encryptPaths) {
|
|
208
|
-
return client.remove(path, {recursive, quiet})
|
|
209
|
-
}
|
|
210
|
-
|
|
211
209
|
if (recursive) {
|
|
212
210
|
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
213
211
|
async function recRm (path) {
|
|
@@ -262,7 +260,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
262
260
|
return true
|
|
263
261
|
}
|
|
264
262
|
}
|
|
265
|
-
return addUtilityFunctions(encClient)
|
|
263
|
+
return addUtilityFunctions(encClient, readOnly)
|
|
266
264
|
}
|
|
267
265
|
|
|
268
266
|
async function readStream(stream, callback, endCallback) {
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -91,6 +91,8 @@ describe('crypto test', () => {
|
|
|
91
91
|
|
|
92
92
|
})
|
|
93
93
|
|
|
94
|
+
const ENC_TESTS = [null, {key: rand(32)}]
|
|
95
|
+
|
|
94
96
|
// To test a single driver:
|
|
95
97
|
// - Uncomment one of the lines below to set driverName to the one you want to test
|
|
96
98
|
// - Comment out the next `for` line, and its closing curly brace (just before EOF)
|
|
@@ -156,6 +158,54 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
156
158
|
})
|
|
157
159
|
})
|
|
158
160
|
|
|
161
|
+
for (const encryption of ENC_TESTS) {
|
|
162
|
+
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
163
|
+
describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
164
|
+
// some random data, plus a bit extra
|
|
165
|
+
const size = 16
|
|
166
|
+
const randomData = rand(size)
|
|
167
|
+
const fileSuffix = '' + Date.now()
|
|
168
|
+
const encryption = {key: rand(32)}
|
|
169
|
+
let fixture
|
|
170
|
+
beforeEach((done) => {
|
|
171
|
+
const name = `test_file_${fileSuffix}`
|
|
172
|
+
const opts = Object.assign({}, config.opts, {readOnly: true})
|
|
173
|
+
connect(driverName, config.key, config.secret, opts, encryption)
|
|
174
|
+
.then(api => { fixture = {api, name, randomData} })
|
|
175
|
+
.catch(err => { throw err })
|
|
176
|
+
.finally(done)
|
|
177
|
+
})
|
|
178
|
+
it("should try to write a file and fail", async () => {
|
|
179
|
+
const response = await writeRandomFile(fixture, size)
|
|
180
|
+
expect(response).to.be.false
|
|
181
|
+
})
|
|
182
|
+
it("should fail to read the file we did not write", async () => {
|
|
183
|
+
try {
|
|
184
|
+
await fixture.api.readFile(fixture.name)
|
|
185
|
+
assert.fail('should not have been able to read non-existent file that we did not write')
|
|
186
|
+
} catch (e) {
|
|
187
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
188
|
+
assert.fail(`unexpected error: ${e} trying to read non-existent file`)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
})
|
|
192
|
+
it("should fail to load metadata on the file we did not write", async () => {
|
|
193
|
+
try {
|
|
194
|
+
await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
|
|
195
|
+
assert.fail('should not have been able to read metadata for non-existent file that we did not write')
|
|
196
|
+
} catch (e) {
|
|
197
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
198
|
+
assert.fail(`unexpected error: ${e} trying to read metadata on non-existent file`)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
})
|
|
202
|
+
it("should fail to delete the file we did not write", async () => {
|
|
203
|
+
const removed = await fixture.api.remove(fixture.name)
|
|
204
|
+
expect(removed).to.be.false
|
|
205
|
+
})
|
|
206
|
+
})
|
|
207
|
+
}
|
|
208
|
+
|
|
159
209
|
describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
|
|
160
210
|
// some random data, plus a bit extra
|
|
161
211
|
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
@@ -168,6 +218,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
168
218
|
const name = `test_file_${fileSuffix}`
|
|
169
219
|
mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
|
|
170
220
|
.then(api => { fixture = {api, name, randomData} })
|
|
221
|
+
.catch((err) => { throw err })
|
|
171
222
|
.finally(done)
|
|
172
223
|
})
|
|
173
224
|
it("should write an encrypted file", async () => {
|
|
@@ -191,7 +242,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
191
242
|
})
|
|
192
243
|
})
|
|
193
244
|
|
|
194
|
-
for (const encryption of
|
|
245
|
+
for (const encryption of ENC_TESTS) {
|
|
195
246
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
196
247
|
describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
|
|
197
248
|
// describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|