mobiletto 1.0.7 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/drivers/local/index.js +1 -3
- package/drivers/s3/index.js +2 -3
- package/index.js +40 -13
- package/package.json +1 -1
- package/test/index.spec.js +56 -20
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
|
|
|
5
5
|
|
|
6
6
|
It supports apps that are agnostic to where files are stored.
|
|
7
7
|
|
|
8
|
-
All drivers are tested for identical behavior with
|
|
8
|
+
All drivers are tested for identical behavior with over 35 tests for each driver.
|
|
9
9
|
|
|
10
10
|
Today the supported drivers are:
|
|
11
11
|
|
|
@@ -22,6 +22,7 @@ Today the supported drivers are:
|
|
|
22
22
|
// * key: base directory
|
|
23
23
|
// * secret: ignored, can be null
|
|
24
24
|
// * opts object:
|
|
25
|
+
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
25
26
|
// * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
|
|
26
27
|
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
|
|
27
28
|
|
|
@@ -29,6 +30,7 @@ Today the supported drivers are:
|
|
|
29
30
|
// * key: AWS access key
|
|
30
31
|
// * secret: AWS secret key
|
|
31
32
|
// * opts object:
|
|
33
|
+
// * readOnly: optional, never change anything on the bucket; default is false
|
|
32
34
|
// * bucket: required, name of the S3 bucket
|
|
33
35
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
34
36
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
@@ -69,6 +71,10 @@ Today the supported drivers are:
|
|
|
69
71
|
local.write(path, generator)
|
|
70
72
|
s3.write(path, generator)
|
|
71
73
|
|
|
74
|
+
// Write an entire file at once (convenience method)
|
|
75
|
+
await local.writeFile(path, bufferOrString) // returns count of bytes written
|
|
76
|
+
await s3.writeFile(path, bufferOrString) // returns count of bytes written
|
|
77
|
+
|
|
72
78
|
// Delete a file
|
|
73
79
|
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
74
80
|
const quiet = true
|
package/drivers/local/index.js
CHANGED
package/drivers/s3/index.js
CHANGED
|
@@ -43,9 +43,8 @@ class StorageClient {
|
|
|
43
43
|
this.client = new S3Client({region: this.region, credentials})
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
46
|
+
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
47
|
+
testConfig = async () => await this._list('', { MaxKeys: 1 })
|
|
49
48
|
|
|
50
49
|
async list (path = '') {
|
|
51
50
|
return (await this._list(path)).map(obj => {
|
package/index.js
CHANGED
|
@@ -32,24 +32,32 @@ function MobilettoNotFoundError (message) {
|
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
const reader = (chunks) => (chunk) => {
|
|
36
|
-
if (chunk) {
|
|
37
|
-
chunks.push(chunk)
|
|
38
|
-
}
|
|
39
|
-
}
|
|
35
|
+
const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
40
36
|
|
|
41
37
|
const UTILITY_FUNCTIONS = {
|
|
42
38
|
readFile: (client) => async (path) => {
|
|
43
39
|
const chunks = []
|
|
44
40
|
await client.read(path, reader(chunks))
|
|
45
41
|
return Buffer.concat(chunks)
|
|
42
|
+
},
|
|
43
|
+
writeFile: (client) => async (path, data) => {
|
|
44
|
+
const readFunc = function* () { yield data }
|
|
45
|
+
return await client.write(path, readFunc)
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
function addUtilityFunctions (client) {
|
|
49
|
+
function addUtilityFunctions (client, readOnly = false) {
|
|
50
50
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
51
51
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
52
52
|
}
|
|
53
|
+
if (readOnly) {
|
|
54
|
+
for (const writeFunc of ['write', 'remove', 'writeFile']) {
|
|
55
|
+
client[writeFunc] = async () => {
|
|
56
|
+
// console.warn(`${writeFunc} not supported in readOnly mode`)
|
|
57
|
+
return false
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
53
61
|
return client
|
|
54
62
|
}
|
|
55
63
|
|
|
@@ -63,8 +71,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
63
71
|
if (!(await client.testConfig())) {
|
|
64
72
|
throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
|
|
65
73
|
}
|
|
74
|
+
const readOnly = opts ? !!opts.readOnly : false
|
|
66
75
|
if (encryption === null) {
|
|
67
|
-
return addUtilityFunctions(client)
|
|
76
|
+
return addUtilityFunctions(client, readOnly)
|
|
68
77
|
}
|
|
69
78
|
const encKey = normalizeKey(encryption.key)
|
|
70
79
|
if (!encKey) {
|
|
@@ -90,10 +99,19 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
90
99
|
}
|
|
91
100
|
return newPath + encrypted
|
|
92
101
|
}
|
|
102
|
+
const metaPath = path => encryptPath(path + ' ~ META')
|
|
93
103
|
const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
|
|
94
104
|
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
95
105
|
|
|
96
106
|
const _metadata = (client) => async (path) => {
|
|
107
|
+
let metaObj
|
|
108
|
+
try {
|
|
109
|
+
let chunks = []
|
|
110
|
+
await client.read(metaPath(path), reader(chunks))
|
|
111
|
+
metaObj = JSON.parse(decrypt(Buffer.concat(chunks).toString(), enc))
|
|
112
|
+
} catch (e) {
|
|
113
|
+
metaObj = {}
|
|
114
|
+
}
|
|
97
115
|
let meta
|
|
98
116
|
try {
|
|
99
117
|
meta = await client.metadata(encryptPath(path))
|
|
@@ -119,7 +137,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
119
137
|
}
|
|
120
138
|
}
|
|
121
139
|
meta.name = path // rewrite name back to plaintext name
|
|
122
|
-
return meta
|
|
140
|
+
return Object.assign({}, meta, metaObj)
|
|
123
141
|
}
|
|
124
142
|
|
|
125
143
|
const _loadMeta = async (dirent, entries) => {
|
|
@@ -139,6 +157,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
139
157
|
const df = direntFile(direntDir(dirname(path)), path);
|
|
140
158
|
await client.remove(df, {recursive: false, quiet: true})
|
|
141
159
|
await client.remove(encryptPath(path), {recursive: false, quiet: true})
|
|
160
|
+
await client.remove(metaPath(path), {recursive: false, quiet: true})
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function stringGenerator (value, enc) {
|
|
164
|
+
return function* () {
|
|
165
|
+
yield encrypt(value, enc)
|
|
166
|
+
}
|
|
142
167
|
}
|
|
143
168
|
|
|
144
169
|
const encClient = {
|
|
@@ -165,9 +190,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
165
190
|
// if encrypting paths, write dirent file(s) for all parent directories
|
|
166
191
|
let p = path
|
|
167
192
|
while (true) {
|
|
168
|
-
const direntGenerator =
|
|
169
|
-
yield encrypt(p + enc.encPathPadding(), enc)
|
|
170
|
-
}
|
|
193
|
+
const direntGenerator = stringGenerator(p + enc.encPathPadding(), enc)
|
|
171
194
|
const dir = direntDir(dirname(p))
|
|
172
195
|
const df = direntFile(dir, p);
|
|
173
196
|
if (!(await client.write(df, direntGenerator()))) {
|
|
@@ -179,9 +202,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
179
202
|
}
|
|
180
203
|
}
|
|
181
204
|
|
|
205
|
+
let bytesRead = 0
|
|
182
206
|
function* cryptGenerator(plaintextGenerator) {
|
|
183
207
|
let chunk = plaintextGenerator.next().value
|
|
184
208
|
while (chunk) {
|
|
209
|
+
bytesRead += chunk.length
|
|
185
210
|
yield cipher.update(chunk)
|
|
186
211
|
chunk = plaintextGenerator.next().value
|
|
187
212
|
}
|
|
@@ -189,7 +214,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
189
214
|
}
|
|
190
215
|
const cipher = crypt.startEncryptStream(enc)
|
|
191
216
|
const realPath = encryptPath(path)
|
|
192
|
-
|
|
217
|
+
await client.write(realPath, cryptGenerator(readFunc))
|
|
218
|
+
await client.write(metaPath(path), stringGenerator(JSON.stringify({ size: bytesRead }), enc)())
|
|
219
|
+
return bytesRead
|
|
193
220
|
},
|
|
194
221
|
remove: async (path, options) => {
|
|
195
222
|
const recursive = (options && options.recursive) || false
|
|
@@ -247,7 +274,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
247
274
|
return true
|
|
248
275
|
}
|
|
249
276
|
}
|
|
250
|
-
return addUtilityFunctions(encClient)
|
|
277
|
+
return addUtilityFunctions(encClient, readOnly)
|
|
251
278
|
}
|
|
252
279
|
|
|
253
280
|
async function readStream(stream, callback, endCallback) {
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -34,20 +34,11 @@ DRIVER_CONFIG = {
|
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
function
|
|
38
|
-
expect(Math.abs(expected - actual)).to.be.lessThan(Math.floor(expected * percent),
|
|
39
|
-
'expected size within 5% of actual (due to encryption)')
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
|
|
37
|
+
async function assertMeta (api, name, expectedSize) {
|
|
43
38
|
const meta = await api.metadata(name)
|
|
44
39
|
should().exist(meta, 'expected return value from metadata call')
|
|
45
40
|
expect(meta.name).equals(name, 'expected name of written file to be correct')
|
|
46
|
-
|
|
47
|
-
closeEnough(expectedSize, meta.size, closeEnoughPercent)
|
|
48
|
-
} else {
|
|
49
|
-
expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
|
|
50
|
-
}
|
|
41
|
+
expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
|
|
51
42
|
}
|
|
52
43
|
|
|
53
44
|
async function assertMetaFail (api, name) {
|
|
@@ -91,6 +82,8 @@ describe('crypto test', () => {
|
|
|
91
82
|
|
|
92
83
|
})
|
|
93
84
|
|
|
85
|
+
const encryptionTests = () => [null, { key: rand(32) }]
|
|
86
|
+
|
|
94
87
|
// To test a single driver:
|
|
95
88
|
// - Uncomment one of the lines below to set driverName to the one you want to test
|
|
96
89
|
// - Comment out the next `for` line, and its closing curly brace (just before EOF)
|
|
@@ -156,23 +149,70 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
156
149
|
})
|
|
157
150
|
})
|
|
158
151
|
|
|
152
|
+
for (const encryption of encryptionTests()) {
|
|
153
|
+
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
154
|
+
describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
155
|
+
// some random data, plus a bit extra
|
|
156
|
+
const size = 16
|
|
157
|
+
const randomData = rand(size)
|
|
158
|
+
const fileSuffix = '' + Date.now()
|
|
159
|
+
let fixture
|
|
160
|
+
beforeEach((done) => {
|
|
161
|
+
const name = `test_file_${fileSuffix}`
|
|
162
|
+
const opts = Object.assign({}, config.opts, {readOnly: true})
|
|
163
|
+
connect(driverName, config.key, config.secret, opts, encryption)
|
|
164
|
+
.then(api => { fixture = {api, name, randomData} })
|
|
165
|
+
.catch(err => { throw err })
|
|
166
|
+
.finally(done)
|
|
167
|
+
})
|
|
168
|
+
it("should try to write a file and fail", async () => {
|
|
169
|
+
const response = await writeRandomFile(fixture, size)
|
|
170
|
+
expect(response).to.be.false
|
|
171
|
+
})
|
|
172
|
+
it("should fail to read the file we did not write", async () => {
|
|
173
|
+
try {
|
|
174
|
+
await fixture.api.readFile(fixture.name)
|
|
175
|
+
assert.fail('should not have been able to read non-existent file that we did not write')
|
|
176
|
+
} catch (e) {
|
|
177
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
178
|
+
assert.fail(`unexpected error: ${e} trying to read non-existent file`)
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
})
|
|
182
|
+
it("should fail to load metadata on the file we did not write", async () => {
|
|
183
|
+
try {
|
|
184
|
+
await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
|
|
185
|
+
assert.fail('should not have been able to read metadata for non-existent file that we did not write')
|
|
186
|
+
} catch (e) {
|
|
187
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
188
|
+
assert.fail(`unexpected error: ${e} trying to read metadata on non-existent file`)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
})
|
|
192
|
+
it("should fail to delete the file we did not write", async () => {
|
|
193
|
+
const removed = await fixture.api.remove(fixture.name)
|
|
194
|
+
expect(removed).to.be.false
|
|
195
|
+
})
|
|
196
|
+
})
|
|
197
|
+
}
|
|
198
|
+
|
|
159
199
|
describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
|
|
160
200
|
// some random data, plus a bit extra
|
|
161
201
|
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
162
202
|
const randomData = rand(size)
|
|
163
203
|
const fileSuffix = '' + Date.now()
|
|
164
204
|
const encryptionKey = rand(32)
|
|
165
|
-
let encryptedByteCount = null
|
|
166
205
|
let fixture
|
|
167
206
|
beforeEach((done) => {
|
|
168
207
|
const name = `test_file_${fileSuffix}`
|
|
169
208
|
mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
|
|
170
209
|
.then(api => { fixture = {api, name, randomData} })
|
|
210
|
+
.catch((err) => { throw err })
|
|
171
211
|
.finally(done)
|
|
172
212
|
})
|
|
173
213
|
it("should write an encrypted file", async () => {
|
|
174
|
-
|
|
175
|
-
|
|
214
|
+
const bytesWritten = await writeRandomFile(fixture, size);
|
|
215
|
+
expect(bytesWritten).is.equal(size, 'expected write API to return correct number of bytes written')
|
|
176
216
|
})
|
|
177
217
|
it("should read the encrypted file we just wrote", async () => {
|
|
178
218
|
const data = await fixture.api.readFile(fixture.name);
|
|
@@ -191,7 +231,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
191
231
|
})
|
|
192
232
|
})
|
|
193
233
|
|
|
194
|
-
for (const encryption of
|
|
234
|
+
for (const encryption of encryptionTests()) {
|
|
195
235
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
196
236
|
describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
|
|
197
237
|
// describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|
|
@@ -218,11 +258,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
218
258
|
|
|
219
259
|
for (let i = 0; i < fileCount; i++) {
|
|
220
260
|
const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
|
|
221
|
-
|
|
222
|
-
closeEnough(READ_SZ, bytesWritten)
|
|
223
|
-
} else {
|
|
224
|
-
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
225
|
-
}
|
|
261
|
+
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
226
262
|
}
|
|
227
263
|
})
|
|
228
264
|
it("should load metadata for one of the new files", async () => {
|