mobiletto 1.0.2 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -16
- package/drivers/local/index.js +26 -26
- package/drivers/s3/index.js +30 -20
- package/index.js +262 -17
- package/package.json +3 -2
- package/test/index.spec.js +140 -71
- package/util/crypt.js +75 -0
package/README.md
CHANGED
|
@@ -1,53 +1,59 @@
|
|
|
1
1
|
Mobiletto
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
Mobiletto is a JavaScript storage abstraction layer.
|
|
4
|
+
Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
|
|
5
5
|
|
|
6
6
|
It supports apps that are agnostic to where files are stored.
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
const { mobiletto } = require('mobiletto')
|
|
8
|
+
All drivers are tested for identical behavior with 30 tests for each driver.
|
|
10
9
|
|
|
11
|
-
|
|
10
|
+
# Basic usage
|
|
11
|
+
const { mobiletto, readFile } = require('mobiletto')
|
|
12
|
+
|
|
13
|
+
// General usage
|
|
12
14
|
const api = await mobiletto(driverName, key, secret, opts)
|
|
13
15
|
|
|
14
|
-
//
|
|
16
|
+
// To use 'local' driver:
|
|
15
17
|
// * key: base directory
|
|
16
18
|
// * secret: ignored, can be null
|
|
17
19
|
// * opts object:
|
|
18
20
|
// * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
|
|
19
21
|
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
|
|
20
22
|
|
|
21
|
-
//
|
|
23
|
+
// To use 's3' driver:
|
|
22
24
|
// * key: AWS access key
|
|
23
25
|
// * secret: AWS secret key
|
|
24
26
|
// * opts object:
|
|
25
27
|
// * bucket: required, name of the S3 bucket
|
|
26
28
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
27
29
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
28
|
-
// * delimiter: optional, directory delimiter, default is '/'
|
|
30
|
+
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
29
31
|
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
30
32
|
|
|
31
|
-
//
|
|
33
|
+
// List files
|
|
32
34
|
local.list() // --> returns an array of file objects
|
|
33
35
|
s3.list() // --> returns an array of file objects
|
|
34
36
|
|
|
35
|
-
//
|
|
37
|
+
// List files in a directory
|
|
36
38
|
const path = 'some/path'
|
|
37
39
|
local.list(path) // --> returns an array of file objects
|
|
38
40
|
s3.list(path) // --> returns an array of file objects
|
|
39
41
|
|
|
40
|
-
//
|
|
42
|
+
// Read metadata for a file
|
|
41
43
|
local.metadata(path)
|
|
42
44
|
s3.metadata(path)
|
|
43
45
|
|
|
44
|
-
//
|
|
46
|
+
// Read a file
|
|
45
47
|
// Provide a callback that writes the data someplace
|
|
46
48
|
const callback = (chunk) => { ... write chunk somewhere ... }
|
|
47
49
|
local.read(path, callback)
|
|
48
50
|
s3.read(path, callback)
|
|
51
|
+
|
|
52
|
+
// Read an entire file at once (convenience method)
|
|
53
|
+
const localData = await local.readFile(path) // returns a byte Buffer of the file contents
|
|
54
|
+
const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
|
|
49
55
|
|
|
50
|
-
//
|
|
56
|
+
// Write a file
|
|
51
57
|
// Provide a generator function that yields chunks of data
|
|
52
58
|
const generator = function* () {
|
|
53
59
|
while ( ... more-data-to-return ... ) {
|
|
@@ -58,17 +64,55 @@ It supports apps that are agnostic to where files are stored.
|
|
|
58
64
|
local.write(path, generator)
|
|
59
65
|
s3.write(path, generator)
|
|
60
66
|
|
|
61
|
-
//
|
|
67
|
+
// Delete a file
|
|
62
68
|
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
63
69
|
const quiet = true
|
|
64
70
|
local.remove(path, {quiet})
|
|
65
71
|
s3.remove(path, {quiet})
|
|
66
72
|
|
|
67
|
-
//
|
|
73
|
+
// Recursively delete a directory
|
|
68
74
|
const recursive = true
|
|
69
75
|
local.remove(path, {recursive, quiet})
|
|
70
76
|
s3.remove(path, {recursive, quiet})
|
|
71
77
|
|
|
78
|
+
# Transparent Encryption
|
|
79
|
+
Enable transparent client-side encryption:
|
|
80
|
+
|
|
81
|
+
// Pass encryption parameters
|
|
82
|
+
const encryption = {
|
|
83
|
+
key: randomstring.generate(128), // required, must be >= 32 chars
|
|
84
|
+
iv: randomstring.generate(128), // optional, default is to derive IV from key
|
|
85
|
+
algo: 'aes-256-cbc', // optional, aes-256-cbc is the default
|
|
86
|
+
encryptPaths: true // optional, default is true
|
|
87
|
+
}
|
|
88
|
+
const api = await mobiletto(driverName, key, secret, opts, encryption)
|
|
89
|
+
|
|
90
|
+
// Subsequent write operations will encrypt data (client side) when writing
|
|
91
|
+
// Subsequent read operations will decrypt data (client side) when reading
|
|
92
|
+
// If `encryptPaths` is true, then path names will be hashed using the encrpytion key
|
|
93
|
+
|
|
94
|
+
Note that when `encryptPaths` is enabled, `list` commands are considerably more inefficient,
|
|
95
|
+
especially for directories with a large number of files.
|
|
96
|
+
|
|
97
|
+
What's happening? A separate "directory entry" directory (encrypted) tracks what files are in that
|
|
98
|
+
directory (aka the dirent directory).
|
|
99
|
+
* The `list` command reads the directory entry files, decrypts each path listed; then returns metadata for each file
|
|
100
|
+
* The `write` command writes dirent files in each parent's dirent directory, recursively; then writes the file
|
|
101
|
+
* The `remove` command removes the corresponding dirent file, and its parent if empty, recursively; then removes the file
|
|
102
|
+
* Note: Recursive removal on large and deep filesystems can be an expensive operation
|
|
103
|
+
|
|
104
|
+
Note that even with `encryptPaths` enabled, an adversary with full visibility into your encrypted storage,
|
|
105
|
+
even without the key, can still see the total number of directories and how many files are in each, and with some
|
|
106
|
+
effort, discover some or all of the overall hierarchical structure. They would not know the names of the
|
|
107
|
+
directories/files unless they also knew your encryption key or had otherwise successfully cracked the encryption.
|
|
108
|
+
All bets are off then!
|
|
109
|
+
|
|
110
|
+
# Alternate usage: import fully scoped module and use `connect`
|
|
111
|
+
|
|
112
|
+
const storage = require('mobiletto')
|
|
113
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
114
|
+
const objectData = await s3.readFile('some/path')
|
|
115
|
+
|
|
72
116
|
# Driver Interface
|
|
73
117
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
|
74
118
|
|
|
@@ -85,6 +129,9 @@ A driver is any JS file that exports a 'storageClient' function with this signat
|
|
|
85
129
|
|
|
86
130
|
The object that the storageClient function returns must define these functions:
|
|
87
131
|
|
|
132
|
+
// Test the driver before using, ensure proper configuration
|
|
133
|
+
async testConfig ()
|
|
134
|
+
|
|
88
135
|
// Find DNS records that match the given domain, type (if provided) and name (if provided)
|
|
89
136
|
async list (path) // path may be omitted
|
|
90
137
|
|
|
@@ -92,10 +139,12 @@ The object that the storageClient function returns must define these functions:
|
|
|
92
139
|
async metadata (path)
|
|
93
140
|
|
|
94
141
|
// Read a file
|
|
95
|
-
|
|
142
|
+
// callback receives a chunk of data. endCallback is called at end-of-stream
|
|
143
|
+
async read (path, callback, endCallback = null)
|
|
96
144
|
|
|
97
145
|
// Write a file
|
|
98
|
-
|
|
146
|
+
// readFunc returns the data you want to write
|
|
147
|
+
async write (path, readFunc)
|
|
99
148
|
|
|
100
149
|
// Remove a file, or recursively delete a directory
|
|
101
150
|
async remove (path, {recursive = false, quiet = false})
|
package/drivers/local/index.js
CHANGED
|
@@ -2,13 +2,23 @@ const { dirname } = require('path')
|
|
|
2
2
|
|
|
3
3
|
const {
|
|
4
4
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
5
|
-
MobilettoError, writeStream, closeStream
|
|
5
|
+
MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
|
|
6
6
|
} = require('../../index')
|
|
7
7
|
|
|
8
8
|
const fs = require('fs')
|
|
9
9
|
|
|
10
10
|
const DEFAULT_MODE = '0700'
|
|
11
11
|
|
|
12
|
+
const isENOENT = (err) => err.code && err.code === 'ENOENT'
|
|
13
|
+
|
|
14
|
+
const ioError = (err, path, method) => isENOENT(err)
|
|
15
|
+
? err instanceof MobilettoNotFoundError
|
|
16
|
+
? err
|
|
17
|
+
: new MobilettoNotFoundError(path)
|
|
18
|
+
: err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
19
|
+
? err
|
|
20
|
+
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
21
|
+
|
|
12
22
|
class StorageClient {
|
|
13
23
|
baseDir
|
|
14
24
|
mode
|
|
@@ -74,7 +84,7 @@ class StorageClient {
|
|
|
74
84
|
return entry
|
|
75
85
|
})
|
|
76
86
|
} catch (err) {
|
|
77
|
-
throw
|
|
87
|
+
throw ioError(err, path, 'list')
|
|
78
88
|
}
|
|
79
89
|
}
|
|
80
90
|
async metadata (path) {
|
|
@@ -82,19 +92,23 @@ class StorageClient {
|
|
|
82
92
|
try {
|
|
83
93
|
const lstat = fs.lstatSync(file)
|
|
84
94
|
if (!lstat) {
|
|
85
|
-
throw new MobilettoError(
|
|
95
|
+
throw new MobilettoError('metadata: lstat error')
|
|
96
|
+
}
|
|
97
|
+
const type = this.fileType(lstat)
|
|
98
|
+
if (type === M_DIR && path !== '') {
|
|
99
|
+
const contents = await this.list(path)
|
|
100
|
+
if (contents.length === 0) {
|
|
101
|
+
throw new MobilettoNotFoundError(path)
|
|
102
|
+
}
|
|
86
103
|
}
|
|
87
104
|
return {
|
|
88
105
|
name: file.startsWith(this.baseDir) ? file.substring(this.baseDir.length) : file,
|
|
89
|
-
type
|
|
106
|
+
type,
|
|
90
107
|
size: lstat.size,
|
|
91
108
|
mtime: lstat.mtimeMs
|
|
92
109
|
}
|
|
93
110
|
} catch (err) {
|
|
94
|
-
|
|
95
|
-
throw new MobilettoNotFoundError(path)
|
|
96
|
-
}
|
|
97
|
-
throw new MobilettoError(`metadata(${path}) error: ${err}`, err)
|
|
111
|
+
throw ioError(err, path, 'metadata')
|
|
98
112
|
}
|
|
99
113
|
}
|
|
100
114
|
|
|
@@ -143,28 +157,14 @@ class StorageClient {
|
|
|
143
157
|
return Promise.resolve(count)
|
|
144
158
|
}
|
|
145
159
|
|
|
146
|
-
async read (path, callback) {
|
|
160
|
+
async read (path, callback, endCallback = null) {
|
|
147
161
|
const file = this.normalizePath(path)
|
|
148
162
|
// console.log(`read: reading path: ${path} - ${file}`)
|
|
149
163
|
try {
|
|
150
164
|
const stream = fs.createReadStream(file)
|
|
151
|
-
|
|
152
|
-
const streamHandler = stream =>
|
|
153
|
-
new Promise((resolve, reject) => {
|
|
154
|
-
stream.on('data', (data) => {
|
|
155
|
-
count += data ? data.length : 0
|
|
156
|
-
callback(data)
|
|
157
|
-
})
|
|
158
|
-
stream.on('error', reject)
|
|
159
|
-
stream.on('end', resolve)
|
|
160
|
-
})
|
|
161
|
-
await streamHandler(stream)
|
|
162
|
-
return count
|
|
165
|
+
return await readStream(stream, callback, endCallback)
|
|
163
166
|
} catch (err) {
|
|
164
|
-
|
|
165
|
-
throw new MobilettoNotFoundError(path)
|
|
166
|
-
}
|
|
167
|
-
throw new MobilettoError(`read(${path}) error: ${err}`, err)
|
|
167
|
+
throw ioError(err, path, 'read')
|
|
168
168
|
}
|
|
169
169
|
}
|
|
170
170
|
|
|
@@ -176,7 +176,7 @@ class StorageClient {
|
|
|
176
176
|
try {
|
|
177
177
|
fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
|
|
178
178
|
} catch (err) {
|
|
179
|
-
if (err
|
|
179
|
+
if (isENOENT(err)) {
|
|
180
180
|
if (!quiet) {
|
|
181
181
|
throw new MobilettoNotFoundError(path)
|
|
182
182
|
}
|
package/drivers/s3/index.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
const {
|
|
1
|
+
const {
|
|
2
|
+
M_DIR, M_FILE,
|
|
3
|
+
MobilettoError, MobilettoNotFoundError, readStream
|
|
4
|
+
} = require('../../index')
|
|
2
5
|
|
|
3
6
|
const {
|
|
4
7
|
S3Client,
|
|
@@ -77,12 +80,14 @@ class StorageClient {
|
|
|
77
80
|
while (truncated) {
|
|
78
81
|
try {
|
|
79
82
|
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
80
|
-
|
|
83
|
+
const hasContents = typeof response.Contents !== 'undefined'
|
|
84
|
+
if (hasContents) {
|
|
81
85
|
response.Contents.forEach((item) => {
|
|
82
86
|
objects.push(item.Key) // todo: transform into standard object
|
|
83
87
|
})
|
|
84
88
|
}
|
|
85
|
-
|
|
89
|
+
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
90
|
+
if (hasCommonPrefixes) {
|
|
86
91
|
response.CommonPrefixes.forEach((item) => {
|
|
87
92
|
objects.push(item.Prefix) // todo: transform into standard object
|
|
88
93
|
})
|
|
@@ -91,13 +96,22 @@ class StorageClient {
|
|
|
91
96
|
// If truncated is true, advance the marker
|
|
92
97
|
if (truncated) {
|
|
93
98
|
bucketParams.Marker = response.NextMarker
|
|
99
|
+
} else if (!hasContents && !hasCommonPrefixes) {
|
|
100
|
+
if (path === '') {
|
|
101
|
+
break
|
|
102
|
+
}
|
|
103
|
+
throw new MobilettoNotFoundError(path)
|
|
94
104
|
}
|
|
95
|
-
// At end of the list, response.truncated is false, and the function exits the while loop.
|
|
96
105
|
} catch (err) {
|
|
97
|
-
|
|
98
|
-
|
|
106
|
+
if (err instanceof MobilettoNotFoundError) {
|
|
107
|
+
throw err
|
|
108
|
+
}
|
|
109
|
+
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
99
110
|
}
|
|
100
111
|
}
|
|
112
|
+
if (recursive && objects.length === 0) {
|
|
113
|
+
throw new MobilettoNotFoundError(path)
|
|
114
|
+
}
|
|
101
115
|
return objects
|
|
102
116
|
}
|
|
103
117
|
|
|
@@ -166,7 +180,7 @@ class StorageClient {
|
|
|
166
180
|
return total
|
|
167
181
|
}
|
|
168
182
|
|
|
169
|
-
async read (path, callback) {
|
|
183
|
+
async read (path, callback, endCallback = null) {
|
|
170
184
|
const Key = this.normalizeKey(path)
|
|
171
185
|
// console.log(`read: reading Key: ${path} - ${Key}`)
|
|
172
186
|
const bucketParams = {
|
|
@@ -177,18 +191,7 @@ class StorageClient {
|
|
|
177
191
|
}
|
|
178
192
|
try {
|
|
179
193
|
const data = await this.client.send(new GetObjectCommand(bucketParams))
|
|
180
|
-
|
|
181
|
-
const streamHandler = stream =>
|
|
182
|
-
new Promise((resolve, reject) => {
|
|
183
|
-
stream.on('data', (data) => {
|
|
184
|
-
count += data ? data.length : 0
|
|
185
|
-
callback(data)
|
|
186
|
-
})
|
|
187
|
-
stream.on('error', reject)
|
|
188
|
-
stream.on('end', resolve)
|
|
189
|
-
})
|
|
190
|
-
await streamHandler(data.Body)
|
|
191
|
-
return count
|
|
194
|
+
return await readStream(data.Body, callback, endCallback)
|
|
192
195
|
} catch (err) {
|
|
193
196
|
throw this.s3error(err, Key, path, 'read')
|
|
194
197
|
}
|
|
@@ -221,7 +224,14 @@ class StorageClient {
|
|
|
221
224
|
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
222
225
|
throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
223
226
|
}
|
|
224
|
-
|
|
227
|
+
try {
|
|
228
|
+
objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
229
|
+
} catch (e) {
|
|
230
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
231
|
+
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
|
|
232
|
+
}
|
|
233
|
+
objects = null
|
|
234
|
+
}
|
|
225
235
|
}
|
|
226
236
|
} else {
|
|
227
237
|
const Key = this.normalizeKey(path)
|
package/index.js
CHANGED
|
@@ -1,3 +1,14 @@
|
|
|
1
|
+
const { basename, dirname } = require('path')
|
|
2
|
+
const shasum = require('shasum')
|
|
3
|
+
const randomstring = require('randomstring')
|
|
4
|
+
|
|
5
|
+
const crypt = require('./util/crypt')
|
|
6
|
+
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt");
|
|
7
|
+
|
|
8
|
+
const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
9
|
+
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
10
|
+
const ENC_PAD_SEP = ' ~ '
|
|
11
|
+
|
|
1
12
|
// adapted from https://stackoverflow.com/a/27724419
|
|
2
13
|
function MobilettoError (message, err) {
|
|
3
14
|
this.message = `${message}`
|
|
@@ -21,13 +32,257 @@ function MobilettoNotFoundError (message) {
|
|
|
21
32
|
}
|
|
22
33
|
}
|
|
23
34
|
|
|
24
|
-
|
|
35
|
+
const reader = (chunks) => (chunk) => {
|
|
36
|
+
if (chunk) {
|
|
37
|
+
chunks.push(chunk)
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const UTILITY_FUNCTIONS = {
|
|
42
|
+
readFile: (client) => async (path) => {
|
|
43
|
+
const chunks = []
|
|
44
|
+
await client.read(path, reader(chunks))
|
|
45
|
+
return Buffer.concat(chunks)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function addUtilityFunctions (client) {
|
|
50
|
+
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
51
|
+
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
52
|
+
}
|
|
53
|
+
return client
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function connect (driverPath, key, secret, opts, encryption = null) {
|
|
57
|
+
return await mobiletto(driverPath, key, secret, opts, encryption)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
25
61
|
const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
|
|
26
62
|
const client = driver.storageClient(key, secret, opts)
|
|
27
63
|
if (!(await client.testConfig())) {
|
|
28
|
-
throw new MobilettoError(`
|
|
64
|
+
throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
|
|
29
65
|
}
|
|
30
|
-
|
|
66
|
+
if (encryption === null) {
|
|
67
|
+
return addUtilityFunctions(client)
|
|
68
|
+
}
|
|
69
|
+
const encKey = normalizeKey(encryption.key)
|
|
70
|
+
if (!encKey) {
|
|
71
|
+
throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption key`)
|
|
72
|
+
}
|
|
73
|
+
const iv = normalizeIV(encryption.iv, encKey)
|
|
74
|
+
if (!iv) {
|
|
75
|
+
throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption IV`)
|
|
76
|
+
}
|
|
77
|
+
const encryptPaths = encryption.encryptPaths || true;
|
|
78
|
+
const enc = {
|
|
79
|
+
key: encKey,
|
|
80
|
+
iv,
|
|
81
|
+
algo: encryption.algo || DEFAULT_CRYPT_ALGO,
|
|
82
|
+
encryptPaths,
|
|
83
|
+
dirLevels: encryption.dirLevels || 4,
|
|
84
|
+
encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
|
|
85
|
+
}
|
|
86
|
+
function encryptPath (path) {
|
|
87
|
+
const encrypted = shasum(enc.key + ' ' + path)
|
|
88
|
+
let newPath = ''
|
|
89
|
+
for (let i = 0; i <= enc.dirLevels; i++) {
|
|
90
|
+
if (newPath.length > 0) newPath += '/'
|
|
91
|
+
newPath += encrypted.substring(i*2, (i*2)+2)
|
|
92
|
+
}
|
|
93
|
+
return newPath + encrypted
|
|
94
|
+
}
|
|
95
|
+
const direntDir = dir => encryptPaths ? encryptPath(dir + DIR_ENT_DIR_SUFFIX) : null
|
|
96
|
+
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
97
|
+
|
|
98
|
+
const _metadata = (client) => async (path) => {
|
|
99
|
+
if (!encryptPaths) {
|
|
100
|
+
return client.metadata(path)
|
|
101
|
+
}
|
|
102
|
+
let meta
|
|
103
|
+
try {
|
|
104
|
+
meta = await client.metadata(encryptPath(path))
|
|
105
|
+
} catch (e) {
|
|
106
|
+
if (e instanceof MobilettoNotFoundError) {
|
|
107
|
+
const dd = direntDir(path);
|
|
108
|
+
try {
|
|
109
|
+
meta = await client.metadata(dd)
|
|
110
|
+
} catch (err) {
|
|
111
|
+
if (err instanceof MobilettoNotFoundError) {
|
|
112
|
+
const contents = await client.list(dd)
|
|
113
|
+
if (Array.isArray(contents) && contents.length > 0) {
|
|
114
|
+
return { name: path, type: M_DIR }
|
|
115
|
+
} else {
|
|
116
|
+
throw err
|
|
117
|
+
}
|
|
118
|
+
} else {
|
|
119
|
+
throw err
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
} else {
|
|
123
|
+
throw e
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
meta.name = path // rewrite name back to plaintext name
|
|
127
|
+
return meta
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const _loadMeta = async (dirent, entries) => {
|
|
131
|
+
const files = []
|
|
132
|
+
for (const entry of entries) {
|
|
133
|
+
const cipherText = []
|
|
134
|
+
await client.read(dirent + '/' + basename(entry.name), reader(cipherText))
|
|
135
|
+
const plain = decrypt(cipherText.toString(), enc)
|
|
136
|
+
const realPath = plain.split(ENC_PAD_SEP)[0]
|
|
137
|
+
const meta = await _metadata(client)(realPath)
|
|
138
|
+
files.push(meta)
|
|
139
|
+
}
|
|
140
|
+
return files
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function removeDirentFile(path) {
|
|
144
|
+
const df = direntFile(direntDir(dirname(path)), path);
|
|
145
|
+
await client.remove(df, {recursive: false, quiet: true})
|
|
146
|
+
await client.remove(encryptPath(path), {recursive: false, quiet: true})
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const encClient = {
|
|
150
|
+
list: async (path) => {
|
|
151
|
+
if (!encryptPaths) {
|
|
152
|
+
return await client.list(path)
|
|
153
|
+
}
|
|
154
|
+
const dirent = direntDir(path)
|
|
155
|
+
const entries = await client.list(dirent)
|
|
156
|
+
if (!entries || entries.length === 0) {
|
|
157
|
+
return []
|
|
158
|
+
}
|
|
159
|
+
return _loadMeta(dirent, entries)
|
|
160
|
+
},
|
|
161
|
+
metadata: _metadata(client),
|
|
162
|
+
read: async (path, callback) => {
|
|
163
|
+
const realPath = encryptPaths ? encryptPath(path) : path
|
|
164
|
+
const cipher = crypt.startDecryptStream(enc)
|
|
165
|
+
return client.read(realPath,
|
|
166
|
+
(chunk) => {
|
|
167
|
+
return callback(crypt.updateCryptStream(cipher, chunk))
|
|
168
|
+
}, () => {
|
|
169
|
+
return callback(crypt.closeCryptStream(cipher))
|
|
170
|
+
})
|
|
171
|
+
},
|
|
172
|
+
write: async (path, readFunc) => {
|
|
173
|
+
// if encrypting paths, write dirent file(s) for all parent directories
|
|
174
|
+
if (encryptPaths) {
|
|
175
|
+
let p = path
|
|
176
|
+
while (true) {
|
|
177
|
+
const direntGenerator = function* () {
|
|
178
|
+
yield encrypt(p + enc.encPathPadding(), enc)
|
|
179
|
+
}
|
|
180
|
+
const dir = direntDir(dirname(p))
|
|
181
|
+
const df = direntFile(dir, p);
|
|
182
|
+
if (!(await client.write(df, direntGenerator()))) {
|
|
183
|
+
throw new MobilettoError('write: error writing dirent file')
|
|
184
|
+
}
|
|
185
|
+
p = dirname(p)
|
|
186
|
+
if (p === '.' || p === '') {
|
|
187
|
+
break
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function* cryptGenerator(plaintextGenerator) {
|
|
193
|
+
let chunk = plaintextGenerator.next().value
|
|
194
|
+
while (chunk) {
|
|
195
|
+
yield cipher.update(chunk)
|
|
196
|
+
chunk = plaintextGenerator.next().value
|
|
197
|
+
}
|
|
198
|
+
yield cipher.final()
|
|
199
|
+
}
|
|
200
|
+
const cipher = crypt.startEncryptStream(enc)
|
|
201
|
+
const realPath = encryptPath(path)
|
|
202
|
+
return client.write(realPath, cryptGenerator(readFunc))
|
|
203
|
+
},
|
|
204
|
+
remove: async (path, options) => {
|
|
205
|
+
const recursive = (options && options.recursive) || false
|
|
206
|
+
const quiet = (options && options.quiet) || false
|
|
207
|
+
if (!encryptPaths) {
|
|
208
|
+
return client.remove(path, {recursive, quiet})
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (recursive) {
|
|
212
|
+
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
213
|
+
async function recRm (path) {
|
|
214
|
+
const dirent = direntDir(path)
|
|
215
|
+
let entries
|
|
216
|
+
try {
|
|
217
|
+
entries = await client.list(dirent)
|
|
218
|
+
} catch (e) {
|
|
219
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
220
|
+
throw e instanceof MobilettoError
|
|
221
|
+
? e
|
|
222
|
+
: new MobilettoError('error listing files for recursive deletion', e)
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (entries && entries.length > 0) {
|
|
226
|
+
const files = await _loadMeta(dirent, entries)
|
|
227
|
+
for (const f of files) {
|
|
228
|
+
await recRm(f.name)
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
await removeDirentFile(path);
|
|
232
|
+
}
|
|
233
|
+
await recRm(path)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// remove the single file/dir
|
|
237
|
+
await removeDirentFile(path);
|
|
238
|
+
|
|
239
|
+
// if we were the last dirent file, then also remove dirent directory, and recursively upwards
|
|
240
|
+
let parent = path
|
|
241
|
+
let dirent = direntDir(parent)
|
|
242
|
+
const df = direntFile(dirent, path)
|
|
243
|
+
await client.remove(df, {recursive: false, quiet: true})
|
|
244
|
+
while (true) {
|
|
245
|
+
try {
|
|
246
|
+
const entries = await client.list(dirent)
|
|
247
|
+
if (entries.length === 0) {
|
|
248
|
+
await removeDirentFile(parent)
|
|
249
|
+
}
|
|
250
|
+
} catch (e) {
|
|
251
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
252
|
+
throw e
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
if (parent === '.' || parent === '/') {
|
|
256
|
+
// do not remove dirent for root dir
|
|
257
|
+
break
|
|
258
|
+
}
|
|
259
|
+
parent = dirname(parent)
|
|
260
|
+
dirent = direntDir(parent)
|
|
261
|
+
}
|
|
262
|
+
return true
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return addUtilityFunctions(encClient)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async function readStream(stream, callback, endCallback) {
|
|
269
|
+
const counter = {count: 0}
|
|
270
|
+
const streamHandler = stream =>
|
|
271
|
+
new Promise((resolve, reject) => {
|
|
272
|
+
stream.on('data', (data) => {
|
|
273
|
+
counter.count += data ? data.length : 0
|
|
274
|
+
callback(data)
|
|
275
|
+
})
|
|
276
|
+
stream.on('error', reject)
|
|
277
|
+
stream.on('end', () => {
|
|
278
|
+
if (endCallback) {
|
|
279
|
+
endCallback()
|
|
280
|
+
}
|
|
281
|
+
resolve()
|
|
282
|
+
})
|
|
283
|
+
})
|
|
284
|
+
await streamHandler(stream)
|
|
285
|
+
return counter.count
|
|
31
286
|
}
|
|
32
287
|
|
|
33
288
|
function writeStream (stream) {
|
|
@@ -35,7 +290,7 @@ function writeStream (stream) {
|
|
|
35
290
|
if (chunk) {
|
|
36
291
|
stream.write(chunk, (err) => {
|
|
37
292
|
if (err) {
|
|
38
|
-
console.
|
|
293
|
+
console.error(`writeStream: error writing: ${err}`)
|
|
39
294
|
throw err
|
|
40
295
|
}
|
|
41
296
|
})
|
|
@@ -46,21 +301,12 @@ function writeStream (stream) {
|
|
|
46
301
|
function closeStream (stream) {
|
|
47
302
|
return () => stream.close((err) => {
|
|
48
303
|
if (err) {
|
|
49
|
-
console.
|
|
304
|
+
console.error(`closeStream: error closing: ${err}`)
|
|
50
305
|
throw err
|
|
51
306
|
}
|
|
52
307
|
})
|
|
53
308
|
}
|
|
54
309
|
|
|
55
|
-
function readCloseHandler (promise, closeHandler) {
|
|
56
|
-
return () => {
|
|
57
|
-
if (closeHandler) {
|
|
58
|
-
closeHandler()
|
|
59
|
-
}
|
|
60
|
-
promise.resolve()
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
310
|
const M_FILE = 'file'
|
|
65
311
|
const M_DIR = 'dir'
|
|
66
312
|
const M_LINK = 'link'
|
|
@@ -68,8 +314,7 @@ const M_SPECIAL = 'special'
|
|
|
68
314
|
|
|
69
315
|
module.exports = {
|
|
70
316
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
71
|
-
mobiletto,
|
|
317
|
+
mobiletto, connect,
|
|
72
318
|
MobilettoError, MobilettoNotFoundError,
|
|
73
|
-
writeStream, closeStream
|
|
319
|
+
readStream, writeStream, closeStream
|
|
74
320
|
}
|
|
75
|
-
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.6",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Jonathan Cobb",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
22
22
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
23
|
-
"node-fetch-commonjs": "^3.1.1"
|
|
23
|
+
"node-fetch-commonjs": "^3.1.1",
|
|
24
|
+
"shasum": "^1.0.2"
|
|
24
25
|
}
|
|
25
26
|
}
|
package/test/index.spec.js
CHANGED
|
@@ -6,17 +6,22 @@ const randomstring = require('randomstring')
|
|
|
6
6
|
|
|
7
7
|
const { expect, should, assert } = require('chai')
|
|
8
8
|
|
|
9
|
-
const { mobiletto } = require(
|
|
10
|
-
const {
|
|
9
|
+
const { mobiletto, connect, MobilettoNotFoundError, M_FILE, M_DIR } = require("../index")
|
|
10
|
+
const { encrypt, decrypt, normalizeKey, normalizeIV, DEFAULT_CRYPT_ALGO } = require("../util/crypt")
|
|
11
11
|
|
|
12
12
|
// chunk size used by generator function, used by driver's 'write' function
|
|
13
13
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
14
14
|
const READ_SZ = 8 * 1024 // xfer data in 8k chunks
|
|
15
15
|
const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
|
|
16
16
|
|
|
17
|
+
// encoded bytes written will differ from actual bytes provided
|
|
18
|
+
// likewise, plaintext bytes read will differ from encoded bytes read
|
|
19
|
+
// our tests give some leeway when filesize differences
|
|
20
|
+
const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
|
|
21
|
+
|
|
17
22
|
DRIVER_CONFIG = {
|
|
18
23
|
local: {
|
|
19
|
-
key: '/tmp'
|
|
24
|
+
key: process.env.LOCAL_STORAGE || '/tmp'
|
|
20
25
|
},
|
|
21
26
|
s3: {
|
|
22
27
|
key: process.env.S3_ACCESS,
|
|
@@ -29,11 +34,20 @@ DRIVER_CONFIG = {
|
|
|
29
34
|
}
|
|
30
35
|
}
|
|
31
36
|
|
|
32
|
-
|
|
37
|
+
function closeEnough (expected, actual, percent = ENC_SIZE_CLOSE_ENOUGH_PERCENT) {
|
|
38
|
+
expect(Math.abs(expected - actual)).to.be.lessThan(Math.floor(expected * percent),
|
|
39
|
+
'expected size within 5% of actual (due to encryption)')
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
|
|
33
43
|
const meta = await api.metadata(name)
|
|
34
44
|
should().exist(meta, 'expected return value from metadata call')
|
|
35
|
-
expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
|
|
36
45
|
expect(meta.name).equals(name, 'expected name of written file to be correct')
|
|
46
|
+
if (closeEnoughPercent) {
|
|
47
|
+
closeEnough(expectedSize, meta.size, closeEnoughPercent)
|
|
48
|
+
} else {
|
|
49
|
+
expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
|
|
50
|
+
}
|
|
37
51
|
}
|
|
38
52
|
|
|
39
53
|
async function assertMetaFail (api, name) {
|
|
@@ -48,10 +62,35 @@ async function assertMetaFail (api, name) {
|
|
|
48
62
|
}
|
|
49
63
|
}
|
|
50
64
|
|
|
51
|
-
|
|
52
|
-
|
|
65
|
+
const rand = count => randomstring.generate(count)
|
|
66
|
+
|
|
67
|
+
async function writeRandomFile(fixture, size) {
|
|
68
|
+
const data = fixture.randomData
|
|
69
|
+
function* dataGenerator() {
|
|
70
|
+
// read in 8k chunks
|
|
71
|
+
for (let i = 0; i < data.length; i += size) {
|
|
72
|
+
const end = (i + size) > data.length ? data.length : (i + size)
|
|
73
|
+
yield data.slice(i, end)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return await fixture.api.write(fixture.name, dataGenerator())
|
|
53
77
|
}
|
|
54
78
|
|
|
79
|
+
describe('crypto test', () => {
|
|
80
|
+
it("should encrypt and decrypt successfully", async () => {
|
|
81
|
+
const enc = {
|
|
82
|
+
key: normalizeKey(randomstring.generate(32)),
|
|
83
|
+
iv: normalizeIV(randomstring.generate(16)),
|
|
84
|
+
algo: DEFAULT_CRYPT_ALGO
|
|
85
|
+
}
|
|
86
|
+
const plaintext = randomstring.generate(1024 + Math.floor(1024*Math.random()))
|
|
87
|
+
const ciphertext = encrypt(plaintext, enc)
|
|
88
|
+
const decrypted = decrypt(ciphertext, enc)
|
|
89
|
+
expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
})
|
|
93
|
+
|
|
55
94
|
// To test a single driver:
|
|
56
95
|
// - Uncomment one of the lines below to set driverName to the one you want to test
|
|
57
96
|
// - Comment out the next `for` line, and its closing curly brace (just before EOF)
|
|
@@ -92,34 +131,17 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
92
131
|
let fixture
|
|
93
132
|
beforeEach((done) => {
|
|
94
133
|
const name = `test_file_${fileSuffix}`
|
|
95
|
-
|
|
134
|
+
connect(driverName, config.key, config.secret, config.opts)
|
|
96
135
|
.then(api => { fixture = {api, name, randomData} })
|
|
97
136
|
.finally(done)
|
|
98
137
|
})
|
|
99
138
|
it("should write a file", async () => {
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
function* dataGenerator() {
|
|
103
|
-
// read in 8k chunks
|
|
104
|
-
for (let i = 0; i < data.length; i += READ_SZ) {
|
|
105
|
-
const end = (i + READ_SZ) > data.length ? data.length : (i + READ_SZ)
|
|
106
|
-
yield data.slice(i, end)
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
const response = await fixture.api.write(fixture.name, dataGenerator())
|
|
111
|
-
expect(response).to.equal(data.length, 'expected write API to return correct number of bytes written')
|
|
139
|
+
const response = await writeRandomFile(fixture, size)
|
|
140
|
+
expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
|
|
112
141
|
})
|
|
113
142
|
it("should read the file we just wrote", async () => {
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
if (chunk) {
|
|
117
|
-
chunks.push(chunk)
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
const response = await fixture.api.read(fixture.name, reader)
|
|
121
|
-
const data = Buffer.concat(chunks)
|
|
122
|
-
expect(response).is.equal(data.length, 'expected read API to return correct number of bytes read')
|
|
143
|
+
const data = await fixture.api.readFile(fixture.name);
|
|
144
|
+
expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
|
|
123
145
|
expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
124
146
|
})
|
|
125
147
|
it("should load metadata on the file we just wrote", async () => {
|
|
@@ -134,59 +156,107 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
134
156
|
})
|
|
135
157
|
})
|
|
136
158
|
|
|
137
|
-
describe(`${driverName} - write
|
|
138
|
-
//
|
|
139
|
-
const
|
|
140
|
-
const
|
|
141
|
-
const
|
|
142
|
-
const
|
|
159
|
+
describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
|
|
160
|
+
// some random data, plus a bit extra
|
|
161
|
+
const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
|
|
162
|
+
const randomData = rand(size)
|
|
163
|
+
const fileSuffix = '' + Date.now()
|
|
164
|
+
const encryptionKey = rand(32)
|
|
165
|
+
let encryptedByteCount = null
|
|
143
166
|
let fixture
|
|
144
167
|
beforeEach((done) => {
|
|
145
|
-
|
|
146
|
-
|
|
168
|
+
const name = `test_file_${fileSuffix}`
|
|
169
|
+
mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
|
|
170
|
+
.then(api => { fixture = {api, name, randomData} })
|
|
147
171
|
.finally(done)
|
|
148
172
|
})
|
|
149
|
-
it(
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
yield rand(READ_SZ)
|
|
153
|
-
}
|
|
154
|
-
for (let i = 0; i < fileCount; i++) {
|
|
155
|
-
const response = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
|
|
156
|
-
expect(response).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
157
|
-
}
|
|
173
|
+
it("should write an encrypted file", async () => {
|
|
174
|
+
encryptedByteCount = await writeRandomFile(fixture, size);
|
|
175
|
+
closeEnough(size, encryptedByteCount)
|
|
158
176
|
})
|
|
159
|
-
it("should
|
|
160
|
-
await
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
})
|
|
167
|
-
it("should see correct types on objects returned from a listing of the subdirectory", async () => {
|
|
168
|
-
const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
|
|
169
|
-
expect(objects).to.have.lengthOf(fileCount)
|
|
170
|
-
for (let i = 0; i < fileCount; i++) {
|
|
171
|
-
// we should find all the files, and they should all have the correct type
|
|
172
|
-
expect(objects
|
|
173
|
-
.find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
|
|
174
|
-
).to.not.be.null
|
|
175
|
-
}
|
|
177
|
+
it("should read the encrypted file we just wrote", async () => {
|
|
178
|
+
const data = await fixture.api.readFile(fixture.name);
|
|
179
|
+
expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
|
|
180
|
+
expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
181
|
+
})
|
|
182
|
+
it("should load metadata on the encrypted file we just wrote", async () => {
|
|
183
|
+
await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
|
|
176
184
|
})
|
|
177
|
-
it("should
|
|
178
|
-
const
|
|
179
|
-
const removed = await fixture.api.remove(randomParent, {recursive})
|
|
185
|
+
it("should delete the encrypted file we just wrote", async () => {
|
|
186
|
+
const removed = await fixture.api.remove(fixture.name)
|
|
180
187
|
expect(removed).to.be.true
|
|
181
188
|
})
|
|
182
|
-
it("loading metadata on the file we wrote now fails", async () => {
|
|
189
|
+
it("loading metadata on the encrypted file we wrote now fails", async () => {
|
|
183
190
|
await assertMetaFail(fixture.api, fixture.name)
|
|
184
191
|
})
|
|
185
|
-
it("loading metadata on the parent dir we wrote now fails", async () => {
|
|
186
|
-
await assertMetaFail(fixture.api, randomParent)
|
|
187
|
-
})
|
|
188
192
|
})
|
|
189
193
|
|
|
194
|
+
for (const encryption of [null, {key: rand(32)}]) {
|
|
195
|
+
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
196
|
+
describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
|
|
197
|
+
// describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|
|
198
|
+
// const encryption = {key: rand(32)}
|
|
199
|
+
// const encryption = null
|
|
200
|
+
// a random directory and file within it
|
|
201
|
+
const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
|
|
202
|
+
const subdirName = `subdir_` + Date.now()
|
|
203
|
+
const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
|
|
204
|
+
//const fileCount = 3 + Math.floor(Math.random() * 10)
|
|
205
|
+
const fileCount = 2
|
|
206
|
+
let fixture
|
|
207
|
+
beforeEach((done) => {
|
|
208
|
+
mobiletto(driverName, config.key, config.secret, config.opts, encryption)
|
|
209
|
+
.then(api => { fixture = {api, name: randomPath} })
|
|
210
|
+
.catch((err) => { throw err })
|
|
211
|
+
.finally(done)
|
|
212
|
+
})
|
|
213
|
+
it(`should write ${fileCount} files in a new directory`, async () => {
|
|
214
|
+
function* dataGenerator() {
|
|
215
|
+
// return one chunk of random data
|
|
216
|
+
yield rand(READ_SZ)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
for (let i = 0; i < fileCount; i++) {
|
|
220
|
+
const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
|
|
221
|
+
if (encryption) {
|
|
222
|
+
closeEnough(READ_SZ, bytesWritten)
|
|
223
|
+
} else {
|
|
224
|
+
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
})
|
|
228
|
+
it("should load metadata for one of the new files", async () => {
|
|
229
|
+
await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
|
|
230
|
+
})
|
|
231
|
+
it("should see correct types on objects returned from a listing of the new directory", async () => {
|
|
232
|
+
const objects = await fixture.api.list(randomParent)
|
|
233
|
+
expect(objects).to.have.lengthOf(1)
|
|
234
|
+
expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
|
|
235
|
+
})
|
|
236
|
+
it("should see correct types on objects returned from a listing of the subdirectory", async () => {
|
|
237
|
+
const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
|
|
238
|
+
expect(objects).to.have.lengthOf(fileCount)
|
|
239
|
+
for (let i = 0; i < fileCount; i++) {
|
|
240
|
+
// we should find all the files, and they should all have the correct type
|
|
241
|
+
expect(objects
|
|
242
|
+
.find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
|
|
243
|
+
).to.not.be.null
|
|
244
|
+
}
|
|
245
|
+
})
|
|
246
|
+
it("should recursively delete the directory and file we just created", async () => {
|
|
247
|
+
const recursive = true
|
|
248
|
+
const removed = await fixture.api.remove(randomParent, {recursive})
|
|
249
|
+
expect(removed).to.be.true
|
|
250
|
+
})
|
|
251
|
+
it("loading metadata on the file we wrote now fails", async () => {
|
|
252
|
+
await assertMetaFail(fixture.api, fixture.name)
|
|
253
|
+
})
|
|
254
|
+
it("loading metadata on the parent dir we wrote now fails", async () => {
|
|
255
|
+
await assertMetaFail(fixture.api, randomParent)
|
|
256
|
+
})
|
|
257
|
+
})
|
|
258
|
+
}
|
|
259
|
+
|
|
190
260
|
describe(`${driverName} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
|
|
191
261
|
it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
|
|
192
262
|
const api = await mobiletto(driverName, config.key, config.secret, config.opts)
|
|
@@ -233,6 +303,5 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
233
303
|
}
|
|
234
304
|
})
|
|
235
305
|
})
|
|
236
|
-
|
|
237
306
|
})
|
|
238
307
|
}
|
package/util/crypt.js
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// adapted from https://stackoverflow.com/a/64136185
|
|
2
|
+
const crypto = require('crypto')
|
|
3
|
+
const shasum = require('shasum')
|
|
4
|
+
|
|
5
|
+
const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
|
|
6
|
+
const MIN_KEY_LEN = 16
|
|
7
|
+
const DEFAULT_CRYPT_ALGO = 'aes-256-cbc';
|
|
8
|
+
|
|
9
|
+
function normalizeKey (k) {
|
|
10
|
+
return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
|
|
11
|
+
? Buffer.from(shasum(k)).subarray(0, 32)
|
|
12
|
+
: null
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function normalizeIV (iv, key) {
|
|
16
|
+
return (typeof iv === 'string')
|
|
17
|
+
? Buffer.from(shasum(iv)).subarray(0, 16)
|
|
18
|
+
: key
|
|
19
|
+
? Buffer.from(shasum(key)).subarray(0, 16)
|
|
20
|
+
: null
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// ensure key long enough for security, and is 32 bytes for AES-256
|
|
24
|
+
let KEY = null
|
|
25
|
+
function setDefaultKey (key) {
|
|
26
|
+
KEY = normalizeKey(key)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// ensure IV is 16 bytes for AES-256
|
|
30
|
+
let CRYPTO_IV = null
|
|
31
|
+
function setDefaultIV (iv) {
|
|
32
|
+
CRYPTO_IV = normalizeIV(iv, KEY)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function getCipher(enc) {
|
|
36
|
+
return crypto.createCipheriv(enc.algo, enc.key, enc.iv);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function encrypt (plainText, encryption, outputEncoding = 'base64') {
|
|
40
|
+
if (!encryption || !encryption.key) {
|
|
41
|
+
if (WARN_PLAINTEXT) {
|
|
42
|
+
console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
|
|
43
|
+
}
|
|
44
|
+
return plainText
|
|
45
|
+
}
|
|
46
|
+
const cipher = getCipher(encryption)
|
|
47
|
+
const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
|
|
48
|
+
return encoded.toString(outputEncoding)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function getDecipher(enc) {
|
|
52
|
+
return crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
|
|
56
|
+
if (!encryption || !encryption.key) {
|
|
57
|
+
return cipherText
|
|
58
|
+
}
|
|
59
|
+
const cipher = getDecipher(encryption)
|
|
60
|
+
const data = Buffer.from(cipherText, inputEncoding)
|
|
61
|
+
return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const startEncryptStream = (enc) => getCipher(enc)
|
|
65
|
+
const startDecryptStream = (enc) => getDecipher(enc)
|
|
66
|
+
const updateCryptStream = (cipher, data) => cipher.update(data)
|
|
67
|
+
const closeCryptStream = (cipher) => cipher.final()
|
|
68
|
+
|
|
69
|
+
module.exports = {
|
|
70
|
+
DEFAULT_CRYPT_ALGO,
|
|
71
|
+
setDefaultKey, setDefaultIV,
|
|
72
|
+
encrypt, decrypt,
|
|
73
|
+
normalizeKey, normalizeIV,
|
|
74
|
+
startEncryptStream, startDecryptStream, updateCryptStream, closeCryptStream
|
|
75
|
+
}
|