mobiletto 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -9
- package/drivers/local/index.js +23 -12
- package/drivers/s3/index.js +24 -6
- package/index.js +200 -40
- package/package.json +1 -1
- package/test/index.spec.js +98 -75
- package/util/crypt.js +10 -9
package/README.md
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
Mobiletto
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
Mobiletto is a JavaScript storage abstraction layer.
|
|
4
|
+
Mobiletto is a JavaScript storage abstraction layer, with optional transparent client-side encryption.
|
|
5
5
|
|
|
6
6
|
It supports apps that are agnostic to where files are stored.
|
|
7
7
|
|
|
8
|
+
All drivers are tested for identical behavior with 30 tests for each driver.
|
|
9
|
+
|
|
8
10
|
# Basic usage
|
|
9
|
-
const { mobiletto } = require('mobiletto')
|
|
11
|
+
const { mobiletto, readFile } = require('mobiletto')
|
|
10
12
|
|
|
11
13
|
// General usage
|
|
12
14
|
const api = await mobiletto(driverName, key, secret, opts)
|
|
@@ -25,7 +27,7 @@ It supports apps that are agnostic to where files are stored.
|
|
|
25
27
|
// * bucket: required, name of the S3 bucket
|
|
26
28
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
27
29
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
28
|
-
// * delimiter: optional, directory delimiter, default is '/'
|
|
30
|
+
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
29
31
|
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
30
32
|
|
|
31
33
|
// List files
|
|
@@ -46,6 +48,10 @@ It supports apps that are agnostic to where files are stored.
|
|
|
46
48
|
const callback = (chunk) => { ... write chunk somewhere ... }
|
|
47
49
|
local.read(path, callback)
|
|
48
50
|
s3.read(path, callback)
|
|
51
|
+
|
|
52
|
+
// Read an entire file at once (convenience method)
|
|
53
|
+
const localData = await local.readFile(path) // returns a byte Buffer of the file contents
|
|
54
|
+
const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
|
|
49
55
|
|
|
50
56
|
// Write a file
|
|
51
57
|
// Provide a generator function that yields chunks of data
|
|
@@ -74,15 +80,38 @@ Enable transparent client-side encryption:
|
|
|
74
80
|
|
|
75
81
|
// Pass encryption parameters
|
|
76
82
|
const encryption = {
|
|
77
|
-
key: randomstring.generate(128), // required, must be >=
|
|
78
|
-
iv: randomstring.generate(128), // optional
|
|
79
|
-
algo: 'aes-256-cbc'
|
|
83
|
+
key: randomstring.generate(128), // required, must be >= 32 chars
|
|
84
|
+
iv: randomstring.generate(128), // optional, default is to derive IV from key
|
|
85
|
+
algo: 'aes-256-cbc', // optional, aes-256-cbc is the default
|
|
86
|
+
encryptPaths: true // optional, default is true
|
|
80
87
|
}
|
|
81
88
|
const api = await mobiletto(driverName, key, secret, opts, encryption)
|
|
82
89
|
|
|
83
90
|
// Subsequent write operations will encrypt data (client side) when writing
|
|
84
91
|
// Subsequent read operations will decrypt data (client side) when reading
|
|
85
|
-
//
|
|
92
|
+
// If `encryptPaths` is true, then path names will be hashed using the encrpytion key
|
|
93
|
+
|
|
94
|
+
Note that when `encryptPaths` is enabled, `list` commands are considerably more inefficient,
|
|
95
|
+
especially for directories with a large number of files.
|
|
96
|
+
|
|
97
|
+
What's happening? A separate "directory entry" directory (encrypted) tracks what files are in that
|
|
98
|
+
directory (aka the dirent directory).
|
|
99
|
+
* The `list` command reads the directory entry files, decrypts each path listed; then returns metadata for each file
|
|
100
|
+
* The `write` command writes dirent files in each parent's dirent directory, recursively; then writes the file
|
|
101
|
+
* The `remove` command removes the corresponding dirent file, and its parent if empty, recursively; then removes the file
|
|
102
|
+
* Note: Recursive removal on large and deep filesystems can be an expensive operation
|
|
103
|
+
|
|
104
|
+
Note that even with `encryptPaths` enabled, an adversary with full visibility into your encrypted storage,
|
|
105
|
+
even without the key, can still see the total number of directories and how many files are in each, and with some
|
|
106
|
+
effort, discover some or all of the overall hierarchical structure. They would not know the names of the
|
|
107
|
+
directories/files unless they also knew your encryption key or had otherwise successfully cracked the encryption.
|
|
108
|
+
All bets are off then!
|
|
109
|
+
|
|
110
|
+
# Alternate usage: import fully scoped module and use `connect`
|
|
111
|
+
|
|
112
|
+
const storage = require('mobiletto')
|
|
113
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
114
|
+
const objectData = await s3.readFile('some/path')
|
|
86
115
|
|
|
87
116
|
# Driver Interface
|
|
88
117
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
|
@@ -110,10 +139,12 @@ The object that the storageClient function returns must define these functions:
|
|
|
110
139
|
async metadata (path)
|
|
111
140
|
|
|
112
141
|
// Read a file
|
|
113
|
-
|
|
142
|
+
// callback receives a chunk of data. endCallback is called at end-of-stream
|
|
143
|
+
async read (path, callback, endCallback = null)
|
|
114
144
|
|
|
115
145
|
// Write a file
|
|
116
|
-
|
|
146
|
+
// readFunc returns the data you want to write
|
|
147
|
+
async write (path, readFunc)
|
|
117
148
|
|
|
118
149
|
// Remove a file, or recursively delete a directory
|
|
119
150
|
async remove (path, {recursive = false, quiet = false})
|
package/drivers/local/index.js
CHANGED
|
@@ -9,6 +9,16 @@ const fs = require('fs')
|
|
|
9
9
|
|
|
10
10
|
const DEFAULT_MODE = '0700'
|
|
11
11
|
|
|
12
|
+
const isENOENT = (err) => err.code && err.code === 'ENOENT'
|
|
13
|
+
|
|
14
|
+
const ioError = (err, path, method) => isENOENT(err)
|
|
15
|
+
? err instanceof MobilettoNotFoundError
|
|
16
|
+
? err
|
|
17
|
+
: new MobilettoNotFoundError(path)
|
|
18
|
+
: err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
19
|
+
? err
|
|
20
|
+
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
21
|
+
|
|
12
22
|
class StorageClient {
|
|
13
23
|
baseDir
|
|
14
24
|
mode
|
|
@@ -74,7 +84,7 @@ class StorageClient {
|
|
|
74
84
|
return entry
|
|
75
85
|
})
|
|
76
86
|
} catch (err) {
|
|
77
|
-
throw
|
|
87
|
+
throw ioError(err, path, 'list')
|
|
78
88
|
}
|
|
79
89
|
}
|
|
80
90
|
async metadata (path) {
|
|
@@ -82,19 +92,23 @@ class StorageClient {
|
|
|
82
92
|
try {
|
|
83
93
|
const lstat = fs.lstatSync(file)
|
|
84
94
|
if (!lstat) {
|
|
85
|
-
throw new MobilettoError(
|
|
95
|
+
throw new MobilettoError('metadata: lstat error')
|
|
96
|
+
}
|
|
97
|
+
const type = this.fileType(lstat)
|
|
98
|
+
if (type === M_DIR && path !== '') {
|
|
99
|
+
const contents = await this.list(path)
|
|
100
|
+
if (contents.length === 0) {
|
|
101
|
+
throw new MobilettoNotFoundError(path)
|
|
102
|
+
}
|
|
86
103
|
}
|
|
87
104
|
return {
|
|
88
105
|
name: file.startsWith(this.baseDir) ? file.substring(this.baseDir.length) : file,
|
|
89
|
-
type
|
|
106
|
+
type,
|
|
90
107
|
size: lstat.size,
|
|
91
108
|
mtime: lstat.mtimeMs
|
|
92
109
|
}
|
|
93
110
|
} catch (err) {
|
|
94
|
-
|
|
95
|
-
throw new MobilettoNotFoundError(path)
|
|
96
|
-
}
|
|
97
|
-
throw new MobilettoError(`metadata(${path}) error: ${err}`, err)
|
|
111
|
+
throw ioError(err, path, 'metadata')
|
|
98
112
|
}
|
|
99
113
|
}
|
|
100
114
|
|
|
@@ -150,10 +164,7 @@ class StorageClient {
|
|
|
150
164
|
const stream = fs.createReadStream(file)
|
|
151
165
|
return await readStream(stream, callback, endCallback)
|
|
152
166
|
} catch (err) {
|
|
153
|
-
|
|
154
|
-
throw new MobilettoNotFoundError(path)
|
|
155
|
-
}
|
|
156
|
-
throw new MobilettoError(`read(${path}) error: ${err}`, err)
|
|
167
|
+
throw ioError(err, path, 'read')
|
|
157
168
|
}
|
|
158
169
|
}
|
|
159
170
|
|
|
@@ -165,7 +176,7 @@ class StorageClient {
|
|
|
165
176
|
try {
|
|
166
177
|
fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
|
|
167
178
|
} catch (err) {
|
|
168
|
-
if (err
|
|
179
|
+
if (isENOENT(err)) {
|
|
169
180
|
if (!quiet) {
|
|
170
181
|
throw new MobilettoNotFoundError(path)
|
|
171
182
|
}
|
package/drivers/s3/index.js
CHANGED
|
@@ -80,12 +80,14 @@ class StorageClient {
|
|
|
80
80
|
while (truncated) {
|
|
81
81
|
try {
|
|
82
82
|
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
83
|
-
|
|
83
|
+
const hasContents = typeof response.Contents !== 'undefined'
|
|
84
|
+
if (hasContents) {
|
|
84
85
|
response.Contents.forEach((item) => {
|
|
85
86
|
objects.push(item.Key) // todo: transform into standard object
|
|
86
87
|
})
|
|
87
88
|
}
|
|
88
|
-
|
|
89
|
+
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
90
|
+
if (hasCommonPrefixes) {
|
|
89
91
|
response.CommonPrefixes.forEach((item) => {
|
|
90
92
|
objects.push(item.Prefix) // todo: transform into standard object
|
|
91
93
|
})
|
|
@@ -94,13 +96,22 @@ class StorageClient {
|
|
|
94
96
|
// If truncated is true, advance the marker
|
|
95
97
|
if (truncated) {
|
|
96
98
|
bucketParams.Marker = response.NextMarker
|
|
99
|
+
} else if (!hasContents && !hasCommonPrefixes) {
|
|
100
|
+
if (path === '') {
|
|
101
|
+
break
|
|
102
|
+
}
|
|
103
|
+
throw new MobilettoNotFoundError(path)
|
|
97
104
|
}
|
|
98
|
-
// At end of the list, response.truncated is false, and the function exits the while loop.
|
|
99
105
|
} catch (err) {
|
|
100
|
-
|
|
101
|
-
|
|
106
|
+
if (err instanceof MobilettoNotFoundError) {
|
|
107
|
+
throw err
|
|
108
|
+
}
|
|
109
|
+
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
102
110
|
}
|
|
103
111
|
}
|
|
112
|
+
if (recursive && objects.length === 0) {
|
|
113
|
+
throw new MobilettoNotFoundError(path)
|
|
114
|
+
}
|
|
104
115
|
return objects
|
|
105
116
|
}
|
|
106
117
|
|
|
@@ -213,7 +224,14 @@ class StorageClient {
|
|
|
213
224
|
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
214
225
|
throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
215
226
|
}
|
|
216
|
-
|
|
227
|
+
try {
|
|
228
|
+
objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
229
|
+
} catch (e) {
|
|
230
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
231
|
+
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
|
|
232
|
+
}
|
|
233
|
+
objects = null
|
|
234
|
+
}
|
|
217
235
|
}
|
|
218
236
|
} else {
|
|
219
237
|
const Key = this.normalizeKey(path)
|
package/index.js
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
|
+
const { basename, dirname } = require('path')
|
|
2
|
+
const shasum = require('shasum')
|
|
3
|
+
const randomstring = require('randomstring')
|
|
4
|
+
|
|
1
5
|
const crypt = require('./util/crypt')
|
|
2
|
-
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV} = require("./util/crypt");
|
|
6
|
+
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt");
|
|
7
|
+
|
|
8
|
+
const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
9
|
+
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
10
|
+
const ENC_PAD_SEP = ' ~ '
|
|
3
11
|
|
|
4
12
|
// adapted from https://stackoverflow.com/a/27724419
|
|
5
13
|
function MobilettoError (message, err) {
|
|
@@ -24,6 +32,31 @@ function MobilettoNotFoundError (message) {
|
|
|
24
32
|
}
|
|
25
33
|
}
|
|
26
34
|
|
|
35
|
+
const reader = (chunks) => (chunk) => {
|
|
36
|
+
if (chunk) {
|
|
37
|
+
chunks.push(chunk)
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const UTILITY_FUNCTIONS = {
|
|
42
|
+
readFile: (client) => async (path) => {
|
|
43
|
+
const chunks = []
|
|
44
|
+
await client.read(path, reader(chunks))
|
|
45
|
+
return Buffer.concat(chunks)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function addUtilityFunctions (client) {
|
|
50
|
+
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
51
|
+
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
52
|
+
}
|
|
53
|
+
return client
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function connect (driverPath, key, secret, opts, encryption = null) {
|
|
57
|
+
return await mobiletto(driverPath, key, secret, opts, encryption)
|
|
58
|
+
}
|
|
59
|
+
|
|
27
60
|
async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
28
61
|
const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
|
|
29
62
|
const client = driver.storageClient(key, secret, opts)
|
|
@@ -31,7 +64,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
31
64
|
throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
|
|
32
65
|
}
|
|
33
66
|
if (encryption === null) {
|
|
34
|
-
return client
|
|
67
|
+
return addUtilityFunctions(client)
|
|
35
68
|
}
|
|
36
69
|
const encKey = normalizeKey(encryption.key)
|
|
37
70
|
if (!encKey) {
|
|
@@ -41,17 +74,95 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
41
74
|
if (!iv) {
|
|
42
75
|
throw new MobilettoError(`mobiletto(${driverPath}): invalid encryption IV`)
|
|
43
76
|
}
|
|
77
|
+
const encryptPaths = encryption.encryptPaths || true;
|
|
44
78
|
const enc = {
|
|
45
79
|
key: encKey,
|
|
46
80
|
iv,
|
|
47
|
-
algo: encryption.algo || DEFAULT_CRYPT_ALGO
|
|
81
|
+
algo: encryption.algo || DEFAULT_CRYPT_ALGO,
|
|
82
|
+
encryptPaths,
|
|
83
|
+
dirLevels: encryption.dirLevels || 4,
|
|
84
|
+
encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
|
|
85
|
+
}
|
|
86
|
+
function encryptPath (path) {
|
|
87
|
+
const encrypted = shasum(enc.key + ' ' + path)
|
|
88
|
+
let newPath = ''
|
|
89
|
+
for (let i = 0; i <= enc.dirLevels; i++) {
|
|
90
|
+
if (newPath.length > 0) newPath += '/'
|
|
91
|
+
newPath += encrypted.substring(i*2, (i*2)+2)
|
|
92
|
+
}
|
|
93
|
+
return newPath + encrypted
|
|
94
|
+
}
|
|
95
|
+
const direntDir = dir => encryptPaths ? encryptPath(dir + DIR_ENT_DIR_SUFFIX) : null
|
|
96
|
+
const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
|
|
97
|
+
|
|
98
|
+
const _metadata = (client) => async (path) => {
|
|
99
|
+
if (!encryptPaths) {
|
|
100
|
+
return client.metadata(path)
|
|
101
|
+
}
|
|
102
|
+
let meta
|
|
103
|
+
try {
|
|
104
|
+
meta = await client.metadata(encryptPath(path))
|
|
105
|
+
} catch (e) {
|
|
106
|
+
if (e instanceof MobilettoNotFoundError) {
|
|
107
|
+
const dd = direntDir(path);
|
|
108
|
+
try {
|
|
109
|
+
meta = await client.metadata(dd)
|
|
110
|
+
} catch (err) {
|
|
111
|
+
if (err instanceof MobilettoNotFoundError) {
|
|
112
|
+
const contents = await client.list(dd)
|
|
113
|
+
if (Array.isArray(contents) && contents.length > 0) {
|
|
114
|
+
return { name: path, type: M_DIR }
|
|
115
|
+
} else {
|
|
116
|
+
throw err
|
|
117
|
+
}
|
|
118
|
+
} else {
|
|
119
|
+
throw err
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
} else {
|
|
123
|
+
throw e
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
meta.name = path // rewrite name back to plaintext name
|
|
127
|
+
return meta
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const _loadMeta = async (dirent, entries) => {
|
|
131
|
+
const files = []
|
|
132
|
+
for (const entry of entries) {
|
|
133
|
+
const cipherText = []
|
|
134
|
+
await client.read(dirent + '/' + basename(entry.name), reader(cipherText))
|
|
135
|
+
const plain = decrypt(cipherText.toString(), enc)
|
|
136
|
+
const realPath = plain.split(ENC_PAD_SEP)[0]
|
|
137
|
+
const meta = await _metadata(client)(realPath)
|
|
138
|
+
files.push(meta)
|
|
139
|
+
}
|
|
140
|
+
return files
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function removeDirentFile(path) {
|
|
144
|
+
const df = direntFile(direntDir(dirname(path)), path);
|
|
145
|
+
await client.remove(df, {recursive: false, quiet: true})
|
|
146
|
+
await client.remove(encryptPath(path), {recursive: false, quiet: true})
|
|
48
147
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
148
|
+
|
|
149
|
+
const encClient = {
|
|
150
|
+
list: async (path) => {
|
|
151
|
+
if (!encryptPaths) {
|
|
152
|
+
return await client.list(path)
|
|
153
|
+
}
|
|
154
|
+
const dirent = direntDir(path)
|
|
155
|
+
const entries = await client.list(dirent)
|
|
156
|
+
if (!entries || entries.length === 0) {
|
|
157
|
+
return []
|
|
158
|
+
}
|
|
159
|
+
return _loadMeta(dirent, entries)
|
|
160
|
+
},
|
|
161
|
+
metadata: _metadata(client),
|
|
52
162
|
read: async (path, callback) => {
|
|
163
|
+
const realPath = encryptPaths ? encryptPath(path) : path
|
|
53
164
|
const cipher = crypt.startDecryptStream(enc)
|
|
54
|
-
return client.read(
|
|
165
|
+
return client.read(realPath,
|
|
55
166
|
(chunk) => {
|
|
56
167
|
return callback(crypt.updateCryptStream(cipher, chunk))
|
|
57
168
|
}, () => {
|
|
@@ -59,6 +170,25 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
59
170
|
})
|
|
60
171
|
},
|
|
61
172
|
write: async (path, readFunc) => {
|
|
173
|
+
// if encrypting paths, write dirent file(s) for all parent directories
|
|
174
|
+
if (encryptPaths) {
|
|
175
|
+
let p = path
|
|
176
|
+
while (true) {
|
|
177
|
+
const direntGenerator = function* () {
|
|
178
|
+
yield encrypt(p + enc.encPathPadding(), enc)
|
|
179
|
+
}
|
|
180
|
+
const dir = direntDir(dirname(p))
|
|
181
|
+
const df = direntFile(dir, p);
|
|
182
|
+
if (!(await client.write(df, direntGenerator()))) {
|
|
183
|
+
throw new MobilettoError('write: error writing dirent file')
|
|
184
|
+
}
|
|
185
|
+
p = dirname(p)
|
|
186
|
+
if (p === '.' || p === '') {
|
|
187
|
+
break
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
62
192
|
function* cryptGenerator(plaintextGenerator) {
|
|
63
193
|
let chunk = plaintextGenerator.next().value
|
|
64
194
|
while (chunk) {
|
|
@@ -68,14 +198,71 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
68
198
|
yield cipher.final()
|
|
69
199
|
}
|
|
70
200
|
const cipher = crypt.startEncryptStream(enc)
|
|
71
|
-
|
|
201
|
+
const realPath = encryptPath(path)
|
|
202
|
+
return client.write(realPath, cryptGenerator(readFunc))
|
|
72
203
|
},
|
|
73
204
|
remove: async (path, options) => {
|
|
74
205
|
const recursive = (options && options.recursive) || false
|
|
75
206
|
const quiet = (options && options.quiet) || false
|
|
76
|
-
|
|
207
|
+
if (!encryptPaths) {
|
|
208
|
+
return client.remove(path, {recursive, quiet})
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (recursive) {
|
|
212
|
+
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
213
|
+
async function recRm (path) {
|
|
214
|
+
const dirent = direntDir(path)
|
|
215
|
+
let entries
|
|
216
|
+
try {
|
|
217
|
+
entries = await client.list(dirent)
|
|
218
|
+
} catch (e) {
|
|
219
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
220
|
+
throw e instanceof MobilettoError
|
|
221
|
+
? e
|
|
222
|
+
: new MobilettoError('error listing files for recursive deletion', e)
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (entries && entries.length > 0) {
|
|
226
|
+
const files = await _loadMeta(dirent, entries)
|
|
227
|
+
for (const f of files) {
|
|
228
|
+
await recRm(f.name)
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
await removeDirentFile(path);
|
|
232
|
+
}
|
|
233
|
+
await recRm(path)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// remove the single file/dir
|
|
237
|
+
await removeDirentFile(path);
|
|
238
|
+
|
|
239
|
+
// if we were the last dirent file, then also remove dirent directory, and recursively upwards
|
|
240
|
+
let parent = path
|
|
241
|
+
let dirent = direntDir(parent)
|
|
242
|
+
const df = direntFile(dirent, path)
|
|
243
|
+
await client.remove(df, {recursive: false, quiet: true})
|
|
244
|
+
while (true) {
|
|
245
|
+
try {
|
|
246
|
+
const entries = await client.list(dirent)
|
|
247
|
+
if (entries.length === 0) {
|
|
248
|
+
await removeDirentFile(parent)
|
|
249
|
+
}
|
|
250
|
+
} catch (e) {
|
|
251
|
+
if (!(e instanceof MobilettoNotFoundError)) {
|
|
252
|
+
throw e
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
if (parent === '.' || parent === '/') {
|
|
256
|
+
// do not remove dirent for root dir
|
|
257
|
+
break
|
|
258
|
+
}
|
|
259
|
+
parent = dirname(parent)
|
|
260
|
+
dirent = direntDir(parent)
|
|
261
|
+
}
|
|
262
|
+
return true
|
|
77
263
|
}
|
|
78
264
|
}
|
|
265
|
+
return addUtilityFunctions(encClient)
|
|
79
266
|
}
|
|
80
267
|
|
|
81
268
|
async function readStream(stream, callback, endCallback) {
|
|
@@ -103,7 +290,7 @@ function writeStream (stream) {
|
|
|
103
290
|
if (chunk) {
|
|
104
291
|
stream.write(chunk, (err) => {
|
|
105
292
|
if (err) {
|
|
106
|
-
console.
|
|
293
|
+
console.error(`writeStream: error writing: ${err}`)
|
|
107
294
|
throw err
|
|
108
295
|
}
|
|
109
296
|
})
|
|
@@ -114,39 +301,12 @@ function writeStream (stream) {
|
|
|
114
301
|
function closeStream (stream) {
|
|
115
302
|
return () => stream.close((err) => {
|
|
116
303
|
if (err) {
|
|
117
|
-
console.
|
|
304
|
+
console.error(`closeStream: error closing: ${err}`)
|
|
118
305
|
throw err
|
|
119
306
|
}
|
|
120
307
|
})
|
|
121
308
|
}
|
|
122
309
|
|
|
123
|
-
async function streamReader (stream, callback, endCallback) {
|
|
124
|
-
let count = 0
|
|
125
|
-
const streamHandler = async (stream) => {
|
|
126
|
-
new Promise((resolve, reject) => {
|
|
127
|
-
stream.on('data', (data) => {
|
|
128
|
-
count += data ? data.length : 0
|
|
129
|
-
callback(data)
|
|
130
|
-
})
|
|
131
|
-
stream.on('error', reject)
|
|
132
|
-
stream.on('end', (resolve) => {
|
|
133
|
-
if (typeof endCallback === 'function') {
|
|
134
|
-
const endData = endCallback()
|
|
135
|
-
if (endData) {
|
|
136
|
-
count += endData.length
|
|
137
|
-
callback(endData)
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
resolve()
|
|
141
|
-
})
|
|
142
|
-
})
|
|
143
|
-
}
|
|
144
|
-
await streamHandler(stream).then(() => {
|
|
145
|
-
// console.log('streamHandler ended')
|
|
146
|
-
})
|
|
147
|
-
return count
|
|
148
|
-
}
|
|
149
|
-
|
|
150
310
|
const M_FILE = 'file'
|
|
151
311
|
const M_DIR = 'dir'
|
|
152
312
|
const M_LINK = 'link'
|
|
@@ -154,7 +314,7 @@ const M_SPECIAL = 'special'
|
|
|
154
314
|
|
|
155
315
|
module.exports = {
|
|
156
316
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
157
|
-
mobiletto,
|
|
317
|
+
mobiletto, connect,
|
|
158
318
|
MobilettoError, MobilettoNotFoundError,
|
|
159
|
-
readStream, writeStream, closeStream
|
|
319
|
+
readStream, writeStream, closeStream
|
|
160
320
|
}
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -6,18 +6,22 @@ const randomstring = require('randomstring')
|
|
|
6
6
|
|
|
7
7
|
const { expect, should, assert } = require('chai')
|
|
8
8
|
|
|
9
|
-
const { mobiletto } = require(
|
|
10
|
-
const {
|
|
9
|
+
const { mobiletto, connect, MobilettoNotFoundError, M_FILE, M_DIR } = require("../index")
|
|
10
|
+
const { encrypt, decrypt, normalizeKey, normalizeIV, DEFAULT_CRYPT_ALGO } = require("../util/crypt")
|
|
11
11
|
|
|
12
12
|
// chunk size used by generator function, used by driver's 'write' function
|
|
13
13
|
// the temp file is also TEMP_SZ_MULTIPLE of this number
|
|
14
14
|
const READ_SZ = 8 * 1024 // xfer data in 8k chunks
|
|
15
15
|
const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
|
|
16
|
-
|
|
16
|
+
|
|
17
|
+
// encoded bytes written will differ from actual bytes provided
|
|
18
|
+
// likewise, plaintext bytes read will differ from encoded bytes read
|
|
19
|
+
// our tests give some leeway when filesize differences
|
|
20
|
+
const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
|
|
17
21
|
|
|
18
22
|
DRIVER_CONFIG = {
|
|
19
23
|
local: {
|
|
20
|
-
key: '/tmp'
|
|
24
|
+
key: process.env.LOCAL_STORAGE || '/tmp'
|
|
21
25
|
},
|
|
22
26
|
s3: {
|
|
23
27
|
key: process.env.S3_ACCESS,
|
|
@@ -30,13 +34,17 @@ DRIVER_CONFIG = {
|
|
|
30
34
|
}
|
|
31
35
|
}
|
|
32
36
|
|
|
37
|
+
function closeEnough (expected, actual, percent = ENC_SIZE_CLOSE_ENOUGH_PERCENT) {
|
|
38
|
+
expect(Math.abs(expected - actual)).to.be.lessThan(Math.floor(expected * percent),
|
|
39
|
+
'expected size within 5% of actual (due to encryption)')
|
|
40
|
+
}
|
|
41
|
+
|
|
33
42
|
async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
|
|
34
43
|
const meta = await api.metadata(name)
|
|
35
44
|
should().exist(meta, 'expected return value from metadata call')
|
|
36
45
|
expect(meta.name).equals(name, 'expected name of written file to be correct')
|
|
37
46
|
if (closeEnoughPercent) {
|
|
38
|
-
|
|
39
|
-
'expected write API to return within 5% of bytes written (due to encryption)')
|
|
47
|
+
closeEnough(expectedSize, meta.size, closeEnoughPercent)
|
|
40
48
|
} else {
|
|
41
49
|
expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
|
|
42
50
|
}
|
|
@@ -68,17 +76,20 @@ async function writeRandomFile(fixture, size) {
|
|
|
68
76
|
return await fixture.api.write(fixture.name, dataGenerator())
|
|
69
77
|
}
|
|
70
78
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
describe('crypto test', () => {
|
|
80
|
+
it("should encrypt and decrypt successfully", async () => {
|
|
81
|
+
const enc = {
|
|
82
|
+
key: normalizeKey(randomstring.generate(32)),
|
|
83
|
+
iv: normalizeIV(randomstring.generate(16)),
|
|
84
|
+
algo: DEFAULT_CRYPT_ALGO
|
|
76
85
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
86
|
+
const plaintext = randomstring.generate(1024 + Math.floor(1024*Math.random()))
|
|
87
|
+
const ciphertext = encrypt(plaintext, enc)
|
|
88
|
+
const decrypted = decrypt(ciphertext, enc)
|
|
89
|
+
expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
})
|
|
82
93
|
|
|
83
94
|
// To test a single driver:
|
|
84
95
|
// - Uncomment one of the lines below to set driverName to the one you want to test
|
|
@@ -120,7 +131,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
120
131
|
let fixture
|
|
121
132
|
beforeEach((done) => {
|
|
122
133
|
const name = `test_file_${fileSuffix}`
|
|
123
|
-
|
|
134
|
+
connect(driverName, config.key, config.secret, config.opts)
|
|
124
135
|
.then(api => { fixture = {api, name, randomData} })
|
|
125
136
|
.finally(done)
|
|
126
137
|
})
|
|
@@ -129,8 +140,8 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
129
140
|
expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
|
|
130
141
|
})
|
|
131
142
|
it("should read the file we just wrote", async () => {
|
|
132
|
-
const
|
|
133
|
-
expect(
|
|
143
|
+
const data = await fixture.api.readFile(fixture.name);
|
|
144
|
+
expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
|
|
134
145
|
expect(data.toString('utf8')).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
135
146
|
})
|
|
136
147
|
it("should load metadata on the file we just wrote", async () => {
|
|
@@ -161,13 +172,12 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
161
172
|
})
|
|
162
173
|
it("should write an encrypted file", async () => {
|
|
163
174
|
encryptedByteCount = await writeRandomFile(fixture, size);
|
|
164
|
-
|
|
165
|
-
'expected write API to return within 5% of bytes written (due to encryption)')
|
|
175
|
+
closeEnough(size, encryptedByteCount)
|
|
166
176
|
})
|
|
167
177
|
it("should read the encrypted file we just wrote", async () => {
|
|
168
|
-
const
|
|
169
|
-
expect(
|
|
170
|
-
expect(data.toString(
|
|
178
|
+
const data = await fixture.api.readFile(fixture.name);
|
|
179
|
+
expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
|
|
180
|
+
expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
171
181
|
})
|
|
172
182
|
it("should load metadata on the encrypted file we just wrote", async () => {
|
|
173
183
|
await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
|
|
@@ -181,58 +191,71 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
181
191
|
})
|
|
182
192
|
})
|
|
183
193
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
194
|
+
for (const encryption of [null, {key: rand(32)}]) {
|
|
195
|
+
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
196
|
+
describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
|
|
197
|
+
// describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|
|
198
|
+
// const encryption = {key: rand(32)}
|
|
199
|
+
// const encryption = null
|
|
200
|
+
// a random directory and file within it
|
|
201
|
+
const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
|
|
202
|
+
const subdirName = `subdir_` + Date.now()
|
|
203
|
+
const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
|
|
204
|
+
//const fileCount = 3 + Math.floor(Math.random() * 10)
|
|
205
|
+
const fileCount = 2
|
|
206
|
+
let fixture
|
|
207
|
+
beforeEach((done) => {
|
|
208
|
+
mobiletto(driverName, config.key, config.secret, config.opts, encryption)
|
|
209
|
+
.then(api => { fixture = {api, name: randomPath} })
|
|
210
|
+
.catch((err) => { throw err })
|
|
211
|
+
.finally(done)
|
|
212
|
+
})
|
|
213
|
+
it(`should write ${fileCount} files in a new directory`, async () => {
|
|
214
|
+
function* dataGenerator() {
|
|
215
|
+
// return one chunk of random data
|
|
216
|
+
yield rand(READ_SZ)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
for (let i = 0; i < fileCount; i++) {
|
|
220
|
+
const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
|
|
221
|
+
if (encryption) {
|
|
222
|
+
closeEnough(READ_SZ, bytesWritten)
|
|
223
|
+
} else {
|
|
224
|
+
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
})
|
|
228
|
+
it("should load metadata for one of the new files", async () => {
|
|
229
|
+
await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
|
|
230
|
+
})
|
|
231
|
+
it("should see correct types on objects returned from a listing of the new directory", async () => {
|
|
232
|
+
const objects = await fixture.api.list(randomParent)
|
|
233
|
+
expect(objects).to.have.lengthOf(1)
|
|
234
|
+
expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
|
|
235
|
+
})
|
|
236
|
+
it("should see correct types on objects returned from a listing of the subdirectory", async () => {
|
|
237
|
+
const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
|
|
238
|
+
expect(objects).to.have.lengthOf(fileCount)
|
|
239
|
+
for (let i = 0; i < fileCount; i++) {
|
|
240
|
+
// we should find all the files, and they should all have the correct type
|
|
241
|
+
expect(objects
|
|
242
|
+
.find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
|
|
243
|
+
).to.not.be.null
|
|
244
|
+
}
|
|
245
|
+
})
|
|
246
|
+
it("should recursively delete the directory and file we just created", async () => {
|
|
247
|
+
const recursive = true
|
|
248
|
+
const removed = await fixture.api.remove(randomParent, {recursive})
|
|
249
|
+
expect(removed).to.be.true
|
|
250
|
+
})
|
|
251
|
+
it("loading metadata on the file we wrote now fails", async () => {
|
|
252
|
+
await assertMetaFail(fixture.api, fixture.name)
|
|
253
|
+
})
|
|
254
|
+
it("loading metadata on the parent dir we wrote now fails", async () => {
|
|
255
|
+
await assertMetaFail(fixture.api, randomParent)
|
|
256
|
+
})
|
|
234
257
|
})
|
|
235
|
-
}
|
|
258
|
+
}
|
|
236
259
|
|
|
237
260
|
describe(`${driverName} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
|
|
238
261
|
it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
|
package/util/crypt.js
CHANGED
|
@@ -36,27 +36,28 @@ function getCipher(enc) {
|
|
|
36
36
|
return crypto.createCipheriv(enc.algo, enc.key, enc.iv);
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
function encrypt (plainText,
|
|
40
|
-
if (!key) {
|
|
39
|
+
function encrypt (plainText, encryption, outputEncoding = 'base64') {
|
|
40
|
+
if (!encryption || !encryption.key) {
|
|
41
41
|
if (WARN_PLAINTEXT) {
|
|
42
|
-
console.warn(` ****** key is undefined, encryption
|
|
42
|
+
console.warn(` ****** encryption.key is undefined, encryption is DISABLED`)
|
|
43
43
|
}
|
|
44
44
|
return plainText
|
|
45
45
|
}
|
|
46
|
-
const cipher = getCipher(
|
|
47
|
-
|
|
46
|
+
const cipher = getCipher(encryption)
|
|
47
|
+
const encoded = Buffer.concat([cipher.update(plainText), cipher.final()])
|
|
48
|
+
return encoded.toString(outputEncoding)
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
function getDecipher(enc) {
|
|
51
52
|
return crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
|
|
52
53
|
}
|
|
53
54
|
|
|
54
|
-
function decrypt (cipherText,
|
|
55
|
-
if (!key) {
|
|
55
|
+
function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
|
|
56
|
+
if (!encryption || !encryption.key) {
|
|
56
57
|
return cipherText
|
|
57
58
|
}
|
|
58
|
-
const cipher = getDecipher(
|
|
59
|
-
const data = Buffer.from(cipherText,
|
|
59
|
+
const cipher = getDecipher(encryption)
|
|
60
|
+
const data = Buffer.from(cipherText, inputEncoding)
|
|
60
61
|
return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
|
|
61
62
|
}
|
|
62
63
|
|