mobiletto 1.0.12 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -10
- package/drivers/local/index.js +111 -43
- package/drivers/s3/index.js +44 -27
- package/index.js +134 -16
- package/package.json +3 -2
- package/test/index.spec.js +69 -15
package/README.md
CHANGED
|
@@ -41,10 +41,18 @@ Today the supported drivers are:
|
|
|
41
41
|
local.list() // --> returns an array of file objects
|
|
42
42
|
s3.list() // --> returns an array of file objects
|
|
43
43
|
|
|
44
|
+
// List files recursively
|
|
45
|
+
local.list({ recursive: true })
|
|
46
|
+
s3.list({ recursive: true })
|
|
47
|
+
|
|
44
48
|
// List files in a directory
|
|
45
49
|
const path = 'some/path'
|
|
46
|
-
local.list(path)
|
|
47
|
-
s3.list(path
|
|
50
|
+
local.list(path)
|
|
51
|
+
s3.list(path, { recursive: true }) // also supports recursive flag
|
|
52
|
+
|
|
53
|
+
// Visit files in a directory -- visitor function must be async
|
|
54
|
+
local.list(path, { visitor: myAsyncFunc })
|
|
55
|
+
s3.list(path, { visitor: myAsyncFunc, recursive: true })
|
|
48
56
|
|
|
49
57
|
// Read metadata for a file
|
|
50
58
|
local.metadata(path)
|
|
@@ -81,12 +89,28 @@ Today the supported drivers are:
|
|
|
81
89
|
local.remove(path, {quiet})
|
|
82
90
|
s3.remove(path, {quiet})
|
|
83
91
|
|
|
84
|
-
// Recursively delete a directory
|
|
92
|
+
// Recursively delete a directory and do it quietly (do not report errors)
|
|
85
93
|
const recursive = true
|
|
94
|
+
const quiet = true
|
|
86
95
|
local.remove(path, {recursive, quiet})
|
|
87
96
|
s3.remove(path, {recursive, quiet})
|
|
88
97
|
|
|
89
|
-
#
|
|
98
|
+
# Mirroring
|
|
99
|
+
|
|
100
|
+
// Copy a local filesystem mobiletto to S3
|
|
101
|
+
s3.mirror(local)
|
|
102
|
+
|
|
103
|
+
// Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
|
|
104
|
+
local.mirror(s3, 'some/local-folder', 'some/s3-folder')
|
|
105
|
+
|
|
106
|
+
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
107
|
+
|
|
108
|
+
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
109
|
+
reader and who is the writer. Imagine it like an assignment statement: the "left-hand mobiletto"
|
|
110
|
+
is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
|
|
111
|
+
argument to the `mirror` method) is the value being assigned (mirrored data is read).
|
|
112
|
+
|
|
113
|
+
# Alternate import style
|
|
90
114
|
Import the fully-scoped module and use the `connect` function:
|
|
91
115
|
|
|
92
116
|
const storage = require('mobiletto')
|
|
@@ -98,9 +122,15 @@ Enable transparent client-side encryption:
|
|
|
98
122
|
|
|
99
123
|
// Pass encryption parameters
|
|
100
124
|
const encryption = {
|
|
101
|
-
key
|
|
102
|
-
|
|
103
|
-
|
|
125
|
+
// key is required, must be >= 16 chars
|
|
126
|
+
key: randomstring.generate(128),
|
|
127
|
+
|
|
128
|
+
// optional, the default is to derive IV from key
|
|
129
|
+
// when set, IV must be >= 16 chars
|
|
130
|
+
iv: randomstring.generate(128),
|
|
131
|
+
|
|
132
|
+
// optional, the default is aes-256-cbc
|
|
133
|
+
algo: 'aes-256-cbc'
|
|
104
134
|
}
|
|
105
135
|
const api = await mobiletto(driverName, key, secret, opts, encryption)
|
|
106
136
|
|
|
@@ -124,6 +154,38 @@ some effort, discover some or all of the overall structure of the directory hier
|
|
|
124
154
|
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
125
155
|
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
126
156
|
|
|
157
|
+
# File metadata
|
|
158
|
+
The `metadata` command returns metadata about a single filesystem entry.
|
|
159
|
+
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
160
|
+
|
|
161
|
+
A metadata object looks like this:
|
|
162
|
+
|
|
163
|
+
{
|
|
164
|
+
"name": "fully/qualified/path/to/file",
|
|
165
|
+
"type": "entry-type",
|
|
166
|
+
"size": size-in-bytes,
|
|
167
|
+
"ctime": creation-time-epoch-millis,
|
|
168
|
+
"mtime": modification-time-epoch-millis
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
172
|
+
|
|
173
|
+
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
174
|
+
should always be present. A subsequent `metadata` command should return all available properties.
|
|
175
|
+
|
|
176
|
+
# Key rotation
|
|
177
|
+
Create a mobiletto with your new key, then mirror the old data into it:
|
|
178
|
+
|
|
179
|
+
const storage = require('mobiletto')
|
|
180
|
+
|
|
181
|
+
const oldEncryption = { key: .... }
|
|
182
|
+
const oldStorage = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'}, oldEncryption)
|
|
183
|
+
|
|
184
|
+
const newEncryption = { key: .... }
|
|
185
|
+
const newStorage = await storage.connect('s3', aws_key, aws_secret, {bucket: 'zz', region: 'us-east-1'}, newEncryption)
|
|
186
|
+
|
|
187
|
+
newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
|
|
188
|
+
|
|
127
189
|
# Driver Interface
|
|
128
190
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
|
129
191
|
|
|
@@ -143,8 +205,11 @@ The object that the storageClient function returns must define these functions:
|
|
|
143
205
|
// Test the driver before using, ensure proper configuration
|
|
144
206
|
async testConfig ()
|
|
145
207
|
|
|
146
|
-
//
|
|
147
|
-
|
|
208
|
+
// List files in path (or from base-directory)
|
|
209
|
+
// If recursive is true, list recursively
|
|
210
|
+
// If visitor is defined, it will be an async function. await the visitor function on each file found
|
|
211
|
+
// Otherwise, perform the listing and return an array of objects
|
|
212
|
+
async list (path, recursive = false, visitor = null) // path may be omitted
|
|
148
213
|
|
|
149
214
|
// Read metadata for a path
|
|
150
215
|
async metadata (path)
|
|
@@ -158,4 +223,4 @@ The object that the storageClient function returns must define these functions:
|
|
|
158
223
|
async write (path, readFunc)
|
|
159
224
|
|
|
160
225
|
// Remove a file, or recursively delete a directory
|
|
161
|
-
async remove (path,
|
|
226
|
+
async remove (path, recursive = false, quiet = false)
|
package/drivers/local/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const { dirname } = require('path')
|
|
2
2
|
|
|
3
3
|
const {
|
|
4
|
-
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
4
|
+
M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
|
|
5
5
|
MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
|
|
6
6
|
} = require('../../index')
|
|
7
7
|
|
|
@@ -9,9 +9,9 @@ const fs = require('fs')
|
|
|
9
9
|
|
|
10
10
|
const DEFAULT_MODE = '0700'
|
|
11
11
|
|
|
12
|
-
const
|
|
12
|
+
const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
|
13
13
|
|
|
14
|
-
const ioError = (err, path, method) =>
|
|
14
|
+
const ioError = (err, path, method) => isNotExistError(err)
|
|
15
15
|
? err instanceof MobilettoNotFoundError
|
|
16
16
|
? err
|
|
17
17
|
: new MobilettoNotFoundError(path)
|
|
@@ -19,6 +19,19 @@ const ioError = (err, path, method) => isENOENT(err)
|
|
|
19
19
|
? err
|
|
20
20
|
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
21
21
|
|
|
22
|
+
const fileType = (stat) => {
|
|
23
|
+
if (stat.isDirectory()) {
|
|
24
|
+
return M_DIR
|
|
25
|
+
}
|
|
26
|
+
if (stat.isFile()) {
|
|
27
|
+
return M_FILE
|
|
28
|
+
}
|
|
29
|
+
if (stat.isSymbolicLink()) {
|
|
30
|
+
return M_LINK
|
|
31
|
+
}
|
|
32
|
+
return M_SPECIAL
|
|
33
|
+
}
|
|
34
|
+
|
|
22
35
|
class StorageClient {
|
|
23
36
|
baseDir
|
|
24
37
|
mode
|
|
@@ -46,41 +59,63 @@ class StorageClient {
|
|
|
46
59
|
return { path, stat }
|
|
47
60
|
}
|
|
48
61
|
testConfig = async () => await this.list()
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
62
|
+
|
|
63
|
+
normalizePath = (path) => path.startsWith(this.baseDir)
|
|
64
|
+
? path
|
|
65
|
+
: this.baseDir + (path.startsWith('/') ? path.substring(1) : path)
|
|
66
|
+
denormalizePath = (path) => path.startsWith(this.baseDir) ? path.substring(this.baseDir.length) : path
|
|
67
|
+
|
|
68
|
+
fileToObject = (dir) => (f) => {
|
|
69
|
+
const stat = fs.lstatSync(dir + '/' + f)
|
|
70
|
+
const type = fileType(stat)
|
|
71
|
+
const entry = {
|
|
72
|
+
name: this.denormalizePath(dir + '/' + f),
|
|
73
|
+
type
|
|
52
74
|
}
|
|
53
|
-
if (
|
|
54
|
-
|
|
75
|
+
if (type === M_LINK) {
|
|
76
|
+
const resolved = this.resolveSymlinks(dir + '/' + f)
|
|
77
|
+
entry.link = resolved.path
|
|
55
78
|
}
|
|
56
|
-
|
|
57
|
-
|
|
79
|
+
return entry
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
readDirFiles = async (dir, recursive, visitor) => {
|
|
83
|
+
if (!recursive) {
|
|
84
|
+
return await this.dirFiles(dir, visitor)
|
|
85
|
+
}
|
|
86
|
+
const files = await this.dirFiles(dir, visitor)
|
|
87
|
+
for (const file of files) {
|
|
88
|
+
if (file.type === M_DIR) {
|
|
89
|
+
await this.readDirFiles(file.name, true, visitor)
|
|
90
|
+
}
|
|
58
91
|
}
|
|
59
|
-
return M_SPECIAL
|
|
60
92
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
93
|
+
|
|
94
|
+
dirFiles = async (dir, visitor) => {
|
|
95
|
+
const norm = this.normalizePath(dir);
|
|
96
|
+
try {
|
|
97
|
+
let names = fs.readdirSync(norm);
|
|
98
|
+
const files = names.map(this.fileToObject(norm))
|
|
99
|
+
for (const f of files) {
|
|
100
|
+
await visitor(f)
|
|
101
|
+
}
|
|
102
|
+
return files
|
|
103
|
+
} catch (e) {
|
|
104
|
+
console.error(`dirFiles: ${e}`)
|
|
105
|
+
throw ioError(e, norm, 'dirFiles')
|
|
64
106
|
}
|
|
65
|
-
return this.baseDir + path
|
|
66
107
|
}
|
|
67
|
-
|
|
108
|
+
|
|
109
|
+
async list (path = '', recursive = false, visitor = null) {
|
|
68
110
|
const dir = this.normalizePath(path)
|
|
69
111
|
try {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
}
|
|
78
|
-
if (type === M_LINK) {
|
|
79
|
-
const resolved = this.resolveSymlinks(dir + '/' + f)
|
|
80
|
-
entry.link = resolved.path
|
|
81
|
-
}
|
|
82
|
-
return entry
|
|
83
|
-
})
|
|
112
|
+
if (visitor === null) {
|
|
113
|
+
const results = []
|
|
114
|
+
await this.readDirFiles(dir, recursive, (obj) => results.push(obj))
|
|
115
|
+
return results
|
|
116
|
+
} else {
|
|
117
|
+
return await this.readDirFiles(dir, recursive, visitor)
|
|
118
|
+
}
|
|
84
119
|
} catch (err) {
|
|
85
120
|
throw ioError(err, path, 'list')
|
|
86
121
|
}
|
|
@@ -92,7 +127,7 @@ class StorageClient {
|
|
|
92
127
|
if (!lstat) {
|
|
93
128
|
throw new MobilettoError('metadata: lstat error')
|
|
94
129
|
}
|
|
95
|
-
const type =
|
|
130
|
+
const type = fileType(lstat)
|
|
96
131
|
if (type === M_DIR && path !== '') {
|
|
97
132
|
const contents = await this.list(path)
|
|
98
133
|
if (contents.length === 0) {
|
|
@@ -100,7 +135,7 @@ class StorageClient {
|
|
|
100
135
|
}
|
|
101
136
|
}
|
|
102
137
|
return {
|
|
103
|
-
name:
|
|
138
|
+
name: this.denormalizePath(file),
|
|
104
139
|
type,
|
|
105
140
|
size: lstat.size,
|
|
106
141
|
mtime: lstat.mtimeMs
|
|
@@ -119,7 +154,7 @@ class StorageClient {
|
|
|
119
154
|
}
|
|
120
155
|
}
|
|
121
156
|
|
|
122
|
-
async
|
|
157
|
+
async normalizePathAndEnsureParentDirs (path) {
|
|
123
158
|
const file = this.normalizePath(path)
|
|
124
159
|
|
|
125
160
|
// console.log(`read: reading path: ${path} - ${file}`)
|
|
@@ -128,7 +163,7 @@ class StorageClient {
|
|
|
128
163
|
try {
|
|
129
164
|
dirStat = fs.lstatSync(parent)
|
|
130
165
|
} catch (err) {
|
|
131
|
-
if (err
|
|
166
|
+
if (isNotExistError(err)) {
|
|
132
167
|
this.mkdirs(parent)
|
|
133
168
|
} else {
|
|
134
169
|
throw new MobilettoError(`write: lstat error on ${parent}: ${err}`, err)
|
|
@@ -139,17 +174,52 @@ class StorageClient {
|
|
|
139
174
|
} else if (!dirStat.isDirectory()) {
|
|
140
175
|
throw new MobilettoError(`write: not a directory: ${parent} (cannot write file ${file})`)
|
|
141
176
|
}
|
|
177
|
+
return file
|
|
178
|
+
}
|
|
142
179
|
|
|
180
|
+
async write (path, generatorOrReadableStream) {
|
|
181
|
+
const file = await this.normalizePathAndEnsureParentDirs(path)
|
|
143
182
|
// console.log(`write: writing path ${path} -> ${file}`)
|
|
144
183
|
const stream = fs.createWriteStream(file, {mode: this.mode})
|
|
145
184
|
const writer = writeStream(stream)
|
|
146
185
|
const closer = closeStream(stream)
|
|
147
186
|
let count = 0
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
187
|
+
|
|
188
|
+
if (isReadable(generatorOrReadableStream)) {
|
|
189
|
+
const readable = generatorOrReadableStream
|
|
190
|
+
const streamHandler = stream =>
|
|
191
|
+
new Promise((resolve, reject) => {
|
|
192
|
+
stream.on('data', (data) => {
|
|
193
|
+
if (data) {
|
|
194
|
+
writer(data)
|
|
195
|
+
count += data.length
|
|
196
|
+
}
|
|
197
|
+
})
|
|
198
|
+
stream.on('error', reject)
|
|
199
|
+
stream.on('end', () => {
|
|
200
|
+
closer()
|
|
201
|
+
resolve()
|
|
202
|
+
})
|
|
203
|
+
})
|
|
204
|
+
await streamHandler(readable)
|
|
205
|
+
return count
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const generator = generatorOrReadableStream
|
|
209
|
+
let chunk = isAsyncGenerator(generator)
|
|
210
|
+
? (await generator.next()).value
|
|
211
|
+
: generator.next().value
|
|
212
|
+
let nullCount = 0
|
|
213
|
+
while (chunk || nullCount < 5) {
|
|
214
|
+
if (chunk) {
|
|
215
|
+
count += chunk.length
|
|
216
|
+
writer(chunk)
|
|
217
|
+
} else {
|
|
218
|
+
nullCount++
|
|
219
|
+
}
|
|
220
|
+
chunk = isAsyncGenerator(generator)
|
|
221
|
+
? (await generator.next()).value
|
|
222
|
+
: generator.next().value
|
|
153
223
|
}
|
|
154
224
|
closer()
|
|
155
225
|
return Promise.resolve(count)
|
|
@@ -166,15 +236,13 @@ class StorageClient {
|
|
|
166
236
|
}
|
|
167
237
|
}
|
|
168
238
|
|
|
169
|
-
async remove (path,
|
|
170
|
-
const recursive = options && options.recursive ? options.recursive : false
|
|
171
|
-
const quiet = options && options.quiet ? options.quiet : false
|
|
239
|
+
async remove (path, recursive, quiet) {
|
|
172
240
|
const file = this.normalizePath(path)
|
|
173
241
|
// console.log(`remove: deleting path: ${path} = ${file}`)
|
|
174
242
|
try {
|
|
175
243
|
fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
|
|
176
244
|
} catch (err) {
|
|
177
|
-
if (
|
|
245
|
+
if (isNotExistError(err)) {
|
|
178
246
|
if (!quiet) {
|
|
179
247
|
throw new MobilettoNotFoundError(path)
|
|
180
248
|
}
|
package/drivers/s3/index.js
CHANGED
|
@@ -44,18 +44,23 @@ class StorageClient {
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
47
|
-
testConfig = async () => await this._list('', { MaxKeys: 1 })
|
|
47
|
+
testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
49
|
+
stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
|
|
50
|
+
|
|
51
|
+
nameToObj = (name) => {
|
|
52
|
+
const relName = this.stripPrefix(name)
|
|
53
|
+
return {
|
|
54
|
+
name: relName,
|
|
55
|
+
type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
56
|
+
}
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
async
|
|
59
|
+
async list (path = '', recursive = false, visitor = null) {
|
|
60
|
+
return await this._list(path, recursive, visitor)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
59
64
|
const logPrefix = `_list(path=${path}):`
|
|
60
65
|
|
|
61
66
|
// Declare truncated as a flag that the while loop is based on.
|
|
@@ -73,6 +78,7 @@ class StorageClient {
|
|
|
73
78
|
bucketParams.Delimiter = this.delimiter
|
|
74
79
|
}
|
|
75
80
|
const objects = []
|
|
81
|
+
let objectCount = 0
|
|
76
82
|
// console.log(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
|
|
77
83
|
|
|
78
84
|
// while loop that runs until 'response.truncated' is false.
|
|
@@ -81,15 +87,25 @@ class StorageClient {
|
|
|
81
87
|
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
82
88
|
const hasContents = typeof response.Contents !== 'undefined'
|
|
83
89
|
if (hasContents) {
|
|
84
|
-
response.Contents
|
|
85
|
-
|
|
86
|
-
|
|
90
|
+
for (const item of response.Contents) {
|
|
91
|
+
const obj = this.nameToObj(item.Key);
|
|
92
|
+
if (visitor) {
|
|
93
|
+
await visitor(obj)
|
|
94
|
+
}
|
|
95
|
+
objects.push(obj)
|
|
96
|
+
objectCount++
|
|
97
|
+
}
|
|
87
98
|
}
|
|
88
99
|
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
89
100
|
if (hasCommonPrefixes) {
|
|
90
|
-
response.CommonPrefixes
|
|
91
|
-
|
|
92
|
-
|
|
101
|
+
for (const item of response.CommonPrefixes) {
|
|
102
|
+
const obj = this.nameToObj(item.Prefix)
|
|
103
|
+
if (visitor) {
|
|
104
|
+
await visitor(obj)
|
|
105
|
+
}
|
|
106
|
+
objects.push(obj)
|
|
107
|
+
objectCount++
|
|
108
|
+
}
|
|
93
109
|
}
|
|
94
110
|
truncated = response.IsTruncated
|
|
95
111
|
// If truncated is true, advance the marker
|
|
@@ -108,16 +124,19 @@ class StorageClient {
|
|
|
108
124
|
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
109
125
|
}
|
|
110
126
|
}
|
|
111
|
-
if (recursive &&
|
|
127
|
+
if (recursive && objectCount === 0 && path !== '') {
|
|
112
128
|
throw new MobilettoNotFoundError(path)
|
|
113
129
|
}
|
|
114
|
-
|
|
130
|
+
const filtered = objects.filter(o => o.name !== path)
|
|
131
|
+
return filtered
|
|
115
132
|
}
|
|
116
133
|
|
|
117
|
-
normalizeKey = (path) =>
|
|
118
|
-
path.startsWith(this.prefix)
|
|
134
|
+
normalizeKey = (path) => {
|
|
135
|
+
const p = (path.startsWith(this.prefix)
|
|
119
136
|
? path
|
|
120
|
-
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path)
|
|
137
|
+
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
|
|
138
|
+
return p.replaceAll(/\/{2,}/g, '/')
|
|
139
|
+
}
|
|
121
140
|
|
|
122
141
|
s3error (err, key, path, method) {
|
|
123
142
|
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
@@ -138,7 +157,7 @@ class StorageClient {
|
|
|
138
157
|
try {
|
|
139
158
|
const head = await this.client.send(new HeadObjectCommand(bucketParams))
|
|
140
159
|
const meta = {
|
|
141
|
-
name: path,
|
|
160
|
+
name: this.stripPrefix(path),
|
|
142
161
|
size: head.ContentLength
|
|
143
162
|
}
|
|
144
163
|
if (head.LastModified) {
|
|
@@ -196,14 +215,12 @@ class StorageClient {
|
|
|
196
215
|
}
|
|
197
216
|
}
|
|
198
217
|
|
|
199
|
-
async remove (path,
|
|
200
|
-
const recursive = options && options.recursive
|
|
201
|
-
const quiet = options && options.quiet
|
|
218
|
+
async remove (path, recursive, quiet) {
|
|
202
219
|
if (recursive) {
|
|
203
|
-
let objects = await this._list(path, { MaxKeys: DELETE_OBJECTS_MAX_KEYS }
|
|
220
|
+
let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
|
|
204
221
|
while (objects && objects.length > 0) {
|
|
205
222
|
const Delete = {
|
|
206
|
-
Objects: objects.map(obj => { return {Key: obj} })
|
|
223
|
+
Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
|
|
207
224
|
}
|
|
208
225
|
if (quiet) {
|
|
209
226
|
Delete.Quiet = true
|
|
@@ -224,7 +241,7 @@ class StorageClient {
|
|
|
224
241
|
throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
225
242
|
}
|
|
226
243
|
try {
|
|
227
|
-
objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
244
|
+
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
228
245
|
} catch (e) {
|
|
229
246
|
if (!(e instanceof MobilettoNotFoundError)) {
|
|
230
247
|
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
|
package/index.js
CHANGED
|
@@ -1,14 +1,35 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
2
|
+
const winston = require('winston')
|
|
1
3
|
const { basename, dirname } = require('path')
|
|
2
4
|
const shasum = require('shasum')
|
|
3
5
|
const randomstring = require('randomstring')
|
|
4
6
|
|
|
5
7
|
const crypt = require('./util/crypt')
|
|
6
|
-
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
|
|
8
|
+
const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
|
|
9
|
+
|
|
10
|
+
const logger = winston.createLogger({
|
|
11
|
+
level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
|
|
12
|
+
format: winston.format.simple(),
|
|
13
|
+
transports: [ new winston.transports.Console() ]
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
const setLogLevel = (level) => { logger.level = level }
|
|
17
|
+
|
|
18
|
+
const setLogTransports = (transports) => {
|
|
19
|
+
logger.transports.splice(0, logger.transports.length)
|
|
20
|
+
logger.transports.push(...transports)
|
|
21
|
+
}
|
|
7
22
|
|
|
8
23
|
const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
9
24
|
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
10
25
|
const ENC_PAD_SEP = ' ~ '
|
|
11
26
|
|
|
27
|
+
const initMobilettoTempDir = () => {
|
|
28
|
+
const dir = process.env.MOBILETTO_TMP || process.env.TMPDIR || '/tmp'
|
|
29
|
+
return dir.endsWith('/') ? dir.substring(0, dir.length - 1) : dir
|
|
30
|
+
}
|
|
31
|
+
const MOBILETTO_TMP = initMobilettoTempDir()
|
|
32
|
+
|
|
12
33
|
// adapted from https://stackoverflow.com/a/27724419
|
|
13
34
|
function MobilettoError (message, err) {
|
|
14
35
|
this.message = `${message}`
|
|
@@ -34,7 +55,23 @@ function MobilettoNotFoundError (message) {
|
|
|
34
55
|
|
|
35
56
|
const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
36
57
|
|
|
58
|
+
const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
|
|
59
|
+
const isReadable = (thing) => thing instanceof fs.ReadStream
|
|
60
|
+
|
|
37
61
|
const UTILITY_FUNCTIONS = {
|
|
62
|
+
list: (client) => async (path, opts) => {
|
|
63
|
+
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
64
|
+
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
65
|
+
if (visitor && typeof visitor !== 'function') {
|
|
66
|
+
throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
|
|
67
|
+
}
|
|
68
|
+
return await client.driver_list(path, recursive, visitor)
|
|
69
|
+
},
|
|
70
|
+
remove: (client) => async (path, opts) => {
|
|
71
|
+
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
72
|
+
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
73
|
+
return await client.driver_remove(path, recursive, quiet)
|
|
74
|
+
},
|
|
38
75
|
readFile: (client) => async (path) => {
|
|
39
76
|
const chunks = []
|
|
40
77
|
await client.read(path, reader(chunks))
|
|
@@ -43,17 +80,75 @@ const UTILITY_FUNCTIONS = {
|
|
|
43
80
|
writeFile: (client) => async (path, data) => {
|
|
44
81
|
const readFunc = function* () { yield data }
|
|
45
82
|
return await client.write(path, readFunc())
|
|
83
|
+
},
|
|
84
|
+
mirror: (client) => async (source, clientPath = '', sourcePath = '') => {
|
|
85
|
+
logger.info(`mirror: starting, sourcePath=${sourcePath} -> clientPath=${clientPath}`)
|
|
86
|
+
const results = {
|
|
87
|
+
success: 0,
|
|
88
|
+
errors: 0
|
|
89
|
+
}
|
|
90
|
+
const visitor = async (obj) => {
|
|
91
|
+
if (obj.type && obj.type === M_FILE) {
|
|
92
|
+
logger.verbose(`mirror: mirroring file: ${obj.name}`)
|
|
93
|
+
const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
|
|
94
|
+
logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
|
|
95
|
+
try {
|
|
96
|
+
// write from source -> write to temp file
|
|
97
|
+
const fd = fs.openSync(tempPath, 'wx', 0o0600)
|
|
98
|
+
const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
|
|
99
|
+
await source.read(obj.name, async (chunk) => {
|
|
100
|
+
if (chunk) { writer.write(chunk) }
|
|
101
|
+
}, () => {
|
|
102
|
+
writer.close((err) => {
|
|
103
|
+
if (err) { throw new MobilettoError(`mirror: error closing temp file: ${err}`) }
|
|
104
|
+
logger.debug(`mirror: finished writing ${obj.name} to temp file ${tempPath}`)
|
|
105
|
+
})
|
|
106
|
+
}).then(async () => {
|
|
107
|
+
// read from temp file -> write to mirror
|
|
108
|
+
const fd = fs.openSync(tempPath, 'r')
|
|
109
|
+
const reader = fs.createReadStream(tempPath, {fd})
|
|
110
|
+
const destName = obj.name.startsWith(sourcePath)
|
|
111
|
+
? obj.name.substring(sourcePath.length)
|
|
112
|
+
: obj.name
|
|
113
|
+
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
114
|
+
await client.write(clientPath + destName, reader)
|
|
115
|
+
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
116
|
+
})
|
|
117
|
+
results.success++
|
|
118
|
+
} catch (e) {
|
|
119
|
+
logger.warn(`mirror: error copying file: ${e}`)
|
|
120
|
+
results.errors++
|
|
121
|
+
} finally {
|
|
122
|
+
logger.verbose(`mirror: file mirrored successfully: ${obj.name}`)
|
|
123
|
+
fs.rmSync(tempPath, {force: true})
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
await mirrorDir(source, sourcePath, visitor)
|
|
128
|
+
return results
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async function mirrorDir (source, sourcePath, visitor) {
|
|
133
|
+
logger.verbose(`mirrorDir: mirroring dir: ${sourcePath}`)
|
|
134
|
+
const listing = await source.list(sourcePath, {recursive: false, visitor})
|
|
135
|
+
for (const obj of listing) {
|
|
136
|
+
if (obj.type === M_DIR) {
|
|
137
|
+
const dir = obj.name.startsWith(sourcePath) ? obj.name : sourcePath + obj.name
|
|
138
|
+
await mirrorDir(source, dir, visitor)
|
|
139
|
+
}
|
|
46
140
|
}
|
|
47
141
|
}
|
|
48
142
|
|
|
49
143
|
function addUtilityFunctions (client, readOnly = false) {
|
|
50
144
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
145
|
+
client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
|
|
51
146
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
52
147
|
}
|
|
53
148
|
if (readOnly) {
|
|
54
149
|
for (const writeFunc of ['write', 'remove', 'writeFile']) {
|
|
55
150
|
client[writeFunc] = async () => {
|
|
56
|
-
|
|
151
|
+
logger.warn(`${writeFunc} not supported in readOnly mode`)
|
|
57
152
|
return false
|
|
58
153
|
}
|
|
59
154
|
}
|
|
@@ -66,22 +161,30 @@ async function connect (driverPath, key, secret, opts, encryption = null) {
|
|
|
66
161
|
}
|
|
67
162
|
|
|
68
163
|
async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
164
|
+
logger.info(`mobiletto: connecting with driver ${driverPath}`)
|
|
69
165
|
const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
|
|
70
166
|
const client = driver.storageClient(key, secret, opts)
|
|
71
167
|
if (!(await client.testConfig())) {
|
|
72
|
-
|
|
168
|
+
const message = `mobiletto(${driverPath}) error: test API call failed`
|
|
169
|
+
logger.error(message)
|
|
170
|
+
throw new MobilettoError(message)
|
|
73
171
|
}
|
|
74
172
|
const readOnly = opts ? !!opts.readOnly : false
|
|
75
173
|
if (encryption === null) {
|
|
174
|
+
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
76
175
|
return addUtilityFunctions(client, readOnly)
|
|
77
176
|
}
|
|
78
177
|
const encKey = normalizeKey(encryption.key)
|
|
79
178
|
if (!encKey) {
|
|
80
|
-
|
|
179
|
+
const message = `mobiletto(${driverPath}): invalid encryption key`
|
|
180
|
+
logger.error(message)
|
|
181
|
+
throw new MobilettoError(message)
|
|
81
182
|
}
|
|
82
183
|
const iv = normalizeIV(encryption.iv, encKey)
|
|
83
184
|
if (!iv) {
|
|
84
|
-
|
|
185
|
+
const message = `mobiletto(${driverPath}): invalid encryption IV`
|
|
186
|
+
logger.error(message)
|
|
187
|
+
throw new MobilettoError(message)
|
|
85
188
|
}
|
|
86
189
|
const enc = {
|
|
87
190
|
key: encKey,
|
|
@@ -136,6 +239,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
136
239
|
throw e
|
|
137
240
|
}
|
|
138
241
|
}
|
|
242
|
+
meta.type ||= M_FILE
|
|
139
243
|
meta.name = path // rewrite name back to plaintext name
|
|
140
244
|
return Object.assign({}, meta, metaObj)
|
|
141
245
|
}
|
|
@@ -155,9 +259,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
155
259
|
|
|
156
260
|
async function removeDirentFile(path) {
|
|
157
261
|
const df = direntFile(direntDir(dirname(path)), path);
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
await client.remove(
|
|
262
|
+
const recursive = false
|
|
263
|
+
const quiet = true
|
|
264
|
+
await client.remove(df, recursive, quiet)
|
|
265
|
+
await client.remove(encryptPath(path), recursive, quiet)
|
|
266
|
+
await client.remove(metaPath(path), recursive, quiet)
|
|
161
267
|
}
|
|
162
268
|
|
|
163
269
|
function stringGenerator (value, enc) {
|
|
@@ -167,13 +273,21 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
167
273
|
}
|
|
168
274
|
|
|
169
275
|
const encClient = {
|
|
170
|
-
list: async (path) => {
|
|
276
|
+
list: async (path, recursive, visitor) => {
|
|
171
277
|
const dirent = direntDir(path)
|
|
172
|
-
const entries = await client.list(dirent)
|
|
278
|
+
const entries = await client.list(dirent, recursive)
|
|
173
279
|
if (!entries || entries.length === 0) {
|
|
174
280
|
return []
|
|
175
281
|
}
|
|
176
|
-
|
|
282
|
+
if (visitor) {
|
|
283
|
+
const objects = await _loadMeta(dirent, entries)
|
|
284
|
+
for (const obj of objects) {
|
|
285
|
+
await visitor(obj)
|
|
286
|
+
}
|
|
287
|
+
return objects
|
|
288
|
+
} else {
|
|
289
|
+
return _loadMeta(dirent, entries)
|
|
290
|
+
}
|
|
177
291
|
},
|
|
178
292
|
metadata: _metadata(client),
|
|
179
293
|
read: async (path, callback) => {
|
|
@@ -215,12 +329,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
215
329
|
const cipher = crypt.startEncryptStream(enc)
|
|
216
330
|
const realPath = encryptPath(path)
|
|
217
331
|
await client.write(realPath, cryptGenerator(readFunc))
|
|
218
|
-
const meta = { name: path, size: bytesRead }
|
|
332
|
+
const meta = { name: path, size: bytesRead, type: M_FILE }
|
|
219
333
|
await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
|
|
220
334
|
return bytesRead
|
|
221
335
|
},
|
|
222
336
|
remove: async (path, options) => {
|
|
223
|
-
const recursive = (options && options.recursive) || false
|
|
337
|
+
const recursive = options === true || (options && options.recursive) || false
|
|
338
|
+
const quiet = (options && options.quiet) || false
|
|
224
339
|
if (recursive) {
|
|
225
340
|
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
226
341
|
async function recRm (path) {
|
|
@@ -253,7 +368,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
253
368
|
let parent = path
|
|
254
369
|
let dirent = direntDir(parent)
|
|
255
370
|
const df = direntFile(dirent, path)
|
|
256
|
-
await client.remove(df,
|
|
371
|
+
await client.remove(df, false, true)
|
|
257
372
|
while (true) {
|
|
258
373
|
try {
|
|
259
374
|
const entries = await client.list(dirent)
|
|
@@ -275,6 +390,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
275
390
|
return true
|
|
276
391
|
}
|
|
277
392
|
}
|
|
393
|
+
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
278
394
|
return addUtilityFunctions(encClient, readOnly)
|
|
279
395
|
}
|
|
280
396
|
|
|
@@ -303,7 +419,7 @@ function writeStream (stream) {
|
|
|
303
419
|
if (chunk) {
|
|
304
420
|
stream.write(chunk, (err) => {
|
|
305
421
|
if (err) {
|
|
306
|
-
|
|
422
|
+
logger.error(`writeStream: error writing: ${err}`)
|
|
307
423
|
throw err
|
|
308
424
|
}
|
|
309
425
|
})
|
|
@@ -314,7 +430,7 @@ function writeStream (stream) {
|
|
|
314
430
|
function closeStream (stream) {
|
|
315
431
|
return () => stream.close((err) => {
|
|
316
432
|
if (err) {
|
|
317
|
-
|
|
433
|
+
logger.error(`closeStream: error closing: ${err}`)
|
|
318
434
|
throw err
|
|
319
435
|
}
|
|
320
436
|
})
|
|
@@ -327,7 +443,9 @@ const M_SPECIAL = 'special'
|
|
|
327
443
|
|
|
328
444
|
module.exports = {
|
|
329
445
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
446
|
+
isAsyncGenerator, isReadable,
|
|
330
447
|
mobiletto, connect,
|
|
448
|
+
setLogLevel, setLogTransports,
|
|
331
449
|
MobilettoError, MobilettoNotFoundError,
|
|
332
450
|
readStream, writeStream, closeStream
|
|
333
451
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.15",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Jonathan Cobb",
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
22
22
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
23
23
|
"node-fetch-commonjs": "^3.1.1",
|
|
24
|
-
"shasum": "^1.0.2"
|
|
24
|
+
"shasum": "^1.0.2",
|
|
25
|
+
"winston": "^3.8.1"
|
|
25
26
|
}
|
|
26
27
|
}
|
package/test/index.spec.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require('dotenv').config()
|
|
4
4
|
|
|
5
5
|
const randomstring = require('randomstring')
|
|
6
|
+
const path = require('path')
|
|
6
7
|
|
|
7
8
|
const { expect, should, assert } = require('chai')
|
|
8
9
|
|
|
@@ -33,6 +34,18 @@ DRIVER_CONFIG = {
|
|
|
33
34
|
}
|
|
34
35
|
}
|
|
35
36
|
}
|
|
37
|
+
DRIVER_NAMES = Object.keys(DRIVER_CONFIG)
|
|
38
|
+
|
|
39
|
+
function pickAnotherDriver (notThisOne) {
|
|
40
|
+
let choice = null
|
|
41
|
+
while (!choice) {
|
|
42
|
+
choice = DRIVER_NAMES[Math.floor(Math.random() * DRIVER_NAMES.length)]
|
|
43
|
+
if (choice === notThisOne) {
|
|
44
|
+
choice = null
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return choice
|
|
48
|
+
}
|
|
36
49
|
|
|
37
50
|
async function assertMeta (api, name, expectedSize) {
|
|
38
51
|
const meta = await api.metadata(name)
|
|
@@ -96,7 +109,7 @@ const encryptionTests = () => [null, { key: rand(32) }]
|
|
|
96
109
|
// the same set of tests, with exactly the same inputs/outputs.
|
|
97
110
|
// This greatly supports developer sanity
|
|
98
111
|
|
|
99
|
-
for (const driverName of
|
|
112
|
+
for (const driverName of DRIVER_NAMES) {
|
|
100
113
|
const config = DRIVER_CONFIG[driverName]
|
|
101
114
|
const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
|
|
102
115
|
const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
|
|
@@ -122,7 +135,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
122
135
|
const randomData = rand(size)
|
|
123
136
|
const fileSuffix = '' + Date.now()
|
|
124
137
|
let fixture
|
|
125
|
-
|
|
138
|
+
before((done) => {
|
|
126
139
|
const name = `test_file_${fileSuffix}`
|
|
127
140
|
connect(driverName, config.key, config.secret, config.opts)
|
|
128
141
|
.then(api => { fixture = {api, name, randomData} })
|
|
@@ -152,12 +165,13 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
152
165
|
for (const encryption of encryptionTests()) {
|
|
153
166
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
154
167
|
describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
155
|
-
|
|
168
|
+
// describe(`${driverName} - ENC fail to write and delete files in readOnly mode`, () => {
|
|
169
|
+
// const encryption = {key: rand(32)}
|
|
156
170
|
const size = 16
|
|
157
171
|
const randomData = rand(size)
|
|
158
172
|
const fileSuffix = '' + Date.now()
|
|
159
173
|
let fixture
|
|
160
|
-
|
|
174
|
+
before((done) => {
|
|
161
175
|
const name = `test_file_${fileSuffix}`
|
|
162
176
|
const opts = Object.assign({}, config.opts, {readOnly: true})
|
|
163
177
|
connect(driverName, config.key, config.secret, opts, encryption)
|
|
@@ -203,7 +217,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
203
217
|
const fileSuffix = '' + Date.now()
|
|
204
218
|
const encryptionKey = rand(32)
|
|
205
219
|
let fixture
|
|
206
|
-
|
|
220
|
+
before((done) => {
|
|
207
221
|
const name = `test_file_${fileSuffix}`
|
|
208
222
|
mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
|
|
209
223
|
.then(api => { fixture = {api, name, randomData} })
|
|
@@ -233,29 +247,40 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
233
247
|
|
|
234
248
|
for (const encryption of encryptionTests()) {
|
|
235
249
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
236
|
-
|
|
250
|
+
const mirrorDriver = pickAnotherDriver(driverName)
|
|
251
|
+
const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
|
|
252
|
+
const mirrorDest = `mirrorDest_${rand(5)}/`
|
|
253
|
+
describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
|
|
237
254
|
// describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|
|
238
255
|
// const encryption = {key: rand(32)}
|
|
239
256
|
// const encryption = null
|
|
240
257
|
// a random directory and file within it
|
|
241
258
|
const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
|
|
242
259
|
const subdirName = `subdir_` + Date.now()
|
|
243
|
-
const
|
|
260
|
+
const fullSubdirPath = `${randomParent}/${subdirName}`
|
|
261
|
+
const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
|
|
244
262
|
//const fileCount = 3 + Math.floor(Math.random() * 10)
|
|
245
263
|
const fileCount = 2
|
|
246
264
|
let fixture
|
|
247
|
-
|
|
265
|
+
before((done) => {
|
|
248
266
|
mobiletto(driverName, config.key, config.secret, config.opts, encryption)
|
|
249
|
-
.then(api => {
|
|
267
|
+
.then(api => {
|
|
268
|
+
// const mirrorEnc = { key: rand(32) }
|
|
269
|
+
const mirrorEnc = null
|
|
270
|
+
mobiletto(mirrorDriver, mirrorConfig.key, mirrorConfig.secret, mirrorConfig.opts, mirrorEnc)
|
|
271
|
+
.then(mirrorApi => {
|
|
272
|
+
fixture = {api, mirrorApi, name: randomPath}
|
|
273
|
+
})
|
|
274
|
+
.catch((err) => {
|
|
275
|
+
throw err
|
|
276
|
+
})
|
|
277
|
+
.finally(done)
|
|
278
|
+
})
|
|
250
279
|
.catch((err) => { throw err })
|
|
251
|
-
.finally(done)
|
|
252
280
|
})
|
|
253
281
|
it(`should write ${fileCount} files in a new directory`, async () => {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
yield rand(READ_SZ)
|
|
257
|
-
}
|
|
258
|
-
|
|
282
|
+
// return one chunk of random data
|
|
283
|
+
function* dataGenerator() { yield rand(READ_SZ) }
|
|
259
284
|
for (let i = 0; i < fileCount; i++) {
|
|
260
285
|
const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
|
|
261
286
|
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
@@ -279,6 +304,31 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
279
304
|
).to.not.be.null
|
|
280
305
|
}
|
|
281
306
|
})
|
|
307
|
+
it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
|
|
308
|
+
const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
|
|
309
|
+
expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
|
|
310
|
+
expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
|
|
311
|
+
})
|
|
312
|
+
it("should see the same files mirrored both places", async () => {
|
|
313
|
+
const originalThings = await fixture.api.list(fullSubdirPath)
|
|
314
|
+
const mirrorPath = `${mirrorDest}${subdirName}`
|
|
315
|
+
const mirrorThings = await fixture.mirrorApi.list(mirrorPath)
|
|
316
|
+
expect(originalThings.length === mirrorThings.length).to.be.true
|
|
317
|
+
|
|
318
|
+
const intersection = originalThings
|
|
319
|
+
.filter(orig => mirrorThings
|
|
320
|
+
.find(m => path.basename(m.name) === path.basename(orig.name)))
|
|
321
|
+
expect(intersection.length === originalThings.length).to.be.true
|
|
322
|
+
})
|
|
323
|
+
it("loading metadata on a mirrored file succeeds", async () => {
|
|
324
|
+
const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
|
|
325
|
+
await assertMeta(fixture.mirrorApi, destPath, READ_SZ)
|
|
326
|
+
})
|
|
327
|
+
it("should recursively delete the mirrored data", async () => {
|
|
328
|
+
const recursive = true
|
|
329
|
+
const removed = await fixture.mirrorApi.remove(mirrorDest, {recursive})
|
|
330
|
+
expect(removed).to.be.true
|
|
331
|
+
})
|
|
282
332
|
it("should recursively delete the directory and file we just created", async () => {
|
|
283
333
|
const recursive = true
|
|
284
334
|
const removed = await fixture.api.remove(randomParent, {recursive})
|
|
@@ -287,6 +337,10 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
287
337
|
it("loading metadata on the file we wrote now fails", async () => {
|
|
288
338
|
await assertMetaFail(fixture.api, fixture.name)
|
|
289
339
|
})
|
|
340
|
+
it("loading metadata on a mirrored file now fails", async () => {
|
|
341
|
+
const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
|
|
342
|
+
await assertMetaFail(fixture.mirrorApi, destPath)
|
|
343
|
+
})
|
|
290
344
|
it("loading metadata on the parent dir we wrote now fails", async () => {
|
|
291
345
|
await assertMetaFail(fixture.api, randomParent)
|
|
292
346
|
})
|