mobiletto 1.0.12 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -7
- package/drivers/local/index.js +111 -43
- package/drivers/s3/index.js +42 -26
- package/index.js +95 -9
- package/package.json +1 -1
- package/test/index.spec.js +69 -15
package/README.md
CHANGED
|
@@ -41,10 +41,18 @@ Today the supported drivers are:
|
|
|
41
41
|
local.list() // --> returns an array of file objects
|
|
42
42
|
s3.list() // --> returns an array of file objects
|
|
43
43
|
|
|
44
|
+
// List files recursively
|
|
45
|
+
local.list({ recursive: true })
|
|
46
|
+
s3.list({ recursive: true })
|
|
47
|
+
|
|
44
48
|
// List files in a directory
|
|
45
49
|
const path = 'some/path'
|
|
46
|
-
local.list(path)
|
|
47
|
-
s3.list(path
|
|
50
|
+
local.list(path)
|
|
51
|
+
s3.list(path, { recursive: true }) // also supports recursive flag
|
|
52
|
+
|
|
53
|
+
// Visit files in a directory -- visitor function must be async
|
|
54
|
+
local.list(path, { visitor: myAsyncFunc })
|
|
55
|
+
s3.list(path, { visitor: myAsyncFunc, recursive: true })
|
|
48
56
|
|
|
49
57
|
// Read metadata for a file
|
|
50
58
|
local.metadata(path)
|
|
@@ -81,12 +89,28 @@ Today the supported drivers are:
|
|
|
81
89
|
local.remove(path, {quiet})
|
|
82
90
|
s3.remove(path, {quiet})
|
|
83
91
|
|
|
84
|
-
// Recursively delete a directory
|
|
92
|
+
// Recursively delete a directory and do it quietly (do not report errors)
|
|
85
93
|
const recursive = true
|
|
94
|
+
const quiet = true
|
|
86
95
|
local.remove(path, {recursive, quiet})
|
|
87
96
|
s3.remove(path, {recursive, quiet})
|
|
88
97
|
|
|
89
|
-
#
|
|
98
|
+
# Mirroring
|
|
99
|
+
|
|
100
|
+
// Copy a local filesystem mobiletto to S3
|
|
101
|
+
s3.mirror(local)
|
|
102
|
+
|
|
103
|
+
// Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
|
|
104
|
+
local.mirror(s3, 'some/local-folder', 'some/s3-folder')
|
|
105
|
+
|
|
106
|
+
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
107
|
+
|
|
108
|
+
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
109
|
+
reader and who is the writer. Imagine it like an assignment statement: the "left-hand mobiletto"
|
|
110
|
+
is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
|
|
111
|
+
argument to the `mirror` method) is the value being assigned (mirrored data is read).
|
|
112
|
+
|
|
113
|
+
# Alternate import style
|
|
90
114
|
Import the fully-scoped module and use the `connect` function:
|
|
91
115
|
|
|
92
116
|
const storage = require('mobiletto')
|
|
@@ -124,6 +148,38 @@ some effort, discover some or all of the overall structure of the directory hier
|
|
|
124
148
|
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
125
149
|
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
126
150
|
|
|
151
|
+
# File metadata
|
|
152
|
+
The `metadata` command returns metadata about a single filesystem entry.
|
|
153
|
+
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
154
|
+
|
|
155
|
+
A metadata object looks like this:
|
|
156
|
+
|
|
157
|
+
{
|
|
158
|
+
"name": "fully/qualified/path/to/file",
|
|
159
|
+
"type": "entry-type",
|
|
160
|
+
"size": size-in-bytes,
|
|
161
|
+
"ctime": creation-time-epoch-millis,
|
|
162
|
+
"mtime": modification-time-epoch-millis
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
166
|
+
|
|
167
|
+
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
168
|
+
should always be present. A subsequent `metadata` command should return all available properties.
|
|
169
|
+
|
|
170
|
+
# Key rotation
|
|
171
|
+
Create a mobiletto with your new key, then mirror the old data into it:
|
|
172
|
+
|
|
173
|
+
const storage = require('mobiletto')
|
|
174
|
+
|
|
175
|
+
const oldEncryption = { key: .... }
|
|
176
|
+
const oldStorage = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'}, oldEncryption)
|
|
177
|
+
|
|
178
|
+
const newEncryption = { key: .... }
|
|
179
|
+
const newStorage = await storage.connect('s3', aws_key, aws_secret, {bucket: 'zz', region: 'us-east-1'}, newEncryption)
|
|
180
|
+
|
|
181
|
+
newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
|
|
182
|
+
|
|
127
183
|
# Driver Interface
|
|
128
184
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
|
129
185
|
|
|
@@ -143,8 +199,11 @@ The object that the storageClient function returns must define these functions:
|
|
|
143
199
|
// Test the driver before using, ensure proper configuration
|
|
144
200
|
async testConfig ()
|
|
145
201
|
|
|
146
|
-
//
|
|
147
|
-
|
|
202
|
+
// List files in path (or from base-directory)
|
|
203
|
+
// If recursive is true, list recursively
|
|
204
|
+
// If visitor is defined, it will be an async function. await the visitor function on each file found
|
|
205
|
+
// Otherwise, perform the listing and return an array of objects
|
|
206
|
+
async list (path, recursive = false, visitor = null) // path may be omitted
|
|
148
207
|
|
|
149
208
|
// Read metadata for a path
|
|
150
209
|
async metadata (path)
|
|
@@ -158,4 +217,4 @@ The object that the storageClient function returns must define these functions:
|
|
|
158
217
|
async write (path, readFunc)
|
|
159
218
|
|
|
160
219
|
// Remove a file, or recursively delete a directory
|
|
161
|
-
async remove (path,
|
|
220
|
+
async remove (path, recursive = false, quiet = false)
|
package/drivers/local/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const { dirname } = require('path')
|
|
2
2
|
|
|
3
3
|
const {
|
|
4
|
-
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
4
|
+
M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
|
|
5
5
|
MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
|
|
6
6
|
} = require('../../index')
|
|
7
7
|
|
|
@@ -9,9 +9,9 @@ const fs = require('fs')
|
|
|
9
9
|
|
|
10
10
|
const DEFAULT_MODE = '0700'
|
|
11
11
|
|
|
12
|
-
const
|
|
12
|
+
const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
|
13
13
|
|
|
14
|
-
const ioError = (err, path, method) =>
|
|
14
|
+
const ioError = (err, path, method) => isNotExistError(err)
|
|
15
15
|
? err instanceof MobilettoNotFoundError
|
|
16
16
|
? err
|
|
17
17
|
: new MobilettoNotFoundError(path)
|
|
@@ -19,6 +19,19 @@ const ioError = (err, path, method) => isENOENT(err)
|
|
|
19
19
|
? err
|
|
20
20
|
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
21
21
|
|
|
22
|
+
const fileType = (stat) => {
|
|
23
|
+
if (stat.isDirectory()) {
|
|
24
|
+
return M_DIR
|
|
25
|
+
}
|
|
26
|
+
if (stat.isFile()) {
|
|
27
|
+
return M_FILE
|
|
28
|
+
}
|
|
29
|
+
if (stat.isSymbolicLink()) {
|
|
30
|
+
return M_LINK
|
|
31
|
+
}
|
|
32
|
+
return M_SPECIAL
|
|
33
|
+
}
|
|
34
|
+
|
|
22
35
|
class StorageClient {
|
|
23
36
|
baseDir
|
|
24
37
|
mode
|
|
@@ -46,41 +59,63 @@ class StorageClient {
|
|
|
46
59
|
return { path, stat }
|
|
47
60
|
}
|
|
48
61
|
testConfig = async () => await this.list()
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
62
|
+
|
|
63
|
+
normalizePath = (path) => path.startsWith(this.baseDir)
|
|
64
|
+
? path
|
|
65
|
+
: this.baseDir + (path.startsWith('/') ? path.substring(1) : path)
|
|
66
|
+
denormalizePath = (path) => path.startsWith(this.baseDir) ? path.substring(this.baseDir.length) : path
|
|
67
|
+
|
|
68
|
+
fileToObject = (dir) => (f) => {
|
|
69
|
+
const stat = fs.lstatSync(dir + '/' + f)
|
|
70
|
+
const type = fileType(stat)
|
|
71
|
+
const entry = {
|
|
72
|
+
name: this.denormalizePath(dir + '/' + f),
|
|
73
|
+
type
|
|
52
74
|
}
|
|
53
|
-
if (
|
|
54
|
-
|
|
75
|
+
if (type === M_LINK) {
|
|
76
|
+
const resolved = this.resolveSymlinks(dir + '/' + f)
|
|
77
|
+
entry.link = resolved.path
|
|
55
78
|
}
|
|
56
|
-
|
|
57
|
-
|
|
79
|
+
return entry
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
readDirFiles = async (dir, recursive, visitor) => {
|
|
83
|
+
if (!recursive) {
|
|
84
|
+
return await this.dirFiles(dir, visitor)
|
|
85
|
+
}
|
|
86
|
+
const files = await this.dirFiles(dir, visitor)
|
|
87
|
+
for (const file of files) {
|
|
88
|
+
if (file.type === M_DIR) {
|
|
89
|
+
await this.readDirFiles(file.name, true, visitor)
|
|
90
|
+
}
|
|
58
91
|
}
|
|
59
|
-
return M_SPECIAL
|
|
60
92
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
93
|
+
|
|
94
|
+
dirFiles = async (dir, visitor) => {
|
|
95
|
+
const norm = this.normalizePath(dir);
|
|
96
|
+
try {
|
|
97
|
+
let names = fs.readdirSync(norm);
|
|
98
|
+
const files = names.map(this.fileToObject(norm))
|
|
99
|
+
for (const f of files) {
|
|
100
|
+
await visitor(f)
|
|
101
|
+
}
|
|
102
|
+
return files
|
|
103
|
+
} catch (e) {
|
|
104
|
+
console.error(`dirFiles: ${e}`)
|
|
105
|
+
throw ioError(e, norm, 'dirFiles')
|
|
64
106
|
}
|
|
65
|
-
return this.baseDir + path
|
|
66
107
|
}
|
|
67
|
-
|
|
108
|
+
|
|
109
|
+
async list (path = '', recursive = false, visitor = null) {
|
|
68
110
|
const dir = this.normalizePath(path)
|
|
69
111
|
try {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
}
|
|
78
|
-
if (type === M_LINK) {
|
|
79
|
-
const resolved = this.resolveSymlinks(dir + '/' + f)
|
|
80
|
-
entry.link = resolved.path
|
|
81
|
-
}
|
|
82
|
-
return entry
|
|
83
|
-
})
|
|
112
|
+
if (visitor === null) {
|
|
113
|
+
const results = []
|
|
114
|
+
await this.readDirFiles(dir, recursive, (obj) => results.push(obj))
|
|
115
|
+
return results
|
|
116
|
+
} else {
|
|
117
|
+
return await this.readDirFiles(dir, recursive, visitor)
|
|
118
|
+
}
|
|
84
119
|
} catch (err) {
|
|
85
120
|
throw ioError(err, path, 'list')
|
|
86
121
|
}
|
|
@@ -92,7 +127,7 @@ class StorageClient {
|
|
|
92
127
|
if (!lstat) {
|
|
93
128
|
throw new MobilettoError('metadata: lstat error')
|
|
94
129
|
}
|
|
95
|
-
const type =
|
|
130
|
+
const type = fileType(lstat)
|
|
96
131
|
if (type === M_DIR && path !== '') {
|
|
97
132
|
const contents = await this.list(path)
|
|
98
133
|
if (contents.length === 0) {
|
|
@@ -100,7 +135,7 @@ class StorageClient {
|
|
|
100
135
|
}
|
|
101
136
|
}
|
|
102
137
|
return {
|
|
103
|
-
name:
|
|
138
|
+
name: this.denormalizePath(file),
|
|
104
139
|
type,
|
|
105
140
|
size: lstat.size,
|
|
106
141
|
mtime: lstat.mtimeMs
|
|
@@ -119,7 +154,7 @@ class StorageClient {
|
|
|
119
154
|
}
|
|
120
155
|
}
|
|
121
156
|
|
|
122
|
-
async
|
|
157
|
+
async normalizePathAndEnsureParentDirs (path) {
|
|
123
158
|
const file = this.normalizePath(path)
|
|
124
159
|
|
|
125
160
|
// console.log(`read: reading path: ${path} - ${file}`)
|
|
@@ -128,7 +163,7 @@ class StorageClient {
|
|
|
128
163
|
try {
|
|
129
164
|
dirStat = fs.lstatSync(parent)
|
|
130
165
|
} catch (err) {
|
|
131
|
-
if (err
|
|
166
|
+
if (isNotExistError(err)) {
|
|
132
167
|
this.mkdirs(parent)
|
|
133
168
|
} else {
|
|
134
169
|
throw new MobilettoError(`write: lstat error on ${parent}: ${err}`, err)
|
|
@@ -139,17 +174,52 @@ class StorageClient {
|
|
|
139
174
|
} else if (!dirStat.isDirectory()) {
|
|
140
175
|
throw new MobilettoError(`write: not a directory: ${parent} (cannot write file ${file})`)
|
|
141
176
|
}
|
|
177
|
+
return file
|
|
178
|
+
}
|
|
142
179
|
|
|
180
|
+
async write (path, generatorOrReadableStream) {
|
|
181
|
+
const file = await this.normalizePathAndEnsureParentDirs(path)
|
|
143
182
|
// console.log(`write: writing path ${path} -> ${file}`)
|
|
144
183
|
const stream = fs.createWriteStream(file, {mode: this.mode})
|
|
145
184
|
const writer = writeStream(stream)
|
|
146
185
|
const closer = closeStream(stream)
|
|
147
186
|
let count = 0
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
187
|
+
|
|
188
|
+
if (isReadable(generatorOrReadableStream)) {
|
|
189
|
+
const readable = generatorOrReadableStream
|
|
190
|
+
const streamHandler = stream =>
|
|
191
|
+
new Promise((resolve, reject) => {
|
|
192
|
+
stream.on('data', (data) => {
|
|
193
|
+
if (data) {
|
|
194
|
+
writer(data)
|
|
195
|
+
count += data.length
|
|
196
|
+
}
|
|
197
|
+
})
|
|
198
|
+
stream.on('error', reject)
|
|
199
|
+
stream.on('end', () => {
|
|
200
|
+
closer()
|
|
201
|
+
resolve()
|
|
202
|
+
})
|
|
203
|
+
})
|
|
204
|
+
await streamHandler(readable)
|
|
205
|
+
return count
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const generator = generatorOrReadableStream
|
|
209
|
+
let chunk = isAsyncGenerator(generator)
|
|
210
|
+
? (await generator.next()).value
|
|
211
|
+
: generator.next().value
|
|
212
|
+
let nullCount = 0
|
|
213
|
+
while (chunk || nullCount < 5) {
|
|
214
|
+
if (chunk) {
|
|
215
|
+
count += chunk.length
|
|
216
|
+
writer(chunk)
|
|
217
|
+
} else {
|
|
218
|
+
nullCount++
|
|
219
|
+
}
|
|
220
|
+
chunk = isAsyncGenerator(generator)
|
|
221
|
+
? (await generator.next()).value
|
|
222
|
+
: generator.next().value
|
|
153
223
|
}
|
|
154
224
|
closer()
|
|
155
225
|
return Promise.resolve(count)
|
|
@@ -166,15 +236,13 @@ class StorageClient {
|
|
|
166
236
|
}
|
|
167
237
|
}
|
|
168
238
|
|
|
169
|
-
async remove (path,
|
|
170
|
-
const recursive = options && options.recursive ? options.recursive : false
|
|
171
|
-
const quiet = options && options.quiet ? options.quiet : false
|
|
239
|
+
async remove (path, recursive, quiet) {
|
|
172
240
|
const file = this.normalizePath(path)
|
|
173
241
|
// console.log(`remove: deleting path: ${path} = ${file}`)
|
|
174
242
|
try {
|
|
175
243
|
fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
|
|
176
244
|
} catch (err) {
|
|
177
|
-
if (
|
|
245
|
+
if (isNotExistError(err)) {
|
|
178
246
|
if (!quiet) {
|
|
179
247
|
throw new MobilettoNotFoundError(path)
|
|
180
248
|
}
|
package/drivers/s3/index.js
CHANGED
|
@@ -44,18 +44,23 @@ class StorageClient {
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
47
|
-
testConfig = async () => await this._list('', { MaxKeys: 1 })
|
|
47
|
+
testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
49
|
+
stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
|
|
50
|
+
|
|
51
|
+
nameToObj = (name) => {
|
|
52
|
+
const relName = this.stripPrefix(name)
|
|
53
|
+
return {
|
|
54
|
+
name: relName,
|
|
55
|
+
type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
56
|
+
}
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
async
|
|
59
|
+
async list (path = '', recursive = false, visitor = null) {
|
|
60
|
+
return await this._list(path, recursive, visitor)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
59
64
|
const logPrefix = `_list(path=${path}):`
|
|
60
65
|
|
|
61
66
|
// Declare truncated as a flag that the while loop is based on.
|
|
@@ -73,6 +78,7 @@ class StorageClient {
|
|
|
73
78
|
bucketParams.Delimiter = this.delimiter
|
|
74
79
|
}
|
|
75
80
|
const objects = []
|
|
81
|
+
let objectCount = 0
|
|
76
82
|
// console.log(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
|
|
77
83
|
|
|
78
84
|
// while loop that runs until 'response.truncated' is false.
|
|
@@ -81,15 +87,25 @@ class StorageClient {
|
|
|
81
87
|
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
82
88
|
const hasContents = typeof response.Contents !== 'undefined'
|
|
83
89
|
if (hasContents) {
|
|
84
|
-
response.Contents
|
|
85
|
-
|
|
86
|
-
|
|
90
|
+
for (const item of response.Contents) {
|
|
91
|
+
const obj = this.nameToObj(item.Key);
|
|
92
|
+
if (visitor) {
|
|
93
|
+
await visitor(obj)
|
|
94
|
+
}
|
|
95
|
+
objects.push(obj)
|
|
96
|
+
objectCount++
|
|
97
|
+
}
|
|
87
98
|
}
|
|
88
99
|
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
89
100
|
if (hasCommonPrefixes) {
|
|
90
|
-
response.CommonPrefixes
|
|
91
|
-
|
|
92
|
-
|
|
101
|
+
for (const item of response.CommonPrefixes) {
|
|
102
|
+
const obj = this.nameToObj(item.Prefix)
|
|
103
|
+
if (visitor) {
|
|
104
|
+
await visitor(obj)
|
|
105
|
+
}
|
|
106
|
+
objects.push(obj)
|
|
107
|
+
objectCount++
|
|
108
|
+
}
|
|
93
109
|
}
|
|
94
110
|
truncated = response.IsTruncated
|
|
95
111
|
// If truncated is true, advance the marker
|
|
@@ -108,16 +124,18 @@ class StorageClient {
|
|
|
108
124
|
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
109
125
|
}
|
|
110
126
|
}
|
|
111
|
-
if (recursive &&
|
|
127
|
+
if (recursive && objectCount === 0 && path !== '') {
|
|
112
128
|
throw new MobilettoNotFoundError(path)
|
|
113
129
|
}
|
|
114
130
|
return objects
|
|
115
131
|
}
|
|
116
132
|
|
|
117
|
-
normalizeKey = (path) =>
|
|
118
|
-
path.startsWith(this.prefix)
|
|
133
|
+
normalizeKey = (path) => {
|
|
134
|
+
const p = (path.startsWith(this.prefix)
|
|
119
135
|
? path
|
|
120
|
-
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path)
|
|
136
|
+
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
|
|
137
|
+
return p.replaceAll(/\/{2,}/g, '/')
|
|
138
|
+
}
|
|
121
139
|
|
|
122
140
|
s3error (err, key, path, method) {
|
|
123
141
|
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
@@ -138,7 +156,7 @@ class StorageClient {
|
|
|
138
156
|
try {
|
|
139
157
|
const head = await this.client.send(new HeadObjectCommand(bucketParams))
|
|
140
158
|
const meta = {
|
|
141
|
-
name: path,
|
|
159
|
+
name: this.stripPrefix(path),
|
|
142
160
|
size: head.ContentLength
|
|
143
161
|
}
|
|
144
162
|
if (head.LastModified) {
|
|
@@ -196,14 +214,12 @@ class StorageClient {
|
|
|
196
214
|
}
|
|
197
215
|
}
|
|
198
216
|
|
|
199
|
-
async remove (path,
|
|
200
|
-
const recursive = options && options.recursive
|
|
201
|
-
const quiet = options && options.quiet
|
|
217
|
+
async remove (path, recursive, quiet) {
|
|
202
218
|
if (recursive) {
|
|
203
|
-
let objects = await this._list(path, { MaxKeys: DELETE_OBJECTS_MAX_KEYS }
|
|
219
|
+
let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
|
|
204
220
|
while (objects && objects.length > 0) {
|
|
205
221
|
const Delete = {
|
|
206
|
-
Objects: objects.map(obj => { return {Key: obj} })
|
|
222
|
+
Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
|
|
207
223
|
}
|
|
208
224
|
if (quiet) {
|
|
209
225
|
Delete.Quiet = true
|
|
@@ -224,7 +240,7 @@ class StorageClient {
|
|
|
224
240
|
throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
225
241
|
}
|
|
226
242
|
try {
|
|
227
|
-
objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
243
|
+
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
228
244
|
} catch (e) {
|
|
229
245
|
if (!(e instanceof MobilettoNotFoundError)) {
|
|
230
246
|
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
|
package/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
1
2
|
const { basename, dirname } = require('path')
|
|
2
3
|
const shasum = require('shasum')
|
|
3
4
|
const randomstring = require('randomstring')
|
|
@@ -9,6 +10,12 @@ const DIR_ENT_DIR_SUFFIX = '__.dirent'
|
|
|
9
10
|
const DIR_ENT_FILE_PREFIX = 'dirent__'
|
|
10
11
|
const ENC_PAD_SEP = ' ~ '
|
|
11
12
|
|
|
13
|
+
const initMobilettoTempDir = () => {
|
|
14
|
+
const dir = process.env.MOBILETTO_TMP || process.env.TMPDIR || '/tmp'
|
|
15
|
+
return dir.endsWith('/') ? dir.substring(0, dir.length - 1) : dir
|
|
16
|
+
}
|
|
17
|
+
const MOBILETTO_TMP = initMobilettoTempDir()
|
|
18
|
+
|
|
12
19
|
// adapted from https://stackoverflow.com/a/27724419
|
|
13
20
|
function MobilettoError (message, err) {
|
|
14
21
|
this.message = `${message}`
|
|
@@ -34,7 +41,23 @@ function MobilettoNotFoundError (message) {
|
|
|
34
41
|
|
|
35
42
|
const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
|
|
36
43
|
|
|
44
|
+
const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
|
|
45
|
+
const isReadable = (thing) => thing instanceof fs.ReadStream
|
|
46
|
+
|
|
37
47
|
const UTILITY_FUNCTIONS = {
|
|
48
|
+
list: (client) => async (path, opts) => {
|
|
49
|
+
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
50
|
+
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
51
|
+
if (visitor && typeof visitor !== 'function') {
|
|
52
|
+
throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
|
|
53
|
+
}
|
|
54
|
+
return await client.driver_list(path, recursive, visitor)
|
|
55
|
+
},
|
|
56
|
+
remove: (client) => async (path, opts) => {
|
|
57
|
+
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
58
|
+
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
59
|
+
return await client.driver_remove(path, recursive, quiet)
|
|
60
|
+
},
|
|
38
61
|
readFile: (client) => async (path) => {
|
|
39
62
|
const chunks = []
|
|
40
63
|
await client.read(path, reader(chunks))
|
|
@@ -43,11 +66,61 @@ const UTILITY_FUNCTIONS = {
|
|
|
43
66
|
writeFile: (client) => async (path, data) => {
|
|
44
67
|
const readFunc = function* () { yield data }
|
|
45
68
|
return await client.write(path, readFunc())
|
|
69
|
+
},
|
|
70
|
+
mirror: (client) => async (source, clientPath = '', sourcePath = '') => {
|
|
71
|
+
const results = {
|
|
72
|
+
success: 0,
|
|
73
|
+
errors: 0
|
|
74
|
+
}
|
|
75
|
+
const visitor = async (obj) => {
|
|
76
|
+
if (obj.type && obj.type === M_FILE) {
|
|
77
|
+
const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
|
|
78
|
+
try {
|
|
79
|
+
// write from source -> write to temp file
|
|
80
|
+
const fd = fs.openSync(tempPath, 'wx', 0o0600)
|
|
81
|
+
const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
|
|
82
|
+
await source.read(obj.name, async (chunk) => {
|
|
83
|
+
if (chunk) { writer.write(chunk) }
|
|
84
|
+
}, () => {
|
|
85
|
+
writer.close((err) => {
|
|
86
|
+
if (err) { throw new MobilettoError(`mirror: error closing temp file: ${err}`) }
|
|
87
|
+
})
|
|
88
|
+
}).then(async () => {
|
|
89
|
+
// read from temp file -> write to mirror
|
|
90
|
+
const fd = fs.openSync(tempPath, 'r')
|
|
91
|
+
const reader = fs.createReadStream(tempPath, {fd})
|
|
92
|
+
const destName = obj.name.startsWith(sourcePath)
|
|
93
|
+
? obj.name.substring(sourcePath.length)
|
|
94
|
+
: obj.name
|
|
95
|
+
await client.write(clientPath + destName, reader)
|
|
96
|
+
})
|
|
97
|
+
results.success++
|
|
98
|
+
} catch (e) {
|
|
99
|
+
console.warn(`mirror: error copying file: ${e}`)
|
|
100
|
+
results.errors++
|
|
101
|
+
} finally {
|
|
102
|
+
fs.rmSync(tempPath, {force: true})
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
await mirrorDir(source, sourcePath, visitor)
|
|
107
|
+
return results
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function mirrorDir (source, sourcePath, visitor) {
|
|
112
|
+
const listing = await source.list(sourcePath, {recursive: false, visitor})
|
|
113
|
+
for (const obj of listing) {
|
|
114
|
+
if (obj.type === M_DIR) {
|
|
115
|
+
const dir = obj.name.startsWith(sourcePath) ? obj.name : sourcePath + obj.name
|
|
116
|
+
await mirrorDir(source, dir, visitor)
|
|
117
|
+
}
|
|
46
118
|
}
|
|
47
119
|
}
|
|
48
120
|
|
|
49
121
|
function addUtilityFunctions (client, readOnly = false) {
|
|
50
122
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
123
|
+
client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
|
|
51
124
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
52
125
|
}
|
|
53
126
|
if (readOnly) {
|
|
@@ -136,6 +209,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
136
209
|
throw e
|
|
137
210
|
}
|
|
138
211
|
}
|
|
212
|
+
meta.type ||= M_FILE
|
|
139
213
|
meta.name = path // rewrite name back to plaintext name
|
|
140
214
|
return Object.assign({}, meta, metaObj)
|
|
141
215
|
}
|
|
@@ -155,9 +229,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
155
229
|
|
|
156
230
|
async function removeDirentFile(path) {
|
|
157
231
|
const df = direntFile(direntDir(dirname(path)), path);
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
await client.remove(
|
|
232
|
+
const recursive = false
|
|
233
|
+
const quiet = true
|
|
234
|
+
await client.remove(df, recursive, quiet)
|
|
235
|
+
await client.remove(encryptPath(path), recursive, quiet)
|
|
236
|
+
await client.remove(metaPath(path), recursive, quiet)
|
|
161
237
|
}
|
|
162
238
|
|
|
163
239
|
function stringGenerator (value, enc) {
|
|
@@ -167,13 +243,21 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
167
243
|
}
|
|
168
244
|
|
|
169
245
|
const encClient = {
|
|
170
|
-
list: async (path) => {
|
|
246
|
+
list: async (path, recursive, visitor) => {
|
|
171
247
|
const dirent = direntDir(path)
|
|
172
|
-
const entries = await client.list(dirent)
|
|
248
|
+
const entries = await client.list(dirent, recursive)
|
|
173
249
|
if (!entries || entries.length === 0) {
|
|
174
250
|
return []
|
|
175
251
|
}
|
|
176
|
-
|
|
252
|
+
if (visitor) {
|
|
253
|
+
const objects = await _loadMeta(dirent, entries)
|
|
254
|
+
for (const obj of objects) {
|
|
255
|
+
await visitor(obj)
|
|
256
|
+
}
|
|
257
|
+
return objects
|
|
258
|
+
} else {
|
|
259
|
+
return _loadMeta(dirent, entries)
|
|
260
|
+
}
|
|
177
261
|
},
|
|
178
262
|
metadata: _metadata(client),
|
|
179
263
|
read: async (path, callback) => {
|
|
@@ -215,12 +299,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
215
299
|
const cipher = crypt.startEncryptStream(enc)
|
|
216
300
|
const realPath = encryptPath(path)
|
|
217
301
|
await client.write(realPath, cryptGenerator(readFunc))
|
|
218
|
-
const meta = { name: path, size: bytesRead }
|
|
302
|
+
const meta = { name: path, size: bytesRead, type: M_FILE }
|
|
219
303
|
await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
|
|
220
304
|
return bytesRead
|
|
221
305
|
},
|
|
222
306
|
remove: async (path, options) => {
|
|
223
|
-
const recursive = (options && options.recursive) || false
|
|
307
|
+
const recursive = options === true || (options && options.recursive) || false
|
|
308
|
+
const quiet = (options && options.quiet) || false
|
|
224
309
|
if (recursive) {
|
|
225
310
|
// ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
|
|
226
311
|
async function recRm (path) {
|
|
@@ -253,7 +338,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
253
338
|
let parent = path
|
|
254
339
|
let dirent = direntDir(parent)
|
|
255
340
|
const df = direntFile(dirent, path)
|
|
256
|
-
await client.remove(df,
|
|
341
|
+
await client.remove(df, false, true)
|
|
257
342
|
while (true) {
|
|
258
343
|
try {
|
|
259
344
|
const entries = await client.list(dirent)
|
|
@@ -327,6 +412,7 @@ const M_SPECIAL = 'special'
|
|
|
327
412
|
|
|
328
413
|
module.exports = {
|
|
329
414
|
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
415
|
+
isAsyncGenerator, isReadable,
|
|
330
416
|
mobiletto, connect,
|
|
331
417
|
MobilettoError, MobilettoNotFoundError,
|
|
332
418
|
readStream, writeStream, closeStream
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require('dotenv').config()
|
|
4
4
|
|
|
5
5
|
const randomstring = require('randomstring')
|
|
6
|
+
const path = require('path')
|
|
6
7
|
|
|
7
8
|
const { expect, should, assert } = require('chai')
|
|
8
9
|
|
|
@@ -33,6 +34,18 @@ DRIVER_CONFIG = {
|
|
|
33
34
|
}
|
|
34
35
|
}
|
|
35
36
|
}
|
|
37
|
+
DRIVER_NAMES = Object.keys(DRIVER_CONFIG)
|
|
38
|
+
|
|
39
|
+
function pickAnotherDriver (notThisOne) {
|
|
40
|
+
let choice = null
|
|
41
|
+
while (!choice) {
|
|
42
|
+
choice = DRIVER_NAMES[Math.floor(Math.random() * DRIVER_NAMES.length)]
|
|
43
|
+
if (choice === notThisOne) {
|
|
44
|
+
choice = null
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return choice
|
|
48
|
+
}
|
|
36
49
|
|
|
37
50
|
async function assertMeta (api, name, expectedSize) {
|
|
38
51
|
const meta = await api.metadata(name)
|
|
@@ -96,7 +109,7 @@ const encryptionTests = () => [null, { key: rand(32) }]
|
|
|
96
109
|
// the same set of tests, with exactly the same inputs/outputs.
|
|
97
110
|
// This greatly supports developer sanity
|
|
98
111
|
|
|
99
|
-
for (const driverName of
|
|
112
|
+
for (const driverName of DRIVER_NAMES) {
|
|
100
113
|
const config = DRIVER_CONFIG[driverName]
|
|
101
114
|
const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
|
|
102
115
|
const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
|
|
@@ -122,7 +135,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
122
135
|
const randomData = rand(size)
|
|
123
136
|
const fileSuffix = '' + Date.now()
|
|
124
137
|
let fixture
|
|
125
|
-
|
|
138
|
+
before((done) => {
|
|
126
139
|
const name = `test_file_${fileSuffix}`
|
|
127
140
|
connect(driverName, config.key, config.secret, config.opts)
|
|
128
141
|
.then(api => { fixture = {api, name, randomData} })
|
|
@@ -152,12 +165,13 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
152
165
|
for (const encryption of encryptionTests()) {
|
|
153
166
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
154
167
|
describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
155
|
-
|
|
168
|
+
// describe(`${driverName} - ENC fail to write and delete files in readOnly mode`, () => {
|
|
169
|
+
// const encryption = {key: rand(32)}
|
|
156
170
|
const size = 16
|
|
157
171
|
const randomData = rand(size)
|
|
158
172
|
const fileSuffix = '' + Date.now()
|
|
159
173
|
let fixture
|
|
160
|
-
|
|
174
|
+
before((done) => {
|
|
161
175
|
const name = `test_file_${fileSuffix}`
|
|
162
176
|
const opts = Object.assign({}, config.opts, {readOnly: true})
|
|
163
177
|
connect(driverName, config.key, config.secret, opts, encryption)
|
|
@@ -203,7 +217,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
203
217
|
const fileSuffix = '' + Date.now()
|
|
204
218
|
const encryptionKey = rand(32)
|
|
205
219
|
let fixture
|
|
206
|
-
|
|
220
|
+
before((done) => {
|
|
207
221
|
const name = `test_file_${fileSuffix}`
|
|
208
222
|
mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
|
|
209
223
|
.then(api => { fixture = {api, name, randomData} })
|
|
@@ -233,29 +247,40 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
233
247
|
|
|
234
248
|
for (const encryption of encryptionTests()) {
|
|
235
249
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
236
|
-
|
|
250
|
+
const mirrorDriver = pickAnotherDriver(driverName)
|
|
251
|
+
const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
|
|
252
|
+
const mirrorDest = `mirrorDest_${rand(5)}/`
|
|
253
|
+
describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
|
|
237
254
|
// describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
|
|
238
255
|
// const encryption = {key: rand(32)}
|
|
239
256
|
// const encryption = null
|
|
240
257
|
// a random directory and file within it
|
|
241
258
|
const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
|
|
242
259
|
const subdirName = `subdir_` + Date.now()
|
|
243
|
-
const
|
|
260
|
+
const fullSubdirPath = `${randomParent}/${subdirName}`
|
|
261
|
+
const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
|
|
244
262
|
//const fileCount = 3 + Math.floor(Math.random() * 10)
|
|
245
263
|
const fileCount = 2
|
|
246
264
|
let fixture
|
|
247
|
-
|
|
265
|
+
before((done) => {
|
|
248
266
|
mobiletto(driverName, config.key, config.secret, config.opts, encryption)
|
|
249
|
-
.then(api => {
|
|
267
|
+
.then(api => {
|
|
268
|
+
// const mirrorEnc = { key: rand(32) }
|
|
269
|
+
const mirrorEnc = null
|
|
270
|
+
mobiletto(mirrorDriver, mirrorConfig.key, mirrorConfig.secret, mirrorConfig.opts, mirrorEnc)
|
|
271
|
+
.then(mirrorApi => {
|
|
272
|
+
fixture = {api, mirrorApi, name: randomPath}
|
|
273
|
+
})
|
|
274
|
+
.catch((err) => {
|
|
275
|
+
throw err
|
|
276
|
+
})
|
|
277
|
+
.finally(done)
|
|
278
|
+
})
|
|
250
279
|
.catch((err) => { throw err })
|
|
251
|
-
.finally(done)
|
|
252
280
|
})
|
|
253
281
|
it(`should write ${fileCount} files in a new directory`, async () => {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
yield rand(READ_SZ)
|
|
257
|
-
}
|
|
258
|
-
|
|
282
|
+
// return one chunk of random data
|
|
283
|
+
function* dataGenerator() { yield rand(READ_SZ) }
|
|
259
284
|
for (let i = 0; i < fileCount; i++) {
|
|
260
285
|
const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
|
|
261
286
|
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
@@ -279,6 +304,31 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
279
304
|
).to.not.be.null
|
|
280
305
|
}
|
|
281
306
|
})
|
|
307
|
+
it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
|
|
308
|
+
const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
|
|
309
|
+
expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
|
|
310
|
+
expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
|
|
311
|
+
})
|
|
312
|
+
it("should see the same files mirrored both places", async () => {
|
|
313
|
+
const originalThings = await fixture.api.list(fullSubdirPath)
|
|
314
|
+
const mirrorPath = `${mirrorDest}${subdirName}`
|
|
315
|
+
const mirrorThings = await fixture.mirrorApi.list(mirrorPath)
|
|
316
|
+
expect(originalThings.length === mirrorThings.length).to.be.true
|
|
317
|
+
|
|
318
|
+
const intersection = originalThings
|
|
319
|
+
.filter(orig => mirrorThings
|
|
320
|
+
.find(m => path.basename(m.name) === path.basename(orig.name)))
|
|
321
|
+
expect(intersection.length === originalThings.length).to.be.true
|
|
322
|
+
})
|
|
323
|
+
it("loading metadata on a mirrored file succeeds", async () => {
|
|
324
|
+
const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
|
|
325
|
+
await assertMeta(fixture.mirrorApi, destPath, READ_SZ)
|
|
326
|
+
})
|
|
327
|
+
it("should recursively delete the mirrored data", async () => {
|
|
328
|
+
const recursive = true
|
|
329
|
+
const removed = await fixture.mirrorApi.remove(mirrorDest, {recursive})
|
|
330
|
+
expect(removed).to.be.true
|
|
331
|
+
})
|
|
282
332
|
it("should recursively delete the directory and file we just created", async () => {
|
|
283
333
|
const recursive = true
|
|
284
334
|
const removed = await fixture.api.remove(randomParent, {recursive})
|
|
@@ -287,6 +337,10 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
|
|
|
287
337
|
it("loading metadata on the file we wrote now fails", async () => {
|
|
288
338
|
await assertMetaFail(fixture.api, fixture.name)
|
|
289
339
|
})
|
|
340
|
+
it("loading metadata on a mirrored file now fails", async () => {
|
|
341
|
+
const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
|
|
342
|
+
await assertMetaFail(fixture.mirrorApi, destPath)
|
|
343
|
+
})
|
|
290
344
|
it("loading metadata on the parent dir we wrote now fails", async () => {
|
|
291
345
|
await assertMetaFail(fixture.api, randomParent)
|
|
292
346
|
})
|