mobiletto 1.0.27 → 1.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/drivers/local/index.js +20 -17
- package/drivers/s3/index.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ Today the supported drivers are:
|
|
|
17
17
|
* [Mobiletto on npm](https://www.npmjs.com/package/mobiletto)
|
|
18
18
|
|
|
19
19
|
## TLDR
|
|
20
|
-
A simple
|
|
20
|
+
A simple example using the mobiletto `s3` driver.
|
|
21
21
|
|
|
22
22
|
This code would run the same if the driver were `local`.
|
|
23
23
|
|
|
@@ -54,6 +54,7 @@ This code would run the same if the driver were `local`.
|
|
|
54
54
|
* [Transparent encryption](#Transparent-encryption)
|
|
55
55
|
* [Key rotation](#Key-rotation)
|
|
56
56
|
* [Driver interface](#Driver-interface)
|
|
57
|
+
* [Logging](#Logging)
|
|
57
58
|
----
|
|
58
59
|
## Basic usage
|
|
59
60
|
const { mobiletto } = require('mobiletto')
|
|
@@ -66,9 +67,10 @@ This code would run the same if the driver were `local`.
|
|
|
66
67
|
// * secret: ignored, can be null
|
|
67
68
|
// * opts object:
|
|
68
69
|
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
69
|
-
// *
|
|
70
|
+
// * fileMode: optional, permissions used when creating new files, default is 0600. can be string or integer
|
|
71
|
+
// * dirMode: optional, permissions used when creating new directories, default is 0700. can be string or integer
|
|
70
72
|
// * cacheSize: optional, LRU cache for `list` results
|
|
71
|
-
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {
|
|
73
|
+
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700', cacheSize: 100})
|
|
72
74
|
|
|
73
75
|
// To use 's3' driver:
|
|
74
76
|
// * key: AWS access key
|
|
@@ -196,6 +198,14 @@ The `mirror` command performs a one-time copy of all files from one mobiletto to
|
|
|
196
198
|
It does not run any process to maintain the mirror over time. Run the `mirror` command again
|
|
197
199
|
to synchronize any missing files.
|
|
198
200
|
|
|
201
|
+
The return value from `mirror` is a simple object with counters for how many files were successfully
|
|
202
|
+
mirrored and how many files had errors:
|
|
203
|
+
|
|
204
|
+
{
|
|
205
|
+
success: count-of-files-mirrored,
|
|
206
|
+
errors: count-of-files-with-errors
|
|
207
|
+
}
|
|
208
|
+
|
|
199
209
|
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
200
210
|
|
|
201
211
|
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
@@ -261,7 +271,7 @@ A driver is any JS file that exports a 'storageClient' function with this signat
|
|
|
261
271
|
* `key`: a string, your API key (for the `local` driver this is the base directory)
|
|
262
272
|
* `secret`: a string, your API secret (can be omitted for the `local` driver)
|
|
263
273
|
* `opts`: an object, the properties are per-driver:
|
|
264
|
-
* For `local`, the `
|
|
274
|
+
* For `local`, the `fileMode` and `dirMode` properties determine how new creating files and directories are created
|
|
265
275
|
* For `s3`, the `bucket` property is required. Optional properties are:
|
|
266
276
|
* `region`: the S3 region, default is us-east-1
|
|
267
277
|
* `prefix`: a prefix to prepend to all S3 paths, default is the empty string
|
|
@@ -292,3 +302,10 @@ The object that the storageClient function returns must define these functions:
|
|
|
292
302
|
// Remove a file, or recursively delete a directory
|
|
293
303
|
// returns a string of a single path removed, or an array of multiple paths removed
|
|
294
304
|
async remove (path, recursive = false, quiet = false)
|
|
305
|
+
|
|
306
|
+
## Logging
|
|
307
|
+
Mobiletto uses the winston logging library. Set the log level with the `MOBILETTO_LOG_LEVEL` environment variable.
|
|
308
|
+
The default level is `warn`. The most verbose level is `silly`, although currently mobiletto does not log at levels
|
|
309
|
+
below `debug`
|
|
310
|
+
|
|
311
|
+
MOBILETTO_LOG_LEVEL=silly
|
package/drivers/local/index.js
CHANGED
|
@@ -7,18 +7,11 @@ const {
|
|
|
7
7
|
|
|
8
8
|
const fs = require('fs')
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
const DEFAULT_FILE_MODE = '0600'
|
|
11
|
+
const DEFAULT_DIR_MODE = '0700'
|
|
11
12
|
|
|
12
13
|
const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
|
13
14
|
|
|
14
|
-
const ioError = (err, path, method) => isNotExistError(err)
|
|
15
|
-
? err instanceof MobilettoNotFoundError
|
|
16
|
-
? err
|
|
17
|
-
: new MobilettoNotFoundError(path)
|
|
18
|
-
: err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
19
|
-
? err
|
|
20
|
-
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
21
|
-
|
|
22
15
|
const fileType = (stat) => {
|
|
23
16
|
if (stat.isDirectory()) {
|
|
24
17
|
return M_DIR
|
|
@@ -34,7 +27,8 @@ const fileType = (stat) => {
|
|
|
34
27
|
|
|
35
28
|
class StorageClient {
|
|
36
29
|
baseDir
|
|
37
|
-
|
|
30
|
+
fileMode
|
|
31
|
+
dirMode
|
|
38
32
|
constructor(baseDir, opts) {
|
|
39
33
|
const resolved = this.resolveSymlinks(baseDir)
|
|
40
34
|
if (!resolved.stat.isDirectory()) {
|
|
@@ -43,7 +37,8 @@ class StorageClient {
|
|
|
43
37
|
}
|
|
44
38
|
const dir = resolved.path
|
|
45
39
|
this.baseDir = dir.endsWith('/') ? dir : dir + '/'
|
|
46
|
-
this.
|
|
40
|
+
this.fileMode = opts && opts.fileMode ? opts.fileMode : DEFAULT_FILE_MODE
|
|
41
|
+
this.dirMode = opts && opts.dirMode ? opts.dirMode : DEFAULT_DIR_MODE
|
|
47
42
|
}
|
|
48
43
|
resolveSymlinks (path) {
|
|
49
44
|
let stat = fs.lstatSync(path)
|
|
@@ -67,6 +62,14 @@ class StorageClient {
|
|
|
67
62
|
const norm = path.startsWith(this.baseDir) ? path.substring(this.baseDir.length) : path
|
|
68
63
|
return norm.startsWith('/') ? norm.substring(1) : norm
|
|
69
64
|
}
|
|
65
|
+
ioError = (err, path, method) => isNotExistError(err)
|
|
66
|
+
? err instanceof MobilettoNotFoundError
|
|
67
|
+
? err
|
|
68
|
+
: new MobilettoNotFoundError(this.denormalizePath(path))
|
|
69
|
+
: err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
70
|
+
? err
|
|
71
|
+
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
72
|
+
|
|
70
73
|
|
|
71
74
|
fileToObject = (dir) => (f) => {
|
|
72
75
|
const stat = fs.lstatSync(dir + '/' + f)
|
|
@@ -105,7 +108,7 @@ class StorageClient {
|
|
|
105
108
|
return files
|
|
106
109
|
} catch (e) {
|
|
107
110
|
console.error(`dirFiles: ${e}`)
|
|
108
|
-
throw ioError(e, norm, 'dirFiles')
|
|
111
|
+
throw this.ioError(e, norm, 'dirFiles')
|
|
109
112
|
}
|
|
110
113
|
}
|
|
111
114
|
|
|
@@ -120,7 +123,7 @@ class StorageClient {
|
|
|
120
123
|
return await this.readDirFiles(dir, recursive, visitor)
|
|
121
124
|
}
|
|
122
125
|
} catch (err) {
|
|
123
|
-
throw ioError(err, path, 'list')
|
|
126
|
+
throw this.ioError(err, path, 'list')
|
|
124
127
|
}
|
|
125
128
|
}
|
|
126
129
|
async metadata (path) {
|
|
@@ -144,14 +147,14 @@ class StorageClient {
|
|
|
144
147
|
mtime: lstat.mtimeMs
|
|
145
148
|
}
|
|
146
149
|
} catch (err) {
|
|
147
|
-
throw ioError(err, path, 'metadata')
|
|
150
|
+
throw this.ioError(err, path, 'metadata')
|
|
148
151
|
}
|
|
149
152
|
}
|
|
150
153
|
|
|
151
154
|
mkdirs (path) {
|
|
152
155
|
try {
|
|
153
156
|
// console.log(`mkdirs: creating directory: ${path}`)
|
|
154
|
-
fs.mkdirSync(path, {recursive: true, mode: this.
|
|
157
|
+
fs.mkdirSync(path, {recursive: true, mode: this.dirMode})
|
|
155
158
|
} catch (err) {
|
|
156
159
|
throw new MobilettoError(`mkdirs: error creating directory ${path}: ${err}`, err)
|
|
157
160
|
}
|
|
@@ -183,7 +186,7 @@ class StorageClient {
|
|
|
183
186
|
async write (path, generatorOrReadableStream) {
|
|
184
187
|
const file = await this.normalizePathAndEnsureParentDirs(path)
|
|
185
188
|
// console.log(`write: writing path ${path} -> ${file}`)
|
|
186
|
-
const stream = fs.createWriteStream(file, {mode: this.
|
|
189
|
+
const stream = fs.createWriteStream(file, {mode: this.fileMode})
|
|
187
190
|
const writer = writeStream(stream)
|
|
188
191
|
const closer = closeStream(stream)
|
|
189
192
|
let count = 0
|
|
@@ -235,7 +238,7 @@ class StorageClient {
|
|
|
235
238
|
const stream = fs.createReadStream(file)
|
|
236
239
|
return await readStream(stream, callback, endCallback)
|
|
237
240
|
} catch (err) {
|
|
238
|
-
throw ioError(err, path, 'read')
|
|
241
|
+
throw this.ioError(err, path, 'read')
|
|
239
242
|
}
|
|
240
243
|
}
|
|
241
244
|
|
package/drivers/s3/index.js
CHANGED
|
@@ -147,7 +147,7 @@ class StorageClient {
|
|
|
147
147
|
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
148
148
|
? err
|
|
149
149
|
: (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
|
|
150
|
-
? new MobilettoNotFoundError(key)
|
|
150
|
+
? new MobilettoNotFoundError(this.denormalizeKey(key))
|
|
151
151
|
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
152
152
|
}
|
|
153
153
|
|