mobiletto 1.0.28 → 1.0.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -6
- package/drivers/local/index.js +8 -5
- package/index.js +3 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -12,12 +12,17 @@ Today the supported drivers are:
|
|
|
12
12
|
* `s3`: read/write an Amazon S3 bucket
|
|
13
13
|
* `local`: read/write to local filesystem
|
|
14
14
|
|
|
15
|
+
## mobiletto-cli
|
|
16
|
+
Mobiletto is intended to be used as a library by other JavaScript code.
|
|
17
|
+
|
|
18
|
+
To work with mobiletto at the command-line, use [mobiletto-cli](https://www.npmjs.com/package/mobiletto-cli)
|
|
19
|
+
|
|
15
20
|
### Source
|
|
16
21
|
* [Mobiletto on GitHub](https://github.com/cobbzilla/mobiletto)
|
|
17
22
|
* [Mobiletto on npm](https://www.npmjs.com/package/mobiletto)
|
|
18
23
|
|
|
19
24
|
## TLDR
|
|
20
|
-
A simple
|
|
25
|
+
A simple example using the mobiletto `s3` driver.
|
|
21
26
|
|
|
22
27
|
This code would run the same if the driver were `local`.
|
|
23
28
|
|
|
@@ -54,6 +59,7 @@ This code would run the same if the driver were `local`.
|
|
|
54
59
|
* [Transparent encryption](#Transparent-encryption)
|
|
55
60
|
* [Key rotation](#Key-rotation)
|
|
56
61
|
* [Driver interface](#Driver-interface)
|
|
62
|
+
* [Logging](#Logging)
|
|
57
63
|
----
|
|
58
64
|
## Basic usage
|
|
59
65
|
const { mobiletto } = require('mobiletto')
|
|
@@ -66,9 +72,10 @@ This code would run the same if the driver were `local`.
|
|
|
66
72
|
// * secret: ignored, can be null
|
|
67
73
|
// * opts object:
|
|
68
74
|
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
69
|
-
// *
|
|
70
|
-
// *
|
|
71
|
-
|
|
75
|
+
// * fileMode: optional, permissions used when creating new files, default is 0600. can be string or integer
|
|
76
|
+
// * dirMode: optional, permissions used when creating new directories, default is 0700. can be string or integer
|
|
77
|
+
// * cacheSize: optional, LRU cache for `list` results, default is no cache
|
|
78
|
+
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700', cacheSize: 100})
|
|
72
79
|
|
|
73
80
|
// To use 's3' driver:
|
|
74
81
|
// * key: AWS access key
|
|
@@ -79,7 +86,7 @@ This code would run the same if the driver were `local`.
|
|
|
79
86
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
80
87
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
81
88
|
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
82
|
-
// * cacheSize: optional, LRU cache for `list` results
|
|
89
|
+
// * cacheSize: optional, LRU cache for `list` results, default is no cache
|
|
83
90
|
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
84
91
|
|
|
85
92
|
// List files
|
|
@@ -196,6 +203,14 @@ The `mirror` command performs a one-time copy of all files from one mobiletto to
|
|
|
196
203
|
It does not run any process to maintain the mirror over time. Run the `mirror` command again
|
|
197
204
|
to synchronize any missing files.
|
|
198
205
|
|
|
206
|
+
The return value from `mirror` is a simple object with counters for how many files were successfully
|
|
207
|
+
mirrored and how many files had errors:
|
|
208
|
+
|
|
209
|
+
{
|
|
210
|
+
success: count-of-files-mirrored,
|
|
211
|
+
errors: count-of-files-with-errors
|
|
212
|
+
}
|
|
213
|
+
|
|
199
214
|
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
200
215
|
|
|
201
216
|
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
@@ -261,7 +276,7 @@ A driver is any JS file that exports a 'storageClient' function with this signat
|
|
|
261
276
|
* `key`: a string, your API key (for the `local` driver this is the base directory)
|
|
262
277
|
* `secret`: a string, your API secret (can be omitted for the `local` driver)
|
|
263
278
|
* `opts`: an object, the properties are per-driver:
|
|
264
|
-
* For `local`, the `
|
|
279
|
+
* For `local`, the `fileMode` and `dirMode` properties determine how new creating files and directories are created
|
|
265
280
|
* For `s3`, the `bucket` property is required. Optional properties are:
|
|
266
281
|
* `region`: the S3 region, default is us-east-1
|
|
267
282
|
* `prefix`: a prefix to prepend to all S3 paths, default is the empty string
|
|
@@ -292,3 +307,27 @@ The object that the storageClient function returns must define these functions:
|
|
|
292
307
|
// Remove a file, or recursively delete a directory
|
|
293
308
|
// returns a string of a single path removed, or an array of multiple paths removed
|
|
294
309
|
async remove (path, recursive = false, quiet = false)
|
|
310
|
+
|
|
311
|
+
## Logging
|
|
312
|
+
Mobiletto uses the [winston](https://www.npmjs.com/package/winston) logging library.
|
|
313
|
+
|
|
314
|
+
Logs **will** contain file paths and error messages, but will **never** contain keys, secrets,
|
|
315
|
+
or any other connection configuration information.
|
|
316
|
+
|
|
317
|
+
### Log level
|
|
318
|
+
Use the `MOBILETTO_LOG_LEVEL` environment variable to set the log level.
|
|
319
|
+
The default level is `warn`. The most verbose level is `silly`, although currently mobiletto
|
|
320
|
+
does not log at levels below `debug`
|
|
321
|
+
|
|
322
|
+
MOBILETTO_LOG_LEVEL=silly
|
|
323
|
+
|
|
324
|
+
### Log file
|
|
325
|
+
By default, the logger writes to the console. To send logs to a file, set the `MOBILETTO_LOG_FILE`
|
|
326
|
+
environment variable. When logging to a file, logs will no longer be written to the console.
|
|
327
|
+
|
|
328
|
+
MOBILETTO_LOG_FILE=/var/my_mobiletto_log
|
|
329
|
+
|
|
330
|
+
To turn off logging:
|
|
331
|
+
|
|
332
|
+
MOBILETTO_LOG_FILE=/dev/null
|
|
333
|
+
MOBILETTO_LOG_LEVEL=emerg # not strictly necessary; minimizes time spent in logging code
|
package/drivers/local/index.js
CHANGED
|
@@ -7,7 +7,8 @@ const {
|
|
|
7
7
|
|
|
8
8
|
const fs = require('fs')
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
const DEFAULT_FILE_MODE = '0600'
|
|
11
|
+
const DEFAULT_DIR_MODE = '0700'
|
|
11
12
|
|
|
12
13
|
const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
|
13
14
|
|
|
@@ -26,7 +27,8 @@ const fileType = (stat) => {
|
|
|
26
27
|
|
|
27
28
|
class StorageClient {
|
|
28
29
|
baseDir
|
|
29
|
-
|
|
30
|
+
fileMode
|
|
31
|
+
dirMode
|
|
30
32
|
constructor(baseDir, opts) {
|
|
31
33
|
const resolved = this.resolveSymlinks(baseDir)
|
|
32
34
|
if (!resolved.stat.isDirectory()) {
|
|
@@ -35,7 +37,8 @@ class StorageClient {
|
|
|
35
37
|
}
|
|
36
38
|
const dir = resolved.path
|
|
37
39
|
this.baseDir = dir.endsWith('/') ? dir : dir + '/'
|
|
38
|
-
this.
|
|
40
|
+
this.fileMode = opts && opts.fileMode ? opts.fileMode : DEFAULT_FILE_MODE
|
|
41
|
+
this.dirMode = opts && opts.dirMode ? opts.dirMode : DEFAULT_DIR_MODE
|
|
39
42
|
}
|
|
40
43
|
resolveSymlinks (path) {
|
|
41
44
|
let stat = fs.lstatSync(path)
|
|
@@ -151,7 +154,7 @@ class StorageClient {
|
|
|
151
154
|
mkdirs (path) {
|
|
152
155
|
try {
|
|
153
156
|
// console.log(`mkdirs: creating directory: ${path}`)
|
|
154
|
-
fs.mkdirSync(path, {recursive: true, mode: this.
|
|
157
|
+
fs.mkdirSync(path, {recursive: true, mode: this.dirMode})
|
|
155
158
|
} catch (err) {
|
|
156
159
|
throw new MobilettoError(`mkdirs: error creating directory ${path}: ${err}`, err)
|
|
157
160
|
}
|
|
@@ -183,7 +186,7 @@ class StorageClient {
|
|
|
183
186
|
async write (path, generatorOrReadableStream) {
|
|
184
187
|
const file = await this.normalizePathAndEnsureParentDirs(path)
|
|
185
188
|
// console.log(`write: writing path ${path} -> ${file}`)
|
|
186
|
-
const stream = fs.createWriteStream(file, {mode: this.
|
|
189
|
+
const stream = fs.createWriteStream(file, {mode: this.fileMode})
|
|
187
190
|
const writer = writeStream(stream)
|
|
188
191
|
const closer = closeStream(stream)
|
|
189
192
|
let count = 0
|
package/index.js
CHANGED
|
@@ -11,7 +11,9 @@ const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = requir
|
|
|
11
11
|
const logger = winston.createLogger({
|
|
12
12
|
level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
|
|
13
13
|
format: winston.format.simple(),
|
|
14
|
-
transports:
|
|
14
|
+
transports: process.env.MOBILETTO_LOG_FILE
|
|
15
|
+
? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]
|
|
16
|
+
: [ new winston.transports.Console() ]
|
|
15
17
|
})
|
|
16
18
|
|
|
17
19
|
const setLogLevel = (level) => { logger.level = level }
|