mobiletto 1.0.26 → 1.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -8
- package/drivers/local/index.js +21 -18
- package/drivers/s3/index.js +12 -2
- package/index.js +1 -0
- package/package.json +1 -1
- package/test/index.spec.js +5 -5
package/README.md
CHANGED
|
@@ -66,9 +66,10 @@ This code would run the same if the driver were `local`.
|
|
|
66
66
|
// * secret: ignored, can be null
|
|
67
67
|
// * opts object:
|
|
68
68
|
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
69
|
-
// *
|
|
69
|
+
// * fileMode: optional, permissions used when creating new files, default is 0600. can be string or integer
|
|
70
|
+
// * dirMode: optional, permissions used when creating new directories, default is 0700. can be string or integer
|
|
70
71
|
// * cacheSize: optional, LRU cache for `list` results
|
|
71
|
-
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {
|
|
72
|
+
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700', cacheSize: 100})
|
|
72
73
|
|
|
73
74
|
// To use 's3' driver:
|
|
74
75
|
// * key: AWS access key
|
|
@@ -137,16 +138,18 @@ This code would run the same if the driver were `local`.
|
|
|
137
138
|
await s3.writeFile(path, bufferOrString) // returns count of bytes written
|
|
138
139
|
|
|
139
140
|
// Delete a file
|
|
140
|
-
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
141
|
+
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
142
|
+
// Always returns a value or throws an error.
|
|
143
|
+
// Return value may be a single string of the file removed, or an array of all files removed (driver-dependent)
|
|
141
144
|
const quiet = true
|
|
142
|
-
local.remove(path, {quiet}) // returns
|
|
143
|
-
s3.remove(path, {quiet}) // returns
|
|
145
|
+
local.remove(path, {quiet}) // returns single path removed
|
|
146
|
+
s3.remove(path, {quiet}) // returns single path removed
|
|
144
147
|
|
|
145
148
|
// Recursively delete a directory and do it quietly (do not report errors)
|
|
146
149
|
const recursive = true
|
|
147
150
|
const quiet = true
|
|
148
|
-
local.remove(path, {recursive, quiet})
|
|
149
|
-
s3.remove(path, {recursive, quiet})
|
|
151
|
+
local.remove(path, {recursive, quiet}) // returns paths removed
|
|
152
|
+
s3.remove(path, {recursive, quiet}) // returns array containing all paths removed
|
|
150
153
|
|
|
151
154
|
## Metadata
|
|
152
155
|
The `metadata` command returns metadata about a single filesystem entry.
|
|
@@ -194,6 +197,14 @@ The `mirror` command performs a one-time copy of all files from one mobiletto to
|
|
|
194
197
|
It does not run any process to maintain the mirror over time. Run the `mirror` command again
|
|
195
198
|
to synchronize any missing files.
|
|
196
199
|
|
|
200
|
+
The return value from `mirror` is a simple object with counters for how many files were successfully
|
|
201
|
+
mirrored and how many files had errors:
|
|
202
|
+
|
|
203
|
+
{
|
|
204
|
+
success: count-of-files-mirrored,
|
|
205
|
+
errors: count-of-files-with-errors
|
|
206
|
+
}
|
|
207
|
+
|
|
197
208
|
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
198
209
|
|
|
199
210
|
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
@@ -259,7 +270,7 @@ A driver is any JS file that exports a 'storageClient' function with this signat
|
|
|
259
270
|
* `key`: a string, your API key (for the `local` driver this is the base directory)
|
|
260
271
|
* `secret`: a string, your API secret (can be omitted for the `local` driver)
|
|
261
272
|
* `opts`: an object, the properties are per-driver:
|
|
262
|
-
* For `local`, the `
|
|
273
|
+
* For `local`, the `fileMode` and `dirMode` properties determine how new creating files and directories are created
|
|
263
274
|
* For `s3`, the `bucket` property is required. Optional properties are:
|
|
264
275
|
* `region`: the S3 region, default is us-east-1
|
|
265
276
|
* `prefix`: a prefix to prepend to all S3 paths, default is the empty string
|
|
@@ -288,4 +299,5 @@ The object that the storageClient function returns must define these functions:
|
|
|
288
299
|
async write (path, generatorOrReadableStream)
|
|
289
300
|
|
|
290
301
|
// Remove a file, or recursively delete a directory
|
|
302
|
+
// returns a string of a single path removed, or an array of multiple paths removed
|
|
291
303
|
async remove (path, recursive = false, quiet = false)
|
package/drivers/local/index.js
CHANGED
|
@@ -7,18 +7,11 @@ const {
|
|
|
7
7
|
|
|
8
8
|
const fs = require('fs')
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
const DEFAULT_FILE_MODE = '0600'
|
|
11
|
+
const DEFAULT_DIR_MODE = '0700'
|
|
11
12
|
|
|
12
13
|
const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
|
13
14
|
|
|
14
|
-
const ioError = (err, path, method) => isNotExistError(err)
|
|
15
|
-
? err instanceof MobilettoNotFoundError
|
|
16
|
-
? err
|
|
17
|
-
: new MobilettoNotFoundError(path)
|
|
18
|
-
: err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
19
|
-
? err
|
|
20
|
-
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
21
|
-
|
|
22
15
|
const fileType = (stat) => {
|
|
23
16
|
if (stat.isDirectory()) {
|
|
24
17
|
return M_DIR
|
|
@@ -34,7 +27,8 @@ const fileType = (stat) => {
|
|
|
34
27
|
|
|
35
28
|
class StorageClient {
|
|
36
29
|
baseDir
|
|
37
|
-
|
|
30
|
+
fileMode
|
|
31
|
+
dirMode
|
|
38
32
|
constructor(baseDir, opts) {
|
|
39
33
|
const resolved = this.resolveSymlinks(baseDir)
|
|
40
34
|
if (!resolved.stat.isDirectory()) {
|
|
@@ -43,7 +37,8 @@ class StorageClient {
|
|
|
43
37
|
}
|
|
44
38
|
const dir = resolved.path
|
|
45
39
|
this.baseDir = dir.endsWith('/') ? dir : dir + '/'
|
|
46
|
-
this.
|
|
40
|
+
this.fileMode = opts && opts.fileMode ? opts.fileMode : DEFAULT_FILE_MODE
|
|
41
|
+
this.dirMode = opts && opts.dirMode ? opts.dirMode : DEFAULT_DIR_MODE
|
|
47
42
|
}
|
|
48
43
|
resolveSymlinks (path) {
|
|
49
44
|
let stat = fs.lstatSync(path)
|
|
@@ -67,6 +62,14 @@ class StorageClient {
|
|
|
67
62
|
const norm = path.startsWith(this.baseDir) ? path.substring(this.baseDir.length) : path
|
|
68
63
|
return norm.startsWith('/') ? norm.substring(1) : norm
|
|
69
64
|
}
|
|
65
|
+
ioError = (err, path, method) => isNotExistError(err)
|
|
66
|
+
? err instanceof MobilettoNotFoundError
|
|
67
|
+
? err
|
|
68
|
+
: new MobilettoNotFoundError(this.denormalizePath(path))
|
|
69
|
+
: err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
70
|
+
? err
|
|
71
|
+
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
72
|
+
|
|
70
73
|
|
|
71
74
|
fileToObject = (dir) => (f) => {
|
|
72
75
|
const stat = fs.lstatSync(dir + '/' + f)
|
|
@@ -105,7 +108,7 @@ class StorageClient {
|
|
|
105
108
|
return files
|
|
106
109
|
} catch (e) {
|
|
107
110
|
console.error(`dirFiles: ${e}`)
|
|
108
|
-
throw ioError(e, norm, 'dirFiles')
|
|
111
|
+
throw this.ioError(e, norm, 'dirFiles')
|
|
109
112
|
}
|
|
110
113
|
}
|
|
111
114
|
|
|
@@ -120,7 +123,7 @@ class StorageClient {
|
|
|
120
123
|
return await this.readDirFiles(dir, recursive, visitor)
|
|
121
124
|
}
|
|
122
125
|
} catch (err) {
|
|
123
|
-
throw ioError(err, path, 'list')
|
|
126
|
+
throw this.ioError(err, path, 'list')
|
|
124
127
|
}
|
|
125
128
|
}
|
|
126
129
|
async metadata (path) {
|
|
@@ -144,14 +147,14 @@ class StorageClient {
|
|
|
144
147
|
mtime: lstat.mtimeMs
|
|
145
148
|
}
|
|
146
149
|
} catch (err) {
|
|
147
|
-
throw ioError(err, path, 'metadata')
|
|
150
|
+
throw this.ioError(err, path, 'metadata')
|
|
148
151
|
}
|
|
149
152
|
}
|
|
150
153
|
|
|
151
154
|
mkdirs (path) {
|
|
152
155
|
try {
|
|
153
156
|
// console.log(`mkdirs: creating directory: ${path}`)
|
|
154
|
-
fs.mkdirSync(path, {recursive: true, mode: this.
|
|
157
|
+
fs.mkdirSync(path, {recursive: true, mode: this.dirMode})
|
|
155
158
|
} catch (err) {
|
|
156
159
|
throw new MobilettoError(`mkdirs: error creating directory ${path}: ${err}`, err)
|
|
157
160
|
}
|
|
@@ -183,7 +186,7 @@ class StorageClient {
|
|
|
183
186
|
async write (path, generatorOrReadableStream) {
|
|
184
187
|
const file = await this.normalizePathAndEnsureParentDirs(path)
|
|
185
188
|
// console.log(`write: writing path ${path} -> ${file}`)
|
|
186
|
-
const stream = fs.createWriteStream(file, {mode: this.
|
|
189
|
+
const stream = fs.createWriteStream(file, {mode: this.fileMode})
|
|
187
190
|
const writer = writeStream(stream)
|
|
188
191
|
const closer = closeStream(stream)
|
|
189
192
|
let count = 0
|
|
@@ -235,7 +238,7 @@ class StorageClient {
|
|
|
235
238
|
const stream = fs.createReadStream(file)
|
|
236
239
|
return await readStream(stream, callback, endCallback)
|
|
237
240
|
} catch (err) {
|
|
238
|
-
throw ioError(err, path, 'read')
|
|
241
|
+
throw this.ioError(err, path, 'read')
|
|
239
242
|
}
|
|
240
243
|
}
|
|
241
244
|
|
|
@@ -253,7 +256,7 @@ class StorageClient {
|
|
|
253
256
|
throw new MobilettoError(`remove(${path}) error: ${err}`, err)
|
|
254
257
|
}
|
|
255
258
|
}
|
|
256
|
-
return
|
|
259
|
+
return path
|
|
257
260
|
}
|
|
258
261
|
}
|
|
259
262
|
|
package/drivers/s3/index.js
CHANGED
|
@@ -137,12 +137,17 @@ class StorageClient {
|
|
|
137
137
|
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
|
|
138
138
|
return p.replaceAll(/\/{2,}/g, '/')
|
|
139
139
|
}
|
|
140
|
+
denormalizeKey = (key) => {
|
|
141
|
+
return key.startsWith(this.prefix)
|
|
142
|
+
? key.substring(this.prefix.length)
|
|
143
|
+
: key
|
|
144
|
+
}
|
|
140
145
|
|
|
141
146
|
s3error (err, key, path, method) {
|
|
142
147
|
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
143
148
|
? err
|
|
144
149
|
: (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
|
|
145
|
-
? new MobilettoNotFoundError(key)
|
|
150
|
+
? new MobilettoNotFoundError(this.denormalizeKey(key))
|
|
146
151
|
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
147
152
|
}
|
|
148
153
|
|
|
@@ -217,6 +222,7 @@ class StorageClient {
|
|
|
217
222
|
|
|
218
223
|
async remove (path, recursive, quiet) {
|
|
219
224
|
if (recursive) {
|
|
225
|
+
const removed = []
|
|
220
226
|
let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
|
|
221
227
|
while (objects && objects.length > 0) {
|
|
222
228
|
const Delete = {
|
|
@@ -240,6 +246,9 @@ class StorageClient {
|
|
|
240
246
|
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
241
247
|
throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
242
248
|
}
|
|
249
|
+
if (response.Deleted) {
|
|
250
|
+
removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
|
|
251
|
+
}
|
|
243
252
|
try {
|
|
244
253
|
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
245
254
|
} catch (e) {
|
|
@@ -249,6 +258,7 @@ class StorageClient {
|
|
|
249
258
|
objects = null
|
|
250
259
|
}
|
|
251
260
|
}
|
|
261
|
+
return removed
|
|
252
262
|
} else {
|
|
253
263
|
const Key = this.normalizeKey(path)
|
|
254
264
|
const bucketParams = {
|
|
@@ -272,8 +282,8 @@ class StorageClient {
|
|
|
272
282
|
} catch (err) {
|
|
273
283
|
throw this.s3error(err, Key, path, 'remove')
|
|
274
284
|
}
|
|
285
|
+
return path
|
|
275
286
|
}
|
|
276
|
-
return true
|
|
277
287
|
}
|
|
278
288
|
}
|
|
279
289
|
|
package/index.js
CHANGED
|
@@ -405,6 +405,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
405
405
|
await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
|
|
406
406
|
return bytesRead
|
|
407
407
|
},
|
|
408
|
+
// todo: remove should return an array of the paths that were actually removed
|
|
408
409
|
remove: async (path, options) => {
|
|
409
410
|
const recursive = options === true || (options && options.recursive) || false
|
|
410
411
|
const quiet = (options && options.quiet) || false
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -155,7 +155,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
155
155
|
})
|
|
156
156
|
it("should delete the file we just wrote", async () => {
|
|
157
157
|
const removed = await fixture.api.remove(fixture.name)
|
|
158
|
-
expect(removed).to.be.true
|
|
158
|
+
expect(!!removed).to.be.true
|
|
159
159
|
})
|
|
160
160
|
it("loading metadata on the file we wrote now fails", async () => {
|
|
161
161
|
await assertMetaFail(fixture.api, fixture.name)
|
|
@@ -205,7 +205,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
205
205
|
})
|
|
206
206
|
it("should fail to delete the file we did not write", async () => {
|
|
207
207
|
const removed = await fixture.api.remove(fixture.name)
|
|
208
|
-
expect(removed).to.be.false
|
|
208
|
+
expect(!!removed).to.be.false
|
|
209
209
|
})
|
|
210
210
|
})
|
|
211
211
|
}
|
|
@@ -238,7 +238,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
238
238
|
})
|
|
239
239
|
it("should delete the encrypted file we just wrote", async () => {
|
|
240
240
|
const removed = await fixture.api.remove(fixture.name)
|
|
241
|
-
expect(removed).to.be.true
|
|
241
|
+
expect(!!removed).to.be.true
|
|
242
242
|
})
|
|
243
243
|
it("loading metadata on the encrypted file we wrote now fails", async () => {
|
|
244
244
|
await assertMetaFail(fixture.api, fixture.name)
|
|
@@ -327,12 +327,12 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
327
327
|
it("should recursively delete the mirrored data", async () => {
|
|
328
328
|
const recursive = true
|
|
329
329
|
const removed = await fixture.mirrorApi.remove(mirrorDest, {recursive})
|
|
330
|
-
expect(removed).to.be.true
|
|
330
|
+
expect(!!removed).to.be.true
|
|
331
331
|
})
|
|
332
332
|
it("should recursively delete the directory and file we just created", async () => {
|
|
333
333
|
const recursive = true
|
|
334
334
|
const removed = await fixture.api.remove(randomParent, {recursive})
|
|
335
|
-
expect(removed).to.be.true
|
|
335
|
+
expect(!!removed).to.be.true
|
|
336
336
|
})
|
|
337
337
|
it("loading metadata on the file we wrote now fails", async () => {
|
|
338
338
|
await assertMetaFail(fixture.api, fixture.name)
|