mobiletto 1.0.26 → 1.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -66,9 +66,10 @@ This code would run the same if the driver were `local`.
66
66
  // * secret: ignored, can be null
67
67
  // * opts object:
68
68
  // * readOnly: optional, never change anything on the filesystem; default is false
69
- // * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
69
+ // * fileMode: optional, permissions used when creating new files, default is 0600. can be string or integer
70
+ // * dirMode: optional, permissions used when creating new directories, default is 0700. can be string or integer
70
71
  // * cacheSize: optional, LRU cache for `list` results
71
- const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700', cacheSize: 100})
72
+ const local = await mobiletto('local', '/home/ubuntu/tmp', null, {fileMode: 0o0600, dirMode: '0700', cacheSize: 100})
72
73
 
73
74
  // To use 's3' driver:
74
75
  // * key: AWS access key
@@ -137,16 +138,18 @@ This code would run the same if the driver were `local`.
137
138
  await s3.writeFile(path, bufferOrString) // returns count of bytes written
138
139
 
139
140
  // Delete a file
140
- // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
141
+ // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
142
+ // Always returns a value or throws an error.
143
+ // Return value may be a single string of the file removed, or an array of all files removed (driver-dependent)
141
144
  const quiet = true
142
- local.remove(path, {quiet}) // returns true upon success
143
- s3.remove(path, {quiet}) // returns true upon success
145
+ local.remove(path, {quiet}) // returns single path removed
146
+ s3.remove(path, {quiet}) // returns single path removed
144
147
 
145
148
  // Recursively delete a directory and do it quietly (do not report errors)
146
149
  const recursive = true
147
150
  const quiet = true
148
- local.remove(path, {recursive, quiet})
149
- s3.remove(path, {recursive, quiet})
151
+ local.remove(path, {recursive, quiet}) // returns paths removed
152
+ s3.remove(path, {recursive, quiet}) // returns array containing all paths removed
150
153
 
151
154
  ## Metadata
152
155
  The `metadata` command returns metadata about a single filesystem entry.
@@ -194,6 +197,14 @@ The `mirror` command performs a one-time copy of all files from one mobiletto to
194
197
  It does not run any process to maintain the mirror over time. Run the `mirror` command again
195
198
  to synchronize any missing files.
196
199
 
200
+ The return value from `mirror` is a simple object with counters for how many files were successfully
201
+ mirrored and how many files had errors:
202
+
203
+ {
204
+ success: count-of-files-mirrored,
205
+ errors: count-of-files-with-errors
206
+ }
207
+
197
208
  WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
198
209
 
199
210
  With the `mirror` call semantics it can sometimes be confusing to understand who is the
@@ -259,7 +270,7 @@ A driver is any JS file that exports a 'storageClient' function with this signat
259
270
  * `key`: a string, your API key (for the `local` driver this is the base directory)
260
271
  * `secret`: a string, your API secret (can be omitted for the `local` driver)
261
272
  * `opts`: an object, the properties are per-driver:
262
- * For `local`, the `mode` property is the file permissions to set when creating new files and directories
273
+ * For `local`, the `fileMode` and `dirMode` properties determine how new creating files and directories are created
263
274
  * For `s3`, the `bucket` property is required. Optional properties are:
264
275
  * `region`: the S3 region, default is us-east-1
265
276
  * `prefix`: a prefix to prepend to all S3 paths, default is the empty string
@@ -288,4 +299,5 @@ The object that the storageClient function returns must define these functions:
288
299
  async write (path, generatorOrReadableStream)
289
300
 
290
301
  // Remove a file, or recursively delete a directory
302
+ // returns a string of a single path removed, or an array of multiple paths removed
291
303
  async remove (path, recursive = false, quiet = false)
@@ -7,18 +7,11 @@ const {
7
7
 
8
8
  const fs = require('fs')
9
9
 
10
- const DEFAULT_MODE = '0700'
10
+ const DEFAULT_FILE_MODE = '0600'
11
+ const DEFAULT_DIR_MODE = '0700'
11
12
 
12
13
  const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
13
14
 
14
- const ioError = (err, path, method) => isNotExistError(err)
15
- ? err instanceof MobilettoNotFoundError
16
- ? err
17
- : new MobilettoNotFoundError(path)
18
- : err instanceof MobilettoError || err instanceof MobilettoNotFoundError
19
- ? err
20
- : new MobilettoError(`${method}(${path}) error: ${err}`, err)
21
-
22
15
  const fileType = (stat) => {
23
16
  if (stat.isDirectory()) {
24
17
  return M_DIR
@@ -34,7 +27,8 @@ const fileType = (stat) => {
34
27
 
35
28
  class StorageClient {
36
29
  baseDir
37
- mode
30
+ fileMode
31
+ dirMode
38
32
  constructor(baseDir, opts) {
39
33
  const resolved = this.resolveSymlinks(baseDir)
40
34
  if (!resolved.stat.isDirectory()) {
@@ -43,7 +37,8 @@ class StorageClient {
43
37
  }
44
38
  const dir = resolved.path
45
39
  this.baseDir = dir.endsWith('/') ? dir : dir + '/'
46
- this.mode = opts && opts.mode ? opts.mode : DEFAULT_MODE
40
+ this.fileMode = opts && opts.fileMode ? opts.fileMode : DEFAULT_FILE_MODE
41
+ this.dirMode = opts && opts.dirMode ? opts.dirMode : DEFAULT_DIR_MODE
47
42
  }
48
43
  resolveSymlinks (path) {
49
44
  let stat = fs.lstatSync(path)
@@ -67,6 +62,14 @@ class StorageClient {
67
62
  const norm = path.startsWith(this.baseDir) ? path.substring(this.baseDir.length) : path
68
63
  return norm.startsWith('/') ? norm.substring(1) : norm
69
64
  }
65
+ ioError = (err, path, method) => isNotExistError(err)
66
+ ? err instanceof MobilettoNotFoundError
67
+ ? err
68
+ : new MobilettoNotFoundError(this.denormalizePath(path))
69
+ : err instanceof MobilettoError || err instanceof MobilettoNotFoundError
70
+ ? err
71
+ : new MobilettoError(`${method}(${path}) error: ${err}`, err)
72
+
70
73
 
71
74
  fileToObject = (dir) => (f) => {
72
75
  const stat = fs.lstatSync(dir + '/' + f)
@@ -105,7 +108,7 @@ class StorageClient {
105
108
  return files
106
109
  } catch (e) {
107
110
  console.error(`dirFiles: ${e}`)
108
- throw ioError(e, norm, 'dirFiles')
111
+ throw this.ioError(e, norm, 'dirFiles')
109
112
  }
110
113
  }
111
114
 
@@ -120,7 +123,7 @@ class StorageClient {
120
123
  return await this.readDirFiles(dir, recursive, visitor)
121
124
  }
122
125
  } catch (err) {
123
- throw ioError(err, path, 'list')
126
+ throw this.ioError(err, path, 'list')
124
127
  }
125
128
  }
126
129
  async metadata (path) {
@@ -144,14 +147,14 @@ class StorageClient {
144
147
  mtime: lstat.mtimeMs
145
148
  }
146
149
  } catch (err) {
147
- throw ioError(err, path, 'metadata')
150
+ throw this.ioError(err, path, 'metadata')
148
151
  }
149
152
  }
150
153
 
151
154
  mkdirs (path) {
152
155
  try {
153
156
  // console.log(`mkdirs: creating directory: ${path}`)
154
- fs.mkdirSync(path, {recursive: true, mode: this.mode})
157
+ fs.mkdirSync(path, {recursive: true, mode: this.dirMode})
155
158
  } catch (err) {
156
159
  throw new MobilettoError(`mkdirs: error creating directory ${path}: ${err}`, err)
157
160
  }
@@ -183,7 +186,7 @@ class StorageClient {
183
186
  async write (path, generatorOrReadableStream) {
184
187
  const file = await this.normalizePathAndEnsureParentDirs(path)
185
188
  // console.log(`write: writing path ${path} -> ${file}`)
186
- const stream = fs.createWriteStream(file, {mode: this.mode})
189
+ const stream = fs.createWriteStream(file, {mode: this.fileMode})
187
190
  const writer = writeStream(stream)
188
191
  const closer = closeStream(stream)
189
192
  let count = 0
@@ -235,7 +238,7 @@ class StorageClient {
235
238
  const stream = fs.createReadStream(file)
236
239
  return await readStream(stream, callback, endCallback)
237
240
  } catch (err) {
238
- throw ioError(err, path, 'read')
241
+ throw this.ioError(err, path, 'read')
239
242
  }
240
243
  }
241
244
 
@@ -253,7 +256,7 @@ class StorageClient {
253
256
  throw new MobilettoError(`remove(${path}) error: ${err}`, err)
254
257
  }
255
258
  }
256
- return Promise.resolve(true)
259
+ return path
257
260
  }
258
261
  }
259
262
 
@@ -137,12 +137,17 @@ class StorageClient {
137
137
  : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
138
138
  return p.replaceAll(/\/{2,}/g, '/')
139
139
  }
140
+ denormalizeKey = (key) => {
141
+ return key.startsWith(this.prefix)
142
+ ? key.substring(this.prefix.length)
143
+ : key
144
+ }
140
145
 
141
146
  s3error (err, key, path, method) {
142
147
  return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
143
148
  ? err
144
149
  : (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
145
- ? new MobilettoNotFoundError(key)
150
+ ? new MobilettoNotFoundError(this.denormalizeKey(key))
146
151
  : new MobilettoError(`${method}(${path}) error: ${err}`, err)
147
152
  }
148
153
 
@@ -217,6 +222,7 @@ class StorageClient {
217
222
 
218
223
  async remove (path, recursive, quiet) {
219
224
  if (recursive) {
225
+ const removed = []
220
226
  let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
221
227
  while (objects && objects.length > 0) {
222
228
  const Delete = {
@@ -240,6 +246,9 @@ class StorageClient {
240
246
  if (!quiet && response.Errors && response.Errors.length > 0) {
241
247
  throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
242
248
  }
249
+ if (response.Deleted) {
250
+ removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
251
+ }
243
252
  try {
244
253
  objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
245
254
  } catch (e) {
@@ -249,6 +258,7 @@ class StorageClient {
249
258
  objects = null
250
259
  }
251
260
  }
261
+ return removed
252
262
  } else {
253
263
  const Key = this.normalizeKey(path)
254
264
  const bucketParams = {
@@ -272,8 +282,8 @@ class StorageClient {
272
282
  } catch (err) {
273
283
  throw this.s3error(err, Key, path, 'remove')
274
284
  }
285
+ return path
275
286
  }
276
- return true
277
287
  }
278
288
  }
279
289
 
package/index.js CHANGED
@@ -405,6 +405,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
405
405
  await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
406
406
  return bytesRead
407
407
  },
408
+ // todo: remove should return an array of the paths that were actually removed
408
409
  remove: async (path, options) => {
409
410
  const recursive = options === true || (options && options.recursive) || false
410
411
  const quiet = (options && options.quiet) || false
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.26",
3
+ "version": "1.0.29",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -155,7 +155,7 @@ for (const driverName of DRIVER_NAMES) {
155
155
  })
156
156
  it("should delete the file we just wrote", async () => {
157
157
  const removed = await fixture.api.remove(fixture.name)
158
- expect(removed).to.be.true
158
+ expect(!!removed).to.be.true
159
159
  })
160
160
  it("loading metadata on the file we wrote now fails", async () => {
161
161
  await assertMetaFail(fixture.api, fixture.name)
@@ -205,7 +205,7 @@ for (const driverName of DRIVER_NAMES) {
205
205
  })
206
206
  it("should fail to delete the file we did not write", async () => {
207
207
  const removed = await fixture.api.remove(fixture.name)
208
- expect(removed).to.be.false
208
+ expect(!!removed).to.be.false
209
209
  })
210
210
  })
211
211
  }
@@ -238,7 +238,7 @@ for (const driverName of DRIVER_NAMES) {
238
238
  })
239
239
  it("should delete the encrypted file we just wrote", async () => {
240
240
  const removed = await fixture.api.remove(fixture.name)
241
- expect(removed).to.be.true
241
+ expect(!!removed).to.be.true
242
242
  })
243
243
  it("loading metadata on the encrypted file we wrote now fails", async () => {
244
244
  await assertMetaFail(fixture.api, fixture.name)
@@ -327,12 +327,12 @@ for (const driverName of DRIVER_NAMES) {
327
327
  it("should recursively delete the mirrored data", async () => {
328
328
  const recursive = true
329
329
  const removed = await fixture.mirrorApi.remove(mirrorDest, {recursive})
330
- expect(removed).to.be.true
330
+ expect(!!removed).to.be.true
331
331
  })
332
332
  it("should recursively delete the directory and file we just created", async () => {
333
333
  const recursive = true
334
334
  const removed = await fixture.api.remove(randomParent, {recursive})
335
- expect(removed).to.be.true
335
+ expect(!!removed).to.be.true
336
336
  })
337
337
  it("loading metadata on the file we wrote now fails", async () => {
338
338
  await assertMetaFail(fixture.api, fixture.name)