mobiletto 1.0.17 → 1.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +53 -41
  2. package/index.js +27 -7
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -5,7 +5,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
5
5
 
6
6
  It supports apps that are agnostic to where files are stored.
7
7
 
8
- All drivers are tested for identical behavior with over 35 tests for each driver.
8
+ All drivers are tested for identical behavior with over 40 tests for each driver.
9
9
 
10
10
  Today the supported drivers are:
11
11
 
@@ -38,8 +38,8 @@ Today the supported drivers are:
38
38
  const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
39
39
 
40
40
  // List files
41
- local.list() // --> returns an array of file objects
42
- s3.list() // --> returns an array of file objects
41
+ local.list() // --> returns an array of metadata objects
42
+ s3.list() // --> returns an array of metadata objects
43
43
 
44
44
  // List files recursively
45
45
  local.list({ recursive: true })
@@ -53,20 +53,28 @@ Today the supported drivers are:
53
53
  // Visit files in a directory -- visitor function must be async
54
54
  local.list(path, { visitor: myAsyncFunc })
55
55
  s3.list(path, { visitor: myAsyncFunc, recursive: true })
56
+
57
+ // The `list` method throws MobilettoNotFoundError if the path does not exist
58
+ // When you call `safeList` on a non-existent path, it returns an empty array
59
+ s3.safeList('/path/that/does/not/exist') // returns []
56
60
 
57
61
  // Read metadata for a file
58
- local.metadata(path)
59
- s3.metadata(path)
62
+ local.metadata(path) // returns metadata object
63
+ s3.metadata(path) // returns metadata object
64
+
65
+ // The `metadata` method throws MobilettoNotFoundError if the path does not exist
66
+ // When you call `safeMetadata` on a non-existent path, it returns null
67
+ local.safeMetadata('/tmp/does_not_exist') // returns null
60
68
 
61
69
  // Read a file
62
70
  // Provide a callback that writes the data someplace
63
71
  const callback = (chunk) => { ... write chunk somewhere ... }
64
- local.read(path, callback)
65
- s3.read(path, callback)
72
+ local.read(path, callback) // returns count of bytes read
73
+ s3.read(path, callback) // returns count of bytes read
66
74
 
67
75
  // Read an entire file at once (convenience method)
68
76
  const localData = await local.readFile(path) // returns a byte Buffer of the file contents
69
- const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
77
+ const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
70
78
 
71
79
  // Write a file
72
80
  // Provide a generator function that yields chunks of data
@@ -76,18 +84,18 @@ Today the supported drivers are:
76
84
  yield data
77
85
  }
78
86
  }
79
- local.write(path, generator)
80
- s3.write(path, generator)
87
+ local.write(path, generator) // returns count of bytes written
88
+ s3.write(path, generator) // returns count of bytes written
81
89
 
82
90
  // Write an entire file at once (convenience method)
83
- await local.writeFile(path, bufferOrString) // returns count of bytes written
91
+ await local.writeFile(path, bufferOrString) // returns count of bytes written
84
92
  await s3.writeFile(path, bufferOrString) // returns count of bytes written
85
93
 
86
94
  // Delete a file
87
95
  // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
88
96
  const quiet = true
89
- local.remove(path, {quiet})
90
- s3.remove(path, {quiet})
97
+ local.remove(path, {quiet}) // returns true upon success
98
+ s3.remove(path, {quiet}) // returns true upon success
91
99
 
92
100
  // Recursively delete a directory and do it quietly (do not report errors)
93
101
  const recursive = true
@@ -95,6 +103,32 @@ Today the supported drivers are:
95
103
  local.remove(path, {recursive, quiet})
96
104
  s3.remove(path, {recursive, quiet})
97
105
 
106
+ # File metadata
107
+ The `metadata` command returns metadata about a single filesystem entry.
108
+ Likewise, the return value from the `list` command is an array of metadata objects.
109
+
110
+ A metadata object looks like this:
111
+
112
+ {
113
+ "name": "fully/qualified/path/to/file",
114
+ "type": "entry-type",
115
+ "size": size-in-bytes,
116
+ "ctime": creation-time-epoch-millis,
117
+ "mtime": modification-time-epoch-millis
118
+ }
119
+
120
+ The `type` property can be `file`, `dir`, `link`, or `special`.
121
+
122
+ Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
123
+ should always be present. A subsequent `metadata` command will return all available properties.
124
+
125
+ # Alternate import style
126
+ Import the fully-scoped module and use the `connect` function:
127
+
128
+ const storage = require('mobiletto')
129
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
130
+ const objectData = await s3.readFile('some/path')
131
+
98
132
  # Mirroring
99
133
 
100
134
  // Copy a local filesystem mobiletto to S3
@@ -103,6 +137,10 @@ Today the supported drivers are:
103
137
  // Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
104
138
  local.mirror(s3, 'some/local-folder', 'some/s3-folder')
105
139
 
140
+ The `mirror` command performs a one-time copy of all files from one mobiletto to another.
141
+ It does not run any process to maintain the mirror over time. Run the `mirror` command again
142
+ to synchronize any missing files.
143
+
106
144
  WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
107
145
 
108
146
  With the `mirror` call semantics it can sometimes be confusing to understand who is the
@@ -110,13 +148,6 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
110
148
  is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
111
149
  argument to the `mirror` method) is the value being assigned (mirrored data is read).
112
150
 
113
- # Alternate import style
114
- Import the fully-scoped module and use the `connect` function:
115
-
116
- const storage = require('mobiletto')
117
- const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
118
- const objectData = await s3.readFile('some/path')
119
-
120
151
  # Transparent Encryption
121
152
  Enable transparent client-side encryption:
122
153
 
@@ -154,25 +185,6 @@ some effort, discover some or all of the overall structure of the directory hier
154
185
  The adversary would not know the names of the directories/files unless they also knew your encryption
155
186
  key or had otherwise successfully cracked the encryption. All bets are off then!
156
187
 
157
- # File metadata
158
- The `metadata` command returns metadata about a single filesystem entry.
159
- Likewise, the return value from the `list` command is an array of metadata objects.
160
-
161
- A metadata object looks like this:
162
-
163
- {
164
- "name": "fully/qualified/path/to/file",
165
- "type": "entry-type",
166
- "size": size-in-bytes,
167
- "ctime": creation-time-epoch-millis,
168
- "mtime": modification-time-epoch-millis
169
- }
170
-
171
- The `type` property can be `file`, `dir`, `link`, or `special`.
172
-
173
- Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
174
- should always be present. A subsequent `metadata` command should return all available properties.
175
-
176
188
  # Key rotation
177
189
  Create a mobiletto with your new key, then mirror the old data into it:
178
190
 
@@ -219,8 +231,8 @@ The object that the storageClient function returns must define these functions:
219
231
  async read (path, callback, endCallback = null)
220
232
 
221
233
  // Write a file
222
- // readFunc returns the data you want to write
223
- async write (path, readFunc)
234
+ // driver must be able to handle a generator or a stream
235
+ async write (path, generatorOrReadableStream)
224
236
 
225
237
  // Remove a file, or recursively delete a directory
226
238
  async remove (path, recursive = false, quiet = false)
package/index.js CHANGED
@@ -81,7 +81,7 @@ const UTILITY_FUNCTIONS = {
81
81
  },
82
82
  safeMetadata: (client) => async (path) => {
83
83
  try {
84
- return await client.driver_metadata(path)
84
+ return await client.metadata(path)
85
85
  } catch (e) {
86
86
  if (e instanceof MobilettoNotFoundError) {
87
87
  return null
@@ -114,7 +114,30 @@ const UTILITY_FUNCTIONS = {
114
114
  logger.verbose(`mirror: mirroring file: ${obj.name}`)
115
115
  const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
116
116
  logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
117
+ const destName = obj.name.startsWith(sourcePath)
118
+ ? obj.name.substring(sourcePath.length)
119
+ : obj.name
120
+ const destFullPath = clientPath + destName;
117
121
  try {
122
+ // if dest already exists and is the same size, don't copy it again
123
+ let srcSize = null
124
+ if (obj.size) {
125
+ srcSize = obj.size
126
+ } else {
127
+ const srcMeta = source.safeMetadata(obj.name)
128
+ if (srcMeta && srcMeta.size) {
129
+ srcSize = srcMeta.size
130
+ }
131
+ }
132
+ // only continue if we could determine the source size
133
+ if (srcSize) {
134
+ const destMeta = client.safeMetadata(destFullPath)
135
+ if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
136
+ logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
137
+ return
138
+ }
139
+ }
140
+
118
141
  // write from source -> write to temp file
119
142
  const fd = fs.openSync(tempPath, 'wx', 0o0600)
120
143
  const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
@@ -129,12 +152,9 @@ const UTILITY_FUNCTIONS = {
129
152
  // read from temp file -> write to mirror
130
153
  const fd = fs.openSync(tempPath, 'r')
131
154
  const reader = fs.createReadStream(tempPath, {fd})
132
- const destName = obj.name.startsWith(sourcePath)
133
- ? obj.name.substring(sourcePath.length)
134
- : obj.name
135
- logger.debug(`mirror: writing temp file ${tempPath} to destination: ${clientPath + destName}`)
136
- await client.write(clientPath + destName, reader)
137
- logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${clientPath + destName}`)
155
+ logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
156
+ await client.write(destFullPath, reader)
157
+ logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
138
158
  })
139
159
  results.success++
140
160
  } catch (e) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.17",
3
+ "version": "1.0.20",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",