mobiletto 1.0.18 → 1.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +98 -47
  2. package/index.js +53 -11
  3. package/package.json +2 -1
package/README.md CHANGED
@@ -12,7 +12,44 @@ Today the supported drivers are:
12
12
  * `s3`: read/write an Amazon S3 bucket
13
13
  * `local`: read/write to local filesystem
14
14
 
15
- # Basic usage
15
+ ## TLDR
16
+ A simple CRUD example using the mobiletto S3 driver.
17
+
18
+ This code would run the same if the driver were `local`.
19
+
20
+ const storage = require('mobiletto')
21
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
22
+
23
+ // returns array of metadata objects
24
+ const listing = await s3.list()
25
+
26
+ // write an entire file in one-shot
27
+ await s3.writeFile('some/path', someBufferOfData)
28
+
29
+ // write a stream
30
+ const bytesWritten = await s3.write('some/path', streamOrGenerator)
31
+
32
+ // read an entire file in one-shot
33
+ const objectData = await s3.readFile('some/path')
34
+
35
+ // read a stream
36
+ const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
37
+
38
+ // remove a file, returns true upon success
39
+ const removed = await s3.remove('some/path')
40
+
41
+ ----
42
+ # Contents
43
+ * [Basic usage](#Basic-usage)
44
+ * [Metadata](#Metadata)
45
+ * [Caching](#Caching)
46
+ * [Mirroring](#Caching)
47
+ * [Alternate import style](#Alternate-import-style)
48
+ * [Transparent encryption](#Transparent-encryption)
49
+ * [Key rotation](#Key-rotation)
50
+ * [Driver interface](#Driver-interface)
51
+ ----
52
+ ## Basic usage
16
53
  const { mobiletto } = require('mobiletto')
17
54
 
18
55
  // General usage
@@ -24,7 +61,8 @@ Today the supported drivers are:
24
61
  // * opts object:
25
62
  // * readOnly: optional, never change anything on the filesystem; default is false
26
63
  // * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
27
- const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
64
+ // * cacheSize: optional, LRU cache for `list` results
65
+ const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700', cacheSize: 100})
28
66
 
29
67
  // To use 's3' driver:
30
68
  // * key: AWS access key
@@ -35,11 +73,12 @@ Today the supported drivers are:
35
73
  // * region: optional, the AWS region to communicate with, default is us-east-1
36
74
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
37
75
  // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
38
- const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
76
+ // * cacheSize: optional, LRU cache for `list` results
77
+ const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
39
78
 
40
79
  // List files
41
- local.list() // --> returns an array of file objects
42
- s3.list() // --> returns an array of file objects
80
+ local.list() // --> returns an array of metadata objects
81
+ s3.list() // --> returns an array of metadata objects
43
82
 
44
83
  // List files recursively
45
84
  local.list({ recursive: true })
@@ -59,8 +98,8 @@ Today the supported drivers are:
59
98
  s3.safeList('/path/that/does/not/exist') // returns []
60
99
 
61
100
  // Read metadata for a file
62
- local.metadata(path)
63
- s3.metadata(path)
101
+ local.metadata(path) // returns metadata object
102
+ s3.metadata(path) // returns metadata object
64
103
 
65
104
  // The `metadata` method throws MobilettoNotFoundError if the path does not exist
66
105
  // When you call `safeMetadata` on a non-existent path, it returns null
@@ -69,12 +108,12 @@ Today the supported drivers are:
69
108
  // Read a file
70
109
  // Provide a callback that writes the data someplace
71
110
  const callback = (chunk) => { ... write chunk somewhere ... }
72
- local.read(path, callback)
73
- s3.read(path, callback)
111
+ local.read(path, callback) // returns count of bytes read
112
+ s3.read(path, callback) // returns count of bytes read
74
113
 
75
114
  // Read an entire file at once (convenience method)
76
115
  const localData = await local.readFile(path) // returns a byte Buffer of the file contents
77
- const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
116
+ const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
78
117
 
79
118
  // Write a file
80
119
  // Provide a generator function that yields chunks of data
@@ -84,18 +123,18 @@ Today the supported drivers are:
84
123
  yield data
85
124
  }
86
125
  }
87
- local.write(path, generator)
88
- s3.write(path, generator)
126
+ local.write(path, generator) // returns count of bytes written
127
+ s3.write(path, generator) // returns count of bytes written
89
128
 
90
129
  // Write an entire file at once (convenience method)
91
- await local.writeFile(path, bufferOrString) // returns count of bytes written
130
+ await local.writeFile(path, bufferOrString) // returns count of bytes written
92
131
  await s3.writeFile(path, bufferOrString) // returns count of bytes written
93
132
 
94
133
  // Delete a file
95
134
  // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
96
135
  const quiet = true
97
- local.remove(path, {quiet})
98
- s3.remove(path, {quiet})
136
+ local.remove(path, {quiet}) // returns true upon success
137
+ s3.remove(path, {quiet}) // returns true upon success
99
138
 
100
139
  // Recursively delete a directory and do it quietly (do not report errors)
101
140
  const recursive = true
@@ -103,7 +142,41 @@ Today the supported drivers are:
103
142
  local.remove(path, {recursive, quiet})
104
143
  s3.remove(path, {recursive, quiet})
105
144
 
106
- # Mirroring
145
+ ## Metadata
146
+ The `metadata` command returns metadata about a single filesystem entry.
147
+ Likewise, the return value from the `list` command is an array of metadata objects.
148
+
149
+ A metadata object looks like this:
150
+
151
+ {
152
+ "name": "fully/qualified/path/to/file",
153
+ "type": "entry-type",
154
+ "size": size-in-bytes,
155
+ "ctime": creation-time-epoch-millis,
156
+ "mtime": modification-time-epoch-millis
157
+ }
158
+
159
+ The `type` property can be `file`, `dir`, `link`, or `special`.
160
+
161
+ Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
162
+ should always be present. A subsequent `metadata` command will return all available properties.
163
+
164
+ ## Alternate import style
165
+ Import the fully-scoped module and use the `connect` function:
166
+
167
+ const storage = require('mobiletto')
168
+ const opts = {bucket: 'bk', region: 'us-east-1', cacheSize: 100}
169
+ const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
170
+ const objectData = await s3.readFile('some/path')
171
+
172
+ ## Caching
173
+ Many apps, by design, will repeatedly call `list` even though the data the app is listing hasn't changed at all.
174
+
175
+ Save bandwidth and increase performance by passing a `cacheSize` option in the `opts` object.
176
+
177
+ Any `write` or `remove` operation will clear the entire cache.
178
+
179
+ ## Mirroring
107
180
 
108
181
  // Copy a local filesystem mobiletto to S3
109
182
  s3.mirror(local)
@@ -111,6 +184,10 @@ Today the supported drivers are:
111
184
  // Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
112
185
  local.mirror(s3, 'some/local-folder', 'some/s3-folder')
113
186
 
187
+ The `mirror` command performs a one-time copy of all files from one mobiletto to another.
188
+ It does not run any process to maintain the mirror over time. Run the `mirror` command again
189
+ to synchronize any missing files.
190
+
114
191
  WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
115
192
 
116
193
  With the `mirror` call semantics it can sometimes be confusing to understand who is the
@@ -118,14 +195,7 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
118
195
  is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
119
196
  argument to the `mirror` method) is the value being assigned (mirrored data is read).
120
197
 
121
- # Alternate import style
122
- Import the fully-scoped module and use the `connect` function:
123
-
124
- const storage = require('mobiletto')
125
- const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
126
- const objectData = await s3.readFile('some/path')
127
-
128
- # Transparent Encryption
198
+ ## Transparent encryption
129
199
  Enable transparent client-side encryption:
130
200
 
131
201
  // Pass encryption parameters
@@ -162,26 +232,7 @@ some effort, discover some or all of the overall structure of the directory hier
162
232
  The adversary would not know the names of the directories/files unless they also knew your encryption
163
233
  key or had otherwise successfully cracked the encryption. All bets are off then!
164
234
 
165
- # File metadata
166
- The `metadata` command returns metadata about a single filesystem entry.
167
- Likewise, the return value from the `list` command is an array of metadata objects.
168
-
169
- A metadata object looks like this:
170
-
171
- {
172
- "name": "fully/qualified/path/to/file",
173
- "type": "entry-type",
174
- "size": size-in-bytes,
175
- "ctime": creation-time-epoch-millis,
176
- "mtime": modification-time-epoch-millis
177
- }
178
-
179
- The `type` property can be `file`, `dir`, `link`, or `special`.
180
-
181
- Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
182
- should always be present. A subsequent `metadata` command should return all available properties.
183
-
184
- # Key rotation
235
+ ## Key rotation
185
236
  Create a mobiletto with your new key, then mirror the old data into it:
186
237
 
187
238
  const storage = require('mobiletto')
@@ -194,7 +245,7 @@ Create a mobiletto with your new key, then mirror the old data into it:
194
245
 
195
246
  newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
196
247
 
197
- # Driver Interface
248
+ ## Driver interface
198
249
  A driver is any JS file that exports a 'storageClient' function with this signature:
199
250
 
200
251
  function storageClient (key, secret, opts)
@@ -227,8 +278,8 @@ The object that the storageClient function returns must define these functions:
227
278
  async read (path, callback, endCallback = null)
228
279
 
229
280
  // Write a file
230
- // readFunc returns the data you want to write
231
- async write (path, readFunc)
281
+ // driver must be able to handle a generator or a stream
282
+ async write (path, generatorOrReadableStream)
232
283
 
233
284
  // Remove a file, or recursively delete a directory
234
285
  async remove (path, recursive = false, quiet = false)
package/index.js CHANGED
@@ -1,5 +1,6 @@
1
1
  const fs = require('fs')
2
2
  const winston = require('winston')
3
+ const LRU = require('lru-cache')
3
4
  const { basename, dirname } = require('path')
4
5
  const shasum = require('shasum')
5
6
  const randomstring = require('randomstring')
@@ -60,12 +61,24 @@ const isReadable = (thing) => thing instanceof fs.ReadStream
60
61
 
61
62
  const UTILITY_FUNCTIONS = {
62
63
  list: (client) => async (path, opts) => {
64
+ let cacheKey
65
+ if (client.listingCache) {
66
+ cacheKey = shasum(path + ' ' + (opts ? JSON.stringify(opts) : '~'))
67
+ const results = client.listingCache.get(cacheKey)
68
+ if (results) {
69
+ return results
70
+ }
71
+ }
63
72
  const recursive = opts && opts.recursive ? opts.recursive : false
64
73
  const visitor = opts && opts.visitor ? opts.visitor : null
65
74
  if (visitor && typeof visitor !== 'function') {
66
75
  throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
67
76
  }
68
- return await client.driver_list(path, recursive, visitor)
77
+ const results = await client.driver_list(path, recursive, visitor)
78
+ if (client.listingCache) {
79
+ client.listingCache.set(cacheKey, results)
80
+ }
81
+ return results
69
82
  },
70
83
  safeList: (client) => async (path, opts) => {
71
84
  const recursive = opts && opts.recursive ? opts.recursive : false
@@ -92,13 +105,20 @@ const UTILITY_FUNCTIONS = {
92
105
  remove: (client) => async (path, opts) => {
93
106
  const recursive = opts && opts.recursive ? opts.recursive : false
94
107
  const quiet = opts && opts.quiet ? opts.quiet : false
95
- return await client.driver_remove(path, recursive, quiet)
108
+ const result = await client.driver_remove(path, recursive, quiet)
109
+ if (client.listingCache) { client.listingCache.clear() }
110
+ return result
96
111
  },
97
112
  readFile: (client) => async (path) => {
98
113
  const chunks = []
99
114
  await client.read(path, reader(chunks))
100
115
  return Buffer.concat(chunks)
101
116
  },
117
+ write: (client) => async (path, data) => {
118
+ const bytesWritten = await client.driver_write(path, data)
119
+ if (client.listingCache) { client.listingCache.clear() }
120
+ return bytesWritten
121
+ },
102
122
  writeFile: (client) => async (path, data) => {
103
123
  const readFunc = function* () { yield data }
104
124
  return await client.write(path, readFunc())
@@ -114,7 +134,30 @@ const UTILITY_FUNCTIONS = {
114
134
  logger.verbose(`mirror: mirroring file: ${obj.name}`)
115
135
  const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
116
136
  logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
137
+ const destName = obj.name.startsWith(sourcePath)
138
+ ? obj.name.substring(sourcePath.length)
139
+ : obj.name
140
+ const destFullPath = clientPath + destName;
117
141
  try {
142
+ // if dest already exists and is the same size, don't copy it again
143
+ let srcSize = null
144
+ if (obj.size) {
145
+ srcSize = obj.size
146
+ } else {
147
+ const srcMeta = source.safeMetadata(obj.name)
148
+ if (srcMeta && srcMeta.size) {
149
+ srcSize = srcMeta.size
150
+ }
151
+ }
152
+ // only continue if we could determine the source size
153
+ if (srcSize) {
154
+ const destMeta = client.safeMetadata(destFullPath)
155
+ if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
156
+ logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
157
+ return
158
+ }
159
+ }
160
+
118
161
  // write from source -> write to temp file
119
162
  const fd = fs.openSync(tempPath, 'wx', 0o0600)
120
163
  const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
@@ -129,12 +172,9 @@ const UTILITY_FUNCTIONS = {
129
172
  // read from temp file -> write to mirror
130
173
  const fd = fs.openSync(tempPath, 'r')
131
174
  const reader = fs.createReadStream(tempPath, {fd})
132
- const destName = obj.name.startsWith(sourcePath)
133
- ? obj.name.substring(sourcePath.length)
134
- : obj.name
135
- logger.debug(`mirror: writing temp file ${tempPath} to destination: ${clientPath + destName}`)
136
- await client.write(clientPath + destName, reader)
137
- logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${clientPath + destName}`)
175
+ logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
176
+ await client.write(destFullPath, reader)
177
+ logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
138
178
  })
139
179
  results.success++
140
180
  } catch (e) {
@@ -162,7 +202,8 @@ async function mirrorDir (source, sourcePath, visitor) {
162
202
  }
163
203
  }
164
204
 
165
- function addUtilityFunctions (client, readOnly = false) {
205
+ function addUtilityFunctions (client, readOnly = false, cacheSize = null) {
206
+ client.listingCache = cacheSize ? new LRU({ max: cacheSize }) : null
166
207
  for (const func of Object.keys(UTILITY_FUNCTIONS)) {
167
208
  client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
168
209
  client[func] = UTILITY_FUNCTIONS[func](client)
@@ -192,9 +233,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
192
233
  throw new MobilettoError(message)
193
234
  }
194
235
  const readOnly = opts ? !!opts.readOnly : false
236
+ const cacheSize = opts && opts.cacheSize ? opts.cacheSize : null
195
237
  if (encryption === null) {
196
238
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
197
- return addUtilityFunctions(client, readOnly)
239
+ return addUtilityFunctions(client, readOnly, cacheSize)
198
240
  }
199
241
  const encKey = normalizeKey(encryption.key)
200
242
  if (!encKey) {
@@ -413,7 +455,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
413
455
  }
414
456
  }
415
457
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
416
- return addUtilityFunctions(encClient, readOnly)
458
+ return addUtilityFunctions(encClient, readOnly, cacheSize)
417
459
  }
418
460
 
419
461
  async function readStream(stream, callback, endCallback) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.18",
3
+ "version": "1.0.21",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -20,6 +20,7 @@
20
20
  "dependencies": {
21
21
  "@aws-sdk/client-s3": "^3.150.0",
22
22
  "@aws-sdk/lib-storage": "^3.150.0",
23
+ "lru-cache": "^7.14.0",
23
24
  "node-fetch-commonjs": "^3.1.1",
24
25
  "shasum": "^1.0.2",
25
26
  "winston": "^3.8.1"