mobiletto 1.0.20 → 1.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +57 -10
  2. package/index.js +27 -5
  3. package/package.json +2 -1
package/README.md CHANGED
@@ -12,7 +12,44 @@ Today the supported drivers are:
12
12
  * `s3`: read/write an Amazon S3 bucket
13
13
  * `local`: read/write to local filesystem
14
14
 
15
- # Basic usage
15
+ ## TLDR
16
+ A simple CRUD example using the mobiletto S3 driver.
17
+
18
+ This code would run the same if the driver were `local`.
19
+
20
+ const storage = require('mobiletto')
21
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
22
+
23
+ // returns array of metadata objects
24
+ const listing = await s3.list()
25
+
26
+ // write an entire file in one-shot
27
+ await s3.writeFile('some/path', someBufferOfData)
28
+
29
+ // write a stream
30
+ const bytesWritten = await s3.write('some/path', streamOrGenerator)
31
+
32
+ // read an entire file in one-shot
33
+ const objectData = await s3.readFile('some/path')
34
+
35
+ // read a stream
36
+ const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
37
+
38
+ // remove a file, returns true upon success
39
+ const removed = await s3.remove('some/path')
40
+
41
+ ----
42
+ # Contents
43
+ * [Basic usage](#Basic-usage)
44
+ * [Metadata](#Metadata)
45
+ * [Caching](#Caching)
46
+ * [Mirroring](#Caching)
47
+ * [Alternate import style](#Alternate-import-style)
48
+ * [Transparent encryption](#Transparent-encryption)
49
+ * [Key rotation](#Key-rotation)
50
+ * [Driver interface](#Driver-interface)
51
+ ----
52
+ ## Basic usage
16
53
  const { mobiletto } = require('mobiletto')
17
54
 
18
55
  // General usage
@@ -24,7 +61,8 @@ Today the supported drivers are:
24
61
  // * opts object:
25
62
  // * readOnly: optional, never change anything on the filesystem; default is false
26
63
  // * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
27
- const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
64
+ // * cacheSize: optional, LRU cache for `list` results
65
+ const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700', cacheSize: 100})
28
66
 
29
67
  // To use 's3' driver:
30
68
  // * key: AWS access key
@@ -35,7 +73,8 @@ Today the supported drivers are:
35
73
  // * region: optional, the AWS region to communicate with, default is us-east-1
36
74
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
37
75
  // * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
38
- const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
76
+ // * cacheSize: optional, LRU cache for `list` results
77
+ const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
39
78
 
40
79
  // List files
41
80
  local.list() // --> returns an array of metadata objects
@@ -103,7 +142,7 @@ Today the supported drivers are:
103
142
  local.remove(path, {recursive, quiet})
104
143
  s3.remove(path, {recursive, quiet})
105
144
 
106
- # File metadata
145
+ ## Metadata
107
146
  The `metadata` command returns metadata about a single filesystem entry.
108
147
  Likewise, the return value from the `list` command is an array of metadata objects.
109
148
 
@@ -122,14 +161,22 @@ The `type` property can be `file`, `dir`, `link`, or `special`.
122
161
  Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
123
162
  should always be present. A subsequent `metadata` command will return all available properties.
124
163
 
125
- # Alternate import style
164
+ ## Alternate import style
126
165
  Import the fully-scoped module and use the `connect` function:
127
166
 
128
167
  const storage = require('mobiletto')
129
- const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
168
+ const opts = {bucket: 'bk', region: 'us-east-1', cacheSize: 100}
169
+ const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
130
170
  const objectData = await s3.readFile('some/path')
131
171
 
132
- # Mirroring
172
+ ## Caching
173
+ Many apps, by design, will repeatedly call `list` even though the data the app is listing hasn't changed at all.
174
+
175
+ Save bandwidth and increase performance by passing a `cacheSize` option in the `opts` object.
176
+
177
+ Any `write` or `remove` operation will clear the entire cache.
178
+
179
+ ## Mirroring
133
180
 
134
181
  // Copy a local filesystem mobiletto to S3
135
182
  s3.mirror(local)
@@ -148,7 +195,7 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
148
195
  is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
149
196
  argument to the `mirror` method) is the value being assigned (mirrored data is read).
150
197
 
151
- # Transparent Encryption
198
+ ## Transparent encryption
152
199
  Enable transparent client-side encryption:
153
200
 
154
201
  // Pass encryption parameters
@@ -185,7 +232,7 @@ some effort, discover some or all of the overall structure of the directory hier
185
232
  The adversary would not know the names of the directories/files unless they also knew your encryption
186
233
  key or had otherwise successfully cracked the encryption. All bets are off then!
187
234
 
188
- # Key rotation
235
+ ## Key rotation
189
236
  Create a mobiletto with your new key, then mirror the old data into it:
190
237
 
191
238
  const storage = require('mobiletto')
@@ -198,7 +245,7 @@ Create a mobiletto with your new key, then mirror the old data into it:
198
245
 
199
246
  newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
200
247
 
201
- # Driver Interface
248
+ ## Driver interface
202
249
  A driver is any JS file that exports a 'storageClient' function with this signature:
203
250
 
204
251
  function storageClient (key, secret, opts)
package/index.js CHANGED
@@ -1,5 +1,6 @@
1
1
  const fs = require('fs')
2
2
  const winston = require('winston')
3
+ const LRU = require('lru-cache')
3
4
  const { basename, dirname } = require('path')
4
5
  const shasum = require('shasum')
5
6
  const randomstring = require('randomstring')
@@ -60,12 +61,24 @@ const isReadable = (thing) => thing instanceof fs.ReadStream
60
61
 
61
62
  const UTILITY_FUNCTIONS = {
62
63
  list: (client) => async (path, opts) => {
64
+ let cacheKey
65
+ if (client.listingCache) {
66
+ cacheKey = shasum(path + ' ' + (opts ? JSON.stringify(opts) : '~'))
67
+ const results = client.listingCache.get(cacheKey)
68
+ if (results) {
69
+ return results
70
+ }
71
+ }
63
72
  const recursive = opts && opts.recursive ? opts.recursive : false
64
73
  const visitor = opts && opts.visitor ? opts.visitor : null
65
74
  if (visitor && typeof visitor !== 'function') {
66
75
  throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
67
76
  }
68
- return await client.driver_list(path, recursive, visitor)
77
+ const results = await client.driver_list(path, recursive, visitor)
78
+ if (client.listingCache) {
79
+ client.listingCache.set(cacheKey, results)
80
+ }
81
+ return results
69
82
  },
70
83
  safeList: (client) => async (path, opts) => {
71
84
  const recursive = opts && opts.recursive ? opts.recursive : false
@@ -92,13 +105,20 @@ const UTILITY_FUNCTIONS = {
92
105
  remove: (client) => async (path, opts) => {
93
106
  const recursive = opts && opts.recursive ? opts.recursive : false
94
107
  const quiet = opts && opts.quiet ? opts.quiet : false
95
- return await client.driver_remove(path, recursive, quiet)
108
+ const result = await client.driver_remove(path, recursive, quiet)
109
+ if (client.listingCache) { client.listingCache.clear() }
110
+ return result
96
111
  },
97
112
  readFile: (client) => async (path) => {
98
113
  const chunks = []
99
114
  await client.read(path, reader(chunks))
100
115
  return Buffer.concat(chunks)
101
116
  },
117
+ write: (client) => async (path, data) => {
118
+ const bytesWritten = await client.driver_write(path, data)
119
+ if (client.listingCache) { client.listingCache.clear() }
120
+ return bytesWritten
121
+ },
102
122
  writeFile: (client) => async (path, data) => {
103
123
  const readFunc = function* () { yield data }
104
124
  return await client.write(path, readFunc())
@@ -182,7 +202,8 @@ async function mirrorDir (source, sourcePath, visitor) {
182
202
  }
183
203
  }
184
204
 
185
- function addUtilityFunctions (client, readOnly = false) {
205
+ function addUtilityFunctions (client, readOnly = false, cacheSize = null) {
206
+ client.listingCache = cacheSize ? new LRU({ max: cacheSize }) : null
186
207
  for (const func of Object.keys(UTILITY_FUNCTIONS)) {
187
208
  client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
188
209
  client[func] = UTILITY_FUNCTIONS[func](client)
@@ -212,9 +233,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
212
233
  throw new MobilettoError(message)
213
234
  }
214
235
  const readOnly = opts ? !!opts.readOnly : false
236
+ const cacheSize = opts && opts.cacheSize ? opts.cacheSize : null
215
237
  if (encryption === null) {
216
238
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
217
- return addUtilityFunctions(client, readOnly)
239
+ return addUtilityFunctions(client, readOnly, cacheSize)
218
240
  }
219
241
  const encKey = normalizeKey(encryption.key)
220
242
  if (!encKey) {
@@ -433,7 +455,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
433
455
  }
434
456
  }
435
457
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
436
- return addUtilityFunctions(encClient, readOnly)
458
+ return addUtilityFunctions(encClient, readOnly, cacheSize)
437
459
  }
438
460
 
439
461
  async function readStream(stream, callback, endCallback) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.20",
3
+ "version": "1.0.21",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -20,6 +20,7 @@
20
20
  "dependencies": {
21
21
  "@aws-sdk/client-s3": "^3.150.0",
22
22
  "@aws-sdk/lib-storage": "^3.150.0",
23
+ "lru-cache": "^7.14.0",
23
24
  "node-fetch-commonjs": "^3.1.1",
24
25
  "shasum": "^1.0.2",
25
26
  "winston": "^3.8.1"