mobiletto 1.0.19 → 1.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -25
- package/index.js +27 -5
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -12,7 +12,44 @@ Today the supported drivers are:
|
|
|
12
12
|
* `s3`: read/write an Amazon S3 bucket
|
|
13
13
|
* `local`: read/write to local filesystem
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
## TLDR
|
|
16
|
+
A simple CRUD example using the mobiletto S3 driver.
|
|
17
|
+
|
|
18
|
+
This code would run the same if the driver were `local`.
|
|
19
|
+
|
|
20
|
+
const storage = require('mobiletto')
|
|
21
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
22
|
+
|
|
23
|
+
// returns array of metadata objects
|
|
24
|
+
const listing = await s3.list()
|
|
25
|
+
|
|
26
|
+
// write an entire file in one-shot
|
|
27
|
+
await s3.writeFile('some/path', someBufferOfData)
|
|
28
|
+
|
|
29
|
+
// write a stream
|
|
30
|
+
const bytesWritten = await s3.write('some/path', streamOrGenerator)
|
|
31
|
+
|
|
32
|
+
// read an entire file in one-shot
|
|
33
|
+
const objectData = await s3.readFile('some/path')
|
|
34
|
+
|
|
35
|
+
// read a stream
|
|
36
|
+
const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
|
|
37
|
+
|
|
38
|
+
// remove a file, returns true upon success
|
|
39
|
+
const removed = await s3.remove('some/path')
|
|
40
|
+
|
|
41
|
+
----
|
|
42
|
+
# Contents
|
|
43
|
+
* [Basic usage](#Basic-usage)
|
|
44
|
+
* [Metadata](#Metadata)
|
|
45
|
+
* [Caching](#Caching)
|
|
46
|
+
* [Mirroring](#Caching)
|
|
47
|
+
* [Alternate import style](#Alternate-import-style)
|
|
48
|
+
* [Transparent encryption](#Transparent-encryption)
|
|
49
|
+
* [Key rotation](#Key-rotation)
|
|
50
|
+
* [Driver interface](#Driver-interface)
|
|
51
|
+
----
|
|
52
|
+
## Basic usage
|
|
16
53
|
const { mobiletto } = require('mobiletto')
|
|
17
54
|
|
|
18
55
|
// General usage
|
|
@@ -24,7 +61,8 @@ Today the supported drivers are:
|
|
|
24
61
|
// * opts object:
|
|
25
62
|
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
26
63
|
// * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
|
|
27
|
-
|
|
64
|
+
// * cacheSize: optional, LRU cache for `list` results
|
|
65
|
+
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700', cacheSize: 100})
|
|
28
66
|
|
|
29
67
|
// To use 's3' driver:
|
|
30
68
|
// * key: AWS access key
|
|
@@ -35,11 +73,12 @@ Today the supported drivers are:
|
|
|
35
73
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
36
74
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
37
75
|
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
38
|
-
|
|
76
|
+
// * cacheSize: optional, LRU cache for `list` results
|
|
77
|
+
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
39
78
|
|
|
40
79
|
// List files
|
|
41
|
-
local.list() // --> returns an array of
|
|
42
|
-
s3.list() // --> returns an array of
|
|
80
|
+
local.list() // --> returns an array of metadata objects
|
|
81
|
+
s3.list() // --> returns an array of metadata objects
|
|
43
82
|
|
|
44
83
|
// List files recursively
|
|
45
84
|
local.list({ recursive: true })
|
|
@@ -59,8 +98,8 @@ Today the supported drivers are:
|
|
|
59
98
|
s3.safeList('/path/that/does/not/exist') // returns []
|
|
60
99
|
|
|
61
100
|
// Read metadata for a file
|
|
62
|
-
local.metadata(path)
|
|
63
|
-
s3.metadata(path)
|
|
101
|
+
local.metadata(path) // returns metadata object
|
|
102
|
+
s3.metadata(path) // returns metadata object
|
|
64
103
|
|
|
65
104
|
// The `metadata` method throws MobilettoNotFoundError if the path does not exist
|
|
66
105
|
// When you call `safeMetadata` on a non-existent path, it returns null
|
|
@@ -69,12 +108,12 @@ Today the supported drivers are:
|
|
|
69
108
|
// Read a file
|
|
70
109
|
// Provide a callback that writes the data someplace
|
|
71
110
|
const callback = (chunk) => { ... write chunk somewhere ... }
|
|
72
|
-
local.read(path, callback)
|
|
73
|
-
s3.read(path, callback)
|
|
111
|
+
local.read(path, callback) // returns count of bytes read
|
|
112
|
+
s3.read(path, callback) // returns count of bytes read
|
|
74
113
|
|
|
75
114
|
// Read an entire file at once (convenience method)
|
|
76
115
|
const localData = await local.readFile(path) // returns a byte Buffer of the file contents
|
|
77
|
-
const s3Data = await s3.readFile(path)
|
|
116
|
+
const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
|
|
78
117
|
|
|
79
118
|
// Write a file
|
|
80
119
|
// Provide a generator function that yields chunks of data
|
|
@@ -84,18 +123,18 @@ Today the supported drivers are:
|
|
|
84
123
|
yield data
|
|
85
124
|
}
|
|
86
125
|
}
|
|
87
|
-
local.write(path, generator)
|
|
88
|
-
s3.write(path, generator)
|
|
126
|
+
local.write(path, generator) // returns count of bytes written
|
|
127
|
+
s3.write(path, generator) // returns count of bytes written
|
|
89
128
|
|
|
90
129
|
// Write an entire file at once (convenience method)
|
|
91
|
-
await local.writeFile(path, bufferOrString)
|
|
130
|
+
await local.writeFile(path, bufferOrString) // returns count of bytes written
|
|
92
131
|
await s3.writeFile(path, bufferOrString) // returns count of bytes written
|
|
93
132
|
|
|
94
133
|
// Delete a file
|
|
95
134
|
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
96
135
|
const quiet = true
|
|
97
|
-
local.remove(path, {quiet})
|
|
98
|
-
s3.remove(path, {quiet})
|
|
136
|
+
local.remove(path, {quiet}) // returns true upon success
|
|
137
|
+
s3.remove(path, {quiet}) // returns true upon success
|
|
99
138
|
|
|
100
139
|
// Recursively delete a directory and do it quietly (do not report errors)
|
|
101
140
|
const recursive = true
|
|
@@ -103,7 +142,7 @@ Today the supported drivers are:
|
|
|
103
142
|
local.remove(path, {recursive, quiet})
|
|
104
143
|
s3.remove(path, {recursive, quiet})
|
|
105
144
|
|
|
106
|
-
|
|
145
|
+
## Metadata
|
|
107
146
|
The `metadata` command returns metadata about a single filesystem entry.
|
|
108
147
|
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
109
148
|
|
|
@@ -120,16 +159,24 @@ A metadata object looks like this:
|
|
|
120
159
|
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
121
160
|
|
|
122
161
|
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
123
|
-
should always be present. A subsequent `metadata` command
|
|
162
|
+
should always be present. A subsequent `metadata` command will return all available properties.
|
|
124
163
|
|
|
125
|
-
|
|
164
|
+
## Alternate import style
|
|
126
165
|
Import the fully-scoped module and use the `connect` function:
|
|
127
166
|
|
|
128
167
|
const storage = require('mobiletto')
|
|
129
|
-
const
|
|
168
|
+
const opts = {bucket: 'bk', region: 'us-east-1', cacheSize: 100}
|
|
169
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
|
|
130
170
|
const objectData = await s3.readFile('some/path')
|
|
131
171
|
|
|
132
|
-
|
|
172
|
+
## Caching
|
|
173
|
+
Many apps, by design, will repeatedly call `list` even though the data the app is listing hasn't changed at all.
|
|
174
|
+
|
|
175
|
+
Save bandwidth and increase performance by passing a `cacheSize` option in the `opts` object.
|
|
176
|
+
|
|
177
|
+
Any `write` or `remove` operation will clear the entire cache.
|
|
178
|
+
|
|
179
|
+
## Mirroring
|
|
133
180
|
|
|
134
181
|
// Copy a local filesystem mobiletto to S3
|
|
135
182
|
s3.mirror(local)
|
|
@@ -148,7 +195,7 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
|
|
|
148
195
|
is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
|
|
149
196
|
argument to the `mirror` method) is the value being assigned (mirrored data is read).
|
|
150
197
|
|
|
151
|
-
|
|
198
|
+
## Transparent encryption
|
|
152
199
|
Enable transparent client-side encryption:
|
|
153
200
|
|
|
154
201
|
// Pass encryption parameters
|
|
@@ -185,7 +232,7 @@ some effort, discover some or all of the overall structure of the directory hier
|
|
|
185
232
|
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
186
233
|
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
187
234
|
|
|
188
|
-
|
|
235
|
+
## Key rotation
|
|
189
236
|
Create a mobiletto with your new key, then mirror the old data into it:
|
|
190
237
|
|
|
191
238
|
const storage = require('mobiletto')
|
|
@@ -198,7 +245,7 @@ Create a mobiletto with your new key, then mirror the old data into it:
|
|
|
198
245
|
|
|
199
246
|
newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
|
|
200
247
|
|
|
201
|
-
|
|
248
|
+
## Driver interface
|
|
202
249
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
|
203
250
|
|
|
204
251
|
function storageClient (key, secret, opts)
|
|
@@ -231,8 +278,8 @@ The object that the storageClient function returns must define these functions:
|
|
|
231
278
|
async read (path, callback, endCallback = null)
|
|
232
279
|
|
|
233
280
|
// Write a file
|
|
234
|
-
//
|
|
235
|
-
async write (path,
|
|
281
|
+
// driver must be able to handle a generator or a stream
|
|
282
|
+
async write (path, generatorOrReadableStream)
|
|
236
283
|
|
|
237
284
|
// Remove a file, or recursively delete a directory
|
|
238
285
|
async remove (path, recursive = false, quiet = false)
|
package/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
2
|
const winston = require('winston')
|
|
3
|
+
const LRU = require('lru-cache')
|
|
3
4
|
const { basename, dirname } = require('path')
|
|
4
5
|
const shasum = require('shasum')
|
|
5
6
|
const randomstring = require('randomstring')
|
|
@@ -60,12 +61,24 @@ const isReadable = (thing) => thing instanceof fs.ReadStream
|
|
|
60
61
|
|
|
61
62
|
const UTILITY_FUNCTIONS = {
|
|
62
63
|
list: (client) => async (path, opts) => {
|
|
64
|
+
let cacheKey
|
|
65
|
+
if (client.listingCache) {
|
|
66
|
+
cacheKey = shasum(path + ' ' + (opts ? JSON.stringify(opts) : '~'))
|
|
67
|
+
const results = client.listingCache.get(cacheKey)
|
|
68
|
+
if (results) {
|
|
69
|
+
return results
|
|
70
|
+
}
|
|
71
|
+
}
|
|
63
72
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
64
73
|
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
65
74
|
if (visitor && typeof visitor !== 'function') {
|
|
66
75
|
throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
|
|
67
76
|
}
|
|
68
|
-
|
|
77
|
+
const results = await client.driver_list(path, recursive, visitor)
|
|
78
|
+
if (client.listingCache) {
|
|
79
|
+
client.listingCache.set(cacheKey, results)
|
|
80
|
+
}
|
|
81
|
+
return results
|
|
69
82
|
},
|
|
70
83
|
safeList: (client) => async (path, opts) => {
|
|
71
84
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
@@ -92,13 +105,20 @@ const UTILITY_FUNCTIONS = {
|
|
|
92
105
|
remove: (client) => async (path, opts) => {
|
|
93
106
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
94
107
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
95
|
-
|
|
108
|
+
const result = await client.driver_remove(path, recursive, quiet)
|
|
109
|
+
if (client.listingCache) { client.listingCache.clear() }
|
|
110
|
+
return result
|
|
96
111
|
},
|
|
97
112
|
readFile: (client) => async (path) => {
|
|
98
113
|
const chunks = []
|
|
99
114
|
await client.read(path, reader(chunks))
|
|
100
115
|
return Buffer.concat(chunks)
|
|
101
116
|
},
|
|
117
|
+
write: (client) => async (path, data) => {
|
|
118
|
+
const bytesWritten = await client.driver_write(path, data)
|
|
119
|
+
if (client.listingCache) { client.listingCache.clear() }
|
|
120
|
+
return bytesWritten
|
|
121
|
+
},
|
|
102
122
|
writeFile: (client) => async (path, data) => {
|
|
103
123
|
const readFunc = function* () { yield data }
|
|
104
124
|
return await client.write(path, readFunc())
|
|
@@ -182,7 +202,8 @@ async function mirrorDir (source, sourcePath, visitor) {
|
|
|
182
202
|
}
|
|
183
203
|
}
|
|
184
204
|
|
|
185
|
-
function addUtilityFunctions (client, readOnly = false) {
|
|
205
|
+
function addUtilityFunctions (client, readOnly = false, cacheSize = null) {
|
|
206
|
+
client.listingCache = cacheSize && cacheSize > 0 ? new LRU({ max: cacheSize }) : null
|
|
186
207
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
187
208
|
client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
|
|
188
209
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
@@ -212,9 +233,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
212
233
|
throw new MobilettoError(message)
|
|
213
234
|
}
|
|
214
235
|
const readOnly = opts ? !!opts.readOnly : false
|
|
236
|
+
const cacheSize = opts && opts.cacheSize ? opts.cacheSize : null
|
|
215
237
|
if (encryption === null) {
|
|
216
238
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
217
|
-
return addUtilityFunctions(client, readOnly)
|
|
239
|
+
return addUtilityFunctions(client, readOnly, cacheSize)
|
|
218
240
|
}
|
|
219
241
|
const encKey = normalizeKey(encryption.key)
|
|
220
242
|
if (!encKey) {
|
|
@@ -433,7 +455,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
433
455
|
}
|
|
434
456
|
}
|
|
435
457
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
436
|
-
return addUtilityFunctions(encClient, readOnly)
|
|
458
|
+
return addUtilityFunctions(encClient, readOnly, cacheSize)
|
|
437
459
|
}
|
|
438
460
|
|
|
439
461
|
async function readStream(stream, callback, endCallback) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.22",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Jonathan Cobb",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
22
22
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
23
|
+
"lru-cache": "^7.14.0",
|
|
23
24
|
"node-fetch-commonjs": "^3.1.1",
|
|
24
25
|
"shasum": "^1.0.2",
|
|
25
26
|
"winston": "^3.8.1"
|