mobiletto 1.0.20 → 1.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -10
- package/index.js +36 -5
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -12,7 +12,44 @@ Today the supported drivers are:
|
|
|
12
12
|
* `s3`: read/write an Amazon S3 bucket
|
|
13
13
|
* `local`: read/write to local filesystem
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
## TLDR
|
|
16
|
+
A simple CRUD example using the mobiletto S3 driver.
|
|
17
|
+
|
|
18
|
+
This code would run the same if the driver were `local`.
|
|
19
|
+
|
|
20
|
+
const storage = require('mobiletto')
|
|
21
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
22
|
+
|
|
23
|
+
// returns array of metadata objects
|
|
24
|
+
const listing = await s3.list()
|
|
25
|
+
|
|
26
|
+
// write an entire file in one-shot
|
|
27
|
+
await s3.writeFile('some/path', someBufferOfData)
|
|
28
|
+
|
|
29
|
+
// write a stream
|
|
30
|
+
const bytesWritten = await s3.write('some/path', streamOrGenerator)
|
|
31
|
+
|
|
32
|
+
// read an entire file in one-shot
|
|
33
|
+
const objectData = await s3.readFile('some/path')
|
|
34
|
+
|
|
35
|
+
// read a stream
|
|
36
|
+
const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
|
|
37
|
+
|
|
38
|
+
// remove a file, returns true upon success
|
|
39
|
+
const removed = await s3.remove('some/path')
|
|
40
|
+
|
|
41
|
+
----
|
|
42
|
+
# Contents
|
|
43
|
+
* [Basic usage](#Basic-usage)
|
|
44
|
+
* [Metadata](#Metadata)
|
|
45
|
+
* [Caching](#Caching)
|
|
46
|
+
* [Mirroring](#Caching)
|
|
47
|
+
* [Alternate import style](#Alternate-import-style)
|
|
48
|
+
* [Transparent encryption](#Transparent-encryption)
|
|
49
|
+
* [Key rotation](#Key-rotation)
|
|
50
|
+
* [Driver interface](#Driver-interface)
|
|
51
|
+
----
|
|
52
|
+
## Basic usage
|
|
16
53
|
const { mobiletto } = require('mobiletto')
|
|
17
54
|
|
|
18
55
|
// General usage
|
|
@@ -24,7 +61,8 @@ Today the supported drivers are:
|
|
|
24
61
|
// * opts object:
|
|
25
62
|
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
26
63
|
// * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
|
|
27
|
-
|
|
64
|
+
// * cacheSize: optional, LRU cache for `list` results
|
|
65
|
+
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700', cacheSize: 100})
|
|
28
66
|
|
|
29
67
|
// To use 's3' driver:
|
|
30
68
|
// * key: AWS access key
|
|
@@ -35,7 +73,8 @@ Today the supported drivers are:
|
|
|
35
73
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
36
74
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
37
75
|
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
38
|
-
|
|
76
|
+
// * cacheSize: optional, LRU cache for `list` results
|
|
77
|
+
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
39
78
|
|
|
40
79
|
// List files
|
|
41
80
|
local.list() // --> returns an array of metadata objects
|
|
@@ -103,7 +142,7 @@ Today the supported drivers are:
|
|
|
103
142
|
local.remove(path, {recursive, quiet})
|
|
104
143
|
s3.remove(path, {recursive, quiet})
|
|
105
144
|
|
|
106
|
-
|
|
145
|
+
## Metadata
|
|
107
146
|
The `metadata` command returns metadata about a single filesystem entry.
|
|
108
147
|
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
109
148
|
|
|
@@ -122,14 +161,22 @@ The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
|
122
161
|
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
123
162
|
should always be present. A subsequent `metadata` command will return all available properties.
|
|
124
163
|
|
|
125
|
-
|
|
164
|
+
## Alternate import style
|
|
126
165
|
Import the fully-scoped module and use the `connect` function:
|
|
127
166
|
|
|
128
167
|
const storage = require('mobiletto')
|
|
129
|
-
const
|
|
168
|
+
const opts = {bucket: 'bk', region: 'us-east-1', cacheSize: 100}
|
|
169
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
|
|
130
170
|
const objectData = await s3.readFile('some/path')
|
|
131
171
|
|
|
132
|
-
|
|
172
|
+
## Caching
|
|
173
|
+
Many apps, by design, will repeatedly call `list` even though the data the app is listing hasn't changed at all.
|
|
174
|
+
|
|
175
|
+
Save bandwidth and increase performance by passing a `cacheSize` option in the `opts` object.
|
|
176
|
+
|
|
177
|
+
Any `write` or `remove` operation will clear the entire cache.
|
|
178
|
+
|
|
179
|
+
## Mirroring
|
|
133
180
|
|
|
134
181
|
// Copy a local filesystem mobiletto to S3
|
|
135
182
|
s3.mirror(local)
|
|
@@ -148,7 +195,7 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
|
|
|
148
195
|
is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
|
|
149
196
|
argument to the `mirror` method) is the value being assigned (mirrored data is read).
|
|
150
197
|
|
|
151
|
-
|
|
198
|
+
## Transparent encryption
|
|
152
199
|
Enable transparent client-side encryption:
|
|
153
200
|
|
|
154
201
|
// Pass encryption parameters
|
|
@@ -185,7 +232,7 @@ some effort, discover some or all of the overall structure of the directory hier
|
|
|
185
232
|
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
186
233
|
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
187
234
|
|
|
188
|
-
|
|
235
|
+
## Key rotation
|
|
189
236
|
Create a mobiletto with your new key, then mirror the old data into it:
|
|
190
237
|
|
|
191
238
|
const storage = require('mobiletto')
|
|
@@ -198,7 +245,7 @@ Create a mobiletto with your new key, then mirror the old data into it:
|
|
|
198
245
|
|
|
199
246
|
newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
|
|
200
247
|
|
|
201
|
-
|
|
248
|
+
## Driver interface
|
|
202
249
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
|
203
250
|
|
|
204
251
|
function storageClient (key, secret, opts)
|
package/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
2
|
const winston = require('winston')
|
|
3
|
+
const LRU = require('lru-cache')
|
|
3
4
|
const { basename, dirname } = require('path')
|
|
4
5
|
const shasum = require('shasum')
|
|
5
6
|
const randomstring = require('randomstring')
|
|
@@ -60,12 +61,33 @@ const isReadable = (thing) => thing instanceof fs.ReadStream
|
|
|
60
61
|
|
|
61
62
|
const UTILITY_FUNCTIONS = {
|
|
62
63
|
list: (client) => async (path, opts) => {
|
|
64
|
+
let cacheKey
|
|
65
|
+
if (client.listingCache) {
|
|
66
|
+
cacheKey = shasum(path + ' ' + (opts ? JSON.stringify(opts) : '~'))
|
|
67
|
+
const results = client.listingCache.get(cacheKey)
|
|
68
|
+
if (results) {
|
|
69
|
+
if (results.cached) { return results }
|
|
70
|
+
throw results
|
|
71
|
+
}
|
|
72
|
+
}
|
|
63
73
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
64
74
|
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
65
75
|
if (visitor && typeof visitor !== 'function') {
|
|
66
76
|
throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
|
|
67
77
|
}
|
|
68
|
-
|
|
78
|
+
try {
|
|
79
|
+
const results = await client.driver_list(path, recursive, visitor)
|
|
80
|
+
if (client.listingCache) {
|
|
81
|
+
results.cached = true
|
|
82
|
+
client.listingCache.set(cacheKey, results)
|
|
83
|
+
}
|
|
84
|
+
return results
|
|
85
|
+
} catch (e) {
|
|
86
|
+
if (client.listingCache && e instanceof MobilettoNotFoundError) {
|
|
87
|
+
client.listingCache.set(cacheKey, e)
|
|
88
|
+
}
|
|
89
|
+
throw e
|
|
90
|
+
}
|
|
69
91
|
},
|
|
70
92
|
safeList: (client) => async (path, opts) => {
|
|
71
93
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
@@ -92,13 +114,20 @@ const UTILITY_FUNCTIONS = {
|
|
|
92
114
|
remove: (client) => async (path, opts) => {
|
|
93
115
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
94
116
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
95
|
-
|
|
117
|
+
const result = await client.driver_remove(path, recursive, quiet)
|
|
118
|
+
if (client.listingCache) { client.listingCache.clear() }
|
|
119
|
+
return result
|
|
96
120
|
},
|
|
97
121
|
readFile: (client) => async (path) => {
|
|
98
122
|
const chunks = []
|
|
99
123
|
await client.read(path, reader(chunks))
|
|
100
124
|
return Buffer.concat(chunks)
|
|
101
125
|
},
|
|
126
|
+
write: (client) => async (path, data) => {
|
|
127
|
+
const bytesWritten = await client.driver_write(path, data)
|
|
128
|
+
if (client.listingCache) { client.listingCache.clear() }
|
|
129
|
+
return bytesWritten
|
|
130
|
+
},
|
|
102
131
|
writeFile: (client) => async (path, data) => {
|
|
103
132
|
const readFunc = function* () { yield data }
|
|
104
133
|
return await client.write(path, readFunc())
|
|
@@ -182,7 +211,8 @@ async function mirrorDir (source, sourcePath, visitor) {
|
|
|
182
211
|
}
|
|
183
212
|
}
|
|
184
213
|
|
|
185
|
-
function addUtilityFunctions (client, readOnly = false) {
|
|
214
|
+
function addUtilityFunctions (client, readOnly = false, cacheSize = null) {
|
|
215
|
+
client.listingCache = cacheSize && cacheSize > 0 ? new LRU({ max: cacheSize }) : null
|
|
186
216
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
187
217
|
client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
|
|
188
218
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
@@ -212,9 +242,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
212
242
|
throw new MobilettoError(message)
|
|
213
243
|
}
|
|
214
244
|
const readOnly = opts ? !!opts.readOnly : false
|
|
245
|
+
const cacheSize = opts && opts.cacheSize ? opts.cacheSize : null
|
|
215
246
|
if (encryption === null) {
|
|
216
247
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
217
|
-
return addUtilityFunctions(client, readOnly)
|
|
248
|
+
return addUtilityFunctions(client, readOnly, cacheSize)
|
|
218
249
|
}
|
|
219
250
|
const encKey = normalizeKey(encryption.key)
|
|
220
251
|
if (!encKey) {
|
|
@@ -433,7 +464,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
433
464
|
}
|
|
434
465
|
}
|
|
435
466
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
436
|
-
return addUtilityFunctions(encClient, readOnly)
|
|
467
|
+
return addUtilityFunctions(encClient, readOnly, cacheSize)
|
|
437
468
|
}
|
|
438
469
|
|
|
439
470
|
async function readStream(stream, callback, endCallback) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.23",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Jonathan Cobb",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
22
22
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
23
|
+
"lru-cache": "^7.14.0",
|
|
23
24
|
"node-fetch-commonjs": "^3.1.1",
|
|
24
25
|
"shasum": "^1.0.2",
|
|
25
26
|
"winston": "^3.8.1"
|