mobiletto 1.0.18 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -47
- package/index.js +53 -11
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -12,7 +12,44 @@ Today the supported drivers are:
|
|
|
12
12
|
* `s3`: read/write an Amazon S3 bucket
|
|
13
13
|
* `local`: read/write to local filesystem
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
## TLDR
|
|
16
|
+
A simple CRUD example using the mobiletto S3 driver.
|
|
17
|
+
|
|
18
|
+
This code would run the same if the driver were `local`.
|
|
19
|
+
|
|
20
|
+
const storage = require('mobiletto')
|
|
21
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
22
|
+
|
|
23
|
+
// returns array of metadata objects
|
|
24
|
+
const listing = await s3.list()
|
|
25
|
+
|
|
26
|
+
// write an entire file in one-shot
|
|
27
|
+
await s3.writeFile('some/path', someBufferOfData)
|
|
28
|
+
|
|
29
|
+
// write a stream
|
|
30
|
+
const bytesWritten = await s3.write('some/path', streamOrGenerator)
|
|
31
|
+
|
|
32
|
+
// read an entire file in one-shot
|
|
33
|
+
const objectData = await s3.readFile('some/path')
|
|
34
|
+
|
|
35
|
+
// read a stream
|
|
36
|
+
const bytesRead = await s3.read('some/path', (chunk) => { ...do something with chunk... } )
|
|
37
|
+
|
|
38
|
+
// remove a file, returns true upon success
|
|
39
|
+
const removed = await s3.remove('some/path')
|
|
40
|
+
|
|
41
|
+
----
|
|
42
|
+
# Contents
|
|
43
|
+
* [Basic usage](#Basic-usage)
|
|
44
|
+
* [Metadata](#Metadata)
|
|
45
|
+
* [Caching](#Caching)
|
|
46
|
+
* [Mirroring](#Caching)
|
|
47
|
+
* [Alternate import style](#Alternate-import-style)
|
|
48
|
+
* [Transparent encryption](#Transparent-encryption)
|
|
49
|
+
* [Key rotation](#Key-rotation)
|
|
50
|
+
* [Driver interface](#Driver-interface)
|
|
51
|
+
----
|
|
52
|
+
## Basic usage
|
|
16
53
|
const { mobiletto } = require('mobiletto')
|
|
17
54
|
|
|
18
55
|
// General usage
|
|
@@ -24,7 +61,8 @@ Today the supported drivers are:
|
|
|
24
61
|
// * opts object:
|
|
25
62
|
// * readOnly: optional, never change anything on the filesystem; default is false
|
|
26
63
|
// * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
|
|
27
|
-
|
|
64
|
+
// * cacheSize: optional, LRU cache for `list` results
|
|
65
|
+
const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700', cacheSize: 100})
|
|
28
66
|
|
|
29
67
|
// To use 's3' driver:
|
|
30
68
|
// * key: AWS access key
|
|
@@ -35,11 +73,12 @@ Today the supported drivers are:
|
|
|
35
73
|
// * region: optional, the AWS region to communicate with, default is us-east-1
|
|
36
74
|
// * prefix: optional, all read/writes within the S3 bucket will be under this prefix
|
|
37
75
|
// * delimiter: optional, directory delimiter, default is '/' (note: always '/' when encryption is enabled)
|
|
38
|
-
|
|
76
|
+
// * cacheSize: optional, LRU cache for `list` results
|
|
77
|
+
const s3 = await mobiletto('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
|
|
39
78
|
|
|
40
79
|
// List files
|
|
41
|
-
local.list() // --> returns an array of
|
|
42
|
-
s3.list() // --> returns an array of
|
|
80
|
+
local.list() // --> returns an array of metadata objects
|
|
81
|
+
s3.list() // --> returns an array of metadata objects
|
|
43
82
|
|
|
44
83
|
// List files recursively
|
|
45
84
|
local.list({ recursive: true })
|
|
@@ -59,8 +98,8 @@ Today the supported drivers are:
|
|
|
59
98
|
s3.safeList('/path/that/does/not/exist') // returns []
|
|
60
99
|
|
|
61
100
|
// Read metadata for a file
|
|
62
|
-
local.metadata(path)
|
|
63
|
-
s3.metadata(path)
|
|
101
|
+
local.metadata(path) // returns metadata object
|
|
102
|
+
s3.metadata(path) // returns metadata object
|
|
64
103
|
|
|
65
104
|
// The `metadata` method throws MobilettoNotFoundError if the path does not exist
|
|
66
105
|
// When you call `safeMetadata` on a non-existent path, it returns null
|
|
@@ -69,12 +108,12 @@ Today the supported drivers are:
|
|
|
69
108
|
// Read a file
|
|
70
109
|
// Provide a callback that writes the data someplace
|
|
71
110
|
const callback = (chunk) => { ... write chunk somewhere ... }
|
|
72
|
-
local.read(path, callback)
|
|
73
|
-
s3.read(path, callback)
|
|
111
|
+
local.read(path, callback) // returns count of bytes read
|
|
112
|
+
s3.read(path, callback) // returns count of bytes read
|
|
74
113
|
|
|
75
114
|
// Read an entire file at once (convenience method)
|
|
76
115
|
const localData = await local.readFile(path) // returns a byte Buffer of the file contents
|
|
77
|
-
const s3Data = await s3.readFile(path)
|
|
116
|
+
const s3Data = await s3.readFile(path) // returns a byte Buffer of the file contents
|
|
78
117
|
|
|
79
118
|
// Write a file
|
|
80
119
|
// Provide a generator function that yields chunks of data
|
|
@@ -84,18 +123,18 @@ Today the supported drivers are:
|
|
|
84
123
|
yield data
|
|
85
124
|
}
|
|
86
125
|
}
|
|
87
|
-
local.write(path, generator)
|
|
88
|
-
s3.write(path, generator)
|
|
126
|
+
local.write(path, generator) // returns count of bytes written
|
|
127
|
+
s3.write(path, generator) // returns count of bytes written
|
|
89
128
|
|
|
90
129
|
// Write an entire file at once (convenience method)
|
|
91
|
-
await local.writeFile(path, bufferOrString)
|
|
130
|
+
await local.writeFile(path, bufferOrString) // returns count of bytes written
|
|
92
131
|
await s3.writeFile(path, bufferOrString) // returns count of bytes written
|
|
93
132
|
|
|
94
133
|
// Delete a file
|
|
95
134
|
// Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
|
|
96
135
|
const quiet = true
|
|
97
|
-
local.remove(path, {quiet})
|
|
98
|
-
s3.remove(path, {quiet})
|
|
136
|
+
local.remove(path, {quiet}) // returns true upon success
|
|
137
|
+
s3.remove(path, {quiet}) // returns true upon success
|
|
99
138
|
|
|
100
139
|
// Recursively delete a directory and do it quietly (do not report errors)
|
|
101
140
|
const recursive = true
|
|
@@ -103,7 +142,41 @@ Today the supported drivers are:
|
|
|
103
142
|
local.remove(path, {recursive, quiet})
|
|
104
143
|
s3.remove(path, {recursive, quiet})
|
|
105
144
|
|
|
106
|
-
|
|
145
|
+
## Metadata
|
|
146
|
+
The `metadata` command returns metadata about a single filesystem entry.
|
|
147
|
+
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
148
|
+
|
|
149
|
+
A metadata object looks like this:
|
|
150
|
+
|
|
151
|
+
{
|
|
152
|
+
"name": "fully/qualified/path/to/file",
|
|
153
|
+
"type": "entry-type",
|
|
154
|
+
"size": size-in-bytes,
|
|
155
|
+
"ctime": creation-time-epoch-millis,
|
|
156
|
+
"mtime": modification-time-epoch-millis
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
160
|
+
|
|
161
|
+
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
162
|
+
should always be present. A subsequent `metadata` command will return all available properties.
|
|
163
|
+
|
|
164
|
+
## Alternate import style
|
|
165
|
+
Import the fully-scoped module and use the `connect` function:
|
|
166
|
+
|
|
167
|
+
const storage = require('mobiletto')
|
|
168
|
+
const opts = {bucket: 'bk', region: 'us-east-1', cacheSize: 100}
|
|
169
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, opts)
|
|
170
|
+
const objectData = await s3.readFile('some/path')
|
|
171
|
+
|
|
172
|
+
## Caching
|
|
173
|
+
Many apps, by design, will repeatedly call `list` even though the data the app is listing hasn't changed at all.
|
|
174
|
+
|
|
175
|
+
Save bandwidth and increase performance by passing a `cacheSize` option in the `opts` object.
|
|
176
|
+
|
|
177
|
+
Any `write` or `remove` operation will clear the entire cache.
|
|
178
|
+
|
|
179
|
+
## Mirroring
|
|
107
180
|
|
|
108
181
|
// Copy a local filesystem mobiletto to S3
|
|
109
182
|
s3.mirror(local)
|
|
@@ -111,6 +184,10 @@ Today the supported drivers are:
|
|
|
111
184
|
// Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
|
|
112
185
|
local.mirror(s3, 'some/local-folder', 'some/s3-folder')
|
|
113
186
|
|
|
187
|
+
The `mirror` command performs a one-time copy of all files from one mobiletto to another.
|
|
188
|
+
It does not run any process to maintain the mirror over time. Run the `mirror` command again
|
|
189
|
+
to synchronize any missing files.
|
|
190
|
+
|
|
114
191
|
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
115
192
|
|
|
116
193
|
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
@@ -118,14 +195,7 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
|
|
|
118
195
|
is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
|
|
119
196
|
argument to the `mirror` method) is the value being assigned (mirrored data is read).
|
|
120
197
|
|
|
121
|
-
|
|
122
|
-
Import the fully-scoped module and use the `connect` function:
|
|
123
|
-
|
|
124
|
-
const storage = require('mobiletto')
|
|
125
|
-
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
126
|
-
const objectData = await s3.readFile('some/path')
|
|
127
|
-
|
|
128
|
-
# Transparent Encryption
|
|
198
|
+
## Transparent encryption
|
|
129
199
|
Enable transparent client-side encryption:
|
|
130
200
|
|
|
131
201
|
// Pass encryption parameters
|
|
@@ -162,26 +232,7 @@ some effort, discover some or all of the overall structure of the directory hier
|
|
|
162
232
|
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
163
233
|
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
164
234
|
|
|
165
|
-
|
|
166
|
-
The `metadata` command returns metadata about a single filesystem entry.
|
|
167
|
-
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
168
|
-
|
|
169
|
-
A metadata object looks like this:
|
|
170
|
-
|
|
171
|
-
{
|
|
172
|
-
"name": "fully/qualified/path/to/file",
|
|
173
|
-
"type": "entry-type",
|
|
174
|
-
"size": size-in-bytes,
|
|
175
|
-
"ctime": creation-time-epoch-millis,
|
|
176
|
-
"mtime": modification-time-epoch-millis
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
180
|
-
|
|
181
|
-
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
182
|
-
should always be present. A subsequent `metadata` command should return all available properties.
|
|
183
|
-
|
|
184
|
-
# Key rotation
|
|
235
|
+
## Key rotation
|
|
185
236
|
Create a mobiletto with your new key, then mirror the old data into it:
|
|
186
237
|
|
|
187
238
|
const storage = require('mobiletto')
|
|
@@ -194,7 +245,7 @@ Create a mobiletto with your new key, then mirror the old data into it:
|
|
|
194
245
|
|
|
195
246
|
newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
|
|
196
247
|
|
|
197
|
-
|
|
248
|
+
## Driver interface
|
|
198
249
|
A driver is any JS file that exports a 'storageClient' function with this signature:
|
|
199
250
|
|
|
200
251
|
function storageClient (key, secret, opts)
|
|
@@ -227,8 +278,8 @@ The object that the storageClient function returns must define these functions:
|
|
|
227
278
|
async read (path, callback, endCallback = null)
|
|
228
279
|
|
|
229
280
|
// Write a file
|
|
230
|
-
//
|
|
231
|
-
async write (path,
|
|
281
|
+
// driver must be able to handle a generator or a stream
|
|
282
|
+
async write (path, generatorOrReadableStream)
|
|
232
283
|
|
|
233
284
|
// Remove a file, or recursively delete a directory
|
|
234
285
|
async remove (path, recursive = false, quiet = false)
|
package/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
2
|
const winston = require('winston')
|
|
3
|
+
const LRU = require('lru-cache')
|
|
3
4
|
const { basename, dirname } = require('path')
|
|
4
5
|
const shasum = require('shasum')
|
|
5
6
|
const randomstring = require('randomstring')
|
|
@@ -60,12 +61,24 @@ const isReadable = (thing) => thing instanceof fs.ReadStream
|
|
|
60
61
|
|
|
61
62
|
const UTILITY_FUNCTIONS = {
|
|
62
63
|
list: (client) => async (path, opts) => {
|
|
64
|
+
let cacheKey
|
|
65
|
+
if (client.listingCache) {
|
|
66
|
+
cacheKey = shasum(path + ' ' + (opts ? JSON.stringify(opts) : '~'))
|
|
67
|
+
const results = client.listingCache.get(cacheKey)
|
|
68
|
+
if (results) {
|
|
69
|
+
return results
|
|
70
|
+
}
|
|
71
|
+
}
|
|
63
72
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
64
73
|
const visitor = opts && opts.visitor ? opts.visitor : null
|
|
65
74
|
if (visitor && typeof visitor !== 'function') {
|
|
66
75
|
throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
|
|
67
76
|
}
|
|
68
|
-
|
|
77
|
+
const results = await client.driver_list(path, recursive, visitor)
|
|
78
|
+
if (client.listingCache) {
|
|
79
|
+
client.listingCache.set(cacheKey, results)
|
|
80
|
+
}
|
|
81
|
+
return results
|
|
69
82
|
},
|
|
70
83
|
safeList: (client) => async (path, opts) => {
|
|
71
84
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
@@ -92,13 +105,20 @@ const UTILITY_FUNCTIONS = {
|
|
|
92
105
|
remove: (client) => async (path, opts) => {
|
|
93
106
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
94
107
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
95
|
-
|
|
108
|
+
const result = await client.driver_remove(path, recursive, quiet)
|
|
109
|
+
if (client.listingCache) { client.listingCache.clear() }
|
|
110
|
+
return result
|
|
96
111
|
},
|
|
97
112
|
readFile: (client) => async (path) => {
|
|
98
113
|
const chunks = []
|
|
99
114
|
await client.read(path, reader(chunks))
|
|
100
115
|
return Buffer.concat(chunks)
|
|
101
116
|
},
|
|
117
|
+
write: (client) => async (path, data) => {
|
|
118
|
+
const bytesWritten = await client.driver_write(path, data)
|
|
119
|
+
if (client.listingCache) { client.listingCache.clear() }
|
|
120
|
+
return bytesWritten
|
|
121
|
+
},
|
|
102
122
|
writeFile: (client) => async (path, data) => {
|
|
103
123
|
const readFunc = function* () { yield data }
|
|
104
124
|
return await client.write(path, readFunc())
|
|
@@ -114,7 +134,30 @@ const UTILITY_FUNCTIONS = {
|
|
|
114
134
|
logger.verbose(`mirror: mirroring file: ${obj.name}`)
|
|
115
135
|
const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
|
|
116
136
|
logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
|
|
137
|
+
const destName = obj.name.startsWith(sourcePath)
|
|
138
|
+
? obj.name.substring(sourcePath.length)
|
|
139
|
+
: obj.name
|
|
140
|
+
const destFullPath = clientPath + destName;
|
|
117
141
|
try {
|
|
142
|
+
// if dest already exists and is the same size, don't copy it again
|
|
143
|
+
let srcSize = null
|
|
144
|
+
if (obj.size) {
|
|
145
|
+
srcSize = obj.size
|
|
146
|
+
} else {
|
|
147
|
+
const srcMeta = source.safeMetadata(obj.name)
|
|
148
|
+
if (srcMeta && srcMeta.size) {
|
|
149
|
+
srcSize = srcMeta.size
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// only continue if we could determine the source size
|
|
153
|
+
if (srcSize) {
|
|
154
|
+
const destMeta = client.safeMetadata(destFullPath)
|
|
155
|
+
if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
|
|
156
|
+
logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
|
|
157
|
+
return
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
118
161
|
// write from source -> write to temp file
|
|
119
162
|
const fd = fs.openSync(tempPath, 'wx', 0o0600)
|
|
120
163
|
const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
|
|
@@ -129,12 +172,9 @@ const UTILITY_FUNCTIONS = {
|
|
|
129
172
|
// read from temp file -> write to mirror
|
|
130
173
|
const fd = fs.openSync(tempPath, 'r')
|
|
131
174
|
const reader = fs.createReadStream(tempPath, {fd})
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
136
|
-
await client.write(clientPath + destName, reader)
|
|
137
|
-
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
175
|
+
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
176
|
+
await client.write(destFullPath, reader)
|
|
177
|
+
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
138
178
|
})
|
|
139
179
|
results.success++
|
|
140
180
|
} catch (e) {
|
|
@@ -162,7 +202,8 @@ async function mirrorDir (source, sourcePath, visitor) {
|
|
|
162
202
|
}
|
|
163
203
|
}
|
|
164
204
|
|
|
165
|
-
function addUtilityFunctions (client, readOnly = false) {
|
|
205
|
+
function addUtilityFunctions (client, readOnly = false, cacheSize = null) {
|
|
206
|
+
client.listingCache = cacheSize ? new LRU({ max: cacheSize }) : null
|
|
166
207
|
for (const func of Object.keys(UTILITY_FUNCTIONS)) {
|
|
167
208
|
client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
|
|
168
209
|
client[func] = UTILITY_FUNCTIONS[func](client)
|
|
@@ -192,9 +233,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
192
233
|
throw new MobilettoError(message)
|
|
193
234
|
}
|
|
194
235
|
const readOnly = opts ? !!opts.readOnly : false
|
|
236
|
+
const cacheSize = opts && opts.cacheSize ? opts.cacheSize : null
|
|
195
237
|
if (encryption === null) {
|
|
196
238
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
|
|
197
|
-
return addUtilityFunctions(client, readOnly)
|
|
239
|
+
return addUtilityFunctions(client, readOnly, cacheSize)
|
|
198
240
|
}
|
|
199
241
|
const encKey = normalizeKey(encryption.key)
|
|
200
242
|
if (!encKey) {
|
|
@@ -413,7 +455,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
413
455
|
}
|
|
414
456
|
}
|
|
415
457
|
logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
|
|
416
|
-
return addUtilityFunctions(encClient, readOnly)
|
|
458
|
+
return addUtilityFunctions(encClient, readOnly, cacheSize)
|
|
417
459
|
}
|
|
418
460
|
|
|
419
461
|
async function readStream(stream, callback, endCallback) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobiletto",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.21",
|
|
4
4
|
"description": "A storage abstraction layer",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Jonathan Cobb",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@aws-sdk/client-s3": "^3.150.0",
|
|
22
22
|
"@aws-sdk/lib-storage": "^3.150.0",
|
|
23
|
+
"lru-cache": "^7.14.0",
|
|
23
24
|
"node-fetch-commonjs": "^3.1.1",
|
|
24
25
|
"shasum": "^1.0.2",
|
|
25
26
|
"winston": "^3.8.1"
|