mobiletto 1.0.16 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -27
- package/index.js +36 -6
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
|
|
|
5
5
|
|
|
6
6
|
It supports apps that are agnostic to where files are stored.
|
|
7
7
|
|
|
8
|
-
All drivers are tested for identical behavior with over
|
|
8
|
+
All drivers are tested for identical behavior with over 40 tests for each driver.
|
|
9
9
|
|
|
10
10
|
Today the supported drivers are:
|
|
11
11
|
|
|
@@ -53,10 +53,18 @@ Today the supported drivers are:
|
|
|
53
53
|
// Visit files in a directory -- visitor function must be async
|
|
54
54
|
local.list(path, { visitor: myAsyncFunc })
|
|
55
55
|
s3.list(path, { visitor: myAsyncFunc, recursive: true })
|
|
56
|
+
|
|
57
|
+
// The `list` method throws MobilettoNotFoundError if the path does not exist
|
|
58
|
+
// When you call `safeList` on a non-existent path, it returns an empty array
|
|
59
|
+
s3.safeList('/path/that/does/not/exist') // returns []
|
|
56
60
|
|
|
57
61
|
// Read metadata for a file
|
|
58
62
|
local.metadata(path)
|
|
59
63
|
s3.metadata(path)
|
|
64
|
+
|
|
65
|
+
// The `metadata` method throws MobilettoNotFoundError if the path does not exist
|
|
66
|
+
// When you call `safeMetadata` on a non-existent path, it returns null
|
|
67
|
+
local.safeMetadata('/tmp/does_not_exist') // returns null
|
|
60
68
|
|
|
61
69
|
// Read a file
|
|
62
70
|
// Provide a callback that writes the data someplace
|
|
@@ -95,6 +103,32 @@ Today the supported drivers are:
|
|
|
95
103
|
local.remove(path, {recursive, quiet})
|
|
96
104
|
s3.remove(path, {recursive, quiet})
|
|
97
105
|
|
|
106
|
+
# File metadata
|
|
107
|
+
The `metadata` command returns metadata about a single filesystem entry.
|
|
108
|
+
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
109
|
+
|
|
110
|
+
A metadata object looks like this:
|
|
111
|
+
|
|
112
|
+
{
|
|
113
|
+
"name": "fully/qualified/path/to/file",
|
|
114
|
+
"type": "entry-type",
|
|
115
|
+
"size": size-in-bytes,
|
|
116
|
+
"ctime": creation-time-epoch-millis,
|
|
117
|
+
"mtime": modification-time-epoch-millis
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
121
|
+
|
|
122
|
+
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
123
|
+
should always be present. A subsequent `metadata` command should return all available properties.
|
|
124
|
+
|
|
125
|
+
# Alternate import style
|
|
126
|
+
Import the fully-scoped module and use the `connect` function:
|
|
127
|
+
|
|
128
|
+
const storage = require('mobiletto')
|
|
129
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
130
|
+
const objectData = await s3.readFile('some/path')
|
|
131
|
+
|
|
98
132
|
# Mirroring
|
|
99
133
|
|
|
100
134
|
// Copy a local filesystem mobiletto to S3
|
|
@@ -103,6 +137,10 @@ Today the supported drivers are:
|
|
|
103
137
|
// Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
|
|
104
138
|
local.mirror(s3, 'some/local-folder', 'some/s3-folder')
|
|
105
139
|
|
|
140
|
+
The `mirror` command performs a one-time copy of all files from one mobiletto to another.
|
|
141
|
+
It does not run any process to maintain the mirror over time. Run the `mirror` command again
|
|
142
|
+
to synchronize any missing files.
|
|
143
|
+
|
|
106
144
|
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
107
145
|
|
|
108
146
|
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
@@ -110,13 +148,6 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
|
|
|
110
148
|
is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
|
|
111
149
|
argument to the `mirror` method) is the value being assigned (mirrored data is read).
|
|
112
150
|
|
|
113
|
-
# Alternate import style
|
|
114
|
-
Import the fully-scoped module and use the `connect` function:
|
|
115
|
-
|
|
116
|
-
const storage = require('mobiletto')
|
|
117
|
-
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
118
|
-
const objectData = await s3.readFile('some/path')
|
|
119
|
-
|
|
120
151
|
# Transparent Encryption
|
|
121
152
|
Enable transparent client-side encryption:
|
|
122
153
|
|
|
@@ -154,25 +185,6 @@ some effort, discover some or all of the overall structure of the directory hier
|
|
|
154
185
|
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
155
186
|
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
156
187
|
|
|
157
|
-
# File metadata
|
|
158
|
-
The `metadata` command returns metadata about a single filesystem entry.
|
|
159
|
-
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
160
|
-
|
|
161
|
-
A metadata object looks like this:
|
|
162
|
-
|
|
163
|
-
{
|
|
164
|
-
"name": "fully/qualified/path/to/file",
|
|
165
|
-
"type": "entry-type",
|
|
166
|
-
"size": size-in-bytes,
|
|
167
|
-
"ctime": creation-time-epoch-millis,
|
|
168
|
-
"mtime": modification-time-epoch-millis
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
172
|
-
|
|
173
|
-
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
174
|
-
should always be present. A subsequent `metadata` command should return all available properties.
|
|
175
|
-
|
|
176
188
|
# Key rotation
|
|
177
189
|
Create a mobiletto with your new key, then mirror the old data into it:
|
|
178
190
|
|
package/index.js
CHANGED
|
@@ -79,6 +79,16 @@ const UTILITY_FUNCTIONS = {
|
|
|
79
79
|
throw e
|
|
80
80
|
}
|
|
81
81
|
},
|
|
82
|
+
safeMetadata: (client) => async (path) => {
|
|
83
|
+
try {
|
|
84
|
+
return await client.metadata(path)
|
|
85
|
+
} catch (e) {
|
|
86
|
+
if (e instanceof MobilettoNotFoundError) {
|
|
87
|
+
return null
|
|
88
|
+
}
|
|
89
|
+
throw e
|
|
90
|
+
}
|
|
91
|
+
},
|
|
82
92
|
remove: (client) => async (path, opts) => {
|
|
83
93
|
const recursive = opts && opts.recursive ? opts.recursive : false
|
|
84
94
|
const quiet = opts && opts.quiet ? opts.quiet : false
|
|
@@ -104,7 +114,30 @@ const UTILITY_FUNCTIONS = {
|
|
|
104
114
|
logger.verbose(`mirror: mirroring file: ${obj.name}`)
|
|
105
115
|
const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
|
|
106
116
|
logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
|
|
117
|
+
const destName = obj.name.startsWith(sourcePath)
|
|
118
|
+
? obj.name.substring(sourcePath.length)
|
|
119
|
+
: obj.name
|
|
120
|
+
const destFullPath = clientPath + destName;
|
|
107
121
|
try {
|
|
122
|
+
// if dest already exists and is the same size, don't copy it again
|
|
123
|
+
let srcSize = null
|
|
124
|
+
if (obj.size) {
|
|
125
|
+
srcSize = obj.size
|
|
126
|
+
} else {
|
|
127
|
+
const srcMeta = source.safeMetadata(obj.name)
|
|
128
|
+
if (srcMeta && srcMeta.size) {
|
|
129
|
+
srcSize = srcMeta.size
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// only continue if we could determine the source size
|
|
133
|
+
if (srcSize) {
|
|
134
|
+
const destMeta = client.safeMetadata(destFullPath)
|
|
135
|
+
if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
|
|
136
|
+
logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
|
|
137
|
+
return
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
108
141
|
// write from source -> write to temp file
|
|
109
142
|
const fd = fs.openSync(tempPath, 'wx', 0o0600)
|
|
110
143
|
const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
|
|
@@ -119,12 +152,9 @@ const UTILITY_FUNCTIONS = {
|
|
|
119
152
|
// read from temp file -> write to mirror
|
|
120
153
|
const fd = fs.openSync(tempPath, 'r')
|
|
121
154
|
const reader = fs.createReadStream(tempPath, {fd})
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
126
|
-
await client.write(clientPath + destName, reader)
|
|
127
|
-
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
155
|
+
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
156
|
+
await client.write(destFullPath, reader)
|
|
157
|
+
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
128
158
|
})
|
|
129
159
|
results.success++
|
|
130
160
|
} catch (e) {
|