mobiletto 1.0.18 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -26
- package/index.js +26 -6
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -103,6 +103,32 @@ Today the supported drivers are:
|
|
|
103
103
|
local.remove(path, {recursive, quiet})
|
|
104
104
|
s3.remove(path, {recursive, quiet})
|
|
105
105
|
|
|
106
|
+
# File metadata
|
|
107
|
+
The `metadata` command returns metadata about a single filesystem entry.
|
|
108
|
+
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
109
|
+
|
|
110
|
+
A metadata object looks like this:
|
|
111
|
+
|
|
112
|
+
{
|
|
113
|
+
"name": "fully/qualified/path/to/file",
|
|
114
|
+
"type": "entry-type",
|
|
115
|
+
"size": size-in-bytes,
|
|
116
|
+
"ctime": creation-time-epoch-millis,
|
|
117
|
+
"mtime": modification-time-epoch-millis
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
121
|
+
|
|
122
|
+
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
123
|
+
should always be present. A subsequent `metadata` command should return all available properties.
|
|
124
|
+
|
|
125
|
+
# Alternate import style
|
|
126
|
+
Import the fully-scoped module and use the `connect` function:
|
|
127
|
+
|
|
128
|
+
const storage = require('mobiletto')
|
|
129
|
+
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
130
|
+
const objectData = await s3.readFile('some/path')
|
|
131
|
+
|
|
106
132
|
# Mirroring
|
|
107
133
|
|
|
108
134
|
// Copy a local filesystem mobiletto to S3
|
|
@@ -111,6 +137,10 @@ Today the supported drivers are:
|
|
|
111
137
|
// Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
|
|
112
138
|
local.mirror(s3, 'some/local-folder', 'some/s3-folder')
|
|
113
139
|
|
|
140
|
+
The `mirror` command performs a one-time copy of all files from one mobiletto to another.
|
|
141
|
+
It does not run any process to maintain the mirror over time. Run the `mirror` command again
|
|
142
|
+
to synchronize any missing files.
|
|
143
|
+
|
|
114
144
|
WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
|
|
115
145
|
|
|
116
146
|
With the `mirror` call semantics it can sometimes be confusing to understand who is the
|
|
@@ -118,13 +148,6 @@ reader and who is the writer. Imagine it like an assignment statement: the "left
|
|
|
118
148
|
is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
|
|
119
149
|
argument to the `mirror` method) is the value being assigned (mirrored data is read).
|
|
120
150
|
|
|
121
|
-
# Alternate import style
|
|
122
|
-
Import the fully-scoped module and use the `connect` function:
|
|
123
|
-
|
|
124
|
-
const storage = require('mobiletto')
|
|
125
|
-
const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'})
|
|
126
|
-
const objectData = await s3.readFile('some/path')
|
|
127
|
-
|
|
128
151
|
# Transparent Encryption
|
|
129
152
|
Enable transparent client-side encryption:
|
|
130
153
|
|
|
@@ -162,25 +185,6 @@ some effort, discover some or all of the overall structure of the directory hier
|
|
|
162
185
|
The adversary would not know the names of the directories/files unless they also knew your encryption
|
|
163
186
|
key or had otherwise successfully cracked the encryption. All bets are off then!
|
|
164
187
|
|
|
165
|
-
# File metadata
|
|
166
|
-
The `metadata` command returns metadata about a single filesystem entry.
|
|
167
|
-
Likewise, the return value from the `list` command is an array of metadata objects.
|
|
168
|
-
|
|
169
|
-
A metadata object looks like this:
|
|
170
|
-
|
|
171
|
-
{
|
|
172
|
-
"name": "fully/qualified/path/to/file",
|
|
173
|
-
"type": "entry-type",
|
|
174
|
-
"size": size-in-bytes,
|
|
175
|
-
"ctime": creation-time-epoch-millis,
|
|
176
|
-
"mtime": modification-time-epoch-millis
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
The `type` property can be `file`, `dir`, `link`, or `special`.
|
|
180
|
-
|
|
181
|
-
Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
|
|
182
|
-
should always be present. A subsequent `metadata` command should return all available properties.
|
|
183
|
-
|
|
184
188
|
# Key rotation
|
|
185
189
|
Create a mobiletto with your new key, then mirror the old data into it:
|
|
186
190
|
|
package/index.js
CHANGED
|
@@ -114,7 +114,30 @@ const UTILITY_FUNCTIONS = {
|
|
|
114
114
|
logger.verbose(`mirror: mirroring file: ${obj.name}`)
|
|
115
115
|
const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
|
|
116
116
|
logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
|
|
117
|
+
const destName = obj.name.startsWith(sourcePath)
|
|
118
|
+
? obj.name.substring(sourcePath.length)
|
|
119
|
+
: obj.name
|
|
120
|
+
const destFullPath = clientPath + destName;
|
|
117
121
|
try {
|
|
122
|
+
// if dest already exists and is the same size, don't copy it again
|
|
123
|
+
let srcSize = null
|
|
124
|
+
if (obj.size) {
|
|
125
|
+
srcSize = obj.size
|
|
126
|
+
} else {
|
|
127
|
+
const srcMeta = source.safeMetadata(obj.name)
|
|
128
|
+
if (srcMeta && srcMeta.size) {
|
|
129
|
+
srcSize = srcMeta.size
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// only continue if we could determine the source size
|
|
133
|
+
if (srcSize) {
|
|
134
|
+
const destMeta = client.safeMetadata(destFullPath)
|
|
135
|
+
if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
|
|
136
|
+
logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
|
|
137
|
+
return
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
118
141
|
// write from source -> write to temp file
|
|
119
142
|
const fd = fs.openSync(tempPath, 'wx', 0o0600)
|
|
120
143
|
const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
|
|
@@ -129,12 +152,9 @@ const UTILITY_FUNCTIONS = {
|
|
|
129
152
|
// read from temp file -> write to mirror
|
|
130
153
|
const fd = fs.openSync(tempPath, 'r')
|
|
131
154
|
const reader = fs.createReadStream(tempPath, {fd})
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
136
|
-
await client.write(clientPath + destName, reader)
|
|
137
|
-
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${clientPath + destName}`)
|
|
155
|
+
logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
156
|
+
await client.write(destFullPath, reader)
|
|
157
|
+
logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
|
|
138
158
|
})
|
|
139
159
|
results.success++
|
|
140
160
|
} catch (e) {
|