mobiletto 1.0.12 → 1.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -41,10 +41,18 @@ Today the supported drivers are:
41
41
  local.list() // --> returns an array of file objects
42
42
  s3.list() // --> returns an array of file objects
43
43
 
44
+ // List files recursively
45
+ local.list({ recursive: true })
46
+ s3.list({ recursive: true })
47
+
44
48
  // List files in a directory
45
49
  const path = 'some/path'
46
- local.list(path) // --> returns an array of file objects
47
- s3.list(path) // --> returns an array of file objects
50
+ local.list(path)
51
+ s3.list(path, { recursive: true }) // also supports recursive flag
52
+
53
+ // Visit files in a directory -- visitor function must be async
54
+ local.list(path, { visitor: myAsyncFunc })
55
+ s3.list(path, { visitor: myAsyncFunc, recursive: true })
48
56
 
49
57
  // Read metadata for a file
50
58
  local.metadata(path)
@@ -81,12 +89,28 @@ Today the supported drivers are:
81
89
  local.remove(path, {quiet})
82
90
  s3.remove(path, {quiet})
83
91
 
84
- // Recursively delete a directory
92
+ // Recursively delete a directory and do it quietly (do not report errors)
85
93
  const recursive = true
94
+ const quiet = true
86
95
  local.remove(path, {recursive, quiet})
87
96
  s3.remove(path, {recursive, quiet})
88
97
 
89
- # Alternate usage
98
+ # Mirroring
99
+
100
+ // Copy a local filesystem mobiletto to S3
101
+ s3.mirror(local)
102
+
103
+ // Mirror a local subdirectory from one mobiletto to an S3 mobiletto, with it's own subdirectory
104
+ local.mirror(s3, 'some/local-folder', 'some/s3-folder')
105
+
106
+ WARNING: Mirroring large data sets can be very time-consuming and bandwidth-intensive
107
+
108
+ With the `mirror` call semantics it can sometimes be confusing to understand who is the
109
+ reader and who is the writer. Imagine it like an assignment statement: the "left-hand mobiletto"
110
+ is the thing being assigned to (mirrored data written), and the "right-hand mobiletto" (the
111
+ argument to the `mirror` method) is the value being assigned (mirrored data is read).
112
+
113
+ # Alternate import style
90
114
  Import the fully-scoped module and use the `connect` function:
91
115
 
92
116
  const storage = require('mobiletto')
@@ -124,6 +148,38 @@ some effort, discover some or all of the overall structure of the directory hier
124
148
  The adversary would not know the names of the directories/files unless they also knew your encryption
125
149
  key or had otherwise successfully cracked the encryption. All bets are off then!
126
150
 
151
+ # File metadata
152
+ The `metadata` command returns metadata about a single filesystem entry.
153
+ Likewise, the return value from the `list` command is an array of metadata objects.
154
+
155
+ A metadata object looks like this:
156
+
157
+ {
158
+ "name": "fully/qualified/path/to/file",
159
+ "type": "entry-type",
160
+ "size": size-in-bytes,
161
+ "ctime": creation-time-epoch-millis,
162
+ "mtime": modification-time-epoch-millis
163
+ }
164
+
165
+ The `type` property can be `file`, `dir`, `link`, or `special`.
166
+
167
+ Depending on the type of driver, a `list` command may not return all fields. The `name` and `type` properties
168
+ should always be present. A subsequent `metadata` command should return all available properties.
169
+
170
+ # Key rotation
171
+ Create a mobiletto with your new key, then mirror the old data into it:
172
+
173
+ const storage = require('mobiletto')
174
+
175
+ const oldEncryption = { key: .... }
176
+ const oldStorage = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1'}, oldEncryption)
177
+
178
+ const newEncryption = { key: .... }
179
+ const newStorage = await storage.connect('s3', aws_key, aws_secret, {bucket: 'zz', region: 'us-east-1'}, newEncryption)
180
+
181
+ newStorage.mirror(oldStorage) // if oldStorage is very large, this may take a looooooong time...
182
+
127
183
  # Driver Interface
128
184
  A driver is any JS file that exports a 'storageClient' function with this signature:
129
185
 
@@ -143,8 +199,11 @@ The object that the storageClient function returns must define these functions:
143
199
  // Test the driver before using, ensure proper configuration
144
200
  async testConfig ()
145
201
 
146
- // Find DNS records that match the given domain, type (if provided) and name (if provided)
147
- async list (path) // path may be omitted
202
+ // List files in path (or from base-directory)
203
+ // If recursive is true, list recursively
204
+ // If visitor is defined, it will be an async function. await the visitor function on each file found
205
+ // Otherwise, perform the listing and return an array of objects
206
+ async list (path, recursive = false, visitor = null) // path may be omitted
148
207
 
149
208
  // Read metadata for a path
150
209
  async metadata (path)
@@ -158,4 +217,4 @@ The object that the storageClient function returns must define these functions:
158
217
  async write (path, readFunc)
159
218
 
160
219
  // Remove a file, or recursively delete a directory
161
- async remove (path, {recursive = false, quiet = false})
220
+ async remove (path, recursive = false, quiet = false)
@@ -1,7 +1,7 @@
1
1
  const { dirname } = require('path')
2
2
 
3
3
  const {
4
- M_FILE, M_DIR, M_LINK, M_SPECIAL,
4
+ M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
5
5
  MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
6
6
  } = require('../../index')
7
7
 
@@ -9,9 +9,9 @@ const fs = require('fs')
9
9
 
10
10
  const DEFAULT_MODE = '0700'
11
11
 
12
- const isENOENT = (err) => err.code && err.code === 'ENOENT'
12
+ const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
13
13
 
14
- const ioError = (err, path, method) => isENOENT(err)
14
+ const ioError = (err, path, method) => isNotExistError(err)
15
15
  ? err instanceof MobilettoNotFoundError
16
16
  ? err
17
17
  : new MobilettoNotFoundError(path)
@@ -19,6 +19,19 @@ const ioError = (err, path, method) => isENOENT(err)
19
19
  ? err
20
20
  : new MobilettoError(`${method}(${path}) error: ${err}`, err)
21
21
 
22
+ const fileType = (stat) => {
23
+ if (stat.isDirectory()) {
24
+ return M_DIR
25
+ }
26
+ if (stat.isFile()) {
27
+ return M_FILE
28
+ }
29
+ if (stat.isSymbolicLink()) {
30
+ return M_LINK
31
+ }
32
+ return M_SPECIAL
33
+ }
34
+
22
35
  class StorageClient {
23
36
  baseDir
24
37
  mode
@@ -46,41 +59,63 @@ class StorageClient {
46
59
  return { path, stat }
47
60
  }
48
61
  testConfig = async () => await this.list()
49
- fileType (stat) {
50
- if (stat.isDirectory()) {
51
- return M_DIR
62
+
63
+ normalizePath = (path) => path.startsWith(this.baseDir)
64
+ ? path
65
+ : this.baseDir + (path.startsWith('/') ? path.substring(1) : path)
66
+ denormalizePath = (path) => path.startsWith(this.baseDir) ? path.substring(this.baseDir.length) : path
67
+
68
+ fileToObject = (dir) => (f) => {
69
+ const stat = fs.lstatSync(dir + '/' + f)
70
+ const type = fileType(stat)
71
+ const entry = {
72
+ name: this.denormalizePath(dir + '/' + f),
73
+ type
52
74
  }
53
- if (stat.isFile()) {
54
- return M_FILE
75
+ if (type === M_LINK) {
76
+ const resolved = this.resolveSymlinks(dir + '/' + f)
77
+ entry.link = resolved.path
55
78
  }
56
- if (stat.isSymbolicLink()) {
57
- return M_LINK
79
+ return entry
80
+ }
81
+
82
+ readDirFiles = async (dir, recursive, visitor) => {
83
+ if (!recursive) {
84
+ return await this.dirFiles(dir, visitor)
85
+ }
86
+ const files = await this.dirFiles(dir, visitor)
87
+ for (const file of files) {
88
+ if (file.type === M_DIR) {
89
+ await this.readDirFiles(file.name, true, visitor)
90
+ }
58
91
  }
59
- return M_SPECIAL
60
92
  }
61
- normalizePath(path) {
62
- if (path.startsWith('/')) {
63
- path = path.substring(1)
93
+
94
+ dirFiles = async (dir, visitor) => {
95
+ const norm = this.normalizePath(dir);
96
+ try {
97
+ let names = fs.readdirSync(norm);
98
+ const files = names.map(this.fileToObject(norm))
99
+ for (const f of files) {
100
+ await visitor(f)
101
+ }
102
+ return files
103
+ } catch (e) {
104
+ console.error(`dirFiles: ${e}`)
105
+ throw ioError(e, norm, 'dirFiles')
64
106
  }
65
- return this.baseDir + path
66
107
  }
67
- async list (path = '') {
108
+
109
+ async list (path = '', recursive = false, visitor = null) {
68
110
  const dir = this.normalizePath(path)
69
111
  try {
70
- const files = fs.readdirSync(dir)
71
- return files.map(f => {
72
- const stat = fs.lstatSync(dir + '/' + f)
73
- const type = this.fileType(stat)
74
- const entry = {
75
- name: f,
76
- type
77
- }
78
- if (type === M_LINK) {
79
- const resolved = this.resolveSymlinks(dir + '/' + f)
80
- entry.link = resolved.path
81
- }
82
- return entry
83
- })
112
+ if (visitor === null) {
113
+ const results = []
114
+ await this.readDirFiles(dir, recursive, (obj) => results.push(obj))
115
+ return results
116
+ } else {
117
+ return await this.readDirFiles(dir, recursive, visitor)
118
+ }
84
119
  } catch (err) {
85
120
  throw ioError(err, path, 'list')
86
121
  }
@@ -92,7 +127,7 @@ class StorageClient {
92
127
  if (!lstat) {
93
128
  throw new MobilettoError('metadata: lstat error')
94
129
  }
95
- const type = this.fileType(lstat)
130
+ const type = fileType(lstat)
96
131
  if (type === M_DIR && path !== '') {
97
132
  const contents = await this.list(path)
98
133
  if (contents.length === 0) {
@@ -100,7 +135,7 @@ class StorageClient {
100
135
  }
101
136
  }
102
137
  return {
103
- name: file.startsWith(this.baseDir) ? file.substring(this.baseDir.length) : file,
138
+ name: this.denormalizePath(file),
104
139
  type,
105
140
  size: lstat.size,
106
141
  mtime: lstat.mtimeMs
@@ -119,7 +154,7 @@ class StorageClient {
119
154
  }
120
155
  }
121
156
 
122
- async write (path, generator) {
157
+ async normalizePathAndEnsureParentDirs (path) {
123
158
  const file = this.normalizePath(path)
124
159
 
125
160
  // console.log(`read: reading path: ${path} - ${file}`)
@@ -128,7 +163,7 @@ class StorageClient {
128
163
  try {
129
164
  dirStat = fs.lstatSync(parent)
130
165
  } catch (err) {
131
- if (err.code && err.code === 'ENOENT') {
166
+ if (isNotExistError(err)) {
132
167
  this.mkdirs(parent)
133
168
  } else {
134
169
  throw new MobilettoError(`write: lstat error on ${parent}: ${err}`, err)
@@ -139,17 +174,52 @@ class StorageClient {
139
174
  } else if (!dirStat.isDirectory()) {
140
175
  throw new MobilettoError(`write: not a directory: ${parent} (cannot write file ${file})`)
141
176
  }
177
+ return file
178
+ }
142
179
 
180
+ async write (path, generatorOrReadableStream) {
181
+ const file = await this.normalizePathAndEnsureParentDirs(path)
143
182
  // console.log(`write: writing path ${path} -> ${file}`)
144
183
  const stream = fs.createWriteStream(file, {mode: this.mode})
145
184
  const writer = writeStream(stream)
146
185
  const closer = closeStream(stream)
147
186
  let count = 0
148
- let chunk = generator.next().value
149
- while (chunk) {
150
- count += chunk.length
151
- writer(chunk)
152
- chunk = generator.next().value
187
+
188
+ if (isReadable(generatorOrReadableStream)) {
189
+ const readable = generatorOrReadableStream
190
+ const streamHandler = stream =>
191
+ new Promise((resolve, reject) => {
192
+ stream.on('data', (data) => {
193
+ if (data) {
194
+ writer(data)
195
+ count += data.length
196
+ }
197
+ })
198
+ stream.on('error', reject)
199
+ stream.on('end', () => {
200
+ closer()
201
+ resolve()
202
+ })
203
+ })
204
+ await streamHandler(readable)
205
+ return count
206
+ }
207
+
208
+ const generator = generatorOrReadableStream
209
+ let chunk = isAsyncGenerator(generator)
210
+ ? (await generator.next()).value
211
+ : generator.next().value
212
+ let nullCount = 0
213
+ while (chunk || nullCount < 5) {
214
+ if (chunk) {
215
+ count += chunk.length
216
+ writer(chunk)
217
+ } else {
218
+ nullCount++
219
+ }
220
+ chunk = isAsyncGenerator(generator)
221
+ ? (await generator.next()).value
222
+ : generator.next().value
153
223
  }
154
224
  closer()
155
225
  return Promise.resolve(count)
@@ -166,15 +236,13 @@ class StorageClient {
166
236
  }
167
237
  }
168
238
 
169
- async remove (path, options) {
170
- const recursive = options && options.recursive ? options.recursive : false
171
- const quiet = options && options.quiet ? options.quiet : false
239
+ async remove (path, recursive, quiet) {
172
240
  const file = this.normalizePath(path)
173
241
  // console.log(`remove: deleting path: ${path} = ${file}`)
174
242
  try {
175
243
  fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
176
244
  } catch (err) {
177
- if (isENOENT(err)) {
245
+ if (isNotExistError(err)) {
178
246
  if (!quiet) {
179
247
  throw new MobilettoNotFoundError(path)
180
248
  }
@@ -44,18 +44,23 @@ class StorageClient {
44
44
  }
45
45
 
46
46
  // noinspection JSUnusedGlobalSymbols -- called by driver init
47
- testConfig = async () => await this._list('', { MaxKeys: 1 })
47
+ testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
48
48
 
49
- async list (path = '') {
50
- return (await this._list(path)).map(obj => {
51
- return {
52
- name: obj,
53
- type: obj.endsWith(this.delimiter) ? M_DIR : M_FILE
54
- }
55
- })
49
+ stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
50
+
51
+ nameToObj = (name) => {
52
+ const relName = this.stripPrefix(name)
53
+ return {
54
+ name: relName,
55
+ type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
56
+ }
56
57
  }
57
58
 
58
- async _list (path, params = {}, recursive = false) {
59
+ async list (path = '', recursive = false, visitor = null) {
60
+ return await this._list(path, recursive, visitor)
61
+ }
62
+
63
+ async _list (path, recursive = false, visitor = null, params = {}) {
59
64
  const logPrefix = `_list(path=${path}):`
60
65
 
61
66
  // Declare truncated as a flag that the while loop is based on.
@@ -73,6 +78,7 @@ class StorageClient {
73
78
  bucketParams.Delimiter = this.delimiter
74
79
  }
75
80
  const objects = []
81
+ let objectCount = 0
76
82
  // console.log(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
77
83
 
78
84
  // while loop that runs until 'response.truncated' is false.
@@ -81,15 +87,25 @@ class StorageClient {
81
87
  const response = await this.client.send(new ListObjectsCommand(bucketParams))
82
88
  const hasContents = typeof response.Contents !== 'undefined'
83
89
  if (hasContents) {
84
- response.Contents.forEach((item) => {
85
- objects.push(item.Key) // todo: transform into standard object
86
- })
90
+ for (const item of response.Contents) {
91
+ const obj = this.nameToObj(item.Key);
92
+ if (visitor) {
93
+ await visitor(obj)
94
+ }
95
+ objects.push(obj)
96
+ objectCount++
97
+ }
87
98
  }
88
99
  const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
89
100
  if (hasCommonPrefixes) {
90
- response.CommonPrefixes.forEach((item) => {
91
- objects.push(item.Prefix) // todo: transform into standard object
92
- })
101
+ for (const item of response.CommonPrefixes) {
102
+ const obj = this.nameToObj(item.Prefix)
103
+ if (visitor) {
104
+ await visitor(obj)
105
+ }
106
+ objects.push(obj)
107
+ objectCount++
108
+ }
93
109
  }
94
110
  truncated = response.IsTruncated
95
111
  // If truncated is true, advance the marker
@@ -108,16 +124,18 @@ class StorageClient {
108
124
  throw new MobilettoError(`${logPrefix} Error: ${err}`)
109
125
  }
110
126
  }
111
- if (recursive && objects.length === 0) {
127
+ if (recursive && objectCount === 0 && path !== '') {
112
128
  throw new MobilettoNotFoundError(path)
113
129
  }
114
130
  return objects
115
131
  }
116
132
 
117
- normalizeKey = (path) =>
118
- path.startsWith(this.prefix)
133
+ normalizeKey = (path) => {
134
+ const p = (path.startsWith(this.prefix)
119
135
  ? path
120
- : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path)
136
+ : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
137
+ return p.replaceAll(/\/{2,}/g, '/')
138
+ }
121
139
 
122
140
  s3error (err, key, path, method) {
123
141
  return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
@@ -138,7 +156,7 @@ class StorageClient {
138
156
  try {
139
157
  const head = await this.client.send(new HeadObjectCommand(bucketParams))
140
158
  const meta = {
141
- name: path,
159
+ name: this.stripPrefix(path),
142
160
  size: head.ContentLength
143
161
  }
144
162
  if (head.LastModified) {
@@ -196,14 +214,12 @@ class StorageClient {
196
214
  }
197
215
  }
198
216
 
199
- async remove (path, options = null) {
200
- const recursive = options && options.recursive
201
- const quiet = options && options.quiet
217
+ async remove (path, recursive, quiet) {
202
218
  if (recursive) {
203
- let objects = await this._list(path, { MaxKeys: DELETE_OBJECTS_MAX_KEYS }, true)
219
+ let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
204
220
  while (objects && objects.length > 0) {
205
221
  const Delete = {
206
- Objects: objects.map(obj => { return {Key: obj} })
222
+ Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
207
223
  }
208
224
  if (quiet) {
209
225
  Delete.Quiet = true
@@ -224,7 +240,7 @@ class StorageClient {
224
240
  throw new MobilettoError(`remove(${path}): DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
225
241
  }
226
242
  try {
227
- objects = await this._list(path, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
243
+ objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
228
244
  } catch (e) {
229
245
  if (!(e instanceof MobilettoNotFoundError)) {
230
246
  throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}): error listing: ${e}`)
package/index.js CHANGED
@@ -1,3 +1,4 @@
1
+ const fs = require('fs')
1
2
  const { basename, dirname } = require('path')
2
3
  const shasum = require('shasum')
3
4
  const randomstring = require('randomstring')
@@ -9,6 +10,12 @@ const DIR_ENT_DIR_SUFFIX = '__.dirent'
9
10
  const DIR_ENT_FILE_PREFIX = 'dirent__'
10
11
  const ENC_PAD_SEP = ' ~ '
11
12
 
13
+ const initMobilettoTempDir = () => {
14
+ const dir = process.env.MOBILETTO_TMP || process.env.TMPDIR || '/tmp'
15
+ return dir.endsWith('/') ? dir.substring(0, dir.length - 1) : dir
16
+ }
17
+ const MOBILETTO_TMP = initMobilettoTempDir()
18
+
12
19
  // adapted from https://stackoverflow.com/a/27724419
13
20
  function MobilettoError (message, err) {
14
21
  this.message = `${message}`
@@ -34,7 +41,23 @@ function MobilettoNotFoundError (message) {
34
41
 
35
42
  const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
36
43
 
44
+ const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
45
+ const isReadable = (thing) => thing instanceof fs.ReadStream
46
+
37
47
  const UTILITY_FUNCTIONS = {
48
+ list: (client) => async (path, opts) => {
49
+ const recursive = opts && opts.recursive ? opts.recursive : false
50
+ const visitor = opts && opts.visitor ? opts.visitor : null
51
+ if (visitor && typeof visitor !== 'function') {
52
+ throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
53
+ }
54
+ return await client.driver_list(path, recursive, visitor)
55
+ },
56
+ remove: (client) => async (path, opts) => {
57
+ const recursive = opts && opts.recursive ? opts.recursive : false
58
+ const quiet = opts && opts.quiet ? opts.quiet : false
59
+ return await client.driver_remove(path, recursive, quiet)
60
+ },
38
61
  readFile: (client) => async (path) => {
39
62
  const chunks = []
40
63
  await client.read(path, reader(chunks))
@@ -43,11 +66,61 @@ const UTILITY_FUNCTIONS = {
43
66
  writeFile: (client) => async (path, data) => {
44
67
  const readFunc = function* () { yield data }
45
68
  return await client.write(path, readFunc())
69
+ },
70
+ mirror: (client) => async (source, clientPath = '', sourcePath = '') => {
71
+ const results = {
72
+ success: 0,
73
+ errors: 0
74
+ }
75
+ const visitor = async (obj) => {
76
+ if (obj.type && obj.type === M_FILE) {
77
+ const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
78
+ try {
79
+ // write from source -> write to temp file
80
+ const fd = fs.openSync(tempPath, 'wx', 0o0600)
81
+ const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
82
+ await source.read(obj.name, async (chunk) => {
83
+ if (chunk) { writer.write(chunk) }
84
+ }, () => {
85
+ writer.close((err) => {
86
+ if (err) { throw new MobilettoError(`mirror: error closing temp file: ${err}`) }
87
+ })
88
+ }).then(async () => {
89
+ // read from temp file -> write to mirror
90
+ const fd = fs.openSync(tempPath, 'r')
91
+ const reader = fs.createReadStream(tempPath, {fd})
92
+ const destName = obj.name.startsWith(sourcePath)
93
+ ? obj.name.substring(sourcePath.length)
94
+ : obj.name
95
+ await client.write(clientPath + destName, reader)
96
+ })
97
+ results.success++
98
+ } catch (e) {
99
+ console.warn(`mirror: error copying file: ${e}`)
100
+ results.errors++
101
+ } finally {
102
+ fs.rmSync(tempPath, {force: true})
103
+ }
104
+ }
105
+ }
106
+ await mirrorDir(source, sourcePath, visitor)
107
+ return results
108
+ }
109
+ }
110
+
111
+ async function mirrorDir (source, sourcePath, visitor) {
112
+ const listing = await source.list(sourcePath, {recursive: false, visitor})
113
+ for (const obj of listing) {
114
+ if (obj.type === M_DIR) {
115
+ const dir = obj.name.startsWith(sourcePath) ? obj.name : sourcePath + obj.name
116
+ await mirrorDir(source, dir, visitor)
117
+ }
46
118
  }
47
119
  }
48
120
 
49
121
  function addUtilityFunctions (client, readOnly = false) {
50
122
  for (const func of Object.keys(UTILITY_FUNCTIONS)) {
123
+ client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
51
124
  client[func] = UTILITY_FUNCTIONS[func](client)
52
125
  }
53
126
  if (readOnly) {
@@ -136,6 +209,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
136
209
  throw e
137
210
  }
138
211
  }
212
+ meta.type ||= M_FILE
139
213
  meta.name = path // rewrite name back to plaintext name
140
214
  return Object.assign({}, meta, metaObj)
141
215
  }
@@ -155,9 +229,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
155
229
 
156
230
  async function removeDirentFile(path) {
157
231
  const df = direntFile(direntDir(dirname(path)), path);
158
- await client.remove(df, {recursive: false, quiet: true})
159
- await client.remove(encryptPath(path), {recursive: false, quiet: true})
160
- await client.remove(metaPath(path), {recursive: false, quiet: true})
232
+ const recursive = false
233
+ const quiet = true
234
+ await client.remove(df, recursive, quiet)
235
+ await client.remove(encryptPath(path), recursive, quiet)
236
+ await client.remove(metaPath(path), recursive, quiet)
161
237
  }
162
238
 
163
239
  function stringGenerator (value, enc) {
@@ -167,13 +243,21 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
167
243
  }
168
244
 
169
245
  const encClient = {
170
- list: async (path) => {
246
+ list: async (path, recursive, visitor) => {
171
247
  const dirent = direntDir(path)
172
- const entries = await client.list(dirent)
248
+ const entries = await client.list(dirent, recursive)
173
249
  if (!entries || entries.length === 0) {
174
250
  return []
175
251
  }
176
- return _loadMeta(dirent, entries)
252
+ if (visitor) {
253
+ const objects = await _loadMeta(dirent, entries)
254
+ for (const obj of objects) {
255
+ await visitor(obj)
256
+ }
257
+ return objects
258
+ } else {
259
+ return _loadMeta(dirent, entries)
260
+ }
177
261
  },
178
262
  metadata: _metadata(client),
179
263
  read: async (path, callback) => {
@@ -215,12 +299,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
215
299
  const cipher = crypt.startEncryptStream(enc)
216
300
  const realPath = encryptPath(path)
217
301
  await client.write(realPath, cryptGenerator(readFunc))
218
- const meta = { name: path, size: bytesRead }
302
+ const meta = { name: path, size: bytesRead, type: M_FILE }
219
303
  await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
220
304
  return bytesRead
221
305
  },
222
306
  remove: async (path, options) => {
223
- const recursive = (options && options.recursive) || false
307
+ const recursive = options === true || (options && options.recursive) || false
308
+ const quiet = (options && options.quiet) || false
224
309
  if (recursive) {
225
310
  // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
226
311
  async function recRm (path) {
@@ -253,7 +338,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
253
338
  let parent = path
254
339
  let dirent = direntDir(parent)
255
340
  const df = direntFile(dirent, path)
256
- await client.remove(df, {recursive: false, quiet: true})
341
+ await client.remove(df, false, true)
257
342
  while (true) {
258
343
  try {
259
344
  const entries = await client.list(dirent)
@@ -327,6 +412,7 @@ const M_SPECIAL = 'special'
327
412
 
328
413
  module.exports = {
329
414
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
415
+ isAsyncGenerator, isReadable,
330
416
  mobiletto, connect,
331
417
  MobilettoError, MobilettoNotFoundError,
332
418
  readStream, writeStream, closeStream
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.12",
3
+ "version": "1.0.13",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -3,6 +3,7 @@
3
3
  require('dotenv').config()
4
4
 
5
5
  const randomstring = require('randomstring')
6
+ const path = require('path')
6
7
 
7
8
  const { expect, should, assert } = require('chai')
8
9
 
@@ -33,6 +34,18 @@ DRIVER_CONFIG = {
33
34
  }
34
35
  }
35
36
  }
37
+ DRIVER_NAMES = Object.keys(DRIVER_CONFIG)
38
+
39
+ function pickAnotherDriver (notThisOne) {
40
+ let choice = null
41
+ while (!choice) {
42
+ choice = DRIVER_NAMES[Math.floor(Math.random() * DRIVER_NAMES.length)]
43
+ if (choice === notThisOne) {
44
+ choice = null
45
+ }
46
+ }
47
+ return choice
48
+ }
36
49
 
37
50
  async function assertMeta (api, name, expectedSize) {
38
51
  const meta = await api.metadata(name)
@@ -96,7 +109,7 @@ const encryptionTests = () => [null, { key: rand(32) }]
96
109
  // the same set of tests, with exactly the same inputs/outputs.
97
110
  // This greatly supports developer sanity
98
111
 
99
- for (const driverName of Object.keys(DRIVER_CONFIG)) {
112
+ for (const driverName of DRIVER_NAMES) {
100
113
  const config = DRIVER_CONFIG[driverName]
101
114
  const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
102
115
  const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
@@ -122,7 +135,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
122
135
  const randomData = rand(size)
123
136
  const fileSuffix = '' + Date.now()
124
137
  let fixture
125
- beforeEach((done) => {
138
+ before((done) => {
126
139
  const name = `test_file_${fileSuffix}`
127
140
  connect(driverName, config.key, config.secret, config.opts)
128
141
  .then(api => { fixture = {api, name, randomData} })
@@ -152,12 +165,13 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
152
165
  for (const encryption of encryptionTests()) {
153
166
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
154
167
  describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
155
- // some random data, plus a bit extra
168
+ // describe(`${driverName} - ENC fail to write and delete files in readOnly mode`, () => {
169
+ // const encryption = {key: rand(32)}
156
170
  const size = 16
157
171
  const randomData = rand(size)
158
172
  const fileSuffix = '' + Date.now()
159
173
  let fixture
160
- beforeEach((done) => {
174
+ before((done) => {
161
175
  const name = `test_file_${fileSuffix}`
162
176
  const opts = Object.assign({}, config.opts, {readOnly: true})
163
177
  connect(driverName, config.key, config.secret, opts, encryption)
@@ -203,7 +217,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
203
217
  const fileSuffix = '' + Date.now()
204
218
  const encryptionKey = rand(32)
205
219
  let fixture
206
- beforeEach((done) => {
220
+ before((done) => {
207
221
  const name = `test_file_${fileSuffix}`
208
222
  mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
209
223
  .then(api => { fixture = {api, name, randomData} })
@@ -233,29 +247,40 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
233
247
 
234
248
  for (const encryption of encryptionTests()) {
235
249
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
236
- describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
250
+ const mirrorDriver = pickAnotherDriver(driverName)
251
+ const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
252
+ const mirrorDest = `mirrorDest_${rand(5)}/`
253
+ describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
237
254
  // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
238
255
  // const encryption = {key: rand(32)}
239
256
  // const encryption = null
240
257
  // a random directory and file within it
241
258
  const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
242
259
  const subdirName = `subdir_` + Date.now()
243
- const randomPath = `${randomParent}/${subdirName}/random_file_${Date.now()}`
260
+ const fullSubdirPath = `${randomParent}/${subdirName}`
261
+ const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
244
262
  //const fileCount = 3 + Math.floor(Math.random() * 10)
245
263
  const fileCount = 2
246
264
  let fixture
247
- beforeEach((done) => {
265
+ before((done) => {
248
266
  mobiletto(driverName, config.key, config.secret, config.opts, encryption)
249
- .then(api => { fixture = {api, name: randomPath} })
267
+ .then(api => {
268
+ // const mirrorEnc = { key: rand(32) }
269
+ const mirrorEnc = null
270
+ mobiletto(mirrorDriver, mirrorConfig.key, mirrorConfig.secret, mirrorConfig.opts, mirrorEnc)
271
+ .then(mirrorApi => {
272
+ fixture = {api, mirrorApi, name: randomPath}
273
+ })
274
+ .catch((err) => {
275
+ throw err
276
+ })
277
+ .finally(done)
278
+ })
250
279
  .catch((err) => { throw err })
251
- .finally(done)
252
280
  })
253
281
  it(`should write ${fileCount} files in a new directory`, async () => {
254
- function* dataGenerator() {
255
- // return one chunk of random data
256
- yield rand(READ_SZ)
257
- }
258
-
282
+ // return one chunk of random data
283
+ function* dataGenerator() { yield rand(READ_SZ) }
259
284
  for (let i = 0; i < fileCount; i++) {
260
285
  const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
261
286
  expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
@@ -279,6 +304,31 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
279
304
  ).to.not.be.null
280
305
  }
281
306
  })
307
+ it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
308
+ const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
309
+ expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
310
+ expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
311
+ })
312
+ it("should see the same files mirrored both places", async () => {
313
+ const originalThings = await fixture.api.list(fullSubdirPath)
314
+ const mirrorPath = `${mirrorDest}${subdirName}`
315
+ const mirrorThings = await fixture.mirrorApi.list(mirrorPath)
316
+ expect(originalThings.length === mirrorThings.length).to.be.true
317
+
318
+ const intersection = originalThings
319
+ .filter(orig => mirrorThings
320
+ .find(m => path.basename(m.name) === path.basename(orig.name)))
321
+ expect(intersection.length === originalThings.length).to.be.true
322
+ })
323
+ it("loading metadata on a mirrored file succeeds", async () => {
324
+ const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
325
+ await assertMeta(fixture.mirrorApi, destPath, READ_SZ)
326
+ })
327
+ it("should recursively delete the mirrored data", async () => {
328
+ const recursive = true
329
+ const removed = await fixture.mirrorApi.remove(mirrorDest, {recursive})
330
+ expect(removed).to.be.true
331
+ })
282
332
  it("should recursively delete the directory and file we just created", async () => {
283
333
  const recursive = true
284
334
  const removed = await fixture.api.remove(randomParent, {recursive})
@@ -287,6 +337,10 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
287
337
  it("loading metadata on the file we wrote now fails", async () => {
288
338
  await assertMetaFail(fixture.api, fixture.name)
289
339
  })
340
+ it("loading metadata on a mirrored file now fails", async () => {
341
+ const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
342
+ await assertMetaFail(fixture.mirrorApi, destPath)
343
+ })
290
344
  it("loading metadata on the parent dir we wrote now fails", async () => {
291
345
  await assertMetaFail(fixture.api, randomParent)
292
346
  })