mobiletto 1.1.9 → 1.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -33,7 +33,7 @@ change storage providers and know that your app's storage layer will behave iden
33
33
 
34
34
  ### Extensive testing
35
35
  All drivers are tested for identical behavior with 60+ tests for each driver.
36
- Each test actually runs *four times* with every (2x2) combination of:
36
+ We test all drivers with every combination of:
37
37
  * Encryption: both enabled and disabled
38
38
  * Redis cache: both enabled and disabled
39
39
 
@@ -70,13 +70,13 @@ This code would run the same if the driver were `b2` or `local`.
70
70
  const dirList = await bucket.list('some/dir/')
71
71
  const everything = await bucket.list('', {recursive: true})
72
72
 
73
- // write an entire file in one-shot
73
+ // write an entire file
74
74
  let bytesWritten = await bucket.writeFile('some/path', someBufferOfData)
75
75
 
76
76
  // write a file from a stream/generator
77
77
  bytesWritten = await bucket.write('some/path', streamOrGenerator)
78
78
 
79
- // read an entire file in one-shot
79
+ // read an entire file
80
80
  // returns null if an exception would otherwise be thrown
81
81
  const bufferOrNull = await bucket.safeReadFile('some/path')
82
82
 
@@ -160,9 +160,13 @@ A much more extensive example, showing most of the features offered:
160
160
  const callback = (chunk) => { ... write chunk somewhere ... }
161
161
  api.read(path, callback) // returns count of bytes read
162
162
 
163
- // Read an entire file at once (convenience method)
163
+ // Read an entire file at once
164
164
  const data = await api.readFile(path) // returns a byte Buffer of the file contents
165
165
 
166
+ // Read an entire file at once
167
+ // returns null if an exception would otherwise be thrown
168
+ const bufferOrNull = await bucket.safeReadFile('some/path')
169
+
166
170
  // Write a file
167
171
  // Provide a generator function that yields chunks of data
168
172
  const generator = function* () {
package/index.js CHANGED
@@ -9,7 +9,7 @@ const randomstring = require('randomstring')
9
9
 
10
10
  const crypt = require('./util/crypt')
11
11
  const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
12
- const LRU = require("lru-cache");
12
+ const LRU = require("lru-cache")
13
13
 
14
14
  const DIR_ENT_DIR_SUFFIX = '__.dirent'
15
15
  const DIR_ENT_FILE_PREFIX = 'dirent__'
@@ -516,36 +516,50 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
516
516
  return thing
517
517
  }
518
518
 
519
- async function tryParentDirForSingleFile (path, visitor, e) {
519
+ async function tryParentDirForSingleFile (p, visitor, e) {
520
520
  // it might be a single file, try listing the parent dir
521
- const parentDirent = direntDir(path.dirname(p));
522
- entries = await client.list(parentDirent, false)
521
+ const parentDirent = direntDir(dirname(p))
522
+ entries = await client.list(parentDirent)
523
523
  const objects = await _loadMeta(parentDirent, entries)
524
524
  const found = objects.find(o => o.name === p)
525
525
  if (found) {
526
526
  if (visitor) {
527
527
  await visitor(found)
528
528
  }
529
- logger.debug(`tryParentDirForSingleFile(${path}) found ${found.name}`)
529
+ logger.debug(`tryParentDirForSingleFile(${p}) found ${found.name}`)
530
530
  return cacheAndReturn([found])
531
531
  }
532
- logger.debug(`tryParentDirForSingleFile(${path}) nothing found! e=${e}`)
533
- throw e === null ? new MobilettoNotFoundError(path) : e
532
+ logger.debug(`tryParentDirForSingleFile(${p}) nothing found! e=${e}`)
533
+ throw e === null ? new MobilettoNotFoundError(p) : e
534
534
  }
535
535
 
536
536
  if (cached) {
537
537
  entries = cached
538
538
  } else {
539
539
  try {
540
- entries = await client.list(dirent, recursive)
540
+ entries = await _loadMeta(dirent, await client.list(dirent))
541
541
  } catch (e) {
542
542
  if (e instanceof MobilettoNotFoundError && p.includes('/')) {
543
- return await tryParentDirForSingleFile(path, visitor, e)
543
+ return await tryParentDirForSingleFile(p, visitor, e)
544
+ }
545
+ }
546
+ if (recursive) {
547
+ const dirs = entries.filter(obj => obj.type === M_DIR)
548
+ while (dirs.length > 0) {
549
+ const dir = dirs.shift()
550
+ const subdir = direntDir(dir.name)
551
+ const subdirListing = await client.list(subdir)
552
+ if (subdirListing && subdirListing.length > 0) {
553
+ const subdirEntries = await _loadMeta(subdir, subdirListing)
554
+ entries.push(...subdirEntries)
555
+ const moreDirs = subdirEntries.filter(obj => obj.type === M_DIR)
556
+ dirs.unshift(...moreDirs)
557
+ }
544
558
  }
545
559
  }
546
560
  }
547
561
  if (entries.length === 0 && p.includes('/')) {
548
- return await tryParentDirForSingleFile(path, visitor, null)
562
+ return await tryParentDirForSingleFile(p, visitor, null)
549
563
  }
550
564
  if (cache) {
551
565
  cache.set(cacheKey, entries).then(
@@ -557,14 +571,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
557
571
  return []
558
572
  }
559
573
  if (visitor) {
560
- const objects = await _loadMeta(dirent, entries)
561
- for (const obj of objects) {
574
+ for (const obj of entries) {
562
575
  await visitor(obj)
563
576
  }
564
- return objects
565
- } else {
566
- return _loadMeta(dirent, entries)
567
577
  }
578
+ return entries
568
579
  },
569
580
  metadata: _metadata(client),
570
581
  read: async (path, callback) => {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.1.9",
3
+ "version": "1.1.10",
4
4
  "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
@@ -8,7 +8,8 @@
8
8
  "s3",
9
9
  "b2",
10
10
  "simple",
11
- "filesystem"
11
+ "filesystem",
12
+ "mobiletto"
12
13
  ],
13
14
  "homepage": "https://github.com/cobbzilla/mobiletto",
14
15
  "author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
package/test/api.spec.js CHANGED
@@ -125,360 +125,370 @@ const redisTests = () => [REDIS_ENABLED, REDIS_DISABLED]
125
125
  // const redisTests = () => [REDIS_DISABLED]
126
126
 
127
127
  for (const redisSetup of redisTests()) {
128
- if (!redisSetup.enabled) continue
128
+ if (!redisSetup.enabled) continue
129
129
 
130
130
  for (const driverName of DRIVER_NAMES) {
131
- // For testing single drivers
132
- // if (driverName !== 'b2') continue
133
-
134
- const driverTest = `${driverName} [${redisSetup.name}]`
135
- const config = redisSetup.config(DRIVER_CONFIG[driverName])
136
- const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
137
- const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
138
- describe(`${driverTest} test`, () => {
139
- beforeEach ((done) => {
140
- redis.flushAll().then(
141
- () => { logger.info(`${driverTest} - flushed redis`) },
142
- (e) => {
143
- logger.error(`${driverTest} - error flushing redis: ${e}`)
144
- throw e
145
- }
146
- ).finally(() => done())
131
+ // For testing single drivers
132
+ // if (driverName !== 'local') continue
133
+
134
+ const driverTest = `${driverName} [${redisSetup.name}]`
135
+ const config = redisSetup.config(DRIVER_CONFIG[driverName])
136
+ const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
137
+ const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
138
+ describe(`${driverTest} test`, () => {
139
+ beforeEach ((done) => {
140
+ redis.flushAll().then(
141
+ () => { logger.info(`${driverTest} - flushed redis`) },
142
+ (e) => {
143
+ logger.error(`${driverTest} - error flushing redis: ${e}`)
144
+ throw e
145
+ }
146
+ ).finally(() => done())
147
+ })
148
+
149
+ describe(`${driverTest} - create api client`, () => {
150
+ it("should validate the config and return an API object", async () => {
151
+ try {
152
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
153
+ should().exist(api, 'expected API object to exist')
154
+ } catch (e) {
155
+ logger.error(`error creating API: ${e} ${JSON.stringify(e)}`)
156
+ throw e
157
+ }
147
158
  })
159
+ })
148
160
 
149
- describe(`${driverTest} - create api client`, () => {
150
- it("should validate the config and return an API object", async () => {
151
- try {
152
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
153
- should().exist(api, 'expected API object to exist')
154
- } catch (e) {
155
- logger.error(`error creating API: ${e} ${JSON.stringify(e)}`)
156
- throw e
157
- }
158
- })
161
+ describe(`${driverTest} - listing with no arguments returns some results`, () => {
162
+ it("should return some results from a default listing", async () => {
163
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
164
+ const results = await api.list()
165
+ expect(results).to.have.lengthOf.greaterThan(0, 'expected some results in the default listing')
159
166
  })
167
+ })
160
168
 
161
- describe(`${driverTest} - listing with no arguments returns some results`, () => {
162
- it("should return some results from a default listing", async () => {
163
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
164
- const results = await api.list()
165
- expect(results).to.have.lengthOf.greaterThan(0, 'expected some results in the default listing')
166
- })
169
+ describe(`${driverTest} - write a file, read file, read metadata, delete file`, () => {
170
+ // some random data, plus a bit extra
171
+ const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
172
+ const randomData = rand(size)
173
+ const fileSuffix = '' + Date.now()
174
+ let fixture
175
+ before((done) => {
176
+ const name = `test_file_${fileSuffix}`
177
+ connect(driverName, config.key, config.secret, config.opts)
178
+ .then(api => { fixture = {api, name, randomData} })
179
+ .finally(done)
180
+ })
181
+ it("should write a file", async () => {
182
+ const response = await writeRandomFile(fixture, size)
183
+ expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
184
+ })
185
+ it("should read the file we just wrote", async () => {
186
+ const data = await fixture.api.readFile(fixture.name)
187
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
188
+ expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
189
+ })
190
+ it("should load metadata on the file we just wrote", async () => {
191
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
192
+ })
193
+ it("should delete the file we just wrote", async () => {
194
+ const removed = await fixture.api.remove(fixture.name)
195
+ expect(!!removed).to.be.true
167
196
  })
197
+ it("loading metadata on the file we wrote now fails", async () => {
198
+ await assertMetaFail(fixture.api, fixture.name)
199
+ })
200
+ it("loading metadata on the file we wrote using safeMeta does not fail but returns null", async () => {
201
+ await assertSafeMetaNull(fixture.api, fixture.name)
202
+ })
203
+ })
168
204
 
169
- describe(`${driverTest} - write a file, read file, read metadata, delete file`, () => {
170
- // some random data, plus a bit extra
171
- const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
205
+ for (const encryption of encryptionTests()) {
206
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
207
+ describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
208
+ // describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
209
+ // const encryption = {key: rand(32)}
210
+ const size = 16
172
211
  const randomData = rand(size)
173
212
  const fileSuffix = '' + Date.now()
174
213
  let fixture
175
214
  before((done) => {
176
215
  const name = `test_file_${fileSuffix}`
177
- connect(driverName, config.key, config.secret, config.opts)
216
+ const opts = Object.assign({}, config.opts, {readOnly: true})
217
+ connect(driverName, config.key, config.secret, opts, encryption)
178
218
  .then(api => { fixture = {api, name, randomData} })
219
+ .catch(err => { throw err })
179
220
  .finally(done)
180
221
  })
181
- it("should write a file", async () => {
222
+ it("should try to write a file and fail", async () => {
182
223
  const response = await writeRandomFile(fixture, size)
183
- expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
224
+ expect(response).to.be.false
184
225
  })
185
- it("should read the file we just wrote", async () => {
186
- const data = await fixture.api.readFile(fixture.name)
187
- expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
188
- expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
226
+ it("should fail to read the file we did not write", async () => {
227
+ try {
228
+ await fixture.api.readFile(fixture.name)
229
+ assert.fail('should not have been able to read non-existent file that we did not write')
230
+ } catch (e) {
231
+ if (!(e instanceof MobilettoNotFoundError)) {
232
+ assert.fail(`unexpected error: ${e} trying to read non-existent file`)
233
+ }
234
+ }
189
235
  })
190
- it("should load metadata on the file we just wrote", async () => {
191
- await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
236
+ it("should fail to load metadata on the file we did not write", async () => {
237
+ try {
238
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
239
+ assert.fail('should not have been able to read metadata for non-existent file that we did not write')
240
+ } catch (e) {
241
+ if (!(e instanceof MobilettoNotFoundError)) {
242
+ assert.fail(`unexpected error: ${e} trying to read metadata on non-existent file`)
243
+ }
244
+ }
192
245
  })
193
- it("should delete the file we just wrote", async () => {
246
+ it("should fail to delete the file we did not write", async () => {
194
247
  const removed = await fixture.api.remove(fixture.name)
195
- expect(!!removed).to.be.true
196
- })
197
- it("loading metadata on the file we wrote now fails", async () => {
198
- await assertMetaFail(fixture.api, fixture.name)
199
- })
200
- it("loading metadata on the file we wrote using safeMeta does not fail but returns null", async () => {
201
- await assertSafeMetaNull(fixture.api, fixture.name)
248
+ expect(!!removed).to.be.false
202
249
  })
203
250
  })
251
+ }
204
252
 
205
- for (const encryption of encryptionTests()) {
206
- const encDesc = encryption ? '(with encryption)' : '(without encryption)'
207
- describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
208
- // describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
209
- // const encryption = {key: rand(32)}
210
- const size = 16
211
- const randomData = rand(size)
212
- const fileSuffix = '' + Date.now()
213
- let fixture
214
- before((done) => {
215
- const name = `test_file_${fileSuffix}`
216
- const opts = Object.assign({}, config.opts, {readOnly: true})
217
- connect(driverName, config.key, config.secret, opts, encryption)
218
- .then(api => { fixture = {api, name, randomData} })
219
- .catch(err => { throw err })
220
- .finally(done)
221
- })
222
- it("should try to write a file and fail", async () => {
223
- const response = await writeRandomFile(fixture, size)
224
- expect(response).to.be.false
225
- })
226
- it("should fail to read the file we did not write", async () => {
227
- try {
228
- await fixture.api.readFile(fixture.name)
229
- assert.fail('should not have been able to read non-existent file that we did not write')
230
- } catch (e) {
231
- if (!(e instanceof MobilettoNotFoundError)) {
232
- assert.fail(`unexpected error: ${e} trying to read non-existent file`)
233
- }
234
- }
235
- })
236
- it("should fail to load metadata on the file we did not write", async () => {
237
- try {
238
- await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
239
- assert.fail('should not have been able to read metadata for non-existent file that we did not write')
240
- } catch (e) {
241
- if (!(e instanceof MobilettoNotFoundError)) {
242
- assert.fail(`unexpected error: ${e} trying to read metadata on non-existent file`)
243
- }
244
- }
245
- })
246
- it("should fail to delete the file we did not write", async () => {
247
- const removed = await fixture.api.remove(fixture.name)
248
- expect(!!removed).to.be.false
249
- })
250
- })
251
- }
253
+ describe(`${driverTest} - write an encrypted file, read file, read metadata, delete file`, () => {
254
+ // some random data, plus a bit extra
255
+ const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
256
+ const randomData = rand(size)
257
+ const fileSuffix = '' + Date.now()
258
+ const encryptionKey = rand(32)
259
+ let fixture
260
+ before((done) => {
261
+ const name = `test_file_${fileSuffix}`
262
+ mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
263
+ .then(api => { fixture = {api, name, randomData} })
264
+ .catch((err) => { throw err })
265
+ .finally(done)
266
+ })
267
+ it("should write an encrypted file", async () => {
268
+ const bytesWritten = await writeRandomFile(fixture, size)
269
+ expect(bytesWritten).is.equal(size, 'expected write API to return correct number of bytes written')
270
+ })
271
+ it("should read the encrypted file we just wrote", async () => {
272
+ const data = await fixture.api.readFile(fixture.name)
273
+ expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
274
+ expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
275
+ })
276
+ it("should load metadata on the encrypted file we just wrote", async () => {
277
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
278
+ })
279
+ it("should delete the encrypted file we just wrote", async () => {
280
+ const removed = await fixture.api.remove(fixture.name)
281
+ expect(!!removed).to.be.true
282
+ })
283
+ it("loading metadata on the encrypted file we wrote now fails", async () => {
284
+ await assertMetaFail(fixture.api, fixture.name)
285
+ })
286
+ })
252
287
 
253
- describe(`${driverTest} - write an encrypted file, read file, read metadata, delete file`, () => {
254
- // some random data, plus a bit extra
255
- const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
256
- const randomData = rand(size)
257
- const fileSuffix = '' + Date.now()
258
- const encryptionKey = rand(32)
288
+ for (const encryption of encryptionTests()) {
289
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
290
+ const mirrorDriver = pickAnotherDriver(driverName)
291
+ // const mirrorDriver = 'local' // for hard-coding the mirror driver
292
+ const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
293
+ const mirrorDest = `mirrorDest_${rand(5)}/`
294
+ describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
295
+ // to force encryption or no encryption, uncomment one of these:
296
+ // const encryption = {key: rand(32)}
297
+ // const encryption = null
298
+
299
+ // a random directory, a subdirectory, and files within it
300
+ const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
301
+ const subdirName = `subdir_` + Date.now()
302
+ const fullSubdirPath = `${randomParent}/${subdirName}`
303
+ const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
304
+ const randomStreamFileSize = READ_SZ * 10
305
+ const randomStreamFileData = crypto.randomBytes(randomStreamFileSize)
306
+ //const fileCount = 3 + Math.floor(Math.random() * 10)
307
+ const fileCount = 2
259
308
  let fixture
260
309
  before((done) => {
261
- const name = `test_file_${fileSuffix}`
262
- mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
263
- .then(api => { fixture = {api, name, randomData} })
310
+ mobiletto(driverName, config.key, config.secret, config.opts, encryption)
311
+ .then(api => {
312
+ // const mirrorEnc = { key: rand(32) }
313
+ const mirrorEnc = null
314
+ mobiletto(mirrorDriver, mirrorConfig.key, mirrorConfig.secret, mirrorConfig.opts, mirrorEnc)
315
+ .then(mirrorApi => {
316
+ fixture = {api, mirrorApi, name: randomPath}
317
+ })
318
+ .catch((err) => {
319
+ throw err
320
+ })
321
+ .finally(done)
322
+ })
264
323
  .catch((err) => { throw err })
265
- .finally(done)
266
324
  })
267
- it("should write an encrypted file", async () => {
268
- const bytesWritten = await writeRandomFile(fixture, size)
269
- expect(bytesWritten).is.equal(size, 'expected write API to return correct number of bytes written')
325
+ it(`should write ${fileCount} files in a new directory`, async () => {
326
+ // return one chunk of random data
327
+ function* dataGenerator() { yield rand(READ_SZ) }
328
+ for (let i = 0; i < fileCount; i++) {
329
+ const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
330
+ expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
331
+ }
270
332
  })
271
- it("should read the encrypted file we just wrote", async () => {
272
- const data = await fixture.api.readFile(fixture.name)
273
- expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
274
- expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
333
+ it('should write a file in a new directory using a stream', async () => {
334
+ // create a random temp file, write it
335
+ const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`
336
+ fs.writeFileSync(tempFile, randomStreamFileData)
337
+ const reader = fs.createReadStream(tempFile)
338
+ const streamFile = fixture.name + '_stream'
339
+ const bytesWritten = await fixture.api.write(streamFile, reader)
340
+ expect(bytesWritten).to.equal(randomStreamFileSize, 'expected write API to return correct number of bytes written for STREAM')
275
341
  })
276
- it("should load metadata on the encrypted file we just wrote", async () => {
277
- await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
342
+ it("should successfully read and then remove the file that was stream-written", async () => {
343
+ const streamFile = fixture.name + '_stream'
344
+ const singleFile = (await fixture.api.readFile(streamFile))
345
+ expect(singleFile).to.have.lengthOf(randomStreamFileSize)
346
+ for (let i = 0; i < singleFile.length; i++) {
347
+ expect(singleFile[i]).to.equal(randomStreamFileData[i])
348
+ }
349
+ const removal = await fixture.api.remove(streamFile)
350
+ expect(removal).to.equal(streamFile, `expected stream file to be deleted, but was: ${removal}`)
278
351
  })
279
- it("should delete the encrypted file we just wrote", async () => {
280
- const removed = await fixture.api.remove(fixture.name)
352
+ it("should successfully list a file individually", async () => {
353
+ const singleFile = await fixture.api.list(fixture.name)
354
+ expect(singleFile).to.have.lengthOf(1)
355
+ expect(singleFile[0]).to.have.property('name', fixture.name)
356
+ })
357
+ it("should successfully list files in the base directory", async () => {
358
+ const baseListing = await fixture.api.list()
359
+ expect(baseListing).to.have.lengthOf.greaterThanOrEqual(1)
360
+ expect(baseListing.find(o => o.name === dirname(randomParent) || o.name === dirname(randomParent)+'/')).to.not.be.null
361
+ })
362
+ it("should successfully list files in a subdirectory, with or without a trailing slash", async () => {
363
+ const noSlash = await fixture.api.list(randomParent)
364
+ expect(noSlash).to.have.lengthOf.greaterThanOrEqual(1)
365
+ const withSlash = await fixture.api.list(randomParent+'/')
366
+ expect(withSlash).to.have.lengthOf.greaterThanOrEqual(1)
367
+ for (const f of noSlash) {
368
+ expect(withSlash.find(o => o.name === f.name)).to.not.be.null
369
+ }
370
+ for (const f of withSlash) {
371
+ expect(noSlash.find(o => o.name === f.name)).to.not.be.null
372
+ }
373
+ })
374
+ it("should successfully recursively list files and see all correct subdirectories and files", async () => {
375
+ const allFiles = await fixture.api.list(randomParent, { recursive: true })
376
+ expect(allFiles).to.have.lengthOf(fileCount + 1)
377
+ expect(allFiles.find(f => f.name === fullSubdirPath && f.type === M_DIR)).to.not.be.null
378
+ expect(allFiles.find(f => f.name === fixture.name && f.type === M_FILE)).to.not.be.null
379
+ for (let i = 0; i < fileCount; i++) {
380
+ expect(allFiles.find(f => f.name === tempFilename(fixture.name, i) && f.type === M_FILE)).to.not.be.null
381
+ }
382
+ })
383
+ it("should load metadata for one of the new files", async () => {
384
+ await assertMeta(fixture.api, fixture.name, READ_SZ)
385
+ })
386
+ it("should see correct types on objects returned from a listing of the new directory", async () => {
387
+ const objects = await fixture.api.list(randomParent)
388
+ expect(objects).to.have.lengthOf(1)
389
+ expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
390
+ })
391
+ it("should see correct types on objects returned from a listing of the subdirectory", async () => {
392
+ const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
393
+ expect(objects).to.have.lengthOf(fileCount)
394
+ for (let i = 0; i < fileCount; i++) {
395
+ // we should find all the files, and they should all have the correct type
396
+ expect(objects
397
+ .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
398
+ ).to.not.be.null
399
+ }
400
+ })
401
+ it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
402
+ const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
403
+ expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
404
+ expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
405
+ })
406
+ it("should see the same files mirrored both places", async () => {
407
+ const originalThings = await fixture.api.list(fullSubdirPath)
408
+ const mirrorPath = `${mirrorDest}${subdirName}`
409
+ const mirrorThings = await fixture.mirrorApi.list(mirrorPath)
410
+ expect(originalThings.length === mirrorThings.length).to.be.true
411
+
412
+ const intersection = originalThings
413
+ .filter(orig => mirrorThings
414
+ .find(m => basename(m.name) === basename(orig.name)))
415
+ expect(intersection.length === originalThings.length).to.be.true
416
+ })
417
+ it("loading metadata on a mirrored file succeeds", async () => {
418
+ const destPath = mirrorDest + subdirName + '/' + basename(fixture.name)
419
+ await assertMeta(fixture.mirrorApi, destPath, READ_SZ)
420
+ })
421
+ it("should recursively delete the mirrored data", async () => {
422
+ const recursive = true
423
+ const removed = await fixture.mirrorApi.remove(mirrorDest, {recursive})
424
+ expect(!!removed).to.be.true
425
+ })
426
+ it("should recursively delete the directory and file we just created", async () => {
427
+ const recursive = true
428
+ const removed = await fixture.api.remove(randomParent, {recursive})
281
429
  expect(!!removed).to.be.true
282
430
  })
283
- it("loading metadata on the encrypted file we wrote now fails", async () => {
431
+ it("loading metadata on the file we wrote now fails", async () => {
284
432
  await assertMetaFail(fixture.api, fixture.name)
285
433
  })
286
- })
287
-
288
- for (const encryption of encryptionTests()) {
289
- const encDesc = encryption ? '(with encryption)' : '(without encryption)'
290
- const mirrorDriver = pickAnotherDriver(driverName)
291
- // const mirrorDriver = 'local' // for hard-coding the mirror driver
292
- const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
293
- const mirrorDest = `mirrorDest_${rand(5)}/`
294
- describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
295
- // describe(`${driverTest} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
296
- // const encryption = {key: rand(32)}
297
- // const encryption = null
298
- // a random directory and file within it
299
- const randomParent = `testRPD_${rand(2)}/rand_${rand(4)}`
300
- const subdirName = `subdir_` + Date.now()
301
- const fullSubdirPath = `${randomParent}/${subdirName}`
302
- const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
303
- const randomStreamFileSize = READ_SZ * 10
304
- const randomStreamFileData = crypto.randomBytes(randomStreamFileSize)
305
- //const fileCount = 3 + Math.floor(Math.random() * 10)
306
- const fileCount = 2
307
- let fixture
308
- before((done) => {
309
- mobiletto(driverName, config.key, config.secret, config.opts, encryption)
310
- .then(api => {
311
- // const mirrorEnc = { key: rand(32) }
312
- const mirrorEnc = null
313
- mobiletto(mirrorDriver, mirrorConfig.key, mirrorConfig.secret, mirrorConfig.opts, mirrorEnc)
314
- .then(mirrorApi => {
315
- fixture = {api, mirrorApi, name: randomPath}
316
- })
317
- .catch((err) => {
318
- throw err
319
- })
320
- .finally(done)
321
- })
322
- .catch((err) => { throw err })
323
- })
324
- it(`should write ${fileCount} files in a new directory`, async () => {
325
- // return one chunk of random data
326
- function* dataGenerator() { yield rand(READ_SZ) }
327
- for (let i = 0; i < fileCount; i++) {
328
- const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
329
- expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
330
- }
331
- })
332
- it('should write a file in a new directory using a stream', async () => {
333
- // create a random temp file, write it
334
- const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`
335
- fs.writeFileSync(tempFile, randomStreamFileData)
336
- const reader = fs.createReadStream(tempFile)
337
- const streamFile = fixture.name + '_stream'
338
- const bytesWritten = await fixture.api.write(streamFile, reader)
339
- expect(bytesWritten).to.equal(randomStreamFileSize, 'expected write API to return correct number of bytes written for STREAM')
340
- })
341
- it("should successfully read the file back that was stream-written", async () => {
342
- const streamFile = fixture.name + '_stream'
343
- const singleFile = (await fixture.api.readFile(streamFile))
344
- expect(singleFile).to.have.lengthOf(randomStreamFileSize)
345
- for (let i = 0; i < singleFile.length; i++) {
346
- expect(singleFile[i]).to.equal(randomStreamFileData[i])
347
- }
348
- const removal = await fixture.api.remove(streamFile)
349
- expect(removal).to.equal(streamFile, `expected stream file to be deleted, but was: ${removal}`)
350
- })
351
- it("should successfully list a file individually", async () => {
352
- const singleFile = await fixture.api.list(fixture.name)
353
- expect(singleFile).to.have.lengthOf(1)
354
- expect(singleFile[0]).to.have.property('name', fixture.name)
355
- })
356
- it("should successfully list files in the base directory", async () => {
357
- const baseListing = await fixture.api.list()
358
- expect(baseListing).to.have.lengthOf.greaterThanOrEqual(1)
359
- expect(baseListing.find(o => o.name === dirname(randomParent) || o.name === dirname(randomParent)+'/')).to.not.be.null
360
- })
361
- it("should successfully list files in a subdirectory, with or without a trailing slash", async () => {
362
- const noSlash = await fixture.api.list(randomParent)
363
- expect(noSlash).to.have.lengthOf.greaterThanOrEqual(1)
364
- const withSlash = await fixture.api.list(randomParent+'/')
365
- expect(withSlash).to.have.lengthOf.greaterThanOrEqual(1)
366
- for (const f of noSlash) {
367
- expect(withSlash.find(o => o.name === f.name)).to.not.be.null
368
- }
369
- for (const f of withSlash) {
370
- expect(noSlash.find(o => o.name === f.name)).to.not.be.null
371
- }
372
- })
373
- it("should load metadata for one of the new files", async () => {
374
- await assertMeta(fixture.api, fixture.name, READ_SZ)
375
- })
376
- it("should see correct types on objects returned from a listing of the new directory", async () => {
377
- const objects = await fixture.api.list(randomParent)
378
- expect(objects).to.have.lengthOf(1)
379
- expect(objects[0]).to.have.property('type', M_DIR, `subdir should have type ${M_DIR}`)
380
- })
381
- it("should see correct types on objects returned from a listing of the subdirectory", async () => {
382
- const objects = await fixture.api.list(`${randomParent}/${subdirName}`)
383
- expect(objects).to.have.lengthOf(fileCount)
384
- for (let i = 0; i < fileCount; i++) {
385
- // we should find all the files, and they should all have the correct type
386
- expect(objects
387
- .find(o => (o.type === M_FILE && o.name === tempFilename(fixture.name, i)))
388
- ).to.not.be.null
389
- }
390
- })
391
- it(`should mirror its subdirectory (${driverName}/${randomParent}) to another place (${mirrorDriver}/${mirrorDest})`, async () => {
392
- const results = await fixture.mirrorApi.mirror(fixture.api, mirrorDest, randomParent)
393
- expect(results).to.have.property('errors', 0, 'expected no errors in mirroring')
394
- expect(results).to.have.property('success').greaterThan(0, 'expected some successes in mirroring')
395
- })
396
- it("should see the same files mirrored both places", async () => {
397
- const originalThings = await fixture.api.list(fullSubdirPath)
398
- const mirrorPath = `${mirrorDest}${subdirName}`
399
- const mirrorThings = await fixture.mirrorApi.list(mirrorPath)
400
- expect(originalThings.length === mirrorThings.length).to.be.true
401
-
402
- const intersection = originalThings
403
- .filter(orig => mirrorThings
404
- .find(m => basename(m.name) === basename(orig.name)))
405
- expect(intersection.length === originalThings.length).to.be.true
406
- })
407
- it("loading metadata on a mirrored file succeeds", async () => {
408
- const destPath = mirrorDest + subdirName + '/' + basename(fixture.name)
409
- await assertMeta(fixture.mirrorApi, destPath, READ_SZ)
410
- })
411
- it("should recursively delete the mirrored data", async () => {
412
- const recursive = true
413
- const removed = await fixture.mirrorApi.remove(mirrorDest, {recursive})
414
- expect(!!removed).to.be.true
415
- })
416
- it("should recursively delete the directory and file we just created", async () => {
417
- const recursive = true
418
- const removed = await fixture.api.remove(randomParent, {recursive})
419
- expect(!!removed).to.be.true
420
- })
421
- it("loading metadata on the file we wrote now fails", async () => {
422
- await assertMetaFail(fixture.api, fixture.name)
423
- })
424
- it("loading metadata on a mirrored file now fails", async () => {
425
- const destPath = mirrorDest + subdirName + '/' + basename(fixture.name)
426
- await assertMetaFail(fixture.mirrorApi, destPath)
427
- })
428
- it("loading metadata on the parent dir we wrote now fails", async () => {
429
- await assertMetaFail(fixture.api, randomParent)
430
- })
434
+ it("loading metadata on a mirrored file now fails", async () => {
435
+ const destPath = mirrorDest + subdirName + '/' + basename(fixture.name)
436
+ await assertMetaFail(fixture.mirrorApi, destPath)
431
437
  })
432
- }
433
-
434
- describe(`${driverTest} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
435
- it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
436
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
437
- try {
438
- await api.read(nonexistentFile, () => {
439
- assert.fail(`ERROR: reading non-existent file should NOT succeed`)
440
- })
441
- } catch (err) {
442
- expect(err).to.be.instanceof(MobilettoNotFoundError, `expected MobilettoNotFoundError, got: ${err}`)
443
- }
438
+ it("loading metadata on the parent dir we wrote now fails", async () => {
439
+ await assertMetaFail(fixture.api, randomParent)
444
440
  })
445
441
  })
442
+ }
446
443
 
447
- describe(`${driverTest} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
448
- it("should throw MobilettoNotFoundError when trying to read metadata for a file that does not exist", async () => {
449
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
450
- try {
451
- const meta = await api.metadata(nonexistentFile)
452
- assert.fail(`ERROR: reading metadata for non-existent file should NOT succeed, but it returned ${meta ? JSON.stringify(meta) : 'null or undefined'}`)
453
- } catch (err) {
454
- expect(err).to.be.instanceof(MobilettoNotFoundError, `expected MobilettoNotFoundError, got: ${err}`)
455
- }
456
- })
444
+ describe(`${driverTest} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
445
+ it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
446
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
447
+ try {
448
+ await api.read(nonexistentFile, () => {
449
+ assert.fail(`ERROR: reading non-existent file should NOT succeed`)
450
+ })
451
+ } catch (err) {
452
+ expect(err).to.be.instanceof(MobilettoNotFoundError, `expected MobilettoNotFoundError, got: ${err}`)
453
+ }
457
454
  })
455
+ })
458
456
 
459
- describe(`${driverTest} - expect MobilettoNotFoundError when removing non-existent file`, () => {
460
- it("should throw MobilettoNotFoundError when trying to remove a file that does not exist", async () => {
461
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
462
- try {
463
- if (await api.remove(nonexistentFile)) {
464
- assert.fail('ERROR: removing non-existent file should NOT succeed')
465
- }
466
- } catch (err) {
467
- expect(err).to.be.instanceof(MobilettoNotFoundError, `expected MobilettoNotFoundError, got: ${err}`)
468
- }
469
- })
457
+ describe(`${driverTest} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
458
+ it("should throw MobilettoNotFoundError when trying to read metadata for a file that does not exist", async () => {
459
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
460
+ try {
461
+ const meta = await api.metadata(nonexistentFile)
462
+ assert.fail(`ERROR: reading metadata for non-existent file should NOT succeed, but it returned ${meta ? JSON.stringify(meta) : 'null or undefined'}`)
463
+ } catch (err) {
464
+ expect(err).to.be.instanceof(MobilettoNotFoundError, `expected MobilettoNotFoundError, got: ${err}`)
465
+ }
470
466
  })
467
+ })
471
468
 
472
- describe(`${driverTest} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
473
- it("should NOT throw MobilettoNotFoundError when quietly removing a file that does not exist", async () => {
474
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
475
- if (!(await api.remove(nonexistentFile, {quiet: true}))) {
476
- assert.fail('ERROR: quietly removing non-existent file should still return true')
469
+ describe(`${driverTest} - expect MobilettoNotFoundError when removing non-existent file`, () => {
470
+ it("should throw MobilettoNotFoundError when trying to remove a file that does not exist", async () => {
471
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
472
+ try {
473
+ if (await api.remove(nonexistentFile)) {
474
+ assert.fail('ERROR: removing non-existent file should NOT succeed')
477
475
  }
478
- })
476
+ } catch (err) {
477
+ expect(err).to.be.instanceof(MobilettoNotFoundError, `expected MobilettoNotFoundError, got: ${err}`)
478
+ }
479
479
  })
480
+ })
480
481
 
482
+ describe(`${driverTest} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
483
+ it("should NOT throw MobilettoNotFoundError when quietly removing a file that does not exist", async () => {
484
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
485
+ if (!(await api.remove(nonexistentFile, {quiet: true}))) {
486
+ assert.fail('ERROR: quietly removing non-existent file should still return true')
487
+ }
488
+ })
481
489
  })
490
+
491
+ })
482
492
  }
483
493
  }
484
494
 
package/util/logger.js CHANGED
@@ -1,7 +1,7 @@
1
1
  const winston = require('winston')
2
2
 
3
3
  const logger = winston.createLogger({
4
- level: process.env.MOBILETTO_LOG_LEVEL || 'debug',
4
+ level: process.env.MOBILETTO_LOG_LEVEL || 'error',
5
5
  format: winston.format.simple(),
6
6
  transports: process.env.MOBILETTO_LOG_FILE
7
7
  ? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]