mobiletto 1.0.46 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  // To run the tests, you need a .env file one level above this directory
2
- // that contains env vars for all the process.env stuff in DRIVER_CONFIG below
2
+ require('dotenv').config()
3
+
3
4
  const fs = require('fs')
4
5
  const { basename, dirname } = require('path')
5
- require('dotenv').config()
6
6
 
7
7
  const randomstring = require('randomstring')
8
8
  const crypto = require('crypto')
@@ -14,25 +14,14 @@ const {
14
14
  M_DIR, M_FILE,
15
15
  mobiletto, connect, MobilettoNotFoundError, readStream
16
16
  } = require("../index")
17
- const {
18
- DEFAULT_CRYPT_ALGO,
19
- normalizeKey, normalizeIV,
20
- encrypt, decrypt,
21
- getCipher, getDecipher
22
- } = require("../util/crypt")
23
- const crypt = require("../util/crypt");
24
- const {logger} = require("../util/logger");
17
+
18
+ const { logger } = require("../util/logger")
25
19
 
26
20
  // chunk size used by generator function, used by driver's 'write' function
27
21
  // the temp file is also TEMP_SZ_MULTIPLE of this number
28
22
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
29
23
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
30
24
 
31
- // encoded bytes written will differ from actual bytes provided
32
- // likewise, plaintext bytes read will differ from encoded bytes read
33
- // our tests give some leeway when filesize differences
34
- const ENC_SIZE_CLOSE_ENOUGH_PERCENT = 0.10
35
-
36
25
  DRIVER_CONFIG = {
37
26
  local: {
38
27
  key: process.env.LOCAL_STORAGE || '/tmp'
@@ -42,9 +31,17 @@ DRIVER_CONFIG = {
42
31
  secret: process.env.S3_SECRET,
43
32
  opts: {
44
33
  bucket: process.env.S3_BUCKET,
45
- prefix: process.env.S3_PREFIX,
34
+ prefix: process.env.S3_PREFIX || 'mobiletto_test',
46
35
  region: process.env.S3_REGION
47
36
  }
37
+ },
38
+ b2: {
39
+ key: process.env.B2_KEY_ID,
40
+ secret: process.env.B2_KEY_SECRET,
41
+ opts: {
42
+ bucket: process.env.B2_BUCKET,
43
+ prefix: process.env.B2_PREFIX || 'mobiletto_test'
44
+ }
48
45
  }
49
46
  }
50
47
  DRIVER_NAMES = Object.keys(DRIVER_CONFIG)
@@ -102,98 +99,63 @@ async function writeRandomFile(fixture, size) {
102
99
  return await fixture.api.write(fixture.name, dataGenerator())
103
100
  }
104
101
 
105
- describe('crypto test', () => {
106
- const enc = {
107
- key: normalizeKey(randomstring.generate(32)),
108
- iv: normalizeIV(randomstring.generate(16)),
109
- algo: DEFAULT_CRYPT_ALGO
110
- }
111
- it("should encrypt and decrypt a data buffer successfully", async () => {
112
- const plaintext = randomstring.generate((64 * 1024) + Math.floor(1024*Math.random()))
113
- const ciphertext = encrypt(plaintext, enc)
114
- const decrypted = decrypt(ciphertext, enc)
115
- expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
116
- })
117
- it("should encrypt and decrypt a stream successfully", async () => {
118
- const plaintext = crypto.randomBytes(81920)
119
- const cipher = getCipher(enc)
120
- const updateData = cipher.update(plaintext)
121
- const finalData = cipher.final()
122
- const encrypted = Buffer.concat([updateData, finalData])
123
-
124
- const decipher = getDecipher(enc)
125
- const decryptUpdate = decipher.update(encrypted)
126
- const decipherFinal = decipher.final()
127
- const decrypted = Buffer.concat([decryptUpdate, decipherFinal])
128
- expect(decrypted.length).to.equal(plaintext.length)
129
- for (let i = 0; i < decrypted.length; i++ ) {
130
- expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
131
- }
132
- })
133
- it("should encrypt and decrypt a stream successfully using readStream", async () => {
134
- const plaintext = crypto.randomBytes(81920)
135
- const cipher = getCipher(enc)
136
- const updateData = cipher.update(plaintext)
137
- const finalData = cipher.final()
138
- const encrypted = Buffer.concat([updateData, finalData])
139
-
140
- const tempFile = `/tmp/stream_foo_${Date.now}.tmp`
141
- fs.writeFileSync(tempFile, encrypted)
142
-
143
- const rs = fs.createReadStream(tempFile)
144
- const decipher = getDecipher(enc)
145
- const buffers = []
146
- await readStream(rs, (chunk) => {
147
- buffers.push(decipher.update(chunk))
148
- }, () => {
149
- buffers.push(decipher.final())
150
- })
151
- const decrypted = Buffer.concat(buffers)
152
- expect(decrypted.length).to.equal(plaintext.length)
153
- for (let i = 0; i < decrypted.length; i++ ) {
154
- expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
155
- }
156
- })
157
-
158
- })
159
-
160
102
  const encryptionTests = () => [null, { key: rand(32) }]
103
+ // const encryptionTests = () => [null]
104
+ // const encryptionTests = () => [{ key: rand(32) }]
161
105
 
162
- // To test a single driver:
163
- // - Uncomment one of the lines below to set driverName to the one you want to test
164
- // - Comment out the next `for` line, and its closing curly brace (just before EOF)
165
-
166
- // const driverName = 'local'
167
- // const driverName = 's3'
106
+ const REDIS_ENABLED = {
107
+ name: 'redis-enabled',
108
+ enabled: true,
109
+ config: cfg => Object.assign({}, cfg)
110
+ }
111
+ const REDIS_DISABLED = {
112
+ name: 'redis-disabled',
113
+ enabled: true,
114
+ config: (cfg) => {
115
+ const newConfig = Object.assign({}, cfg)
116
+ if (!newConfig.opts) { newConfig.opts = {} }
117
+ if (!newConfig.opts.redisConfig) { newConfig.opts.redisConfig = {} }
118
+ newConfig.opts.redisConfig.enabled = false
119
+ return newConfig
120
+ }
121
+ }
122
+ const redisTests = () => [REDIS_ENABLED, REDIS_DISABLED]
168
123
 
169
- // When you're NOT testing a single driver, it's important that *all* tests remain
170
- // within the singular 'for' loop below. This ensures that all drivers pass exactly
171
- // the same set of tests, with exactly the same inputs/outputs.
172
- // This greatly supports developer sanity
124
+ for (const redisSetup of redisTests()) {
125
+ if (!redisSetup.enabled) continue
173
126
 
174
127
  for (const driverName of DRIVER_NAMES) {
175
- const config = DRIVER_CONFIG[driverName]
128
+ // For testing single drivers
129
+ if (driverName !== 'b2') continue
130
+
131
+ const driverTest = `${driverName} [${redisSetup.name}]`
132
+ const config = redisSetup.config(DRIVER_CONFIG[driverName])
176
133
  const nonexistentFile = 'random_file_that_does_not_exist_' + rand(100) + '_' + Date.now()
177
134
  const tempFilename = (name, i) => name + (i > 0 ? '_' + i : '')
178
- describe(`${driverName} test`, () => {
135
+ describe(`${driverTest} test`, () => {
179
136
  beforeEach ((done) => {
180
- redis.flushMobiletto().then(
181
- () => { logger.info(`${driverName} - flushed redis`) },
137
+ redis.flushAll().then(
138
+ () => { logger.info(`${driverTest} - flushed redis`) },
182
139
  (e) => {
183
- logger.error(`${driverName} - error flushing redis: ${e}`)
140
+ logger.error(`${driverTest} - error flushing redis: ${e}`)
184
141
  throw e
185
142
  }
186
143
  ).finally(() => done())
187
144
  })
188
145
 
189
- describe(`${driverName} - create api client`, () => {
146
+ describe(`${driverTest} - create api client`, () => {
190
147
  it("should validate the config and return an API object", async () => {
191
- const api = await mobiletto(driverName, config.key, config.secret, config.opts)
192
- should().exist(api, 'expected API object to exist')
148
+ try {
149
+ const api = await mobiletto(driverName, config.key, config.secret, config.opts)
150
+ should().exist(api, 'expected API object to exist')
151
+ } catch (e) {
152
+ logger.error(`error creating API: ${e} ${JSON.stringify(e)}`)
153
+ throw e
154
+ }
193
155
  })
194
156
  })
195
157
 
196
- describe(`${driverName} - listing with no arguments returns some results`, () => {
158
+ describe(`${driverTest} - listing with no arguments returns some results`, () => {
197
159
  it("should return some results from a default listing", async () => {
198
160
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
199
161
  const results = await api.list()
@@ -201,7 +163,7 @@ for (const driverName of DRIVER_NAMES) {
201
163
  })
202
164
  })
203
165
 
204
- describe(`${driverName} - write a file, read file, read metadata, delete file`, () => {
166
+ describe(`${driverTest} - write a file, read file, read metadata, delete file`, () => {
205
167
  // some random data, plus a bit extra
206
168
  const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
207
169
  const randomData = rand(size)
@@ -218,12 +180,12 @@ for (const driverName of DRIVER_NAMES) {
218
180
  expect(response).to.equal(size, 'expected write API to return correct number of bytes written')
219
181
  })
220
182
  it("should read the file we just wrote", async () => {
221
- const data = await fixture.api.readFile(fixture.name);
183
+ const data = await fixture.api.readFile(fixture.name)
222
184
  expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
223
185
  expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
224
186
  })
225
187
  it("should load metadata on the file we just wrote", async () => {
226
- await assertMeta (fixture.api, fixture.name, fixture.randomData.length)
188
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
227
189
  })
228
190
  it("should delete the file we just wrote", async () => {
229
191
  const removed = await fixture.api.remove(fixture.name)
@@ -239,8 +201,8 @@ for (const driverName of DRIVER_NAMES) {
239
201
 
240
202
  for (const encryption of encryptionTests()) {
241
203
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
242
- describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
243
- // describe(`${driverName} - ENC fail to write and delete files in readOnly mode`, () => {
204
+ describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
205
+ // describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
244
206
  // const encryption = {key: rand(32)}
245
207
  const size = 16
246
208
  const randomData = rand(size)
@@ -285,7 +247,7 @@ for (const driverName of DRIVER_NAMES) {
285
247
  })
286
248
  }
287
249
 
288
- describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
250
+ describe(`${driverTest} - write an encrypted file, read file, read metadata, delete file`, () => {
289
251
  // some random data, plus a bit extra
290
252
  const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
291
253
  const randomData = rand(size)
@@ -300,16 +262,16 @@ for (const driverName of DRIVER_NAMES) {
300
262
  .finally(done)
301
263
  })
302
264
  it("should write an encrypted file", async () => {
303
- const bytesWritten = await writeRandomFile(fixture, size);
265
+ const bytesWritten = await writeRandomFile(fixture, size)
304
266
  expect(bytesWritten).is.equal(size, 'expected write API to return correct number of bytes written')
305
267
  })
306
268
  it("should read the encrypted file we just wrote", async () => {
307
- const data = await fixture.api.readFile(fixture.name);
269
+ const data = await fixture.api.readFile(fixture.name)
308
270
  expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
309
271
  expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
310
272
  })
311
273
  it("should load metadata on the encrypted file we just wrote", async () => {
312
- await assertMeta(fixture.api, fixture.name, fixture.randomData.length, ENC_SIZE_CLOSE_ENOUGH_PERCENT)
274
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
313
275
  })
314
276
  it("should delete the encrypted file we just wrote", async () => {
315
277
  const removed = await fixture.api.remove(fixture.name)
@@ -322,11 +284,12 @@ for (const driverName of DRIVER_NAMES) {
322
284
 
323
285
  for (const encryption of encryptionTests()) {
324
286
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
325
- const mirrorDriver = pickAnotherDriver(driverName)
287
+ // const mirrorDriver = pickAnotherDriver(driverName)
288
+ const mirrorDriver = 'local' // for hard-coding the mirror driver
326
289
  const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
327
290
  const mirrorDest = `mirrorDest_${rand(5)}/`
328
- describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
329
- // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
291
+ describe(`${driverTest} - ${encDesc} write files in a new dir, read metadata, mirror to another place (${mirrorDriver}), verify mirror, recursively delete from both`, () => {
292
+ // describe(`${driverTest} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
330
293
  // const encryption = {key: rand(32)}
331
294
  // const encryption = null
332
295
  // a random directory and file within it
@@ -334,7 +297,7 @@ for (const driverName of DRIVER_NAMES) {
334
297
  const subdirName = `subdir_` + Date.now()
335
298
  const fullSubdirPath = `${randomParent}/${subdirName}`
336
299
  const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
337
- const randomStreamFileSize = READ_SZ * 10;
300
+ const randomStreamFileSize = READ_SZ * 10
338
301
  const randomStreamFileData = crypto.randomBytes(randomStreamFileSize)
339
302
  //const fileCount = 3 + Math.floor(Math.random() * 10)
340
303
  const fileCount = 2
@@ -365,7 +328,7 @@ for (const driverName of DRIVER_NAMES) {
365
328
  })
366
329
  it('should write a file in a new directory using a stream', async () => {
367
330
  // create a random temp file, write it
368
- const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`;
331
+ const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`
369
332
  fs.writeFileSync(tempFile, randomStreamFileData)
370
333
  const reader = fs.createReadStream(tempFile)
371
334
  const streamFile = fixture.name + '_stream'
@@ -373,13 +336,14 @@ for (const driverName of DRIVER_NAMES) {
373
336
  expect(bytesWritten).to.equal(randomStreamFileSize, 'expected write API to return correct number of bytes written for STREAM')
374
337
  })
375
338
  it("should successfully read the file back that was stream-written", async () => {
376
- const streamFile = fixture.name + '_stream';
339
+ const streamFile = fixture.name + '_stream'
377
340
  const singleFile = (await fixture.api.readFile(streamFile))
378
341
  expect(singleFile).to.have.lengthOf(randomStreamFileSize)
379
342
  for (let i = 0; i < singleFile.length; i++) {
380
343
  expect(singleFile[i]).to.equal(randomStreamFileData[i])
381
344
  }
382
- expect(!!fixture.api.remove(streamFile)).to.be.true
345
+ const removal = await fixture.api.remove(streamFile)
346
+ expect(removal).to.equal(streamFile, `expected stream file to be deleted, but was: ${removal}`)
383
347
  })
384
348
  it("should successfully list a file individually", async () => {
385
349
  const singleFile = await fixture.api.list(fixture.name)
@@ -389,7 +353,7 @@ for (const driverName of DRIVER_NAMES) {
389
353
  it("should successfully list files in the base directory", async () => {
390
354
  const baseListing = await fixture.api.list()
391
355
  expect(baseListing).to.have.lengthOf.greaterThanOrEqual(1)
392
- expect(baseListing.find(o => o.name === dirname(randomParent))).to.not.be.null
356
+ expect(baseListing.find(o => o.name === dirname(randomParent) || o.name === dirname(randomParent)+'/')).to.not.be.null
393
357
  })
394
358
  it("should successfully list files in a subdirectory, with or without a trailing slash", async () => {
395
359
  const noSlash = await fixture.api.list(randomParent)
@@ -404,7 +368,7 @@ for (const driverName of DRIVER_NAMES) {
404
368
  }
405
369
  })
406
370
  it("should load metadata for one of the new files", async () => {
407
- await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
371
+ await assertMeta(fixture.api, fixture.name, READ_SZ)
408
372
  })
409
373
  it("should see correct types on objects returned from a listing of the new directory", async () => {
410
374
  const objects = await fixture.api.list(randomParent)
@@ -464,7 +428,7 @@ for (const driverName of DRIVER_NAMES) {
464
428
  })
465
429
  }
466
430
 
467
- describe(`${driverName} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
431
+ describe(`${driverTest} - expect MobilettoNotFoundError when reading nonexistent file `, () => {
468
432
  it("should throw MobilettoNotFoundError when trying to read a file that does not exist", async () => {
469
433
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
470
434
  try {
@@ -477,7 +441,7 @@ for (const driverName of DRIVER_NAMES) {
477
441
  })
478
442
  })
479
443
 
480
- describe(`${driverName} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
444
+ describe(`${driverTest} - expect MobilettoNotFoundError when reading metadata for nonexistent file`, () => {
481
445
  it("should throw MobilettoNotFoundError when trying to read metadata for a file that does not exist", async () => {
482
446
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
483
447
  try {
@@ -489,7 +453,7 @@ for (const driverName of DRIVER_NAMES) {
489
453
  })
490
454
  })
491
455
 
492
- describe(`${driverName} - expect MobilettoNotFoundError when removing non-existent file`, () => {
456
+ describe(`${driverTest} - expect MobilettoNotFoundError when removing non-existent file`, () => {
493
457
  it("should throw MobilettoNotFoundError when trying to remove a file that does not exist", async () => {
494
458
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
495
459
  try {
@@ -502,7 +466,7 @@ for (const driverName of DRIVER_NAMES) {
502
466
  })
503
467
  })
504
468
 
505
- describe(`${driverName} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
469
+ describe(`${driverTest} - quietly removing non-existent file does not throw MobilettoNotFoundError`, () => {
506
470
  it("should NOT throw MobilettoNotFoundError when quietly removing a file that does not exist", async () => {
507
471
  const api = await mobiletto(driverName, config.key, config.secret, config.opts)
508
472
  if (!(await api.remove(nonexistentFile, {quiet: true}))) {
@@ -510,16 +474,12 @@ for (const driverName of DRIVER_NAMES) {
510
474
  }
511
475
  })
512
476
  })
477
+
513
478
  })
514
479
  }
480
+ }
515
481
 
516
- after ((done) => {
517
- try {
518
- logger.info('all tests finished, tearing down redis...')
519
- redis.teardown()
520
- } catch (e) {
521
- logger.warn(`error calling redis.teardown: ${e}`)
522
- } finally {
523
- done()
524
- }
482
+ after ( (done) => {
483
+ logger.info('all tests finished, tearing down redis...')
484
+ redis.teardown().finally(done)
525
485
  })
@@ -0,0 +1,67 @@
1
+ const fs = require('fs')
2
+ const { expect } = require('chai')
3
+ const randomstring = require('randomstring')
4
+ const crypto = require('crypto')
5
+
6
+ const {
7
+ DEFAULT_CRYPT_ALGO,
8
+ normalizeKey, normalizeIV,
9
+ encrypt, decrypt,
10
+ getCipher, getDecipher
11
+ } = require("../util/crypt")
12
+
13
+ const { readStream } = require("../index")
14
+
15
+ describe('crypto test', () => {
16
+ const enc = {
17
+ key: normalizeKey(randomstring.generate(32)),
18
+ iv: normalizeIV(randomstring.generate(16)),
19
+ algo: DEFAULT_CRYPT_ALGO
20
+ }
21
+ it("should encrypt and decrypt a data buffer successfully", async () => {
22
+ const plaintext = randomstring.generate((64 * 1024) + Math.floor(1024*Math.random()))
23
+ const ciphertext = encrypt(plaintext, enc)
24
+ const decrypted = decrypt(ciphertext, enc)
25
+ expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
26
+ })
27
+ it("should encrypt and decrypt a stream successfully", async () => {
28
+ const plaintext = crypto.randomBytes(81920)
29
+ const cipher = getCipher(enc)
30
+ const updateData = cipher.update(plaintext)
31
+ const finalData = cipher.final()
32
+ const encrypted = Buffer.concat([updateData, finalData])
33
+
34
+ const decipher = getDecipher(enc)
35
+ const decryptUpdate = decipher.update(encrypted)
36
+ const decipherFinal = decipher.final()
37
+ const decrypted = Buffer.concat([decryptUpdate, decipherFinal])
38
+ expect(decrypted.length).to.equal(plaintext.length)
39
+ for (let i = 0; i < decrypted.length; i++ ) {
40
+ expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
41
+ }
42
+ })
43
+ it("should encrypt and decrypt a stream successfully using readStream", async () => {
44
+ const plaintext = crypto.randomBytes(81920)
45
+ const cipher = getCipher(enc)
46
+ const updateData = cipher.update(plaintext)
47
+ const finalData = cipher.final()
48
+ const encrypted = Buffer.concat([updateData, finalData])
49
+
50
+ const tempFile = `/tmp/stream_foo_${Date.now}.tmp`
51
+ fs.writeFileSync(tempFile, encrypted)
52
+
53
+ const rs = fs.createReadStream(tempFile)
54
+ const decipher = getDecipher(enc)
55
+ const buffers = []
56
+ await readStream(rs, (chunk) => {
57
+ buffers.push(decipher.update(chunk))
58
+ }, () => {
59
+ buffers.push(decipher.final())
60
+ })
61
+ const decrypted = Buffer.concat(buffers)
62
+ expect(decrypted.length).to.equal(plaintext.length)
63
+ for (let i = 0; i < decrypted.length; i++ ) {
64
+ expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
65
+ }
66
+ })
67
+ })
package/util/logger.js CHANGED
@@ -1,7 +1,7 @@
1
1
  const winston = require('winston')
2
2
 
3
3
  const logger = winston.createLogger({
4
- level: process.env.MOBILETTO_LOG_LEVEL || 'warn',
4
+ level: process.env.MOBILETTO_LOG_LEVEL || 'debug',
5
5
  format: winston.format.simple(),
6
6
  transports: process.env.MOBILETTO_LOG_FILE
7
7
  ? [ new winston.transports.File({ filename: process.env.MOBILETTO_LOG_FILE }) ]