mobiletto 1.0.40 → 1.0.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -5,7 +5,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
5
5
 
6
6
  It supports apps that are agnostic to where files are stored.
7
7
 
8
- All drivers are tested for identical behavior with over 50 tests for each driver.
8
+ All drivers are tested for identical behavior with over 55 tests for each driver.
9
9
 
10
10
  Today the supported drivers are:
11
11
 
@@ -27,7 +27,7 @@ A simple example using the mobiletto `s3` driver.
27
27
  This code would run the same if the driver were `local`.
28
28
 
29
29
  const storage = require('mobiletto')
30
- const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk', region: 'us-east-1', cacheSize: 100})
30
+ const s3 = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
31
31
 
32
32
  // list objects: returns array of metadata objects
33
33
  const listing = await s3.list()
@@ -1,7 +1,7 @@
1
1
  const { dirname } = require('path')
2
2
 
3
3
  const {
4
- M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
4
+ M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable, logger,
5
5
  MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
6
6
  } = require('../../index')
7
7
 
@@ -107,7 +107,7 @@ class StorageClient {
107
107
  }
108
108
  return files
109
109
  } catch (e) {
110
- console.error(`dirFiles: ${e}`)
110
+ logger.error(`dirFiles: ${e}`)
111
111
  throw this.ioError(e, norm, 'dirFiles')
112
112
  }
113
113
  }
@@ -167,7 +167,7 @@ class StorageClient {
167
167
 
168
168
  mkdirs (path) {
169
169
  try {
170
- // console.log(`mkdirs: creating directory: ${path}`)
170
+ logger.debug(`mkdirs: creating directory: ${path}`)
171
171
  fs.mkdirSync(path, {recursive: true, mode: this.dirMode})
172
172
  } catch (err) {
173
173
  throw new MobilettoError(`mkdirs: error creating directory ${path}: ${err}`, err)
@@ -177,7 +177,7 @@ class StorageClient {
177
177
  async normalizePathAndEnsureParentDirs (path) {
178
178
  const file = this.normalizePath(path)
179
179
 
180
- // console.log(`read: reading path: ${path} - ${file}`)
180
+ logger.debug(`read: reading path: ${path} - ${file}`)
181
181
  const parent = dirname(file)
182
182
  let dirStat
183
183
  try {
@@ -199,7 +199,7 @@ class StorageClient {
199
199
 
200
200
  async write (path, generatorOrReadableStream) {
201
201
  const file = await this.normalizePathAndEnsureParentDirs(path)
202
- // console.log(`write: writing path ${path} -> ${file}`)
202
+ logger.debug(`write: writing path ${path} -> ${file}`)
203
203
  const stream = fs.createWriteStream(file, {mode: this.fileMode})
204
204
  const writer = writeStream(stream)
205
205
  const closer = closeStream(stream)
@@ -247,7 +247,7 @@ class StorageClient {
247
247
 
248
248
  async read (path, callback, endCallback = null) {
249
249
  const file = this.normalizePath(path)
250
- // console.log(`read: reading path: ${path} - ${file}`)
250
+ logger.debug(`read: reading path: ${path} - ${file}`)
251
251
  try {
252
252
  const stream = fs.createReadStream(file)
253
253
  return await readStream(stream, callback, endCallback)
@@ -258,7 +258,7 @@ class StorageClient {
258
258
 
259
259
  async remove (path, recursive, quiet) {
260
260
  const file = this.normalizePath(path)
261
- // console.log(`remove: deleting path: ${path} = ${file}`)
261
+ logger.debug(`remove: deleting path: ${path} = ${file}`)
262
262
  try {
263
263
  fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
264
264
  } catch (err) {
@@ -1,5 +1,5 @@
1
1
  const {
2
- M_DIR, M_FILE,
2
+ M_DIR, M_FILE, logger,
3
3
  MobilettoError, MobilettoNotFoundError, readStream
4
4
  } = require('../../index')
5
5
 
@@ -93,7 +93,7 @@ class StorageClient {
93
93
  }
94
94
  const objects = []
95
95
  let objectCount = 0
96
- // console.log(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
96
+ logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
97
97
 
98
98
  // while loop that runs until 'response.truncated' is false.
99
99
  while (truncated) {
@@ -207,7 +207,7 @@ class StorageClient {
207
207
  })
208
208
  let total = 0
209
209
  uploader.on('httpUploadProgress', (progress) => {
210
- // console.log(`write(${bucketParams.Key}): ${JSON.stringify(progress)}`)
210
+ logger.debug(`write(${bucketParams.Key}): ${JSON.stringify(progress)}`)
211
211
  total += progress.loaded
212
212
  })
213
213
  const response = await uploader.done()
@@ -219,7 +219,7 @@ class StorageClient {
219
219
 
220
220
  async read (path, callback, endCallback = null) {
221
221
  const Key = this.normalizeKey(path)
222
- // console.log(`read: reading Key: ${path} - ${Key}`)
222
+ logger.debug(`read: reading Key: ${path} - ${Key}`)
223
223
  const bucketParams = {
224
224
  Region: this.region,
225
225
  Bucket: this.bucket,
@@ -250,7 +250,7 @@ class StorageClient {
250
250
  Bucket: this.bucket,
251
251
  Delete
252
252
  }
253
- // console.log(`remove(${path}): deleting objects: ${JSON.stringify(objects)}`)
253
+ logger.debug(`remove(${path}): deleting objects: ${JSON.stringify(objects)}`)
254
254
  const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
255
255
  let statusCode = response.$metadata.httpStatusCode
256
256
  let statusClass = Math.floor(statusCode / 100)
package/index.js CHANGED
@@ -1,5 +1,6 @@
1
1
  const fs = require('fs')
2
2
  const path = require('path')
3
+ const { Readable, Transform } = require('stream')
3
4
  const winston = require('winston')
4
5
  const LRU = require('lru-cache')
5
6
  const { basename, dirname } = require('path')
@@ -60,7 +61,7 @@ function MobilettoNotFoundError (message) {
60
61
  const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
61
62
 
62
63
  const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
63
- const isReadable = (thing) => thing instanceof fs.ReadStream
64
+ const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
64
65
 
65
66
  const UTILITY_FUNCTIONS = {
66
67
  list: (client) => async (path, opts) => {
@@ -383,12 +384,12 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
383
384
  metadata: _metadata(client),
384
385
  read: async (path, callback) => {
385
386
  const realPath = encryptPath(path)
386
- const cipher = crypt.startDecryptStream(enc)
387
- return client.read(realPath,
387
+ const cipher = crypt.getDecipher(enc)
388
+ return await client.read(realPath,
388
389
  (chunk) => {
389
- return callback(crypt.updateCryptStream(cipher, chunk))
390
+ return callback(cipher.update(chunk))
390
391
  }, () => {
391
- return callback(crypt.closeCryptStream(cipher))
392
+ callback(cipher.final())
392
393
  })
393
394
  },
394
395
  write: async (path, readFunc) => {
@@ -407,22 +408,42 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
407
408
  }
408
409
  }
409
410
 
411
+ const cipher = crypt.getCipher(enc)
412
+ const realPath = encryptPath(path)
410
413
  let generatorBytes = 0
411
- function* cryptGenerator(plaintextGenerator) {
412
- let chunk = plaintextGenerator.next().value
413
- while (chunk) {
414
- generatorBytes += chunk.length
415
- yield cipher.update(chunk)
416
- chunk = plaintextGenerator.next().value
414
+ const createEncryptStream = (input) =>
415
+ input.pipe(new Transform({
416
+ transform(chunk, encoding, callback) {
417
+ generatorBytes += chunk.length
418
+ this.push(cipher.update(chunk))
419
+ callback()
420
+ },
421
+ flush(callback) {
422
+ this.push(cipher.final())
423
+ callback()
424
+ }
425
+ }))
426
+
427
+ // If we were passed a Readable, call write with a transform method
428
+ if (isReadable(readFunc)) {
429
+ await client.write(realPath, createEncryptStream(readFunc))
430
+ } else {
431
+ // If we were passed a generator, wrap it with another generator that encrypts
432
+ function* cryptGenerator(plaintextGenerator) {
433
+ let chunk = plaintextGenerator.next().value
434
+ while (chunk) {
435
+ generatorBytes += chunk.length
436
+ yield cipher.update(chunk)
437
+ chunk = plaintextGenerator.next().value
438
+ }
439
+ yield cipher.final()
417
440
  }
418
- yield cipher.final()
441
+ await client.write(realPath, cryptGenerator(readFunc))
419
442
  }
420
- const cipher = crypt.startEncryptStream(enc)
421
- const realPath = encryptPath(path)
422
- const streamBytes = await client.write(realPath, isReadable(readFunc) ? readFunc : cryptGenerator(readFunc))
443
+ // write metadata
423
444
  const meta = { name: path, size: generatorBytes, type: M_FILE }
424
445
  await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
425
- return isReadable(readFunc) ? streamBytes : generatorBytes
446
+ return generatorBytes
426
447
  },
427
448
  // todo: remove should return an array of the paths that were actually removed
428
449
  remove: async (path, options) => {
@@ -536,7 +557,7 @@ const M_SPECIAL = 'special'
536
557
  module.exports = {
537
558
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
538
559
  isAsyncGenerator, isReadable,
539
- mobiletto, connect,
560
+ mobiletto, connect, logger,
540
561
  setLogLevel, setLogTransports,
541
562
  MobilettoError, MobilettoNotFoundError,
542
563
  readStream, writeStream, closeStream
package/package.json CHANGED
@@ -1,9 +1,19 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.40",
3
+ "version": "1.0.43",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
- "author": "Jonathan Cobb",
6
+ "keywords": ["storage", "s3", "simple", "filesystem"],
7
+ "homepage": "https://github.com/cobbzilla/mobiletto",
8
+ "author": "Jonathan Cobb <bqppl0m2@duck.com> (https://github.com/cobbzilla)",
9
+ "funding": {
10
+ "type" : "patreon",
11
+ "url" : "https://www.patreon.com/cobbzilla"
12
+ },
13
+ "repository": {
14
+ "type" : "git",
15
+ "url" : "https://github.com/cobbzilla/mobiletto"
16
+ },
7
17
  "license": "Apache-2.0",
8
18
  "scripts": {
9
19
  "test": "mocha --timeout 10000"
@@ -1,15 +1,25 @@
1
1
  // To run the tests, you need a .env file one level above this directory
2
2
  // that contains env vars for all the process.env stuff in DRIVER_CONFIG below
3
3
  const fs = require('fs')
4
+ const { basename, dirname } = require('path')
4
5
  require('dotenv').config()
5
6
 
6
7
  const randomstring = require('randomstring')
7
- const path = require('path')
8
+ const crypto = require('crypto')
8
9
 
9
10
  const { expect, should, assert } = require('chai')
10
11
 
11
- const { mobiletto, connect, MobilettoNotFoundError, M_FILE, M_DIR } = require("../index")
12
- const { encrypt, decrypt, normalizeKey, normalizeIV, DEFAULT_CRYPT_ALGO } = require("../util/crypt")
12
+ const {
13
+ M_DIR, M_FILE,
14
+ mobiletto, connect, MobilettoNotFoundError, readStream
15
+ } = require("../index")
16
+ const {
17
+ DEFAULT_CRYPT_ALGO,
18
+ normalizeKey, normalizeIV,
19
+ encrypt, decrypt,
20
+ getCipher, getDecipher
21
+ } = require("../util/crypt")
22
+ const crypt = require("../util/crypt");
13
23
 
14
24
  // chunk size used by generator function, used by driver's 'write' function
15
25
  // the temp file is also TEMP_SZ_MULTIPLE of this number
@@ -82,17 +92,57 @@ async function writeRandomFile(fixture, size) {
82
92
  }
83
93
 
84
94
  describe('crypto test', () => {
85
- it("should encrypt and decrypt successfully", async () => {
86
- const enc = {
87
- key: normalizeKey(randomstring.generate(32)),
88
- iv: normalizeIV(randomstring.generate(16)),
89
- algo: DEFAULT_CRYPT_ALGO
90
- }
91
- const plaintext = randomstring.generate(1024 + Math.floor(1024*Math.random()))
95
+ const enc = {
96
+ key: normalizeKey(randomstring.generate(32)),
97
+ iv: normalizeIV(randomstring.generate(16)),
98
+ algo: DEFAULT_CRYPT_ALGO
99
+ }
100
+ it("should encrypt and decrypt a data buffer successfully", async () => {
101
+ const plaintext = randomstring.generate((64 * 1024) + Math.floor(1024*Math.random()))
92
102
  const ciphertext = encrypt(plaintext, enc)
93
103
  const decrypted = decrypt(ciphertext, enc)
94
104
  expect(decrypted).to.equal(plaintext, 'decrypted data did not match plaintext')
95
105
  })
106
+ it("should encrypt and decrypt a stream successfully", async () => {
107
+ const plaintext = crypto.randomBytes(81920)
108
+ const cipher = getCipher(enc)
109
+ const updateData = cipher.update(plaintext)
110
+ const finalData = cipher.final()
111
+ const encrypted = Buffer.concat([updateData, finalData])
112
+
113
+ const decipher = getDecipher(enc)
114
+ const decryptUpdate = decipher.update(encrypted)
115
+ const decipherFinal = decipher.final()
116
+ const decrypted = Buffer.concat([decryptUpdate, decipherFinal])
117
+ expect(decrypted.length).to.equal(plaintext.length)
118
+ for (let i = 0; i < decrypted.length; i++ ) {
119
+ expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
120
+ }
121
+ })
122
+ it("should encrypt and decrypt a stream successfully using readStream", async () => {
123
+ const plaintext = crypto.randomBytes(81920)
124
+ const cipher = getCipher(enc)
125
+ const updateData = cipher.update(plaintext)
126
+ const finalData = cipher.final()
127
+ const encrypted = Buffer.concat([updateData, finalData])
128
+
129
+ const tempFile = `/tmp/stream_foo_${Date.now}.tmp`
130
+ fs.writeFileSync(tempFile, encrypted)
131
+
132
+ const rs = fs.createReadStream(tempFile)
133
+ const decipher = getDecipher(enc)
134
+ const buffers = []
135
+ await readStream(rs, (chunk) => {
136
+ buffers.push(decipher.update(chunk))
137
+ }, () => {
138
+ buffers.push(decipher.final())
139
+ })
140
+ const decrypted = Buffer.concat(buffers)
141
+ expect(decrypted.length).to.equal(plaintext.length)
142
+ for (let i = 0; i < decrypted.length; i++ ) {
143
+ expect(decrypted[i]).to.equal(plaintext[i], `decrypted data did not match plaintext at position ${i}`)
144
+ }
145
+ })
96
146
 
97
147
  })
98
148
 
@@ -260,6 +310,8 @@ for (const driverName of DRIVER_NAMES) {
260
310
  const subdirName = `subdir_` + Date.now()
261
311
  const fullSubdirPath = `${randomParent}/${subdirName}`
262
312
  const randomPath = `${fullSubdirPath}/random_file_${Date.now()}`
313
+ const randomStreamFileSize = READ_SZ * 10;
314
+ const randomStreamFileData = crypto.randomBytes(randomStreamFileSize)
263
315
  //const fileCount = 3 + Math.floor(Math.random() * 10)
264
316
  const fileCount = 2
265
317
  let fixture
@@ -289,16 +341,21 @@ for (const driverName of DRIVER_NAMES) {
289
341
  })
290
342
  it('should write a file in a new directory using a stream', async () => {
291
343
  // create a random temp file, write it
292
- const data = randomstring.generate(READ_SZ)
293
344
  const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`;
294
- fs.writeFileSync(tempFile, data)
345
+ fs.writeFileSync(tempFile, randomStreamFileData)
295
346
  const reader = fs.createReadStream(tempFile)
296
- const streamFile = fixture.name + '_stream';
347
+ const streamFile = fixture.name + '_stream'
297
348
  const bytesWritten = await fixture.api.write(streamFile, reader)
298
- expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written for STREAM')
299
- // remove stream file
300
- const removed = await fixture.api.remove(streamFile)
301
- expect(!!removed).to.be.true
349
+ expect(bytesWritten).to.equal(randomStreamFileSize, 'expected write API to return correct number of bytes written for STREAM')
350
+ })
351
+ it("should successfully read the file back that was stream-written", async () => {
352
+ const streamFile = fixture.name + '_stream';
353
+ const singleFile = (await fixture.api.readFile(streamFile))
354
+ expect(singleFile).to.have.lengthOf(randomStreamFileSize)
355
+ for (let i = 0; i < singleFile.length; i++) {
356
+ expect(singleFile[i]).to.equal(randomStreamFileData[i])
357
+ }
358
+ expect(!!fixture.api.remove(streamFile)).to.be.true
302
359
  })
303
360
  it("should successfully list a file individually", async () => {
304
361
  const singleFile = await fixture.api.list(fixture.name)
@@ -308,7 +365,7 @@ for (const driverName of DRIVER_NAMES) {
308
365
  it("should successfully list files in the base directory", async () => {
309
366
  const baseListing = await fixture.api.list()
310
367
  expect(baseListing).to.have.lengthOf.greaterThanOrEqual(1)
311
- expect(baseListing.find(o => o.name === path.dirname(randomParent))).to.not.be.null
368
+ expect(baseListing.find(o => o.name === dirname(randomParent))).to.not.be.null
312
369
  })
313
370
  it("should successfully list files in a subdirectory, with or without a trailing slash", async () => {
314
371
  const noSlash = await fixture.api.list(randomParent)
@@ -353,11 +410,11 @@ for (const driverName of DRIVER_NAMES) {
353
410
 
354
411
  const intersection = originalThings
355
412
  .filter(orig => mirrorThings
356
- .find(m => path.basename(m.name) === path.basename(orig.name)))
413
+ .find(m => basename(m.name) === basename(orig.name)))
357
414
  expect(intersection.length === originalThings.length).to.be.true
358
415
  })
359
416
  it("loading metadata on a mirrored file succeeds", async () => {
360
- const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
417
+ const destPath = mirrorDest + subdirName + '/' + basename(fixture.name)
361
418
  await assertMeta(fixture.mirrorApi, destPath, READ_SZ)
362
419
  })
363
420
  it("should recursively delete the mirrored data", async () => {
@@ -374,7 +431,7 @@ for (const driverName of DRIVER_NAMES) {
374
431
  await assertMetaFail(fixture.api, fixture.name)
375
432
  })
376
433
  it("loading metadata on a mirrored file now fails", async () => {
377
- const destPath = mirrorDest + subdirName + '/' + path.basename(fixture.name)
434
+ const destPath = mirrorDest + subdirName + '/' + basename(fixture.name)
378
435
  await assertMetaFail(fixture.mirrorApi, destPath)
379
436
  })
380
437
  it("loading metadata on the parent dir we wrote now fails", async () => {
package/util/crypt.js CHANGED
@@ -35,7 +35,9 @@ function setDefaultIV (iv) {
35
35
 
36
36
  function getCipher(enc) {
37
37
  try {
38
- return crypto.createCipheriv(enc.algo, enc.key, enc.iv);
38
+ let c = crypto.createCipheriv(enc.algo, enc.key, enc.iv);
39
+ c.setAutoPadding(true)
40
+ return c
39
41
  } catch (e) {
40
42
  winston.error(`getCipher(${enc}): ${e}`)
41
43
  throw e
@@ -55,7 +57,9 @@ function encrypt (plainText, encryption, outputEncoding = 'base64') {
55
57
  }
56
58
 
57
59
  function getDecipher(enc) {
58
- return crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
60
+ let c = crypto.createDecipheriv(enc.algo, enc.key, enc.iv);
61
+ c.setAutoPadding(true)
62
+ return c
59
63
  }
60
64
 
61
65
  function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding = 'base64') {
@@ -67,15 +71,10 @@ function decrypt (cipherText, encryption, outputEncoding = 'utf8', inputEncoding
67
71
  return Buffer.concat([cipher.update(data), cipher.final()]).toString(outputEncoding)
68
72
  }
69
73
 
70
- const startEncryptStream = (enc) => getCipher(enc)
71
- const startDecryptStream = (enc) => getDecipher(enc)
72
- const updateCryptStream = (cipher, data) => cipher.update(data)
73
- const closeCryptStream = (cipher) => cipher.final()
74
-
75
74
  module.exports = {
76
75
  DEFAULT_CRYPT_ALGO,
77
76
  setDefaultKey, setDefaultIV,
78
77
  encrypt, decrypt,
79
78
  normalizeKey, normalizeIV,
80
- startEncryptStream, startDecryptStream, updateCryptStream, closeCryptStream
79
+ getCipher, getDecipher
81
80
  }