bfj 4.2.0 → 4.2.4

package/HISTORY.md

@@ -1,5 +1,23 @@
 # History
 
+## 4.2.4
+
+* chore: update deps (c3eeeb4)
+
+## 4.2.3
+
+* fix: eliminate costly string concatenation (42998d7)
+* fix: micro-optimise eventify::proceed::after (98a2519)
+* fix: micro-optimise walk::character (8d1c4cf)
+
+## 4.2.2
+
+* fix: silence obnoxious unhandled rejection warnings (1d4a902)
+
+## 4.2.1
+
+* refactor: discard chunks more aggressively (970a964)
+
 ## 4.2.0
 
 * chore: add a unit test for parallel object references (e8f3895)

package/README.md

@@ -548,14 +548,6 @@ of an object,
   is analagous to the
   [reviver parameter for JSON.parse][reviver].
 
-* `options.discard`:
-  The number of characters
-  to process before
-  discarding them
-  to save memory.
-  The default value
-  is `1048576`.
-
 * `options.yieldRate`:
   The number of data items to process
   before yielding to the event loop.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bfj",
-  "version": "4.2.0",
+  "version": "4.2.4",
   "description": "Big-friendly JSON. Asynchronous streaming functions for large JSON data sets.",
   "homepage": "https://github.com/philbooth/bfj",
   "bugs": "https://github.com/philbooth/bfj/issues",
@@ -30,14 +30,14 @@
   },
   "dependencies": {
     "check-types": "^7.3.0",
-    "hoopy": "^0.1.1",
-    "trier": "^0.3.6"
+    "hoopy": "^0.1.2",
+    "tryer": "^1.0.0"
   },
   "devDependencies": {
-    "eslint": "3.19.x",
-    "mocha": "3.2.x",
-    "chai": "3.5.x",
-    "mockery": "2.0.x",
+    "eslint": "4.6.x",
+    "mocha": "3.5.x",
+    "chai": "4.1.x",
+    "mockery": "2.1.x",
     "spooks": "2.0.x",
     "please-release-me": "1.0.x",
     "request": "2.81.x"

package/src/eventify.js

@@ -114,21 +114,20 @@ function eventify (data, options) {
         return literal(coerced)
       }
 
-      const type = typeof coerced
-
-      if (type === 'number') {
-        return value(coerced, type)
-      }
-
-      if (type === 'string') {
-        return value(escapeString(coerced), type)
-      }
-
       if (Array.isArray(coerced)) {
         return array(coerced)
       }
 
-      return object(coerced)
+      const type = typeof coerced
+
+      switch (type) {
+        case 'number':
+          return value(coerced, type)
+        case 'string':
+          return value(escapeString(coerced), type)
+        default:
+          return object(coerced)
+      }
     }
   }
 

package/src/parse.js

@@ -17,9 +17,6 @@ module.exports = parse
  *
  * @option reviver:   Transformation function, invoked depth-first.
  *
- * @option discard:   The number of characters to process before discarding
- *                    them to save memory. The default value is `1048576`.
- *
  * @option yieldRate: The number of data items to process per timeslice,
  *                    default is 16384.
  **/

package/src/read.js

@@ -16,9 +16,6 @@ module.exports = read
  *
  * @option reviver:   Transformation function, invoked depth-first.
  *
- * @option discard:   The number of characters to process before discarding them
- *                    to save memory. The default value is `1048576`.
- *
  * @option yieldRate: The number of data items to process per timeslice,
  *                    default is 16384.
  **/

package/src/streamify.js

@@ -5,7 +5,7 @@ const eventify = require('./eventify')
 const events = require('./events')
 const JsonStream = require('./jsonstream')
 const Hoopy = require('hoopy')
-const trier = require('trier')
+const tryer = require('tryer')
 
 const DEFAULT_BUFFER_LENGTH = 1048576
 
@@ -167,7 +167,7 @@ function streamify (data, options = {}) {
     isPaused = true
     return new Promise(resolve => {
       const unpause = emitter.pause()
-      trier.attempt({
+      tryer({
         interval: -10,
         until () {
           return length + 1 <= json.length

package/src/stringify.js

@@ -34,7 +34,7 @@ function stringify (data, options) {
   let resolve, reject
 
   const stream = streamify(data, options)
-  let json = ''
+  const json = []
 
   stream.on('data', read)
   stream.on('end', end)
@@ -46,11 +46,11 @@ function stringify (data, options) {
   })
 
   function read (chunk) {
-    json += chunk
+    json.push(chunk)
   }
 
   function end () {
-    resolve(json)
+    resolve(json.join(''))
   }
 
   function error (e) {

package/src/walk.js

@@ -33,9 +33,6 @@ module.exports = initialise
  *
  * @param stream:     Readable instance representing the incoming JSON.
  *
- * @option discard:   The number of characters to process before discarding
- *                    them to save memory. The default value is `1048576`.
- *
  * @option yieldRate: The number of data items to process per timeslice,
  *                    default is 16384.
  **/
@@ -44,7 +41,6 @@ function initialise (stream, options) {
 
   options = options || {}
 
-  let json = ''
   let index = 0
   let isStreamEnded = false
   let isWalkBegun = false
@@ -52,7 +48,10 @@ function initialise (stream, options) {
   let isWalkingString = false
   let count = 0
   let resumeFn
+  let cachedCharacter
 
+  const json = []
+  const lengths = []
   const currentPosition = {
     line: 1,
     column: 1
@@ -64,7 +63,6 @@ function initialise (stream, options) {
     obj: property
   }
   const emitter = new EventEmitter()
-  const discardThreshold = options.discard || 1048576
   const yieldRate = options.yieldRate || 16384
 
   stream.setEncoding('utf8')
@@ -74,7 +72,7 @@ function initialise (stream, options) {
   return emitter
 
   function readStream (chunk) {
-    json += chunk
+    addChunk(chunk)
 
     if (!isWalkBegun) {
       isWalkBegun = true
@@ -84,6 +82,26 @@ function initialise (stream, options) {
     return resume()
   }
 
+  function addChunk (chunk) {
+    json.push(chunk)
+
+    const chunkLength = chunk.length
+    lengths.push({
+      item: chunkLength,
+      aggregate: length() + chunkLength
+    })
+  }
+
+  function length () {
+    const chunkCount = lengths.length
+
+    if (chunkCount === 0) {
+      return 0
+    }
+
+    return lengths[chunkCount - 1].aggregate
+  }
+
   function value () {
     /* eslint-disable no-underscore-dangle */
     if (++count % yieldRate !== 0) {
@@ -121,7 +139,7 @@ function initialise (stream, options) {
   function awaitCharacter () {
     let resolve, reject
 
-    if (index < json.length) {
+    if (index < length()) {
       return Promise.resolve()
     }
 
@@ -138,7 +156,7 @@ function initialise (stream, options) {
     })
 
     function after () {
-      if (index < json.length) {
+      if (index < length()) {
         return resolve()
       }
 
@@ -151,7 +169,21 @@ function initialise (stream, options) {
   }
 
   function character () {
-    return json[index]
+    if (cachedCharacter) {
+      return cachedCharacter
+    }
+
+    if (lengths[0].item > index) {
+      return cachedCharacter = json[0][index]
+    }
+
+    const len = lengths.length
+    for (let i = 1; i < len; ++i) {
+      const { aggregate, item } = lengths[i]
+      if (aggregate > index) {
+        return cachedCharacter = json[i][index + item - aggregate]
+      }
+    }
   }
 
   function next () {
@@ -160,6 +192,7 @@ function initialise (stream, options) {
     function after () {
       const result = character()
 
+      cachedCharacter = null
       index += 1
       previousPosition.line = currentPosition.line
       previousPosition.column = currentPosition.column
@@ -171,9 +204,13 @@ function initialise (stream, options) {
         currentPosition.column += 1
       }
 
-      if (index === discardThreshold) {
-        json = json.substring(index)
-        index = 0
+      if (index > lengths[0].aggregate) {
+        json.shift()
+
+        const difference = lengths.shift().item
+        index -= difference
+
+        lengths.forEach(len => len.aggregate -= difference)
       }
 
       return result
@@ -320,7 +357,7 @@ function initialise (stream, options) {
 
   function walkString (event) {
     let isEscaping = false
-    let str = ''
+    const str = []
 
     isWalkingString = true
 
@@ -331,7 +368,7 @@ function initialise (stream, options) {
         isEscaping = false
 
         return escape(char).then(escaped => {
-          str += escaped
+          str.push(escaped)
           return next().then(step)
         })
       }
@@ -342,12 +379,12 @@ function initialise (stream, options) {
       }
 
       if (char !== '"') {
-        str += char
+        str.push(char)
         return next().then(step)
       }
 
       isWalkingString = false
-      emitter.emit(event, str)
+      emitter.emit(event, str.join(''))
     }
   }
 
@@ -365,19 +402,21 @@ function initialise (stream, options) {
   }
 
   function escapeHex () {
-    let hexits = ''
+    let hexits = []
 
     return next().then(step.bind(null, 0))
 
     function step (idx, char) {
       if (isHexit(char)) {
-        hexits += char
+        hexits.push(char)
       }
 
       if (idx < 3) {
         return next().then(step.bind(null, idx + 1))
       }
 
+      hexits = hexits.join('')
+
       if (hexits.length === 4) {
         return String.fromCharCode(parseInt(hexits, 16))
       }
@@ -393,12 +432,12 @@ function initialise (stream, options) {
   }
 
   function number (firstCharacter) {
-    let digits = firstCharacter
+    let digits = [ firstCharacter ]
 
     return walkDigits().then(addDigits.bind(null, checkDecimalPlace))
 
     function addDigits (step, result) {
-      digits += result.digits
+      digits = digits.concat(result.digits)
 
       if (result.atEnd) {
         return endNumber()
@@ -410,7 +449,7 @@ function initialise (stream, options) {
     function checkDecimalPlace () {
       if (character() === '.') {
         return next().then(char => {
-          digits += char
+          digits.push(char)
           walkDigits().then(addDigits.bind(null, checkExponent))
         })
       }
@@ -421,7 +460,7 @@ function initialise (stream, options) {
     function checkExponent () {
       if (character() === 'e' || character() === 'E') {
         return next().then(char => {
-          digits += char
+          digits.push(char)
           awaitCharacter()
             .then(checkSign)
             .catch(fail.bind(null, 'EOF', 'exponent', currentPosition))
@@ -434,7 +473,7 @@ function initialise (stream, options) {
     function checkSign () {
       if (character() === '+' || character() === '-') {
         return next().then(char => {
-          digits += char
+          digits.push(char)
           readExponent()
         })
       }
@@ -447,13 +486,13 @@ function initialise (stream, options) {
     }
 
     function endNumber () {
-      emitter.emit(events.number, parseFloat(digits))
+      emitter.emit(events.number, parseFloat(digits.join('')))
       return endValue()
     }
   }
 
   function walkDigits () {
-    let digits = ''
+    const digits = []
 
     return wait()
 
@@ -466,7 +505,7 @@ function initialise (stream, options) {
     function step () {
       if (isDigit(character())) {
         return next().then(char => {
-          digits += char
+          digits.push(char)
           return wait()
         })
       }

package/test/unit/read.js

@@ -148,7 +148,7 @@ suite('read with error thrown by fs.createReadStream:', () => {
 
   test('read does not throw', () => {
     assert.doesNotThrow(() => {
-      read()
+      read().catch(() => {})
     })
   })
 

package/test/unit/streamify.js

@@ -9,7 +9,7 @@ const modulePath = '../../src/streamify'
 mockery.registerAllowable(modulePath)
 mockery.registerAllowable('hoopy')
 mockery.registerAllowable('check-types')
-mockery.registerAllowable('trier')
+mockery.registerAllowable('tryer')
 mockery.registerAllowable('./events')
 
 suite('streamify:', () => {

package/test/unit/write.js

@@ -290,7 +290,7 @@ suite('write with error thrown by fs.createWriteStream:', () => {
 
   test('write does not throw', () => {
     assert.doesNotThrow(() => {
-      write()
+      write().catch(() => {})
     })
   })