bfj 5.3.0 → 6.1.1

package/src/match.js

@@ -0,0 +1,218 @@
+'use strict'
+
+const check = require('check-types')
+const DataStream = require('./datastream')
+const events = require('./events')
+const Hoopy = require('hoopy')
+const walk = require('./walk')
+
+const DEFAULT_BUFFER_LENGTH = 1024
+
+module.exports = match
+
+/**
+ * Public function `match`.
+ *
+ * Asynchronously parses a stream of JSON data, returning a stream of items
+ * that match the argument. Note that if a value is `null`, it won't be matched
+ * because `null` is used to signify end-of-stream in node.
+ *
+ * @param stream:         Readable instance representing the incoming JSON.
+ *
+ * @param selector:       Regular expression, string or predicate function used to
+ *                        identify matches. If a regular expression or string is
+ *                        passed, only property keys are tested. If a predicate is
+ *                        passed, both the key and the value are passed to it as
+ *                        arguments.
+ *
+ * @option numbers:       Boolean, indicating whether numerical keys (e.g. array
+ *                        indices) should be coerced to strings before testing the
+ *                        match. Only applies if the `selector` argument is a string
+ *                        or regular expression.
+ *
+ * @option ndjson:        Set this to true to parse newline-delimited JSON,
+ *                        default is `false`.
+ *
+ * @option yieldRate:     The number of data items to process per timeslice,
+ *                        default is 16384.
+ *
+ * @option bufferLength:  The length of the match buffer, default is 1024.
+ *
+ * @option highWaterMark: If set, will be passed to the readable stream constructor
+ *                        as the value for the highWaterMark option.
+ *
+ * @option Promise:       The promise constructor to use, defaults to bluebird.
+ **/
+function match (stream, selector, options = {}) {
+  const scopes = []
+  const properties = []
+  const emitter = walk(stream, options)
+  const matches = new Hoopy(options.bufferLength || DEFAULT_BUFFER_LENGTH)
+  let streamOptions
+  const { highWaterMark } = options
+  if (highWaterMark) {
+    streamOptions = { highWaterMark }
+  }
+  const results = new DataStream(read, streamOptions)
+
+  let selectorFunction, selectorString, resume
+  let coerceNumbers = false
+  let awaitPush = true
+  let isEnded = false
+  let length = 0
+  let index = 0
+
+  if (check.function(selector)) {
+    selectorFunction = selector
+    selector = null
+  } else {
+    coerceNumbers = !! options.numbers
+
+    if (check.string(selector)) {
+      check.assert.nonEmptyString(selector)
+      selectorString = selector
+      selector = null
+    } else {
+      check.assert.instanceStrict(selector, RegExp)
+    }
+  }
+
+  emitter.on(events.array, array)
+  emitter.on(events.object, object)
+  emitter.on(events.property, property)
+  emitter.on(events.endArray, endScope)
+  emitter.on(events.endObject, endScope)
+  emitter.on(events.string, value)
+  emitter.on(events.number, value)
+  emitter.on(events.literal, value)
+  emitter.on(events.end, end)
+  emitter.on(events.error, error)
+  emitter.on(events.dataError, dataError)
+
+  return results
+
+  function read () {
+    if (awaitPush) {
+      awaitPush = false
+
+      if (isEnded) {
+        if (length > 0) {
+          after()
+        }
+
+        return endResults()
+      }
+    }
+
+    if (resume) {
+      const resumeCopy = resume
+      resume = null
+      resumeCopy()
+      after()
+    }
+  }
+
+  function after () {
+    if (awaitPush || resume) {
+      return
+    }
+
+    let i
+
+    for (i = 0; i < length && ! resume; ++i) {
+      if (! results.push(matches[i + index])) {
+        pause()
+      }
+    }
+
+    if (i === length) {
+      index = length = 0
+    } else {
+      length -= i
+      index += i
+    }
+  }
+
+  function pause () {
+    resume = emitter.pause()
+  }
+
+  function endResults () {
+    if (! awaitPush) {
+      results.push(null)
+    }
+  }
+
+  function array () {
+    scopes.push([])
+  }
+
+  function object () {
+    scopes.push({})
+  }
+
+  function property (name) {
+    properties.push(name)
+  }
+
+  function endScope () {
+    value(scopes.pop())
+  }
+
+  function value (v) {
+    let key
+
+    if (scopes.length > 0) {
+      const scope = scopes[scopes.length - 1]
+
+      if (Array.isArray(scope)) {
+        key = scope.length
+      } else {
+        key = properties.pop()
+      }
+
+      scope[key] = v
+    }
+
+    if (v === null) {
+      return
+    }
+
+    if (selectorFunction) {
+      if (selectorFunction(key, v, scopes.length)) {
+        push(v)
+      }
+    } else {
+      if (coerceNumbers && typeof key === 'number') {
+        key = key.toString()
+      }
+
+      if ((selectorString && selectorString === key) || (selector && selector.test(key))) {
+        push(v)
+      }
+    }
+  }
+
+  function push (v) {
+    if (length + 1 === matches.length) {
+      pause()
+    }
+
+    matches[index + length++] = v
+
+    after()
+  }
+
+  function end () {
+    isEnded = true
+    endResults()
+  }
+
+  function error (e) {
+    results.emit('error', e)
+  }
+
+  function dataError (e) {
+    results.emit('dataError', e)
+  }
+}

package/src/parse.js

@@ -65,6 +65,7 @@ function parse (stream, options = {}) {
   emitter.on(events.endObject, endScope)
   emitter.on(events.end, end)
   emitter.on(events.error, error)
+  emitter.on(events.dataError, error)
 
   if (shouldHandleNdjson) {
     emitter.on(events.endLine, endLine)

package/src/stream.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const util = require('util')
+const Readable = require('stream').Readable
+const check = require('check-types')
+
+util.inherits(BfjStream, Readable)
+
+module.exports = BfjStream
+
+function BfjStream (read, options) {
+  if (check.not.instanceStrict(this, BfjStream)) {
+    return new BfjStream(read)
+  }
+
+  check.assert.function(read, 'Invalid read implementation')
+
+  this._read = function () { // eslint-disable-line no-underscore-dangle
+    read()
+  }
+
+  return Readable.call(this, options)
+}

package/src/streamify.js

@@ -18,34 +18,42 @@ module.exports = streamify
  * Asynchronously serialises a data structure to a stream of JSON
  * data. Sanely handles promises, buffers, maps and other iterables.
  *
- * @param data:         The data to transform.
+ * @param data:           The data to transform.
  *
- * @option space:       Indentation string, or the number of spaces
- *                      to indent each nested level by.
+ * @option space:         Indentation string, or the number of spaces
+ *                        to indent each nested level by.
  *
- * @option promises:    'resolve' or 'ignore', default is 'resolve'.
+ * @option promises:      'resolve' or 'ignore', default is 'resolve'.
  *
- * @option buffers:     'toString' or 'ignore', default is 'toString'.
+ * @option buffers:       'toString' or 'ignore', default is 'toString'.
  *
- * @option maps:        'object' or 'ignore', default is 'object'.
+ * @option maps:          'object' or 'ignore', default is 'object'.
  *
- * @option iterables:   'array' or 'ignore', default is 'array'.
+ * @option iterables:     'array' or 'ignore', default is 'array'.
  *
- * @option circular:    'error' or 'ignore', default is 'error'.
+ * @option circular:      'error' or 'ignore', default is 'error'.
  *
- * @option yieldRate:    The number of data items to process per timeslice,
- *                       default is 16384.
+ * @option yieldRate:     The number of data items to process per timeslice,
+ *                        default is 16384.
  *
- * @option bufferLength: The length of the buffer, default is 1024.
+ * @option bufferLength:  The length of the buffer, default is 1024.
  *
- * @option Promise:      The promise constructor to use, defaults to bluebird.
+ * @option highWaterMark: If set, will be passed to the readable stream constructor
+ *                        as the value for the highWaterMark option.
+ *
+ * @option Promise:       The promise constructor to use, defaults to bluebird.
  **/
 function streamify (data, options = {}) {
   const emitter = eventify(data, options)
   const json = new Hoopy(options.bufferLength || DEFAULT_BUFFER_LENGTH)
   const Promise = promise(options)
   const space = normaliseSpace(options)
-  const stream = new JsonStream(read)
+  let streamOptions
+  const { highWaterMark } = options
+  if (highWaterMark) {
+    streamOptions = { highWaterMark }
+  }
+  const stream = new JsonStream(read, streamOptions)
 
   let awaitPush = true
   let index = 0
@@ -67,6 +75,7 @@ function streamify (data, options = {}) {
   emitter.on(events.endObject, noRacing(endObject))
   emitter.on(events.end, noRacing(end))
   emitter.on(events.error, noRacing(error))
+  emitter.on(events.dataError, noRacing(dataError))
 
   return stream
 
@@ -255,6 +264,10 @@ function streamify (data, options = {}) {
   }
 
   function error (err) {
+    stream.emit('error', err)
+  }
+
+  function dataError (err) {
     stream.emit('dataError', err)
   }
 }

package/src/stringify.js

@@ -11,27 +11,30 @@ module.exports = stringify
  * Returns a promise and asynchronously serialises a data structure to a
  * JSON string. Sanely handles promises, buffers, maps and other iterables.
  *
- * @param data:         The data to transform
+ * @param data:          The data to transform
  *
- * @option space:       Indentation string, or the number of spaces
- *                      to indent each nested level by.
+ * @option space:        Indentation string, or the number of spaces
+ *                       to indent each nested level by.
  *
- * @option promises:    'resolve' or 'ignore', default is 'resolve'.
+ * @option promises:     'resolve' or 'ignore', default is 'resolve'.
  *
- * @option buffers:     'toString' or 'ignore', default is 'toString'.
+ * @option buffers:      'toString' or 'ignore', default is 'toString'.
  *
- * @option maps:        'object' or 'ignore', default is 'object'.
+ * @option maps:         'object' or 'ignore', default is 'object'.
  *
- * @option iterables:   'array' or 'ignore', default is 'array'.
+ * @option iterables:    'array' or 'ignore', default is 'array'.
  *
- * @option circular:    'error' or 'ignore', default is 'error'.
+ * @option circular:     'error' or 'ignore', default is 'error'.
  *
- * @option yieldRate:    The number of data items to process per timeslice,
- *                       default is 16384.
+ * @option yieldRate:     The number of data items to process per timeslice,
+ *                        default is 16384.
  *
- * @option bufferLength: The length of the buffer, default is 1024.
+ * @option bufferLength:  The length of the buffer, default is 1024.
  *
- * @option Promise:      The promise constructor to use, defaults to bluebird.
+ * @option highWaterMark: If set, will be passed to the readable stream constructor
+ *                        as the value for the highWaterMark option.
+ *
+ * @option Promise:       The promise constructor to use, defaults to bluebird.
  **/
 function stringify (data, options) {
   const json = []
@@ -42,6 +45,7 @@ function stringify (data, options) {
 
   stream.on('data', read)
   stream.on('end', end)
+  stream.on('error', error)
   stream.on('dataError', error)
 
   return new Promise((res, rej) => {

package/src/unpipe.js

@@ -29,7 +29,7 @@ function unpipe (callback, options) {
 
   const jsonstream = new stream.PassThrough()
 
-  parse(jsonstream, options)
+  parse(jsonstream, Object.assign({}, options, { ndjson: false }))
     .then(data => callback(null, data))
     .catch(error => callback(error))
 

package/src/walk.js

@@ -374,7 +374,7 @@ function initialise (stream, options = {}) {
 
       return endScope(scp)
         .then(() => {
-          if (! shouldHandleNdjson || scopes.length > 0) {
+          if (scopes.length > 0) {
             return checkCharacter(character(), ',', currentPosition)
           }
         })
@@ -389,7 +389,7 @@ function initialise (stream, options = {}) {
 
   function fail (actual, expected, position) {
     return emit(
-      events.error,
+      events.dataError,
       error.create(
         actual,
         expected,

package/src/write.js

@@ -13,29 +13,32 @@ module.exports = write
  * JSON file on disk. Sanely handles promises, buffers, maps and other
  * iterables.
  *
- * @param path:         Path to the JSON file.
+ * @param path:           Path to the JSON file.
  *
- * @param data:         The data to transform.
+ * @param data:           The data to transform.
  *
- * @option space:       Indentation string, or the number of spaces
- *                      to indent each nested level by.
+ * @option space:         Indentation string, or the number of spaces
+ *                        to indent each nested level by.
  *
- * @option promises:    'resolve' or 'ignore', default is 'resolve'.
+ * @option promises:      'resolve' or 'ignore', default is 'resolve'.
  *
- * @option buffers:     'toString' or 'ignore', default is 'toString'.
+ * @option buffers:       'toString' or 'ignore', default is 'toString'.
  *
- * @option maps:        'object' or 'ignore', default is 'object'.
+ * @option maps:          'object' or 'ignore', default is 'object'.
  *
- * @option iterables:   'array' or 'ignore', default is 'array'.
+ * @option iterables:     'array' or 'ignore', default is 'array'.
  *
- * @option circular:    'error' or 'ignore', default is 'error'.
+ * @option circular:      'error' or 'ignore', default is 'error'.
  *
- * @option yieldRate:    The number of data items to process per timeslice,
- *                       default is 16384.
+ * @option yieldRate:     The number of data items to process per timeslice,
+ *                        default is 16384.
  *
- * @option bufferLength: The length of the buffer, default is 1024.
+ * @option bufferLength:  The length of the buffer, default is 1024.
  *
- * @option Promise:      The promise constructor to use, defaults to bluebird.
+ * @option highWaterMark: If set, will be passed to the readable stream constructor
+ *                        as the value for the highWaterMark option.
+ *
+ * @option Promise:       The promise constructor to use, defaults to bluebird.
  **/
 function write (path, data, options) {
   const Promise = promise(options)

package/test/integration.js

@@ -41,6 +41,14 @@ suite('integration:', () => {
       assert.lengthOf(bfj.walk, 1)
     })
 
+    test('match function is exported', () => {
+      assert.isFunction(bfj.match)
+    })
+
+    test('match expects two arguments', () => {
+      assert.lengthOf(bfj.match, 2)
+    })
+
     test('parse function is exported', () => {
       assert.isFunction(bfj.parse)
     })
@@ -161,61 +169,6 @@ suite('integration:', () => {
       })
     })
 
-    suite('parse NDJSON:', () => {
-      let failed, file, results
-
-      setup(() => {
-        failed = false
-        file = path.join(__dirname, 'data.ndjson')
-        results = []
-        fs.writeFileSync(file, [
-          JSON.stringify([ 'b', 'a', 'r' ]),
-          JSON.stringify(null),
-          '',
-          '',
-          JSON.stringify('wibble')
-        ].join('\n'))
-        const stream = fs.createReadStream(file)
-        return bfj.parse(stream, { ndjson: true })
-          .then(result => {
-            results.push(result)
-            return bfj.parse(stream, { ndjson: true })
-          })
-          .then(result => {
-            results.push(result)
-            return bfj.parse(stream, { ndjson: true })
-          })
-          .then(result => {
-            results.push(result)
-            return bfj.parse(stream, { ndjson: true })
-          })
-          .then(result => {
-            results.push(result)
-            return bfj.parse(stream, { ndjson: true })
-          })
-          .then(result => results.push(result))
-          .catch(e => {
-            failed = true
-          })
-      })
-
-      teardown(() => {
-        fs.unlinkSync(file)
-      })
-
-      test('results were correct', () => {
-        assert.isFalse(failed)
-        assert.lengthOf(results, 5)
-        assert.deepEqual(results, [
-          [ 'b', 'a', 'r' ],
-          null,
-          'wibble',
-          undefined,
-          undefined
-        ])
-      })
-    })
-
     suite('read error:', () => {
       let failed, file, result, error
 
@@ -264,12 +217,95 @@ suite('integration:', () => {
       })
     })
 
+    suite('match predicate:', () => {
+      let file, results, errors
+
+      setup(done => {
+        file = path.join(__dirname, 'data.json')
+        fs.writeFileSync(file, JSON.stringify({
+          foo: 'bar',
+          baz: 'qux',
+          wibble: 'blee'
+        }))
+        results = []
+        errors = []
+        const datastream = bfj.match(fs.createReadStream(file), (k, v) => k === 'baz' || v === 'blee')
+        datastream.on('data', item => results.push(item))
+        datastream.on('error', error => errors.push(error))
+        datastream.on('end', done)
+      })
+
+      test('the correct properties were matched', () => {
+        assert.deepEqual([ 'qux', 'blee' ], results)
+      })
+
+      test('no errors occurred', () => {
+        assert.deepEqual(errors, [])
+      })
+    })
+
+    suite('match nested:', () => {
+      let file, results, errors
+
+      setup(done => {
+        file = path.join(__dirname, 'data.json')
+        fs.writeFileSync(file, JSON.stringify({
+          foo: {
+            bar: 'baz'
+          }
+        }))
+        results = []
+        errors = []
+        const datastream = bfj.match(fs.createReadStream(file), () => true)
+        datastream.on('data', item => results.push(item))
+        datastream.on('error', error => errors.push(error))
+        datastream.on('end', done)
+      })
+
+      test('the correct properties were matched', () => {
+        assert.deepEqual([ 'baz', { bar: 'baz' }, { foo: { bar: 'baz' } } ], results)
+      })
+
+      test('no errors occurred', () => {
+        assert.deepEqual(errors, [])
+      })
+    })
+
+    suite('match ndjson:', () => {
+      let file, results, errors
+
+      setup(done => {
+        file = path.join(__dirname, 'data.ndjson')
+        fs.writeFileSync(file, [
+          JSON.stringify([ 'a', 'b' ]),
+          JSON.stringify(null),
+          '',
+          '',
+          JSON.stringify('wibble')
+        ].join('\n'))
+        results = []
+        errors = []
+        const datastream = bfj.match(fs.createReadStream(file), () => true, { ndjson: true })
+        datastream.on('data', item => results.push(item))
+        datastream.on('error', error => errors.push(error))
+        datastream.on('end', done)
+      })
+
+      test('the correct properties were matched', () => {
+        assert.deepEqual([ 'a', 'b', [ 'a', 'b' ], 'wibble' ], results)
+      })
+
+      test('no errors occurred', () => {
+        assert.deepEqual(errors, [])
+      })
+    })
+
     suite('parse request:', () => {
       let error, result
 
       setup(done => {
         const jsonstream = new stream.PassThrough()
-        request({ url: 'https://raw.githubusercontent.com/philbooth/bfj/master/package.json' })
+        request({ url: 'https://gitlab.com/philbooth/bfj/raw/master/package.json' })
           .pipe(bfj.unpipe((err, res) => {
             error = err
             result = res
@@ -283,6 +319,61 @@ suite('integration:', () => {
       })
     })
 
+    suite('parse NDJSON:', () => {
+      let failed, file, results
+
+      setup(() => {
+        failed = false
+        file = path.join(__dirname, 'data.ndjson')
+        results = []
+        fs.writeFileSync(file, [
+          JSON.stringify([ 'b', 'a', 'r' ]),
+          JSON.stringify(null),
+          '',
+          '',
+          JSON.stringify('wibble')
+        ].join('\n'))
+        const stream = fs.createReadStream(file)
+        return bfj.parse(stream, { ndjson: true })
+          .then(result => {
+            results.push(result)
+            return bfj.parse(stream, { ndjson: true })
+          })
+          .then(result => {
+            results.push(result)
+            return bfj.parse(stream, { ndjson: true })
+          })
+          .then(result => {
+            results.push(result)
+            return bfj.parse(stream, { ndjson: true })
+          })
+          .then(result => {
+            results.push(result)
+            return bfj.parse(stream, { ndjson: true })
+          })
+          .then(result => results.push(result))
+          .catch(e => {
+            failed = true
+          })
+      })
+
+      teardown(() => {
+        fs.unlinkSync(file)
+      })
+
+      test('results were correct', () => {
+        assert.isFalse(failed)
+        assert.lengthOf(results, 5)
+        assert.deepEqual(results, [
+          [ 'b', 'a', 'r' ],
+          null,
+          'wibble',
+          undefined,
+          undefined
+        ])
+      })
+    })
+
     suite('stringify value:', () => {
       let result