bfj 7.0.2 → 7.1.0

package/HISTORY.md

@@ -1,5 +1,16 @@
 # History
 
+## 7.1.0
+
+### New features
+
+* match: support jsonpath expressions (96e3a20eea18efd982867dd894c2f1c2a649321e)
+
+### Other changes
+
+* deps: upgrade dependencies (f6e8664281931e0eeea26d62059bfbc9e9a8c37d)
+* tests: add perf test for bfj.match (f345bc83e5fbd9204cab02169a2ae9f4c02d1450)
+
 ## 7.0.2
 
 ### Other changes

package/README.md

@@ -243,7 +243,7 @@ request({ url }).pipe(bfj.unpipe((error, data) => {
 ```js
 const bfj = require('bfj');
 
-// Call match with your stream and a selector predicate/regex/string
+// Call match with your stream and a selector predicate/regex/JSONPath/string
 const dataStream = bfj.match(jsonStream, selector, options);
 
 // Get data out of the returned stream with event handlers
@@ -264,7 +264,7 @@ It takes three arguments:
 a [readable stream][readable]
 from which the JSON will be parsed;
 a selector argument for determining matches,
-which may be a string, a regular expression or a predicate function;
+which may be a string, a regular expression, a JSONPath expression, or a predicate function;
 and an [options](#options-for-parsing-functions) object.
 
 If the selector is a string,
@@ -275,6 +275,9 @@ If it is a regular expression,
 the comparison will be made
 by calling the [RegExp `test` method][regexp-test]
 with the property key.
+If it is a JSONPath expression,
+it must start with `$.` to identify the root node
+and only use `child` scope expressions for subsequent nodes.
 Predicate functions will be called with three arguments:
 `key`, `value` and `depth`.
 If the result of the predicate is a truthy value

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bfj",
-  "version": "7.0.2",
+  "version": "7.1.0",
   "description": "Big-friendly JSON. Asynchronous streaming functions for large JSON data sets.",
   "homepage": "https://gitlab.com/philbooth/bfj",
   "bugs": "https://gitlab.com/philbooth/bfj/issues",
@@ -29,18 +29,19 @@
     "node": ">= 8.0.0"
   },
   "dependencies": {
-    "bluebird": "^3.5.5",
-    "check-types": "^11.1.1",
+    "bluebird": "^3.7.2",
+    "check-types": "^11.2.3",
     "hoopy": "^0.1.4",
+    "jsonpath": "^1.1.1",
     "tryer": "^1.0.1"
   },
   "devDependencies": {
-    "chai": "^4.2.0",
-    "eslint": "^6.0.1",
-    "mocha": "^6.1.4",
-    "please-release-me": "^2.1.2",
-    "proxyquire": "^2.1.0",
-    "request": "^2.88.0",
+    "axios": "^1.5.0",
+    "chai": "^4.3.8",
+    "eslint": "^8.48.0",
+    "mocha": "^10.2.0",
+    "please-release-me": "^2.1.4",
+    "proxyquire": "^2.1.3",
     "spooks": "^2.0.0"
   },
   "scripts": {

package/src/match.js

@@ -4,6 +4,7 @@ const check = require('check-types')
 const DataStream = require('./datastream')
 const events = require('./events')
 const Hoopy = require('hoopy')
+const jsonpath = require('jsonpath')
 const walk = require('./walk')
 
 const DEFAULT_BUFFER_LENGTH = 1024
@@ -49,6 +50,7 @@ module.exports = match
  * @option Promise:       The promise constructor to use, defaults to bluebird.
  **/
 function match (stream, selector, options = {}) {
+  const keys = []
   const scopes = []
   const properties = []
   const emitter = walk(stream, options)
@@ -60,7 +62,7 @@ function match (stream, selector, options = {}) {
   }
   const results = new DataStream(read, streamOptions)
 
-  let selectorFunction, selectorString, resume
+  let selectorFunction, selectorPath, selectorString, resume
   let coerceNumbers = false
   let awaitPush = true
   let isEnded = false
@@ -73,16 +75,29 @@ function match (stream, selector, options = {}) {
   if (check.function(selector)) {
     selectorFunction = selector
     selector = null
-  } else {
-    coerceNumbers = !! options.numbers
-
-    if (check.string(selector)) {
-      check.assert.nonEmptyString(selector)
-      selectorString = selector
-      selector = null
+  } else if (check.string(selector)) {
+    check.assert.nonEmptyString(selector)
+
+    if (selector.startsWith('$.')) {
+      selectorPath = jsonpath.parse(selector)
+      check.assert.identical(selectorPath.shift(), {
+        expression: {
+          type: 'root',
+          value: '$',
+        },
+      })
+      selectorPath.forEach((part) => {
+        check.assert.equal(part.scope, 'child')
+      })
     } else {
-      check.assert.instanceStrict(selector, RegExp)
+      selectorString = selector
+      coerceNumbers = !! options.numbers
     }
+
+    selector = null
+  } else {
+    check.assert.instanceStrict(selector, RegExp)
+    coerceNumbers = !! options.numbers
   }
 
   emitter.on(events.array, array)
@@ -160,6 +175,8 @@ function match (stream, selector, options = {}) {
   }
 
   function property (name) {
+    keys.push(name)
+
     if (scopes.length < minDepth) {
       return
     }
@@ -168,6 +185,9 @@ function match (stream, selector, options = {}) {
   }
 
   function endScope () {
+    if (selectorPath) {
+      keys.pop()
+    }
     value(scopes.pop())
   }
 
@@ -198,6 +218,10 @@ function match (stream, selector, options = {}) {
       if (selectorFunction(key, v, scopes.length)) {
         push(v)
       }
+    } else if (selectorPath) {
+      if (isSelectorPathSatisfied([ ...keys, key ])) {
+        push(v)
+      }
     } else {
       if (coerceNumbers && typeof key === 'number') {
         key = key.toString()
@@ -209,6 +233,34 @@ function match (stream, selector, options = {}) {
     }
   }
 
+  function isSelectorPathSatisfied (path) {
+    if (selectorPath.length !== path.length) {
+      return false
+    }
+
+    return selectorPath.every(({ expression, operation }, i) => {
+      if (
+        (operation === 'member' && expression.type === 'identifier') ||
+        (operation === 'subscript' && (
+          expression.type === 'string_literal' ||
+          expression.type === 'numeric_literal'
+        ))
+      ) {
+        return path[i] === expression.value
+      }
+
+      if (
+        operation === 'subscript' &&
+        expression.type === 'wildcard' &&
+        expression.value === '*'
+      ) {
+        return true
+      }
+
+      return false
+    })
+  }
+
   function push (v) {
     if (length + 1 === matches.length) {
       pause()

package/test/integration.js

@@ -1,10 +1,10 @@
 'use strict'
 
 const assert = require('chai').assert
+const axios = require('axios')
 const fs = require('fs')
 const path = require('path')
 const Promise = require('bluebird')
-const request = require('request')
 const stream = require('stream')
 
 const modulePath = '../src'
@@ -307,14 +307,16 @@ suite('integration:', () => {
     suite('parse request:', () => {
       let error, result
 
-      setup(done => {
+      setup((done) => {
         const jsonstream = new stream.PassThrough()
-        request({ url: 'https://gitlab.com/philbooth/bfj/raw/master/package.json' })
-          .pipe(bfj.unpipe((err, res) => {
-            error = err
-            result = res
-            done()
-          }))
+        axios({
+          responseType: 'stream',
+          url: 'https://gitlab.com/philbooth/bfj/raw/master/package.json',
+        }).then((response) => response.data.pipe(bfj.unpipe((err, res) => {
+          error = err
+          result = res
+          done()
+        })))
       })
 
       test('result was correct', () => {

package/test/performance.js

@@ -7,18 +7,41 @@ const path = require('path')
 const check = require('check-types')
 const bfj = require('../src')
 
-console.log('reading json')
+const inPath = getDataPath('.json');
 
 let time = process.hrtime()
 
-bfj.read(getDataPath('.json'))
-  .then(data => {
+if (process.argv.length === 4) {
+  const stuff = []
+  const stream = bfj.match(fs.createReadStream(inPath), process.argv[3])
+  stream.on('data', thing => stuff.push(thing))
+  stream.on('end', () => {
     reportTime()
-    console.log('writing json')
-    return bfj.write(getDataPath('-result.json'), data)
+    console.log('hooray!', stuff.length)
+    fs.writeFileSync(getDataPath('-result.ndjson'), stuff.map(s => JSON.stringify(s)).join('\n'), {
+      encoding: 'utf8',
+    })
+    process.exit(0)
   })
-  .then(() => done('succeeded'))
-  .catch(error => done(error.stack, 1))
+  stream.on('error', error => {
+    console.error('error!', error.stack)
+    process.exit(1)
+  })
+  stream.on('dataError', error => {
+    console.error('dataError!', error.stack)
+    process.exit(2)
+  })
+} else {
+  console.log('reading json')
+  bfj.read(inPath)
+    .then(data => {
+      reportTime()
+      console.log('writing json')
+      return bfj.write(getDataPath('-result.json'), data)
+    })
+    .then(() => done('succeeded'))
+    .catch(error => done(error.stack, 1))
+}
 
 function getDataPath (suffix) {
   return path.resolve(__dirname, process.argv[2] + suffix)

package/test/unit/match.js

@@ -915,6 +915,65 @@ suite('match:', () => {
       })
     })
 
+    suite('match with jsonpath expression:', () => {
+      let stream, options, result
+
+      setup(() => {
+        result = match({}, '$.foo.bar[*]', {})
+      })
+
+      test('DataStream was called once', () => {
+        assert.strictEqual(log.counts.DataStream, 1)
+      })
+
+      test('walk was called once', () => {
+        assert.strictEqual(log.counts.walk, 1)
+      })
+
+      test('EventEmitter.on was called eleven times', () => {
+        assert.strictEqual(log.counts.on, 11)
+      })
+
+      suite('read events:', () => {
+        setup(() => {
+          log.args.DataStream[0][0]()
+          // { "foo": { "bar": [ "baz", "qux" ], "wibble": "blee" } }
+          log.args.on[1][1]()
+          log.args.on[2][1]('foo')
+          log.args.on[1][1]()
+          log.args.on[2][1]('bar')
+          log.args.on[0][1]()
+          log.args.on[5][1]('baz')
+          log.args.on[5][1]('qux')
+          log.args.on[3][1]()
+          log.args.on[2][1]('wibble')
+          log.args.on[5][1]('blee')
+          log.args.on[4][1]()
+          log.args.on[8][1]()
+        })
+
+        test('results.push was called three times', () => {
+          assert.strictEqual(log.counts.push, 3)
+        })
+
+        test('results.push was called correctly first time', () => {
+          assert.strictEqual(log.args.push[0][0], 'baz')
+        })
+
+        test('results.push was called correctly second time', () => {
+          assert.strictEqual(log.args.push[1][0], 'qux')
+        })
+
+        test('results.push was called correctly third time', () => {
+          assert.isNull(log.args.push[2][0])
+        })
+
+        test('results.emit was not called', () => {
+          assert.strictEqual(log.counts.emit, 0)
+        })
+      })
+    })
+
     suite('match with numbers=true:', () => {
       let stream, options, result