bfj 9.0.2 → 9.1.0

package/HISTORY.md

@@ -1,5 +1,12 @@
 # History
 
+## 9.1.0
+
+### New features
+
+* match: implement `recursive` option (acdd744c2eeedad3b5b3df6bc9cf4c48272b6677)
+* walk: implement `stringChunkSize` option and `stringChunk` event (ea8bafcdcbc9a45d177c3407d6e7e88e1e22072a)
+
 ## 9.0.2
 
 ### Bug fixes

package/README.md

@@ -24,6 +24,8 @@ Big-Friendly JSON. Asynchronous streaming functions for large JSON data sets.
   * [Options for parsing functions](#options-for-parsing-functions)
   * [Options for serialisation functions](#options-for-serialisation-functions)
 * [Is it possible to pause parsing or serialisation from calling code?](#is-it-possible-to-pause-parsing-or-serialisation-from-calling-code)
+* [Can it break long strings into chunks?](#can-it-break-long-strings-into-chunks)
+* [Can it recursively parse JSON nested inside a JSON string?](#can-it-recursively-parse-json-nested-inside-a-json-string)
 * [Can it handle newline-delimited JSON (NDJSON)?](#can-it-handle-newline-delimited-json-ndjson)
 * [Is there a change log?](#is-there-a-change-log)
 * [How do I set up the dev environment?](#how-do-i-set-up-the-dev-environment)
@@ -475,6 +477,16 @@ of an object,
   the value
   as its argument.
 
+* `bfj.events.stringChunk`
+  indicates that
+  a string chunk
+  has been encountered
+  if the `stringChunkSize` [option](#options-for-parsing-functions) was set.
+  The listener
+  will be passed
+  the chunk
+  as its argument.
+
 * `bfj.events.number`
   indicates that
   a number
@@ -687,6 +699,19 @@ of an object,
   discrete chunks of JSON.
   See [NDJSON](#can-it-handle-newline-delimited-json-ndjson) for more information.
 
+* `options.stringChunkSize`:
+  For `bfj.walk` only,
+  set this to the character count
+  at which you wish to chunk strings.
+  Each chunk will be emitted as a `bfj.events.stringChunk` event,
+  followed by the regular `bfj.events.string` event
+  after all chunks are emitted.
+
+* `options.recursive`:
+  For `bfj.match` only,
+  set this to `true`
+  if you wish to match against recursively JSON-parsed strings.
+
 * `options.numbers`:
   For `bfj.match` only,
   set this to `true`
@@ -821,6 +846,29 @@ const resume = emitter.pause();
 resume();
 ```
 
+## Can it break long strings into chunks?
+
+Yes.
+If you pass the `stringChunkSize` [option](#options-for-parsing-functions)
+to `bfj.walk`,
+it will emit a `bfj.events.stringChunk` event
+for each chunk of the string.
+The regular `bfj.events.string` event
+will still be emitted
+after all the chunks.
+
+## Can it recursively parse JSON nested inside a JSON string?
+
+Yes.
+If you pass the `recursive` [option](#options-for-parsing-functions)
+to `bfj.match`,
+it will recursively parse any string values
+that satisfy the `selector` argument.
+Note the same selector is applied
+to every level of recursion,
+so this works best in combination
+with selectors that are predicate functions.
+
 ## Can it handle [newline-delimited JSON (NDJSON)](http://ndjson.org/)?
 
 Yes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bfj",
-  "version": "9.0.2",
+  "version": "9.1.0",
   "description": "Big-friendly JSON. Asynchronous streaming functions for large JSON data sets.",
   "homepage": "https://gitlab.com/philbooth/bfj",
   "bugs": "https://gitlab.com/philbooth/bfj/issues",

package/src/events.js

@@ -5,6 +5,7 @@ module.exports = {
   object: 'obj',
   property: 'pro',
   string: 'str',
+  stringChunk: 'str-chunk',
   number: 'num',
   literal: 'lit',
   endPrefix: 'end-',

package/src/match.js

@@ -5,6 +5,7 @@ const DataStream = require('./datastream')
 const events = require('./events')
 const Hoopy = require('hoopy')
 const jsonpath = require('jsonpath')
+const { PassThrough } = require('node:stream')
 const walk = require('./walk')
 
 const DEFAULT_BUFFER_LENGTH = 256
@@ -39,6 +40,10 @@ module.exports = match
  * @option ndjson:        Set this to true to parse newline-delimited JSON,
  *                        default is `false`.
  *
+ * @option recursive:     Set this to true to recursively parse
+ *                        matched string values,
+ *                        default is `false`.
+ *
  * @option yieldRate:     The number of data items to process per timeslice,
  *                        default is 1024.
  *
@@ -51,18 +56,17 @@ function match (stream, selector, options = {}) {
   const keys = []
   const scopes = []
   const properties = []
-  const emitter = walk(stream, options)
   const matches = new Hoopy(options.bufferLength || DEFAULT_BUFFER_LENGTH)
   let streamOptions
   const { highWaterMark } = options
   if (highWaterMark) {
     streamOptions = { highWaterMark }
   }
-  const results = new DataStream(read, streamOptions)
+  const results = options.results || new DataStream(read, streamOptions)
 
-  let selectorFunction, selectorPath, selectorString, resume
+  let chunkStream, selectorFunction, selectorPath, selectorString, resume
   let coerceNumbers = false
-  let awaitPush = true
+  let awaitPush = ! options.results
   let isEnded = false
   let length = 0
   let index = 0
@@ -70,6 +74,9 @@ function match (stream, selector, options = {}) {
   const minDepth = options.minDepth || 0
   check.assert.greaterOrEqual(minDepth, 0)
 
+  const recursive = options.recursive || false
+  check.assert.boolean(recursive)
+
   if (check.function(selector)) {
     selectorFunction = selector
     selector = null
@@ -98,6 +105,11 @@ function match (stream, selector, options = {}) {
     coerceNumbers = !! options.numbers
   }
 
+  const emitter = walk(stream, {
+    ...options,
+    streamChunkSize: recursive ? matches.length : null,
+  })
+
   emitter.on(events.array, array)
   emitter.on(events.object, object)
   emitter.on(events.property, property)
@@ -110,6 +122,10 @@ function match (stream, selector, options = {}) {
   emitter.on(events.error, error)
   emitter.on(events.dataError, dataError)
 
+  if (recursive) {
+    emitter.on(events.stringChunk, stringChunk)
+  }
+
   return results
 
   function read () {
@@ -190,12 +206,16 @@ function match (stream, selector, options = {}) {
   }
 
   function value (v) {
-    let key
+    if (chunkStream) {
+      chunkStream = null
+    }
 
     if (scopes.length < minDepth) {
       return
     }
 
+    let key
+
     if (scopes.length > 0) {
       const scope = scopes[scopes.length - 1]
 
@@ -269,6 +289,19 @@ function match (stream, selector, options = {}) {
     after()
   }
 
+  function stringChunk (chunk) {
+    if (!chunkStream) {
+      chunkStream = new PassThrough(streamOptions)
+      match(
+        chunkStream,
+        selectorFunction || selectorPath || selectorString || selector,
+        { ...options, results },
+      )
+    }
+
+    chunkStream.write(chunk)
+  }
+
   function end () {
     isEnded = true
     endResults()

package/src/walk.js

@@ -34,17 +34,23 @@ module.exports = initialise
  * emitting events as it encounters tokens. The event emitter is decorated
  * with a `pause` method that can be called to pause processing.
  *
- * @param stream:     Readable instance representing the incoming JSON.
+ * @param stream:        Readable instance representing the incoming JSON.
  *
- * @option ndjson:    Set this to true to parse newline-delimited JSON.
+ * @option bufferLength:    The length of the walk buffer, default is 256.
  *
- * @option yieldRate: The number of data items to process per timeslice,
- *                    default is 1024.
+ * @option ndjson:          Set this to true to parse newline-delimited JSON.
  *
- * @option bufferLength:  The length of the walk buffer, default is 256.
+ * @option stringChunkSize: The size at which to chunk long strings, emitting
+ *                          `string-chunk` events for each chunk followed by a
+ *                          regular `string` event when the complete string has
+ *                          been walked. Default is disabled.
+ *
+ * @option yieldRate:       The number of data items to process per timeslice,
+ *                          default is 1024.
  **/
 function initialise (stream, options = {}) {
   check.assert.instanceStrict(stream, require('stream').Readable, 'Invalid stream argument')
+  check.assert.maybe.greater(options.stringChunkSize, 0, 'Invalid stringChunkSize option')
 
   const currentPosition = {
     line: 1,
@@ -62,6 +68,7 @@ function initialise (stream, options = {}) {
   const shouldHandleNdjson = !! options.ndjson
   const yieldRate = options.yieldRate || 1024
   const bufferLength = options.bufferLength || DEFAULT_BUFFER_LENGTH
+  const stringChunkSize = options.stringChunkSize || Number.POSITIVE_INFINITY
 
   let index = 0
   let isStreamEnded = false
@@ -71,6 +78,8 @@ function initialise (stream, options = {}) {
   let isWalkingString = false
   let hasEndedLine = true
   let count = 0
+  let stringChunkCount = 0
+  let stringChunkStart = 0
   let resumeFn
   let pause
   let cachedCharacter
@@ -433,6 +442,9 @@ function initialise (stream, options = {}) {
 
     if (isEscaping) {
       str.push(await escape(char))
+      if (++stringChunkCount >= stringChunkSize) {
+        await walkStringChunk(event, str)
+      }
       return walkStringContinue(event, str, false)
     }
 
@@ -442,13 +454,29 @@ function initialise (stream, options = {}) {
 
     if (char !== '"') {
       str.push(char)
+      if (++stringChunkCount >= stringChunkSize) {
+        await walkStringChunk(event, str)
+      }
       return walkStringContinue(event, str, isEscaping)
     }
 
     isWalkingString = false
+
+    await walkStringChunk(event, str)
+    stringChunkStart = 0
+
     return emit(event, str.join(''))
   }
 
+  function walkStringChunk (event, str) {
+    if (event === events.string) {
+      const chunk = str.slice(stringChunkStart).join('')
+      stringChunkStart = str.length
+      stringChunkCount = 0
+      return emit(events.stringChunk, chunk)
+    }
+  }
+
   async function escape (char) {
     if (escapes[char]) {
       return escapes[char]

package/test/integration.js

@@ -303,6 +303,35 @@ suite('integration:', () => {
       })
     })
 
+    suite('match recursive:', () => {
+      let file, results, errors
+
+      setup(done => {
+        file = path.join(__dirname, 'data.json')
+        fs.writeFileSync(file, JSON.stringify({
+          foo: 'bar',
+          wibble: {
+            foo: 'baz',
+            wibble: JSON.stringify({ foo: 'qux' }),
+          },
+        }))
+        results = []
+        errors = []
+        const datastream = bfj.match(fs.createReadStream(file), (k) => k === 'foo', { recursive: true })
+        datastream.on('data', item => results.push(item))
+        datastream.on('error', error => errors.push(error))
+        datastream.on('end', done)
+      })
+
+      test('the correct properties were matched', () => {
+        assert.deepEqual([ 'bar', 'baz', 'qux' ], results)
+      })
+
+      test('no errors occurred', () => {
+        assert.deepEqual(errors, [])
+      })
+    })
+
     suite('parse request:', () => {
       let error, result
 

package/test/unit/match.js

@@ -122,8 +122,11 @@ suite('match:', () => {
         assert.lengthOf(log.args.walk[0], 2)
         assert.strictEqual(log.args.walk[0][0], stream)
         assert.lengthOf(Object.keys(log.args.walk[0][0]), 0)
-        assert.strictEqual(log.args.walk[0][1], options)
-        assert.lengthOf(Object.keys(log.args.walk[0][1]), 2)
+        assert.deepEqual(log.args.walk[0][1], {
+          ...options,
+          streamChunkSize: null,
+        })
+        assert.lengthOf(Object.keys(log.args.walk[0][1]), 3)
       })
 
       test('EventEmitter.on was called eleven times', () => {
@@ -1253,5 +1256,72 @@ suite('match:', () => {
         })
       })
     })
+
+    suite('match recursive:', () => {
+      let stream, predicate, options, result
+
+      setup(() => {
+        stream = {}
+        predicate = spooks.fn({ name: 'predicate', log, results: [ true ] })
+        options = { recursive: true }
+        result = match(stream, predicate, options)
+      })
+
+      test('DataStream was called once', () => {
+        assert.strictEqual(log.counts.DataStream, 1)
+      })
+
+      test('walk was called once', () => {
+        assert.strictEqual(log.counts.walk, 1)
+      })
+
+      test('EventEmitter.on was called twelve times', () => {
+        assert.strictEqual(log.counts.on, 12)
+      })
+
+      test('EventEmitter.on was called correctly twelfth time', () => {
+        assert.lengthOf(log.args.on[11], 2)
+        assert.strictEqual(log.args.on[11][0], 'str-chunk')
+        assert.isFunction(log.args.on[11][1])
+        assert.strictEqual(log.these.on[11], results.walk[0])
+      })
+
+      suite('array with string chunks:', () => {
+        setup(() => {
+          log.args.on[0][1]()
+          log.args.on[11][1]('{"foo":"')
+          log.args.on[11][1]('bar",')
+          log.args.on[11][1]('"baz"')
+          log.args.on[11][1](':"qux"')
+          log.args.on[5][1]('{"foo":"bar","baz":"qux"}')
+          log.args.on[3][1]()
+          log.args.DataStream[0][0]()
+        })
+
+        test('predicate was called twice', () => {
+          assert.strictEqual(log.counts.predicate, 2)
+        })
+
+        test('predicate was called correctly first time', () => {
+          assert.strictEqual(log.args.predicate[0][0], 0)
+          assert.strictEqual(log.args.predicate[0][1], '{"foo":"bar","baz":"qux"}')
+          assert.strictEqual(log.args.predicate[0][2], 1)
+        })
+
+        test('predicate was called correctly second time', () => {
+          assert.isUndefined(log.args.predicate[1][0])
+          assert.deepEqual(log.args.predicate[1][1], ['{"foo":"bar","baz":"qux"}'])
+          assert.strictEqual(log.args.predicate[1][2], 0)
+        })
+
+        test('results.push was not called', () => {
+          assert.strictEqual(log.counts.push, 0)
+        })
+
+        test('results.emit was not called', () => {
+          assert.strictEqual(log.counts.emit, 0)
+        })
+      })
+    })
   })
 })

package/test/unit/walk.js

@@ -103,6 +103,10 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.string, 0)
       })
 
+      test('stringChunk event did not occur', () => {
+        assert.strictEqual(log.counts.stringChunk, 0)
+      })
+
       test('number event did not occur', () => {
         assert.strictEqual(log.counts.number, 0)
       })
@@ -190,6 +194,10 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.string, 0)
       })
 
+      test('stringChunk event did not occur', () => {
+        assert.strictEqual(log.counts.stringChunk, 0)
+      })
+
       test('number event did not occur', () => {
         assert.strictEqual(log.counts.number, 0)
       })
@@ -273,6 +281,10 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.string, 0)
       })
 
+      test('stringChunk event did not occur', () => {
+        assert.strictEqual(log.counts.stringChunk, 0)
+      })
+
       test('number event did not occur', () => {
         assert.strictEqual(log.counts.number, 0)
       })
@@ -349,6 +361,109 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.property, 0)
       })
 
+      test('stringChunk event occured once', () => {
+        assert.strictEqual(log.counts.stringChunk, 1)
+      })
+
+      test('number event did not occur', () => {
+        assert.strictEqual(log.counts.number, 0)
+      })
+
+      test('literal event did not occur', () => {
+        assert.strictEqual(log.counts.literal, 0)
+      })
+
+      test('endArray event did not occur', () => {
+        assert.strictEqual(log.counts.endArray, 0)
+      })
+
+      test('endObject event did not occur', () => {
+        assert.strictEqual(log.counts.endObject, 0)
+      })
+
+      test('error event did not occur', () => {
+        assert.strictEqual(log.counts.error, 0)
+      })
+
+      test('dataError event did not occur', () => {
+        assert.strictEqual(log.counts.dataError, 0)
+      })
+
+      test('endLine event did not occur', () => {
+        assert.strictEqual(log.counts.endLine, 0)
+      })
+
+      test('endPrefix event did not occur', () => {
+        assert.strictEqual(log.counts.endPrefix, 0)
+      })
+    })
+
+    suite('string (with chunk size):', () => {
+      let stream, emitter
+
+      setup(done => {
+        stream = new Readable()
+        stream._read = () => {}
+
+        emitter = walk(stream, { stringChunkSize: 8 })
+
+        stream.push('"\\"the quick brown fox\r\n\\tjumps\\u00a0over the lazy\\u1680dog\\""')
+        stream.push(null)
+
+        Object.entries(events).forEach(([ key, value ]) => {
+          emitter.on(value, spooks.fn({
+            name: key,
+            log: log
+          }))
+        })
+
+        emitter.on(events.end, done)
+      })
+
+      test('stringChunk event occurred seven times', () => {
+        assert.strictEqual(log.counts.stringChunk, 6)
+      })
+
+      test('stringChunk event was dispatched correctly', () => {
+        assert.lengthOf(log.args.stringChunk[0], 1)
+        assert.strictEqual(log.args.stringChunk[0][0], '"the qui')
+        assert.lengthOf(log.args.stringChunk[1], 1)
+        assert.strictEqual(log.args.stringChunk[1][0], 'ck brown')
+        assert.lengthOf(log.args.stringChunk[2], 1)
+        assert.strictEqual(log.args.stringChunk[2][0], ' fox\r\n\tj')
+        assert.lengthOf(log.args.stringChunk[3], 1)
+        assert.strictEqual(log.args.stringChunk[3][0], 'umps\u00a0ove')
+        assert.lengthOf(log.args.stringChunk[4], 1)
+        assert.strictEqual(log.args.stringChunk[4][0], 'r the la')
+        assert.lengthOf(log.args.stringChunk[5], 1)
+        assert.strictEqual(log.args.stringChunk[5][0], 'zy\u1680dog"')
+      })
+
+      test('string event occurred once', () => {
+        assert.strictEqual(log.counts.string, 1)
+      })
+
+      test('string event was dispatched correctly', () => {
+        assert.lengthOf(log.args.string[0], 1)
+        assert.strictEqual(log.args.string[0][0], '"the quick brown fox\r\n\tjumps\u00a0over the lazy\u1680dog"')
+      })
+
+      test('end event occurred once', () => {
+        assert.strictEqual(log.counts.end, 1)
+      })
+
+      test('array event did not occur', () => {
+        assert.strictEqual(log.counts.array, 0)
+      })
+
+      test('object event did not occur', () => {
+        assert.strictEqual(log.counts.object, 0)
+      })
+
+      test('property event did not occur', () => {
+        assert.strictEqual(log.counts.property, 0)
+      })
+
       test('number event did not occur', () => {
         assert.strictEqual(log.counts.number, 0)
       })
@@ -433,6 +548,10 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.string, 0)
       })
 
+      test('stringChunk event did not occur', () => {
+        assert.strictEqual(log.counts.stringChunk, 0)
+      })
+
       test('literal event did not occur', () => {
         assert.strictEqual(log.counts.literal, 0)
       })
@@ -513,6 +632,10 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.string, 0)
       })
 
+      test('stringChunk event did not occur', () => {
+        assert.strictEqual(log.counts.stringChunk, 0)
+      })
+
       test('number event did not occur', () => {
         assert.strictEqual(log.counts.number, 0)
       })
@@ -592,6 +715,10 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.string, 0)
       })
 
+      test('stringChunk event did not occur', () => {
+        assert.strictEqual(log.counts.stringChunk, 0)
+      })
+
       test('number event did not occur', () => {
         assert.strictEqual(log.counts.number, 0)
       })
@@ -671,6 +798,10 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.string, 0)
       })
 
+      test('stringChunk event did not occur', () => {
+        assert.strictEqual(log.counts.stringChunk, 0)
+      })
+
       test('number event did not occur', () => {
         assert.strictEqual(log.counts.number, 0)
       })
@@ -2892,8 +3023,8 @@ suite('walk:', () => {
         assert.strictEqual(log.counts.endObject, 1)
       })
 
-      test('error event occurred eleven times', () => {
-        assert.strictEqual(log.counts.error, 11)
+      test('error event occurred thirteen times', () => {
+        assert.strictEqual(log.counts.error, 13)
       })
 
       test('end event occurred once', () => {