bfj 7.1.0 → 9.0.0

package/.eslintrc

@@ -45,7 +45,7 @@ rules:
   lines-around-comment: 0
   max-depth: [ 1, 3 ]
   max-nested-callbacks: [ 1, 2 ]
-  max-params: [ 1, 4 ]
+  max-params: [ 1, 8 ]
   max-statements: 0
   new-cap: 2
   new-parens: 2

package/.gitlab-ci.yml

@@ -12,14 +12,10 @@ before_script:
     - npm run lint
     - npm test
 
-test:node8:
-  image: node:8
+test:node18:
+  image: node:18
   <<: *npm_test
 
-test:node10:
-  image: node:10
-  <<: *npm_test
-
-test:node12:
-  image: node:12
+test:node20:
+  image: node:20
   <<: *npm_test

package/HISTORY.md

@@ -1,6 +1,25 @@
 # History
 
-## 7.1.0
+## 9.0.0
+
+### Breaking changes
+
+* options: ditch bluebird promises and the `Promise` option (7b8292977c3ea1ebf0b0abdfb91a428b0159f374)
+
+### Other changes
+
+* docs: remove some old information from the readme (7665213e6f24c3cf287d361100513288d0f8b1cf)
+* deps: update dev dependencies (d8a1e3c4028d300a4b292efb82b1bb5d02cff62b)
+* tests: add perf test for walk (ac48de9bb613133a62bd6f67b8f7baccc3808609)
+* walk: restrict json buffer length when parsing (59d8f5f17081a8610f94c614929e3665d5ac511d)
+* options: more modest default options (ff8d18d0c57120cc2a16c66de17908ce85a7f028)
+* perf: add test command for match perf testing (2d4c48040fdc43f7838a565913ec577da921938d)
+
+## 8.0.0
+
+### Breaking changes
+
+* platform: only support node 18+ (45062680c52dc5ee09b136babcb3c333e3409409)
 
 ### New features
 
@@ -8,6 +27,8 @@
 
 ### Other changes
 
+* docs: change downloads metric to weekly in readme (869b78f6629cbfcf9f4f6035c8bef52f1efaf0d1)
+* ci: run tests in node 16, 18 and 20 (a688782892f72f554215e181d7925fb19de934ff)
 * deps: upgrade dependencies (f6e8664281931e0eeea26d62059bfbc9e9a8c37d)
 * tests: add perf test for bfj.match (f345bc83e5fbd9204cab02169a2ae9f4c02d1450)
 

package/README.md

@@ -2,7 +2,7 @@
 
 [![Build status](https://gitlab.com/philbooth/bfj/badges/master/pipeline.svg)](https://gitlab.com/philbooth/bfj/pipelines)
 [![Package status](https://img.shields.io/npm/v/bfj.svg)](https://www.npmjs.com/package/bfj)
-[![Downloads](https://img.shields.io/npm/dm/bfj.svg)](https://www.npmjs.com/package/bfj)
+[![Downloads](https://img.shields.io/npm/dw/bfj.svg)](https://www.npmjs.com/package/bfj)
 [![License](https://img.shields.io/npm/l/bfj.svg)](https://opensource.org/licenses/MIT)
 
 Big-Friendly JSON. Asynchronous streaming functions for large JSON data sets.
@@ -25,7 +25,6 @@ Big-Friendly JSON. Asynchronous streaming functions for large JSON data sets.
   * [Options for serialisation functions](#options-for-serialisation-functions)
 * [Is it possible to pause parsing or serialisation from calling code?](#is-it-possible-to-pause-parsing-or-serialisation-from-calling-code)
 * [Can it handle newline-delimited JSON (NDJSON)?](#can-it-handle-newline-delimited-json-ndjson)
-* [Why does it default to bluebird promises?](#why-does-it-default-to-bluebird-promises)
 * [Can I specify a different promise implementation?](#can-i-specify-a-different-promise-implementation)
 * [Is there a change log?](#is-there-a-change-log)
 * [How do I set up the dev environment?](#how-do-i-set-up-the-dev-environment)
@@ -159,7 +158,7 @@ bfj.read(path, options)
   });
 ```
 
-`read` returns a [bluebird promise][promise] and
+`read` returns a promise and
 asynchronously parses
 a JSON file
 from disk.
@@ -200,7 +199,7 @@ request({ url }).pipe(bfj.unpipe((error, data) => {
 }))
 ```
 
-* `parse` returns a [bluebird promise][promise]
+* `parse` returns a promise
   and asynchronously parses
   a stream of JSON data.
 
@@ -313,7 +312,7 @@ bfj.write(path, data, options)
   });
 ```
 
-`write` returns a [bluebird promise][promise]
+`write` returns a promise
 and asynchronously serialises a data structure
 to a JSON file on disk.
 The promise is resolved
@@ -373,7 +372,7 @@ bfj.stringify(data, options)
   });
 ```
 
-`stringify` returns a [bluebird promise][promise] and
+`stringify` returns a promise and
 asynchronously serialises a data structure
 to a JSON string.
 The promise is resolved
@@ -680,18 +679,12 @@ of an object,
   but the overall parsing time will be slower.
   Larger values yield to the event loop less often,
   meaning slower tick times but faster overall parsing time.
-  The default value is `16384`.
+  The default value is `1024`.
 
 * `options.Promise`:
   Promise constructor that will be used
   for promises returned by all methods.
-  If you set this option,
-  please be aware that some promise implementations
-  (including native promises)
-  may cause your process to die
-  with out-of-memory exceptions.
-  Defaults to [bluebird's implementation][promise],
-  which does not have that problem.
+  Defaults to JS builtin `Promise`.
 
 * `options.ndjson`:
   If set to `true`,
@@ -712,7 +705,7 @@ of an object,
   the length of the match buffer.
   Smaller values use less memory
   but may result in a slower parse time.
-  The default value is `1024`.
+  The default value is `256`.
 
 * `options.highWaterMark`:
   For `bfj.match` only,
@@ -789,7 +782,7 @@ of an object,
   The length of the write buffer.
   Smaller values use less memory
   but may result in a slower serialisation time.
-  The default value is `1024`.
+  The default value is `256`.
 
 * `options.highWaterMark`:
   Set this if you would like to
@@ -804,18 +797,12 @@ of an object,
   but the overall serialisation time will be slower.
   Larger values yield to the event loop less often,
   meaning slower tick times but faster overall serialisation time.
-  The default value is `16384`.
+  The default value is `1024`.
 
 * `options.Promise`:
   Promise constructor that will be used
   for promises returned by all methods.
-  If you set this option,
-  please be aware that some promise implementations
-  (including native promises)
-  may cause your process to die
-  with out-of-memory exceptions.
-  Defaults to [bluebird's implementation][promise],
-  which does not have that problem.
+  Defaults to JS builtin `Promise`.
 
 ## Is it possible to pause parsing or serialisation from calling code?
 
@@ -873,31 +860,14 @@ discrete JSON values:
 
 `bfj.unpipe` and `bfj.read` will not parse NDJSON.
 
-## Why does it default to bluebird promises?
-
-Until version `4.2.4`,
-native promises were used.
-But they were found
-to cause out-of-memory errors
-when serialising large amounts of data to JSON,
-due to [well-documented problems
-with the native promise implementation](https://alexn.org/blog/2017/10/11/javascript-promise-leaks-memory.html).
-So in version `5.0.0`,
-bluebird promises were used instead.
-In version `5.1.0`,
-an option was added
-that enables callers to specify
-the promise constructor to use.
-Use it at your own risk.
-
 ## Can I specify a different promise implementation?
 
 Yes.
 Just pass the `Promise` option
 to any method.
-If you get out-of-memory errors
-when using that option,
-consider changing your promise implementation.
+You might want to try this
+if you get any out-of-memory errors
+when parsing huge files.
 
 ## Is there a change log?
 
@@ -935,8 +905,8 @@ with the command
 
 ## What versions of Node.js does it support?
 
-As of [version `7.0.0`](HISTORY.md#700),
-only Node.js versions 8 or greater
+As of [version `8.0.0`](HISTORY.md#700),
+only Node.js versions 18 or greater
 are supported.
 
 Between versions [`3.0.0`](HISTORY.md#300)
@@ -955,7 +925,6 @@ were supported.
 [ci-image]: https://secure.travis-ci.org/philbooth/bfj.png?branch=master
 [ci-status]: http://travis-ci.org/#!/philbooth/bfj
 [sax]: http://en.wikipedia.org/wiki/Simple_API_for_XML
-[promise]: http://bluebirdjs.com/docs/api-reference.html
 [bfj-collections]: https://github.com/hash-bang/bfj-collections
 [eventemitter]: https://nodejs.org/api/events.html#events_class_eventemitter
 [readable]: https://nodejs.org/api/stream.html#stream_readable_streams

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bfj",
-  "version": "7.1.0",
+  "version": "9.0.0",
   "description": "Big-friendly JSON. Asynchronous streaming functions for large JSON data sets.",
   "homepage": "https://gitlab.com/philbooth/bfj",
   "bugs": "https://gitlab.com/philbooth/bfj/issues",
@@ -26,20 +26,19 @@
     "url": "https://gitlab.com/philbooth/bfj.git"
   },
   "engines": {
-    "node": ">= 8.0.0"
+    "node": ">= 18.0.0"
   },
   "dependencies": {
-    "bluebird": "^3.7.2",
     "check-types": "^11.2.3",
     "hoopy": "^0.1.4",
     "jsonpath": "^1.1.1",
     "tryer": "^1.0.1"
   },
   "devDependencies": {
-    "axios": "^1.5.0",
-    "chai": "^4.3.8",
-    "eslint": "^8.48.0",
-    "mocha": "^10.2.0",
+    "axios": "^1.7.7",
+    "chai": "^4.5.0",
+    "eslint": "^8.57.1",
+    "mocha": "^10.7.3",
     "please-release-me": "^2.1.4",
     "proxyquire": "^2.1.3",
     "spooks": "^2.0.0"
@@ -49,6 +48,7 @@
     "test": "npm run unit && npm run integration",
     "unit": "mocha --ui tdd --reporter spec --recursive --colors --slow 120 test/unit",
     "integration": "mocha --ui tdd --reporter spec --colors test/integration",
-    "perf": "wget -O test/mtg.json http://mtgjson.com/json/AllSets-x.json && node test/performance mtg"
+    "perf": "wget -O test/mtg.json https://mtgjson.com/api/v5/AllPrices.json && node test/performance mtg",
+    "perf-match": "wget -O test/mtg.json https://mtgjson.com/api/v5/AllPrices.json && node test/performance mtg currency"
   }
 }

package/src/eventify.js

@@ -3,7 +3,6 @@
 const check = require('check-types')
 const EventEmitter = require('events').EventEmitter
 const events = require('./events')
-const promise = require('./promise')
 
 const invalidTypes = {
   undefined: true, // eslint-disable-line no-undefined
@@ -34,14 +33,11 @@ module.exports = eventify
  * @option circular:   'error' or 'ignore', default is 'error'.
  *
  * @option yieldRate:  The number of data items to process per timeslice,
- *                     default is 16384.
- *
- * @option Promise:      The promise constructor to use, defaults to bluebird.
+ *                     default is 1024.
  **/
 function eventify (data, options = {}) {
   const coercions = {}
   const emitter = new EventEmitter()
-  const Promise = promise(options)
   const references = new Map()
 
   let count = 0
@@ -53,7 +49,7 @@ function eventify (data, options = {}) {
 
   emitter.pause = () => {
     let resolve
-    pause = new Promise(res => resolve = res)
+    pause = new Promise((res) => resolve = res)
     return () => {
       pause = null
       count = 0
@@ -80,7 +76,7 @@ function eventify (data, options = {}) {
     }
 
     check.assert.maybe.positive(options.yieldRate)
-    yieldRate = options.yieldRate || 16384
+    yieldRate = options.yieldRate || 1024
   }
 
   function parseCoercionOption (key) {
@@ -89,59 +85,31 @@ function eventify (data, options = {}) {
     }
   }
 
-  function begin () {
-    return proceed(data)
-      .catch(error => emit(events.error, error))
-      .then(() => emit(events.end))
+  async function begin () {
+    try {
+      await proceed(data)
+    } catch (error) {
+      await emit(events.error, error)
+    } finally {
+      await emit(events.end)
+    }
   }
 
-  function proceed (datum) {
+  async function proceed (datum) {
     if (++count % yieldRate !== 0) {
-      return coerce(datum).then(after)
+      return afterCoercion(await coerce(datum))
     }
 
-    return new Promise((resolve, reject) => {
-      setImmediate(() => {
-        coerce(datum)
-          .then(after)
-          .then(resolve)
-          .catch(reject)
-      })
-    })
-
-    function after (coerced) {
-      if (isInvalid(coerced)) {
-        return
-      }
-
-      if (coerced === false || coerced === true || coerced === null) {
-        return literal(coerced)
-      }
-
-      if (Array.isArray(coerced)) {
-        return array(coerced)
-      }
-
-      const type = typeof coerced
-
-      switch (type) {
-        case 'number':
-          return value(coerced, type)
-        case 'string':
-          return value(escapeString(coerced), type)
-        default:
-          return object(coerced)
-      }
-    }
+    return new Promise(yieldThenProceed.bind(null, datum))
   }
 
-  function coerce (datum) {
+  async function coerce (datum) {
     if (disableCoercions || check.primitive(datum)) {
-      return Promise.resolve(datum)
+      return datum
     }
 
     if (check.thenable(datum)) {
-      return coerceThing(datum, 'promises', coercePromise).then(coerce)
+      return coerce(await coerceThing(datum, 'promises', coercePromise))
     }
 
     if (check.instanceStrict(datum, Buffer)) {
@@ -161,10 +129,10 @@ function eventify (data, options = {}) {
     }
 
     if (check.function(datum.toJSON)) {
-      return Promise.resolve(datum.toJSON())
+      return datum.toJSON()
     }
 
-    return Promise.resolve(datum)
+    return datum
   }
 
   function coerceThing (datum, thing, fn) {
@@ -186,8 +154,8 @@ function eventify (data, options = {}) {
   function coerceMap (map) {
     const result = {}
 
-    return coerceCollection(map, result, (item, key) => {
-      result[key] = item
+    return coerceCollection(map, result, (val, key) => {
+      result[key] = val
     })
   }
 
@@ -200,11 +168,36 @@ function eventify (data, options = {}) {
   function coerceIterable (iterable) {
     const result = []
 
-    return coerceCollection(iterable, result, item => {
-      result.push(item)
+    return coerceCollection(iterable, result, (val) => {
+      result.push(val)
     })
   }
 
+  function afterCoercion (coerced) {
+    if (isInvalid(coerced)) {
+      return
+    }
+
+    if (coerced === false || coerced === true || coerced === null) {
+      return literal(coerced)
+    }
+
+    if (Array.isArray(coerced)) {
+      return array(coerced)
+    }
+
+    const type = typeof coerced
+
+    switch (type) {
+      case 'number':
+        return value(coerced, type)
+      case 'string':
+        return value(escapeString(coerced), type)
+      default:
+        return object(coerced)
+    }
+  }
+
   function isInvalid (datum) {
     const type = typeof datum
     return !! invalidTypes[type] || (
@@ -216,89 +209,104 @@ function eventify (data, options = {}) {
     return datum > Number.NEGATIVE_INFINITY && datum < Number.POSITIVE_INFINITY
   }
 
-  function literal (datum) {
-    return value(datum, 'literal')
-  }
-
-  function value (datum, type) {
-    return emit(events[type], datum)
+  async function yieldThenProceed (datum, resolve, reject) {
+    setImmediate(async () => {
+      try {
+        resolve(await afterCoercion(await coerce(datum)))
+      } catch (error) {
+        reject(error)
+      }
+    })
   }
 
-  function emit (event, eventData) {
-    return (pause || Promise.resolve())
-      .then(() => emitter.emit(event, eventData))
-      .catch(err => {
-        try {
-          emitter.emit(events.error, err)
-        } catch (_) {
-          // When calling user code, anything is possible
-        }
-      })
+  function literal (datum) {
+    return value(datum, 'literal')
   }
 
   function array (datum) {
     // For an array, collection:object and collection:array are the same.
-    return collection(datum, datum, 'array', item => {
-      if (isInvalid(item)) {
-        return proceed(null)
+    return collection(datum, datum, 'array', async (val) => {
+      if (isInvalid(val)) {
+        await proceed(null)
+      } else {
+        await proceed(val)
       }
-
-      return proceed(item)
     })
   }
 
-  function collection (obj, arr, type, action) {
+  async function collection (obj, arr, type, action) {
     let ignoreThisItem
 
-    return Promise.resolve()
-      .then(() => {
-        if (references.has(obj)) {
-          ignoreThisItem = ignoreItems = true
-
-          if (! ignoreCircularReferences) {
-            return emit(events.dataError, new Error('Circular reference.'))
-          }
-        } else {
-          references.set(obj, true)
-        }
-      })
-      .then(() => emit(events[type]))
-      .then(() => item(0))
-
-    function item (index) {
-      if (index >= arr.length) {
-        if (ignoreThisItem) {
-          ignoreItems = false
-        }
-
-        if (ignoreItems) {
-          return Promise.resolve()
-        }
-
-        return emit(events.endPrefix + events[type])
-          .then(() => references.delete(obj))
+    if (references.has(obj)) {
+      ignoreThisItem = ignoreItems = true
+
+      if (! ignoreCircularReferences) {
+        return emit(events.dataError, new Error('Circular reference.'))
+      }
+    } else {
+      references.set(obj, true)
+    }
+
+    await emit(events[type])
+
+    await item(obj, arr, type, action, ignoreThisItem, 0)
+  }
+
+  async function emit (event, eventData) {
+    try {
+      await pause
+      emitter.emit(event, eventData)
+    } catch (error) {
+      try {
+        emitter.emit(events.error, error)
+      } catch (_) {
+        // When calling user code, anything is possible
+      }
+    }
+  }
+
+  async function item (obj, arr, type, action, ignoreThisItem, index) {
+    if (index >= arr.length) {
+      if (ignoreThisItem) {
+        ignoreItems = false
       }
 
       if (ignoreItems) {
-        return item(index + 1)
+        return
       }
 
-      return action(arr[index])
-        .then(() => item(index + 1))
+      await emit(events.endPrefix + events[type])
+
+      references.delete(obj)
+
+      return
+    }
+
+    if (ignoreItems) {
+      return item(obj, arr, type, action, ignoreThisItem, index + 1)
     }
+
+    await action(arr[index])
+
+    await item(obj, arr, type, action, ignoreThisItem, index + 1)
+  }
+
+  function value (datum, type) {
+    return emit(events[type], datum)
   }
 
   function object (datum) {
     // For an object, collection:object and collection:array are different.
-    return collection(datum, Object.keys(datum), 'object', key => {
-      const item = datum[key]
+    return collection(datum, Object.keys(datum), 'object', async (key) => {
+      const val = datum[key]
 
-      if (isInvalid(item)) {
-        return Promise.resolve()
+      if (isInvalid(val)) {
+        return
       }
 
-      return emit(events.property, escapeString(key))
-        .then(() => proceed(item))
+      await emit(events.property, escapeString(key))
+
+      await proceed(val)
     })
   }
 

package/src/match.js

@@ -7,7 +7,7 @@ const Hoopy = require('hoopy')
 const jsonpath = require('jsonpath')
 const walk = require('./walk')
 
-const DEFAULT_BUFFER_LENGTH = 1024
+const DEFAULT_BUFFER_LENGTH = 256
 
 module.exports = match
 
@@ -40,14 +40,12 @@ module.exports = match
  *                        default is `false`.
  *
  * @option yieldRate:     The number of data items to process per timeslice,
- *                        default is 16384.
+ *                        default is 1024.
  *
- * @option bufferLength:  The length of the match buffer, default is 1024.
+ * @option bufferLength:  The length of the match buffer, default is 256.
  *
  * @option highWaterMark: If set, will be passed to the readable stream constructor
  *                        as the value for the highWaterMark option.
- *
- * @option Promise:       The promise constructor to use, defaults to bluebird.
  **/
 function match (stream, selector, options = {}) {
   const keys = []

package/src/parse.js

@@ -2,7 +2,6 @@
 
 const check = require('check-types')
 const events = require('./events')
-const promise = require('./promise')
 const walk = require('./walk')
 
 module.exports = parse
@@ -21,9 +20,7 @@ const NDJSON_STATE = new Map()
  * @option reviver:   Transformation function, invoked depth-first.
  *
  * @option yieldRate: The number of data items to process per timeslice,
- *                    default is 16384.
- *
- * @option Promise:   The promise constructor to use, defaults to bluebird.
+ *                    default is 1024.
  *
  * @option ndjson:    Set this to true to parse newline-delimited JSON. In
  *                    this case, each call will be resolved with one value
@@ -31,14 +28,8 @@ const NDJSON_STATE = new Map()
  *                    should be made sequentially one-at-a-time until the
  *                    returned promise resolves to `undefined`.
  **/
-function parse (stream, options = {}) {
-  const Promise = promise(options)
-
-  try {
-    check.assert.maybe.function(options.reviver, 'Invalid reviver option')
-  } catch (err) {
-    return Promise.reject(err)
-  }
+async function parse (stream, options = {}) {
+  check.assert.maybe.function(options.reviver, 'Invalid reviver option')
 
   const errors = []
   const scopes = []

package/src/read.js

@@ -17,9 +17,7 @@ module.exports = read
  * @option reviver:   Transformation function, invoked depth-first.
  *
  * @option yieldRate: The number of data items to process per timeslice,
- *                    default is 16384.
- *
- * @option Promise:   The promise constructor to use, defaults to bluebird.
+ *                    default is 1024.
  **/
 function read (path, options) {
   return parse(fs.createReadStream(path, options), { ...options, ndjson: false })