@naturalcycles/nodejs-lib 13.19.0 → 13.21.0

package/dist/csv/transformToCSV.js

@@ -13,6 +13,7 @@ function transformToCSV(opt) {
     return new node_stream_1.Transform({
         writableObjectMode: true,
         readableObjectMode: false,
+        readableHighWaterMark: 64 * 1024,
         transform(chunk, _, cb) {
             try {
                 let s = '';

package/dist/stream/ndjson/transformJsonParse.js

@@ -21,6 +21,9 @@ function transformJsonParse(opt = {}) {
     return new node_stream_1.Transform({
         writableObjectMode: false,
         readableObjectMode: true,
+        // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
+        // todo: it'll be default in Node 22, then we can remove this
+        writableHighWaterMark: 64 * 1024,
         transform(chunk, _, cb) {
             try {
                 const data = JSON.parse(chunk, reviver);

package/dist/stream/ndjson/transformToNDJson.js

@@ -11,6 +11,7 @@ function transformToNDJson(opt = {}) {
     return new node_stream_1.Transform({
         writableObjectMode: true,
         readableObjectMode: false,
+        readableHighWaterMark: 64 * 1024,
         transform(chunk, _, cb) {
             try {
                 if (sortObjects) {

package/dist/stream/transform/transformMap.d.ts

@@ -18,7 +18,10 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
      * Number of concurrently pending promises returned by `mapper`.
      *
-     * @default 16 (to match default highWatermark option for objectMode streams)
+     * Default is 32.
+     * It was recently changed up from 16, after some testing that shown that
+     * for simple low-cpu mapper functions 32 produces almost 2x throughput.
+     * For example, in scenarios like streaming a query from Datastore.
      */
     concurrency?: number;
     /**
@@ -83,7 +86,7 @@ export interface TransformMapStatsSummary extends TransformMapStats {
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 16.
+ * Concurrency defaults to 32.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */

package/dist/stream/transform/transformMap.js

@@ -15,12 +15,12 @@ const stream_util_1 = require("../stream.util");
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 16.
+ * Concurrency defaults to 32.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
+    const { concurrency = 32, predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, onDone, metric = 'stream', logger = console, } = opt;
     const started = Date.now();
     let index = -1;

package/dist/stream/transform/transformSplit.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transformSplit = void 0;
+// https://github.com/max-mapper/binary-split
+// todo: test its newer version that doesn't have `through2` dependency
+// todo: test writableHighWaterMark of 64k
 const _binarySplit = require('binary-split');
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.19.0",
+  "version": "13.21.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/csv/transformToCSV.ts

@@ -30,6 +30,7 @@ export function transformToCSV<IN extends AnyObject = any>(
   return new Transform({
     writableObjectMode: true,
     readableObjectMode: false,
+    readableHighWaterMark: 64 * 1024,
     transform(chunk: IN, _, cb) {
       try {
         let s = ''

package/src/stream/ndjson/transformJsonParse.ts

@@ -35,6 +35,9 @@ export function transformJsonParse<OUT = any>(
   return new Transform({
     writableObjectMode: false,
     readableObjectMode: true,
+    // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
+    // todo: it'll be default in Node 22, then we can remove this
+    writableHighWaterMark: 64 * 1024,
     transform(chunk: string, _, cb) {
       try {
         const data = JSON.parse(chunk, reviver)

package/src/stream/ndjson/transformToNDJson.ts

@@ -34,6 +34,7 @@ export function transformToNDJson<IN = any>(
   return new Transform({
     writableObjectMode: true,
     readableObjectMode: false,
+    readableHighWaterMark: 64 * 1024,
     transform(chunk: IN, _, cb) {
       try {
         if (sortObjects) {

package/src/stream/transform/transformMap.ts

@@ -40,7 +40,10 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
    * Number of concurrently pending promises returned by `mapper`.
    *
-   * @default 16 (to match default highWatermark option for objectMode streams)
+   * Default is 32.
+   * It was recently changed up from 16, after some testing that shown that
+   * for simple low-cpu mapper functions 32 produces almost 2x throughput.
+   * For example, in scenarios like streaming a query from Datastore.
    */
   concurrency?: number
 
@@ -117,7 +120,7 @@ export interface TransformMapStatsSummary extends TransformMapStats {
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 16.
+ * Concurrency defaults to 32.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
@@ -126,7 +129,7 @@ export function transformMap<IN = any, OUT = IN>(
   opt: TransformMapOptions<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
   const {
-    concurrency = 16,
+    concurrency = 32,
     predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,

package/src/stream/transform/transformSplit.ts

@@ -1,5 +1,8 @@
 import { TransformTyped } from '../stream.model'
 
+// https://github.com/max-mapper/binary-split
+// todo: test its newer version that doesn't have `through2` dependency
+// todo: test writableHighWaterMark of 64k
 const _binarySplit = require('binary-split')
 
 /**