@naturalcycles/nodejs-lib 12.54.0 → 12.55.0

package/dist/stream/transform/beta/transformMap2.d.ts

@@ -0,0 +1,17 @@
+import { AsyncMapper } from '@naturalcycles/js-lib';
+import { TransformTyped } from '../../stream.model';
+import { TransformMapOptions } from '../transformMap';
+export declare function notNullishPredicate(item: any): boolean;
+/**
+ * Like pMap, but for streams.
+ * Inspired by `through2`.
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
+ * Using this allows native stream .pipe() to work and use backpressure.
+ *
+ * Only works in objectMode (due to through2Concurrent).
+ *
+ * Concurrency defaults to 16.
+ *
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ */
+export declare function transformMap2<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/beta/transformMap2.js

@@ -0,0 +1,104 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transformMap2 = exports.notNullishPredicate = void 0;
+const stream_1 = require("stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
+const colors_1 = require("../../../colors");
+function notNullishPredicate(item) {
+    return item !== undefined && item !== null;
+}
+exports.notNullishPredicate = notNullishPredicate;
+/**
+ * Like pMap, but for streams.
+ * Inspired by `through2`.
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
+ * Using this allows native stream .pipe() to work and use backpressure.
+ *
+ * Only works in objectMode (due to through2Concurrent).
+ *
+ * Concurrency defaults to 16.
+ *
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ */
+function transformMap2(mapper, opt = {}) {
+    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
+    let index = -1;
+    let isRejected = false;
+    let errors = 0;
+    const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
+    const q = new js_lib_1.PQueue({
+        concurrency,
+        resolveOn: 'start',
+        // debug: true,
+    });
+    return new stream_1.Transform({
+        objectMode: true,
+        async final(cb) {
+            // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
+            // wait for the current inFlight jobs to complete and push their results
+            await q.onIdle();
+            logErrorStats(logger, true);
+            await beforeFinal?.(); // call beforeFinal if defined
+            if (collectedErrors.length) {
+                // emit Aggregated error
+                cb(new js_lib_1.AggregatedError(collectedErrors));
+            }
+            else {
+                // emit no error
+                cb();
+            }
+        },
+        async transform(chunk, _encoding, cb) {
+            index++;
+            // console.log('transform', {index})
+            // Stop processing if THROW_IMMEDIATELY mode is used
+            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
+                return cb();
+            let calledBack = false;
+            // Allow up to 1 x concurrency "buffer" (aka highWatermark)
+            if (q.queueSize < concurrency) {
+                cb();
+                calledBack = true;
+            }
+            await q.push(async () => {
+                try {
+                    const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+                    const res = await mapper(chunk, currentIndex);
+                    const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
+                    passedResults.forEach(r => this.push(r));
+                }
+                catch (err) {
+                    logger.error(err);
+                    errors++;
+                    logErrorStats(logger);
+                    if (onError) {
+                        try {
+                            onError(err, chunk);
+                        }
+                        catch { }
+                    }
+                    if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+                        isRejected = true;
+                        // Emit error immediately
+                        // return cb(err as Error)
+                        return this.emit('error', err);
+                    }
+                    if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
+                        collectedErrors.push(err);
+                    }
+                }
+            });
+            // Resolved, which means it STARTED processing
+            // This means we can take more load
+            if (!calledBack) {
+                cb();
+            }
+        },
+    });
+    function logErrorStats(logger, final = false) {
+        if (!errors)
+            return;
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
+    }
+}
+exports.transformMap2 = transformMap2;

package/dist/stream/transform/transformMap.d.ts

@@ -1,4 +1,4 @@
-import { AsyncMapper, AsyncPredicate, ErrorMode } from '@naturalcycles/js-lib';
+import { AsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -41,6 +41,7 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * If defined - called BEFORE `final()` callback is called.
      */
     beforeFinal?: () => any;
+    logger?: CommonLogger;
 }
 export declare function notNullishPredicate(item: any): boolean;
 /**

package/dist/stream/transform/transformMap.js

@@ -21,7 +21,7 @@ exports.notNullishPredicate = notNullishPredicate;
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', } = opt;
+    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
     let index = -1;
     let isRejected = false;
     let errors = 0;
@@ -31,7 +31,7 @@ function transformMap(mapper, opt = {}) {
         // autoDestroy: true,
         async final(cb) {
             // console.log('transformMap final')
-            logErrorStats(true);
+            logErrorStats(logger, true);
             await beforeFinal?.(); // call beforeFinal if defined
             if (collectedErrors.length) {
                 // emit Aggregated error
@@ -64,9 +64,9 @@ function transformMap(mapper, opt = {}) {
             }
         }
         catch (err) {
-            console.error(err);
+            logger.error(err);
             errors++;
-            logErrorStats();
+            logErrorStats(logger);
             if (onError) {
                 try {
                     onError(err, chunk);
@@ -85,10 +85,10 @@ function transformMap(mapper, opt = {}) {
             cb();
         }
     });
-    function logErrorStats(final = false) {
+    function logErrorStats(logger, final = false) {
         if (!errors)
             return;
-        console.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
     }
 }
 exports.transformMap = transformMap;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.54.0",
+  "version": "12.55.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",

package/src/stream/transform/beta/transformMap2.ts

@@ -0,0 +1,142 @@
+import { Transform } from 'stream'
+import {
+  AggregatedError,
+  AsyncMapper,
+  CommonLogger,
+  ErrorMode,
+  pFilter,
+  PQueue,
+} from '@naturalcycles/js-lib'
+import { yellow } from '../../../colors'
+import { TransformTyped } from '../../stream.model'
+import { TransformMapOptions } from '../transformMap'
+
+export function notNullishPredicate(item: any): boolean {
+  return item !== undefined && item !== null
+}
+
+/**
+ * Like pMap, but for streams.
+ * Inspired by `through2`.
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
+ * Using this allows native stream .pipe() to work and use backpressure.
+ *
+ * Only works in objectMode (due to through2Concurrent).
+ *
+ * Concurrency defaults to 16.
+ *
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ */
+export function transformMap2<IN = any, OUT = IN>(
+  mapper: AsyncMapper<IN, OUT>,
+  opt: TransformMapOptions<IN, OUT> = {},
+): TransformTyped<IN, OUT> {
+  const {
+    concurrency = 16,
+    predicate = notNullishPredicate,
+    errorMode = ErrorMode.THROW_IMMEDIATELY,
+    flattenArrayOutput,
+    onError,
+    beforeFinal,
+    metric = 'stream',
+    logger = console,
+  } = opt
+
+  let index = -1
+  let isRejected = false
+  let errors = 0
+  const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
+
+  const q = new PQueue({
+    concurrency,
+    resolveOn: 'start',
+    // debug: true,
+  })
+
+  return new Transform({
+    objectMode: true,
+
+    async final(cb) {
+      // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
+
+      // wait for the current inFlight jobs to complete and push their results
+      await q.onIdle()
+
+      logErrorStats(logger, true)
+
+      await beforeFinal?.() // call beforeFinal if defined
+
+      if (collectedErrors.length) {
+        // emit Aggregated error
+        cb(new AggregatedError(collectedErrors))
+      } else {
+        // emit no error
+        cb()
+      }
+    },
+
+    async transform(this: Transform, chunk: IN, _encoding, cb) {
+      index++
+      // console.log('transform', {index})
+
+      // Stop processing if THROW_IMMEDIATELY mode is used
+      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
+
+      let calledBack = false
+
+      // Allow up to 1 x concurrency "buffer" (aka highWatermark)
+      if (q.queueSize < concurrency) {
+        cb()
+        calledBack = true
+      }
+
+      await q.push(async () => {
+        try {
+          const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+          const res = await mapper(chunk, currentIndex)
+          const passedResults = await pFilter(
+            flattenArrayOutput && Array.isArray(res) ? res : [res],
+            async r => await predicate(r, currentIndex),
+          )
+
+          passedResults.forEach(r => this.push(r))
+        } catch (err) {
+          logger.error(err)
+
+          errors++
+
+          logErrorStats(logger)
+
+          if (onError) {
+            try {
+              onError(err, chunk)
+            } catch {}
+          }
+
+          if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+            isRejected = true
+            // Emit error immediately
+            // return cb(err as Error)
+            return this.emit('error', err as Error)
+          }
+
+          if (errorMode === ErrorMode.THROW_AGGREGATED) {
+            collectedErrors.push(err as Error)
+          }
+        }
+      })
+
+      // Resolved, which means it STARTED processing
+      // This means we can take more load
+      if (!calledBack) {
+        cb()
+      }
+    },
+  })
+
+  function logErrorStats(logger: CommonLogger, final = false): void {
+    if (!errors) return
+
+    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
+  }
+}

package/src/stream/transform/transformMap.ts

@@ -3,6 +3,7 @@ import {
   AggregatedError,
   AsyncMapper,
   AsyncPredicate,
+  CommonLogger,
   ErrorMode,
   pFilter,
 } from '@naturalcycles/js-lib'
@@ -57,6 +58,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * If defined - called BEFORE `final()` callback is called.
    */
   beforeFinal?: () => any
+
+  logger?: CommonLogger
 }
 
 export function notNullishPredicate(item: any): boolean {
@@ -87,6 +90,7 @@ export function transformMap<IN = any, OUT = IN>(
     onError,
     beforeFinal,
     metric = 'stream',
+    logger = console,
   } = opt
 
   let index = -1
@@ -101,7 +105,7 @@ export function transformMap<IN = any, OUT = IN>(
       async final(cb) {
         // console.log('transformMap final')
 
-        logErrorStats(true)
+        logErrorStats(logger, true)
 
         await beforeFinal?.() // call beforeFinal if defined
 
@@ -144,11 +148,11 @@ export function transformMap<IN = any, OUT = IN>(
           cb() // done processing
         }
       } catch (err) {
-        console.error(err)
+        logger.error(err)
 
         errors++
 
-        logErrorStats()
+        logErrorStats(logger)
 
         if (onError) {
           try {
@@ -172,9 +176,9 @@ export function transformMap<IN = any, OUT = IN>(
     },
   )
 
-  function logErrorStats(final = false): void {
+  function logErrorStats(logger: CommonLogger, final = false): void {
     if (!errors) return
 
-    console.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
+    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
 }