@naturalcycles/nodejs-lib 12.55.1 → 12.58.0

package/src/stream/transform/legacy/transformMap.ts

@@ -0,0 +1,133 @@
+import { Transform } from 'stream'
+import {
+  AggregatedError,
+  AsyncMapper,
+  CommonLogger,
+  ErrorMode,
+  pFilter,
+} from '@naturalcycles/js-lib'
+import through2Concurrent = require('through2-concurrent')
+import { yellow } from '../../../colors'
+import { TransformTyped } from '../../stream.model'
+import { TransformMapOptions } from '../transformMap'
+
+export function notNullishPredicate(item: any): boolean {
+  return item !== undefined && item !== null
+}
+
+/**
+ * Like pMap, but for streams.
+ * Inspired by `through2`.
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
+ * Using this allows native stream .pipe() to work and use backpressure.
+ *
+ * Only works in objectMode (due to through2Concurrent).
+ *
+ * Concurrency defaults to 16.
+ *
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ */
+export function transformMapLegacy<IN = any, OUT = IN>(
+  mapper: AsyncMapper<IN, OUT>,
+  opt: TransformMapOptions<IN, OUT> = {},
+): TransformTyped<IN, OUT> {
+  const {
+    concurrency = 16,
+    predicate = notNullishPredicate,
+    errorMode = ErrorMode.THROW_IMMEDIATELY,
+    flattenArrayOutput,
+    onError,
+    beforeFinal,
+    metric = 'stream',
+    logger = console,
+  } = opt
+
+  let index = -1
+  let isRejected = false
+  let errors = 0
+  const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
+
+  return through2Concurrent.obj(
+    {
+      maxConcurrency: concurrency,
+      // autoDestroy: true,
+      async final(cb) {
+        // console.log('transformMap final')
+
+        logErrorStats(logger, true)
+
+        await beforeFinal?.() // call beforeFinal if defined
+
+        if (collectedErrors.length) {
+          // emit Aggregated error
+          cb(new AggregatedError(collectedErrors))
+        } else {
+          // emit no error
+          cb()
+        }
+      },
+    },
+    async function transformMapFn(
+      this: Transform,
+      chunk: IN,
+      _encoding: any,
+      cb: (...args: any[]) => any,
+    ) {
+      index++
+      // console.log({chunk, _encoding})
+
+      // Stop processing if THROW_IMMEDIATELY mode is used
+      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
+
+      try {
+        const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+        const res = await mapper(chunk, currentIndex)
+        const passedResults = await pFilter(
+          flattenArrayOutput && Array.isArray(res) ? res : [res],
+          async r => await predicate(r, currentIndex),
+        )
+
+        if (passedResults.length === 0) {
+          cb() // 0 results
+        } else {
+          passedResults.forEach(r => {
+            this.push(r)
+            // cb(null, r)
+          })
+          cb() // done processing
+        }
+      } catch (err) {
+        logger.error(err)
+
+        errors++
+
+        logErrorStats(logger)
+
+        if (onError) {
+          try {
+            onError(err, chunk)
+          } catch {}
+        }
+
+        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+          isRejected = true
+          // Emit error immediately
+          return cb(err)
+        }
+
+        if (errorMode === ErrorMode.THROW_AGGREGATED) {
+          collectedErrors.push(err as Error)
+        }
+
+        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+        cb()
+      }
+    },
+  )
+
+  function logErrorStats(logger: CommonLogger, final = false): void {
+    if (!errors) return
+
+    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
+  }
+}

package/src/stream/transform/transformLimit.ts

@@ -1,13 +1,19 @@
 import { Transform } from 'stream'
+import { CommonLogger } from '@naturalcycles/js-lib'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
+export interface TransformLimitOptions extends TransformOptions {
+  logger?: CommonLogger
+}
+
 /**
  * 0 or falsy value means "no limit"
  */
 export function transformLimit<IN>(
   limit?: number,
-  opt: TransformOptions = {},
+  opt: TransformLimitOptions = {},
 ): TransformTyped<IN, IN> {
+  const { logger = console } = opt
   let index = 0
   let ended = false
 
@@ -25,7 +31,7 @@ export function transformLimit<IN>(
 
       if (limit && index === limit) {
         ended = true
-        console.log(`transformLimit: limit of ${limit} reached`)
+        logger.log(`transformLimit: limit of ${limit} reached`)
         // this.emit('end') // this makes it "halt" on Node 14 lts
       }
     },

package/src/stream/transform/transformLogProgress.ts

@@ -1,6 +1,6 @@
 import { Transform } from 'stream'
 import { inspect, InspectOptions } from 'util'
-import { SimpleMovingAverage, _mb, _since, AnyObject } from '@naturalcycles/js-lib'
+import { SimpleMovingAverage, _mb, _since, AnyObject, CommonLogger } from '@naturalcycles/js-lib'
 import { dayjs } from '@naturalcycles/time-lib'
 import { boldWhite, dimGrey, white, yellow } from '../../colors'
 import { hasColors } from '../../colors/colors'
@@ -86,6 +86,8 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    */
   logEvery?: number
 
+  logger?: CommonLogger
+
   /**
    * Function to return extra properties to the "progress object".
    *
@@ -124,6 +126,7 @@ export function transformLogProgress<IN = any>(
     logEvery = 1000,
     batchSize = 1,
     extra,
+    logger = console,
   } = opt
   const logProgress = opt.logProgress !== false && logEvery !== 0 // true by default
   const logEvery10 = logEvery * 10
@@ -172,7 +175,7 @@ export function transformLogProgress<IN = any>(
     const rps10 = Math.round(sma.push(lastRPS))
     if (mem.rss > peakRSS) peakRSS = mem.rss
 
-    console.log(
+    logger.log(
       inspect(
         {
           [final ? `${metric}_final` : metric]: batchedProgress,
@@ -202,13 +205,13 @@ export function transformLogProgress<IN = any>(
         perHour = Math.round(perHour / 1000) + 'K'
       }
 
-      console.log(
+      logger.log(
         `${dimGrey(dayjs().toPretty())} ${white(metric)} took ${yellow(
           _since(started),
         )} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`,
       )
     } else if (final) {
-      console.log(
+      logger.log(
         `${boldWhite(metric)} took ${yellow(_since(started))} to process ${yellow(
           batchedProgress,
         )} rows with total RPS of ${yellow(rpsTotal)}`,

package/src/stream/transform/transformMap.ts

@@ -6,10 +6,11 @@ import {
   CommonLogger,
   ErrorMode,
   pFilter,
+  PQueue,
 } from '@naturalcycles/js-lib'
-import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors'
 import { TransformTyped } from '../stream.model'
+import { transformMapLegacy } from './legacy/transformMap'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
@@ -66,6 +67,11 @@ export function notNullishPredicate(item: any): boolean {
   return item !== undefined && item !== null
 }
 
+/**
+ * Temporary export legacy transformMap, to debug 503 errors
+ */
+export const transformMap = transformMapLegacy
+
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -78,7 +84,7 @@ export function notNullishPredicate(item: any): boolean {
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-export function transformMap<IN = any, OUT = IN>(
+export function transformMapNew<IN = any, OUT = IN>(
   mapper: AsyncMapper<IN, OUT>,
   opt: TransformMapOptions<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
@@ -98,83 +104,89 @@ export function transformMap<IN = any, OUT = IN>(
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  return through2Concurrent.obj(
-    {
-      maxConcurrency: concurrency,
-      // autoDestroy: true,
-      async final(cb) {
-        // console.log('transformMap final')
+  const q = new PQueue({
+    concurrency,
+    resolveOn: 'start',
+    // debug: true,
+  })
 
-        logErrorStats(logger, true)
+  return new Transform({
+    objectMode: true,
 
-        await beforeFinal?.() // call beforeFinal if defined
+    async final(cb) {
+      // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
 
-        if (collectedErrors.length) {
-          // emit Aggregated error
-          cb(new AggregatedError(collectedErrors))
-        } else {
-          // emit no error
-          cb()
-        }
-      },
-    },
-    async function transformMapFn(
-      this: Transform,
-      chunk: IN,
-      _encoding: any,
-      cb: (...args: any[]) => any,
-    ) {
-      index++
-      // console.log({chunk, _encoding})
-
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
+      // wait for the current inFlight jobs to complete and push their results
+      await q.onIdle()
 
-      try {
-        const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-        const res = await mapper(chunk, currentIndex)
-        const passedResults = await pFilter(
-          flattenArrayOutput && Array.isArray(res) ? res : [res],
-          async r => await predicate(r, currentIndex),
-        )
-
-        if (passedResults.length === 0) {
-          cb() // 0 results
-        } else {
-          passedResults.forEach(r => {
-            this.push(r)
-            // cb(null, r)
-          })
-          cb() // done processing
-        }
-      } catch (err) {
-        logger.error(err)
+      logErrorStats(logger, true)
 
-        errors++
+      await beforeFinal?.() // call beforeFinal if defined
 
-        logErrorStats(logger)
+      if (collectedErrors.length) {
+        // emit Aggregated error
+        // For the same reason, magically, let's not call `cb`, but emit an error event instead
+        // this.emit('error', new AggregatedError(collectedErrors))
+        cb(new AggregatedError(collectedErrors))
+      } else {
+        // emit no error
+        // It is truly a mistery, but calling cb() here was causing ERR_MULTIPLE_CALLBACK ?!
+        // Commenting it out seems to work ?!
+        // ?!
+        // cb()
+      }
+    },
 
-        if (onError) {
-          try {
-            onError(err, chunk)
-          } catch {}
-        }
+    async transform(this: Transform, chunk: IN, _encoding, cb) {
+      index++
+      // console.log('transform', {index})
 
-        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-          isRejected = true
-          // Emit error immediately
-          return cb(err)
-        }
+      // Stop processing if THROW_IMMEDIATELY mode is used
+      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
 
-        if (errorMode === ErrorMode.THROW_AGGREGATED) {
-          collectedErrors.push(err as Error)
+      // It resolves when it is successfully STARTED execution.
+      // If it's queued instead - it'll wait and resolve only upon START.
+      await q.push(async () => {
+        try {
+          const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+          const res = await mapper(chunk, currentIndex)
+          const passedResults = await pFilter(
+            flattenArrayOutput && Array.isArray(res) ? res : [res],
+            async r => await predicate(r, currentIndex),
+          )
+
+          passedResults.forEach(r => this.push(r))
+        } catch (err) {
+          logger.error(err)
+
+          errors++
+
+          logErrorStats(logger)
+
+          if (onError) {
+            try {
+              onError(err, chunk)
+            } catch {}
+          }
+
+          if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+            isRejected = true
+            // Emit error immediately
+            // return cb(err as Error)
+            return this.emit('error', err as Error)
+          }
+
+          if (errorMode === ErrorMode.THROW_AGGREGATED) {
+            collectedErrors.push(err as Error)
+          }
         }
+      })
 
-        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
-        cb()
-      }
+      // Resolved, which means it STARTED processing
+      // This means we can take more load
+      cb()
     },
-  )
+  })
 
   function logErrorStats(logger: CommonLogger, final = false): void {
     if (!errors) return

package/src/stream/transform/transformMapSimple.ts

@@ -1,5 +1,5 @@
 import { Transform } from 'stream'
-import { ErrorMode, Mapper } from '@naturalcycles/js-lib'
+import { CommonLogger, ErrorMode, Mapper } from '@naturalcycles/js-lib'
 import { TransformTyped } from '../stream.model'
 
 export interface TransformMapSimpleOptions {
@@ -9,6 +9,8 @@ export interface TransformMapSimpleOptions {
    * @default ErrorMode.THROW_IMMEDIATELY
    */
   errorMode?: ErrorMode.THROW_IMMEDIATELY | ErrorMode.SUPPRESS
+
+  logger?: CommonLogger
 }
 
 /**
@@ -25,7 +27,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
   opt: TransformMapSimpleOptions = {},
 ): TransformTyped<IN, OUT> {
   let index = -1
-  const { errorMode = ErrorMode.THROW_IMMEDIATELY } = opt
+  const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console } = opt
 
   return new Transform({
     objectMode: true,
@@ -33,7 +35,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
       try {
         cb(null, mapper(chunk, ++index))
       } catch (err) {
-        console.error(err)
+        logger.error(err)
 
         if (errorMode === ErrorMode.SUPPRESS) {
           cb() // suppress the error

package/src/stream/transform/transformMapSync.ts

@@ -1,5 +1,5 @@
 import { Transform } from 'stream'
-import { AggregatedError, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib'
+import { AggregatedError, CommonLogger, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib'
 import { yellow } from '../../colors'
 import { TransformTyped } from '../stream.model'
 import { notNullishPredicate } from './transformMap'
@@ -43,6 +43,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
    * @default `stream`
    */
   metric?: string
+
+  logger?: CommonLogger
 }
 
 /**
@@ -62,6 +64,7 @@ export function transformMapSync<IN = any, OUT = IN>(
     onError,
     metric = 'stream',
     objectMode = true,
+    logger = console,
   } = opt
   let isRejected = false
   let errors = 0
@@ -92,7 +95,7 @@ export function transformMapSync<IN = any, OUT = IN>(
           cb(null, v)
         }
       } catch (err) {
-        console.error(err)
+        logger.error(err)
         errors++
 
         logErrorStats()
@@ -134,6 +137,6 @@ export function transformMapSync<IN = any, OUT = IN>(
   function logErrorStats(final = false): void {
     if (!errors) return
 
-    console.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
+    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
 }

package/src/stream/transform/transformTap.ts

@@ -1,7 +1,11 @@
 import { Transform } from 'stream'
-import { AsyncMapper } from '@naturalcycles/js-lib'
+import { AsyncMapper, CommonLogger } from '@naturalcycles/js-lib'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
+export interface TransformTapOptions extends TransformOptions {
+  logger?: CommonLogger
+}
+
 /**
  * Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
  * Item is passed through to the output.
@@ -10,8 +14,9 @@ import { TransformOptions, TransformTyped } from '../stream.model'
  */
 export function transformTap<IN>(
   fn: AsyncMapper<IN, any>,
-  opt: TransformOptions = {},
+  opt: TransformTapOptions = {},
 ): TransformTyped<IN, IN> {
+  const { logger = console } = opt
   let index = 0
 
   return new Transform({
@@ -23,7 +28,7 @@ export function transformTap<IN>(
       try {
         await fn(chunk, index++)
       } catch (err) {
-        console.error(err)
+        logger.error(err)
         // suppressed error
       }
 

package/src/stream/writable/writableForEach.ts

@@ -1,5 +1,5 @@
-import { AsyncMapper, _passNothingPredicate } from '@naturalcycles/js-lib'
-import { transformMap, TransformMapOptions } from '../..'
+import { AsyncMapper, _passNothingPredicate, Mapper } from '@naturalcycles/js-lib'
+import { transformMap, TransformMapOptions, transformMapSync } from '../..'
 import { WritableTyped } from '../stream.model'
 
 /**
@@ -11,3 +11,13 @@ export function writableForEach<IN = any>(
 ): WritableTyped<IN> {
   return transformMap<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
 }
+
+/**
+ * Just an alias to transformMap that declares OUT as void.
+ */
+export function writableForEachSync<IN = any>(
+  mapper: Mapper<IN, void>,
+  opt: TransformMapOptions<IN, void> = {},
+): WritableTyped<IN> {
+  return transformMapSync<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
+}

package/src/stream/writable/writableFork.ts

@@ -7,6 +7,8 @@ import { TransformOptions, WritableTyped } from '../stream.model'
  * Currently does NOT (!) maintain backpressure.
  * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
  * Will wait until all forked pipelines are completed before completing the stream.
+ *
+ * @experimental
  */
 export function writableFork<T>(
   chains: NodeJS.WritableStream[][],

package/dist/stream/transform/beta/transformMap2.js

@@ -1,98 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMap2 = exports.notNullishPredicate = void 0;
-const stream_1 = require("stream");
-const js_lib_1 = require("@naturalcycles/js-lib");
-const colors_1 = require("../../../colors");
-function notNullishPredicate(item) {
-    return item !== undefined && item !== null;
-}
-exports.notNullishPredicate = notNullishPredicate;
-/**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
- *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
- */
-function transformMap2(mapper, opt = {}) {
-    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
-    let index = -1;
-    let isRejected = false;
-    let errors = 0;
-    const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    const q = new js_lib_1.PQueue({
-        concurrency,
-        resolveOn: 'start',
-        // debug: true,
-    });
-    return new stream_1.Transform({
-        objectMode: true,
-        async final(cb) {
-            // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
-            // wait for the current inFlight jobs to complete and push their results
-            await q.onIdle();
-            logErrorStats(logger, true);
-            await beforeFinal?.(); // call beforeFinal if defined
-            if (collectedErrors.length) {
-                // emit Aggregated error
-                cb(new js_lib_1.AggregatedError(collectedErrors));
-            }
-            else {
-                // emit no error
-                cb();
-            }
-        },
-        async transform(chunk, _encoding, cb) {
-            index++;
-            // console.log('transform', {index})
-            // Stop processing if THROW_IMMEDIATELY mode is used
-            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
-                return cb();
-            // It resolves when it is successfully STARTED execution.
-            // If it's queued instead - it'll wait and resolve only upon START.
-            await q.push(async () => {
-                try {
-                    const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-                    const res = await mapper(chunk, currentIndex);
-                    const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
-                    passedResults.forEach(r => this.push(r));
-                }
-                catch (err) {
-                    logger.error(err);
-                    errors++;
-                    logErrorStats(logger);
-                    if (onError) {
-                        try {
-                            onError(err, chunk);
-                        }
-                        catch { }
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                        isRejected = true;
-                        // Emit error immediately
-                        // return cb(err as Error)
-                        return this.emit('error', err);
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
-                        collectedErrors.push(err);
-                    }
-                }
-            });
-            // Resolved, which means it STARTED processing
-            // This means we can take more load
-            cb();
-        },
-    });
-    function logErrorStats(logger, final = false) {
-        if (!errors)
-            return;
-        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
-    }
-}
-exports.transformMap2 = transformMap2;