@naturalcycles/nodejs-lib 12.58.0 → 12.59.0

package/src/stream/transform/transformLimit.ts

@@ -1,38 +1,84 @@
-import { Transform } from 'stream'
+import { Readable } from 'stream'
 import { CommonLogger } from '@naturalcycles/js-lib'
+import { AbortableTransform, transformNoOp } from '../../index'
 import { TransformOptions, TransformTyped } from '../stream.model'
+import { pipelineClose } from '../stream.util'
 
 export interface TransformLimitOptions extends TransformOptions {
+  /**
+   * Nullish value (e.g 0 or undefined) would mean "no limit"
+   */
+  limit?: number
+
+  /**
+   * If provided (recommended!) - it will call readable.destroy() on limit.
+   * Without it - it will only stop the downstream consumers, but won't stop
+   * the Readable ("source" of the stream).
+   * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+   */
+  sourceReadable?: Readable
+
+  /**
+   * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
+   * to be able to wait for Consumers before calling `readable.destroy`.
+   * Has no effect if `readable` is not provided.
+   */
+  streamDone?: Promise<void>
+
   logger?: CommonLogger
+
+  /**
+   * Set to true to enable additional debug messages, e.g it'll log
+   * when readable still emits values after the limit is reached.
+   */
+  debug?: boolean
 }
 
+/**
+ * Class only exists to be able to do `instanceof TransformLimit`
+ * and to set sourceReadable+streamDone to it in `_pipeline`.
+ */
+export class TransformLimit extends AbortableTransform {}
+
 /**
  * 0 or falsy value means "no limit"
  */
-export function transformLimit<IN>(
-  limit?: number,
-  opt: TransformLimitOptions = {},
-): TransformTyped<IN, IN> {
-  const { logger = console } = opt
-  let index = 0
-  let ended = false
+export function transformLimit<IN>(opt: TransformLimitOptions = {}): TransformTyped<IN, IN> {
+  const { logger = console, limit, debug } = opt
 
-  return new Transform({
+  if (!limit) {
+    // No limit - returning pass-through transform
+    return transformNoOp()
+  }
+
+  let i = 0 // so we start first chunk with 1
+  let ended = false
+  return new TransformLimit({
     objectMode: true,
     ...opt,
-    transform(this: Transform, chunk: IN, _encoding, cb) {
-      index++
-
-      if (!ended) {
-        cb(null, chunk) // pass through the item
-      } else {
-        cb(null) // pass-through empty
-      }
+    transform(this: TransformLimit, chunk, _, cb) {
+      i++
 
-      if (limit && index === limit) {
+      if (i === limit) {
         ended = true
-        logger.log(`transformLimit: limit of ${limit} reached`)
-        // this.emit('end') // this makes it "halt" on Node 14 lts
+        logger.log(`transformLimit of ${limit} reached`)
+        this.push(chunk)
+
+        pipelineClose(
+          'transformLimit',
+          this,
+          opt.sourceReadable || this.sourceReadable,
+          opt.streamDone || this.streamDone,
+          logger,
+        )
+
+        cb() // after pause
+      } else if (!ended) {
+        cb(null, chunk)
+      } else {
+        if (debug) logger.log(`transformLimit.transform after limit`, i)
+        // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
+        cb() // ended, don't emit anything
       }
     },
   })

package/src/stream/transform/transformMap.ts

@@ -1,16 +1,18 @@
-import { Transform } from 'stream'
 import {
+  AbortableAsyncMapper,
   AggregatedError,
-  AsyncMapper,
   AsyncPredicate,
   CommonLogger,
+  END,
   ErrorMode,
   pFilter,
-  PQueue,
+  SKIP,
 } from '@naturalcycles/js-lib'
+import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors'
+import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
-import { transformMapLegacy } from './legacy/transformMap'
+import { pipelineClose } from '../stream.util'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
@@ -55,11 +57,6 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    */
   metric?: string
 
-  /**
-   * If defined - called BEFORE `final()` callback is called.
-   */
-  beforeFinal?: () => any
-
   logger?: CommonLogger
 }
 
@@ -67,10 +64,8 @@ export function notNullishPredicate(item: any): boolean {
   return item !== undefined && item !== null
 }
 
-/**
- * Temporary export legacy transformMap, to debug 503 errors
- */
-export const transformMap = transformMapLegacy
+// doesn't work, cause here we don't construct our Transform instance ourselves
+// export class TransformMap extends AbortableTransform {}
 
 /**
  * Like pMap, but for streams.
@@ -84,8 +79,8 @@ export const transformMap = transformMapLegacy
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-export function transformMapNew<IN = any, OUT = IN>(
-  mapper: AsyncMapper<IN, OUT>,
+export function transformMap<IN = any, OUT = IN>(
+  mapper: AbortableAsyncMapper<IN, OUT>,
   opt: TransformMapOptions<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
   const {
@@ -94,103 +89,89 @@ export function transformMapNew<IN = any, OUT = IN>(
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,
     onError,
-    beforeFinal,
     metric = 'stream',
     logger = console,
   } = opt
 
   let index = -1
-  let isRejected = false
+  let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  const q = new PQueue({
-    concurrency,
-    resolveOn: 'start',
-    // debug: true,
-  })
+  return through2Concurrent.obj(
+    {
+      maxConcurrency: concurrency,
+      async final(cb) {
+        // console.log('transformMap final')
 
-  return new Transform({
-    objectMode: true,
+        logErrorStats(true)
 
-    async final(cb) {
-      // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
-
-      // wait for the current inFlight jobs to complete and push their results
-      await q.onIdle()
+        if (collectedErrors.length) {
+          // emit Aggregated error
+          cb(new AggregatedError(collectedErrors))
+        } else {
+          // emit no error
+          cb()
+        }
+      },
+    },
+    async function transformMapFn(this: AbortableTransform, chunk: IN, _, cb) {
+      index++
+      // console.log({chunk, _encoding})
+
+      // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
+      if (isSettled) return cb()
+
+      try {
+        const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+        const res = await mapper(chunk, currentIndex)
+        const passedResults = await pFilter(
+          flattenArrayOutput && Array.isArray(res) ? res : [res],
+          async r => {
+            if (r === END) {
+              isSettled = true // will be checked later
+              return false
+            }
+            return r !== SKIP && (await predicate(r, currentIndex))
+          },
+        )
+
+        passedResults.forEach(r => this.push(r))
+
+        if (isSettled) {
+          logger.log(`transformMap END received at index ${index}`)
+          pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
+        }
 
-      logErrorStats(logger, true)
+        cb() // done processing
+      } catch (err) {
+        logger.error(err)
+        errors++
+        logErrorStats()
 
-      await beforeFinal?.() // call beforeFinal if defined
+        if (onError) {
+          try {
+            onError(err, chunk)
+          } catch {}
+        }
 
-      if (collectedErrors.length) {
-        // emit Aggregated error
-        // For the same reason, magically, let's not call `cb`, but emit an error event instead
-        // this.emit('error', new AggregatedError(collectedErrors))
-        cb(new AggregatedError(collectedErrors))
-      } else {
-        // emit no error
-        // It is truly a mistery, but calling cb() here was causing ERR_MULTIPLE_CALLBACK ?!
-        // Commenting it out seems to work ?!
-        // ?!
-        // cb()
-      }
-    },
+        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+          isSettled = true
+          return cb(err) // Emit error immediately
+        }
 
-    async transform(this: Transform, chunk: IN, _encoding, cb) {
-      index++
-      // console.log('transform', {index})
-
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
-
-      // It resolves when it is successfully STARTED execution.
-      // If it's queued instead - it'll wait and resolve only upon START.
-      await q.push(async () => {
-        try {
-          const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-          const res = await mapper(chunk, currentIndex)
-          const passedResults = await pFilter(
-            flattenArrayOutput && Array.isArray(res) ? res : [res],
-            async r => await predicate(r, currentIndex),
-          )
-
-          passedResults.forEach(r => this.push(r))
-        } catch (err) {
-          logger.error(err)
-
-          errors++
-
-          logErrorStats(logger)
-
-          if (onError) {
-            try {
-              onError(err, chunk)
-            } catch {}
-          }
-
-          if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-            isRejected = true
-            // Emit error immediately
-            // return cb(err as Error)
-            return this.emit('error', err as Error)
-          }
-
-          if (errorMode === ErrorMode.THROW_AGGREGATED) {
-            collectedErrors.push(err as Error)
-          }
+        if (errorMode === ErrorMode.THROW_AGGREGATED) {
+          collectedErrors.push(err as Error)
         }
-      })
 
-      // Resolved, which means it STARTED processing
-      // This means we can take more load
-      cb()
+        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+        cb()
+      }
     },
-  })
+  )
 
-  function logErrorStats(logger: CommonLogger, final = false): void {
+  function logErrorStats(final = false): void {
     if (!errors) return
-
     logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
 }

package/src/stream/transform/worker/workerClassProxy.js

@@ -10,6 +10,7 @@ if (!workerFile) {
 // console.log(`worker#${workerIndex} created`)
 
 try {
+  // require('esbuild-register') // alternative
   require('ts-node/register/transpile-only')
   require('tsconfig-paths/register')
 } catch {} // require if exists

package/src/stream/writable/writableLimit.ts

@@ -0,0 +1,28 @@
+import { Readable, Writable } from 'stream'
+import { WritableTyped } from '../stream.model'
+
+/**
+ * Allows to stop the Readable stream after the pipeline has processed X number of rows.
+ * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
+ * It ensures that everything has been processed before issuing a STOP on the readable.
+ */
+export function writableLimit<T>(readable: Readable, limit: number): WritableTyped<T> {
+  let i = 0
+
+  return new Writable({
+    objectMode: true,
+    write(chunk, _, cb) {
+      if (limit === 0) return cb() // no limit, just passthrough
+
+      i++
+
+      if (i === limit) {
+        console.log(`writableLimit of ${limit} reached`)
+        readable.destroy()
+        cb() // do we need it?
+      } else {
+        cb() // passthrough
+      }
+    },
+  })
+}

package/src/stream/writable/writableVoid.ts

@@ -1,17 +1,29 @@
 import { Writable } from 'stream'
+import { DeferredPromise } from '@naturalcycles/js-lib'
 import { TransformOptions } from '../stream.model'
 
+export interface WritableVoidOptions extends TransformOptions {
+  /**
+   * If set - it will be Resolved when the Stream is done (after final.cb)
+   */
+  streamDone?: DeferredPromise
+}
+
 /**
  * Use as a "null-terminator" of stream.pipeline.
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-export function writableVoid(opt?: TransformOptions): Writable {
+export function writableVoid(opt: WritableVoidOptions = {}): Writable {
   return new Writable({
     objectMode: true,
     ...opt,
     write(chunk, _encoding, cb) {
       cb()
     },
+    final(cb) {
+      cb()
+      opt.streamDone?.resolve()
+    },
   })
 }

package/dist/stream/transform/legacy/transformMap.d.ts

@@ -1,17 +0,0 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
-import { TransformTyped } from '../../stream.model';
-import { TransformMapOptions } from '../transformMap';
-export declare function notNullishPredicate(item: any): boolean;
-/**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
- *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
- */
-export declare function transformMapLegacy<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/legacy/transformMap.js

@@ -1,94 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMapLegacy = exports.notNullishPredicate = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-const through2Concurrent = require("through2-concurrent");
-const colors_1 = require("../../../colors");
-function notNullishPredicate(item) {
-    return item !== undefined && item !== null;
-}
-exports.notNullishPredicate = notNullishPredicate;
-/**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
- *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
- */
-function transformMapLegacy(mapper, opt = {}) {
-    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
-    let index = -1;
-    let isRejected = false;
-    let errors = 0;
-    const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    return through2Concurrent.obj({
-        maxConcurrency: concurrency,
-        // autoDestroy: true,
-        async final(cb) {
-            // console.log('transformMap final')
-            logErrorStats(logger, true);
-            await beforeFinal?.(); // call beforeFinal if defined
-            if (collectedErrors.length) {
-                // emit Aggregated error
-                cb(new js_lib_1.AggregatedError(collectedErrors));
-            }
-            else {
-                // emit no error
-                cb();
-            }
-        },
-    }, async function transformMapFn(chunk, _encoding, cb) {
-        index++;
-        // console.log({chunk, _encoding})
-        // Stop processing if THROW_IMMEDIATELY mode is used
-        if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
-            return cb();
-        try {
-            const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-            const res = await mapper(chunk, currentIndex);
-            const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
-            if (passedResults.length === 0) {
-                cb(); // 0 results
-            }
-            else {
-                passedResults.forEach(r => {
-                    this.push(r);
-                    // cb(null, r)
-                });
-                cb(); // done processing
-            }
-        }
-        catch (err) {
-            logger.error(err);
-            errors++;
-            logErrorStats(logger);
-            if (onError) {
-                try {
-                    onError(err, chunk);
-                }
-                catch { }
-            }
-            if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                isRejected = true;
-                // Emit error immediately
-                return cb(err);
-            }
-            if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
-                collectedErrors.push(err);
-            }
-            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
-            cb();
-        }
-    });
-    function logErrorStats(logger, final = false) {
-        if (!errors)
-            return;
-        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
-    }
-}
-exports.transformMapLegacy = transformMapLegacy;

package/src/stream/transform/legacy/transformMap.ts

@@ -1,133 +0,0 @@
-import { Transform } from 'stream'
-import {
-  AggregatedError,
-  AsyncMapper,
-  CommonLogger,
-  ErrorMode,
-  pFilter,
-} from '@naturalcycles/js-lib'
-import through2Concurrent = require('through2-concurrent')
-import { yellow } from '../../../colors'
-import { TransformTyped } from '../../stream.model'
-import { TransformMapOptions } from '../transformMap'
-
-export function notNullishPredicate(item: any): boolean {
-  return item !== undefined && item !== null
-}
-
-/**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
- *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
- */
-export function transformMapLegacy<IN = any, OUT = IN>(
-  mapper: AsyncMapper<IN, OUT>,
-  opt: TransformMapOptions<IN, OUT> = {},
-): TransformTyped<IN, OUT> {
-  const {
-    concurrency = 16,
-    predicate = notNullishPredicate,
-    errorMode = ErrorMode.THROW_IMMEDIATELY,
-    flattenArrayOutput,
-    onError,
-    beforeFinal,
-    metric = 'stream',
-    logger = console,
-  } = opt
-
-  let index = -1
-  let isRejected = false
-  let errors = 0
-  const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
-
-  return through2Concurrent.obj(
-    {
-      maxConcurrency: concurrency,
-      // autoDestroy: true,
-      async final(cb) {
-        // console.log('transformMap final')
-
-        logErrorStats(logger, true)
-
-        await beforeFinal?.() // call beforeFinal if defined
-
-        if (collectedErrors.length) {
-          // emit Aggregated error
-          cb(new AggregatedError(collectedErrors))
-        } else {
-          // emit no error
-          cb()
-        }
-      },
-    },
-    async function transformMapFn(
-      this: Transform,
-      chunk: IN,
-      _encoding: any,
-      cb: (...args: any[]) => any,
-    ) {
-      index++
-      // console.log({chunk, _encoding})
-
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
-
-      try {
-        const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-        const res = await mapper(chunk, currentIndex)
-        const passedResults = await pFilter(
-          flattenArrayOutput && Array.isArray(res) ? res : [res],
-          async r => await predicate(r, currentIndex),
-        )
-
-        if (passedResults.length === 0) {
-          cb() // 0 results
-        } else {
-          passedResults.forEach(r => {
-            this.push(r)
-            // cb(null, r)
-          })
-          cb() // done processing
-        }
-      } catch (err) {
-        logger.error(err)
-
-        errors++
-
-        logErrorStats(logger)
-
-        if (onError) {
-          try {
-            onError(err, chunk)
-          } catch {}
-        }
-
-        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-          isRejected = true
-          // Emit error immediately
-          return cb(err)
-        }
-
-        if (errorMode === ErrorMode.THROW_AGGREGATED) {
-          collectedErrors.push(err as Error)
-        }
-
-        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
-        cb()
-      }
-    },
-  )
-
-  function logErrorStats(logger: CommonLogger, final = false): void {
-    if (!errors) return
-
-    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
-  }
-}