@naturalcycles/nodejs-lib 12.56.1 → 12.60.0

package/src/stream/stream.util.ts

@@ -0,0 +1,29 @@
+import { Readable } from 'stream'
+import { CommonLogger } from '@naturalcycles/js-lib'
+
+export function pipelineClose(
+  name: string,
+  readableDownstream: Readable,
+  sourceReadable: Readable | undefined,
+  streamDone: Promise<void> | undefined,
+  logger: CommonLogger,
+): void {
+  readableDownstream.push(null) // this closes the stream, so downstream Readable will receive `end` and won't write anything
+
+  if (!sourceReadable) {
+    logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`)
+  } else {
+    logger.log(`${name} is calling readable.unpipe() to pause the stream`)
+    sourceReadable.unpipe() // it is expected to pause the stream
+
+    if (!streamDone) {
+      logger.log(`${name} streamDone is not provided, will do readable.destroy right away`)
+      sourceReadable.destroy()
+    } else {
+      void streamDone.then(() => {
+        logger.log(`${name} streamDone, calling readable.destroy()`)
+        sourceReadable.destroy() // this throws ERR_STREAM_PREMATURE_CLOSE
+      })
+    }
+  }
+}

package/src/stream/transform/transformBuffer.ts

@@ -18,7 +18,7 @@ export function transformBuffer<IN = any>(opt: TransformBufferOptions): Transfor
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk, _encoding, cb) {
+    transform(chunk, _, cb) {
       buf.push(chunk)
 
       if (buf.length >= batchSize) {

package/src/stream/transform/transformFilter.ts

@@ -1,32 +1,18 @@
 import { Transform } from 'stream'
 import { AsyncPredicate, Predicate } from '@naturalcycles/js-lib'
 import { TransformOptions, TransformTyped } from '../stream.model'
+import { transformMap, TransformMapOptions } from './transformMap'
 
 /**
- * Note, that currently it's NOT concurrent! (concurrency = 1)
- * So, it's recommended to use transformMap instead, that is both concurrent and has
- * filtering feature by default.
+ * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 export function transformFilter<IN = any>(
   predicate: AsyncPredicate<IN>,
-  opt: TransformOptions = {},
+  opt: TransformMapOptions = {},
 ): TransformTyped<IN, IN> {
-  let index = 0
-
-  return new Transform({
-    objectMode: true,
+  return transformMap(v => v, {
+    predicate,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
-      try {
-        if (await predicate(chunk, index++)) {
-          cb(null, chunk) // pass through
-        } else {
-          cb() // signal that we've finished processing, but emit no output here
-        }
-      } catch (err) {
-        cb(err as Error)
-      }
-    },
   })
 }
 
@@ -42,7 +28,7 @@ export function transformFilterSync<IN = any>(
   return new Transform({
     objectMode: true,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         if (predicate(chunk, index++)) {
           cb(null, chunk) // pass through

package/src/stream/transform/transformLimit.ts

@@ -1,32 +1,84 @@
-import { Transform } from 'stream'
+import { Readable } from 'stream'
+import { CommonLogger } from '@naturalcycles/js-lib'
+import { AbortableTransform, transformNoOp } from '../../index'
 import { TransformOptions, TransformTyped } from '../stream.model'
+import { pipelineClose } from '../stream.util'
+
+export interface TransformLimitOptions extends TransformOptions {
+  /**
+   * Nullish value (e.g 0 or undefined) would mean "no limit"
+   */
+  limit?: number
+
+  /**
+   * If provided (recommended!) - it will call readable.destroy() on limit.
+   * Without it - it will only stop the downstream consumers, but won't stop
+   * the Readable ("source" of the stream).
+   * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+   */
+  sourceReadable?: Readable
+
+  /**
+   * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
+   * to be able to wait for Consumers before calling `readable.destroy`.
+   * Has no effect if `readable` is not provided.
+   */
+  streamDone?: Promise<void>
+
+  logger?: CommonLogger
+
+  /**
+   * Set to true to enable additional debug messages, e.g it'll log
+   * when readable still emits values after the limit is reached.
+   */
+  debug?: boolean
+}
+
+/**
+ * Class only exists to be able to do `instanceof TransformLimit`
+ * and to set sourceReadable+streamDone to it in `_pipeline`.
+ */
+export class TransformLimit extends AbortableTransform {}
 
 /**
  * 0 or falsy value means "no limit"
  */
-export function transformLimit<IN>(
-  limit?: number,
-  opt: TransformOptions = {},
-): TransformTyped<IN, IN> {
-  let index = 0
-  let ended = false
+export function transformLimit<IN>(opt: TransformLimitOptions = {}): TransformTyped<IN, IN> {
+  const { logger = console, limit, debug } = opt
+
+  if (!limit) {
+    // No limit - returning pass-through transform
+    return transformNoOp()
+  }
 
-  return new Transform({
+  let i = 0 // so we start first chunk with 1
+  let ended = false
+  return new TransformLimit({
     objectMode: true,
     ...opt,
-    transform(this: Transform, chunk: IN, _encoding, cb) {
-      index++
+    transform(this: TransformLimit, chunk, _, cb) {
+      i++
 
-      if (!ended) {
-        cb(null, chunk) // pass through the item
-      } else {
-        cb(null) // pass-through empty
-      }
-
-      if (limit && index === limit) {
+      if (i === limit) {
         ended = true
-        console.log(`transformLimit: limit of ${limit} reached`)
-        // this.emit('end') // this makes it "halt" on Node 14 lts
+        logger.log(`transformLimit of ${limit} reached`)
+        this.push(chunk)
+
+        pipelineClose(
+          'transformLimit',
+          this,
+          opt.sourceReadable || this.sourceReadable,
+          opt.streamDone || this.streamDone,
+          logger,
+        )
+
+        cb() // after pause
+      } else if (!ended) {
+        cb(null, chunk)
+      } else {
+        if (debug) logger.log(`transformLimit.transform after limit`, i)
+        // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
+        cb() // ended, don't emit anything
       }
     },
   })

package/src/stream/transform/transformLogProgress.ts

@@ -1,9 +1,10 @@
 import { Transform } from 'stream'
 import { inspect, InspectOptions } from 'util'
-import { SimpleMovingAverage, _mb, _since, AnyObject } from '@naturalcycles/js-lib'
+import { SimpleMovingAverage, _mb, _since, AnyObject, CommonLogger } from '@naturalcycles/js-lib'
 import { dayjs } from '@naturalcycles/time-lib'
 import { boldWhite, dimGrey, white, yellow } from '../../colors'
 import { hasColors } from '../../colors/colors'
+import { SizeStack } from '../sizeStack'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
@@ -86,6 +87,8 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    */
   logEvery?: number
 
+  logger?: CommonLogger
+
   /**
    * Function to return extra properties to the "progress object".
    *
@@ -101,6 +104,41 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    * Defaults to 1.
    */
   batchSize?: number
+
+  /**
+   * Experimental logging of item (shunk) sizes, when json-stringified.
+   *
+   * Defaults to false.
+   *
+   * @experimental
+   */
+  logSizes?: boolean
+
+  /**
+   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+   * Defaults to 100_000.
+   * Cannot be Infinity.
+   */
+  logSizesBuffer?: number
+
+  /**
+   * Works in addition to `logSizes`. Adds "zipped sizes".
+   *
+   * @experimental
+   */
+  logZippedSizes?: boolean
+}
+
+interface LogItem extends AnyObject {
+  heapUsed?: number
+  heapTotal?: number
+  rss?: number
+  peakRSS?: number
+  rssMinusHeap?: number
+  external?: number
+  arrayBuffers?: number
+  rps10?: number
+  rpsTotal?: number
 }
 
 const inspectOpt: InspectOptions = {
@@ -122,8 +160,12 @@ export function transformLogProgress<IN = any>(
     peakRSS: logPeakRSS = true,
     logRPS = true,
     logEvery = 1000,
+    logSizes = false,
+    logSizesBuffer = 100_000,
+    logZippedSizes = false,
     batchSize = 1,
     extra,
+    logger = console,
   } = opt
   const logProgress = opt.logProgress !== false && logEvery !== 0 // true by default
   const logEvery10 = logEvery * 10
@@ -135,15 +177,23 @@ export function transformLogProgress<IN = any>(
   let progress = 0
   let peakRSS = 0
 
+  const sizes = logSizes ? new SizeStack('json', logSizesBuffer) : undefined
+  const sizesZipped = logZippedSizes ? new SizeStack('json.gz', logSizesBuffer) : undefined
+
   logStats() // initial
 
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       progress++
       processedLastSecond++
 
+      if (sizes) {
+        // Check it, cause gzipping might be delayed here..
+        void SizeStack.countItem(chunk, logger, sizes, sizesZipped)
+      }
+
       if (logProgress && progress % logEvery === 0) {
         logStats(chunk, false, progress % logEvery10 === 0)
       }
@@ -172,28 +222,30 @@ export function transformLogProgress<IN = any>(
     const rps10 = Math.round(sma.push(lastRPS))
     if (mem.rss > peakRSS) peakRSS = mem.rss
 
-    console.log(
-      inspect(
-        {
-          [final ? `${metric}_final` : metric]: batchedProgress,
-          ...(extra ? extra(chunk, progress) : {}),
-          ...(logHeapUsed ? { heapUsed: _mb(mem.heapUsed) } : {}),
-          ...(logHeapTotal ? { heapTotal: _mb(mem.heapTotal) } : {}),
-          ...(logRss ? { rss: _mb(mem.rss) } : {}),
-          ...(logPeakRSS ? { peakRSS: _mb(peakRSS) } : {}),
-          ...(opt.rssMinusHeap ? { rssMinusHeap: _mb(mem.rss - mem.heapTotal) } : {}),
-          ...(opt.external ? { external: _mb(mem.external) } : {}),
-          ...(opt.arrayBuffers ? { arrayBuffers: _mb(mem.arrayBuffers || 0) } : {}),
-          ...(logRPS
-            ? {
-                rps10,
-                rpsTotal,
-              }
-            : {}),
-        },
-        inspectOpt,
-      ),
-    )
+    const o: LogItem = {
+      [final ? `${metric}_final` : metric]: batchedProgress,
+    }
+
+    if (extra) Object.assign(o, extra(chunk, progress))
+    if (logHeapUsed) o.heapUsed = _mb(mem.heapUsed)
+    if (logHeapTotal) o.heapTotal = _mb(mem.heapTotal)
+    if (logRss) o.rss = _mb(mem.rss)
+    if (logPeakRSS) o.peakRSS = _mb(peakRSS)
+    if (opt.rssMinusHeap) o.rssMinusHeap = _mb(mem.rss - mem.heapTotal)
+    if (opt.external) o.external = _mb(mem.external)
+    if (opt.arrayBuffers) o.arrayBuffers = _mb(mem.arrayBuffers || 0)
+
+    if (logRPS) Object.assign(o, { rps10, rpsTotal })
+
+    logger.log(inspect(o, inspectOpt))
+
+    if (sizes?.items.length) {
+      logger.log(sizes.getStats())
+
+      if (sizesZipped?.items.length) {
+        logger.log(sizesZipped.getStats())
+      }
+    }
 
     if (tenx) {
       let perHour: number | string =
@@ -202,13 +254,13 @@ export function transformLogProgress<IN = any>(
         perHour = Math.round(perHour / 1000) + 'K'
       }
 
-      console.log(
+      logger.log(
         `${dimGrey(dayjs().toPretty())} ${white(metric)} took ${yellow(
           _since(started),
         )} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`,
       )
     } else if (final) {
-      console.log(
+      logger.log(
         `${boldWhite(metric)} took ${yellow(_since(started))} to process ${yellow(
           batchedProgress,
         )} rows with total RPS of ${yellow(rpsTotal)}`,

package/src/stream/transform/transformMap.ts

@@ -1,15 +1,18 @@
-import { Transform } from 'stream'
 import {
+  AbortableAsyncMapper,
   AggregatedError,
-  AsyncMapper,
   AsyncPredicate,
   CommonLogger,
+  END,
   ErrorMode,
   pFilter,
-  PQueue,
+  SKIP,
 } from '@naturalcycles/js-lib'
+import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors'
+import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
+import { pipelineClose } from '../stream.util'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
@@ -23,9 +26,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
    *
-   * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
-   *
-   * @default to filter out undefined/null values, but pass anything else
+   * Defaults to "pass everything" (including null, undefined, etc).
+   * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
    */
   predicate?: AsyncPredicate<OUT>
 
@@ -54,17 +56,11 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    */
   metric?: string
 
-  /**
-   * If defined - called BEFORE `final()` callback is called.
-   */
-  beforeFinal?: () => any
-
   logger?: CommonLogger
 }
 
-export function notNullishPredicate(item: any): boolean {
-  return item !== undefined && item !== null
-}
+// doesn't work, cause here we don't construct our Transform instance ourselves
+// export class TransformMap extends AbortableTransform {}
 
 /**
  * Like pMap, but for streams.
@@ -79,112 +75,96 @@ export function notNullishPredicate(item: any): boolean {
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export function transformMap<IN = any, OUT = IN>(
-  mapper: AsyncMapper<IN, OUT>,
+  mapper: AbortableAsyncMapper<IN, OUT>,
   opt: TransformMapOptions<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
   const {
     concurrency = 16,
-    predicate = notNullishPredicate,
+    predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,
     onError,
-    beforeFinal,
     metric = 'stream',
     logger = console,
   } = opt
 
   let index = -1
-  let isRejected = false
+  let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  const q = new PQueue({
-    concurrency,
-    resolveOn: 'start',
-    // debug: true,
-  })
-
-  return new Transform({
-    objectMode: true,
+  return through2Concurrent.obj(
+    {
+      maxConcurrency: concurrency,
+      async final(cb) {
+        // console.log('transformMap final')
 
-    async final(cb) {
-      // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
+        logErrorStats(true)
 
-      // wait for the current inFlight jobs to complete and push their results
-      await q.onIdle()
+        if (collectedErrors.length) {
+          // emit Aggregated error
+          cb(new AggregatedError(collectedErrors))
+        } else {
+          // emit no error
+          cb()
+        }
+      },
+    },
+    async function transformMapFn(this: AbortableTransform, chunk: IN, _, cb) {
+      // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
+      if (isSettled) return cb()
+
+      const currentIndex = ++index
+
+      try {
+        const res = await mapper(chunk, currentIndex)
+        const passedResults = await pFilter(
+          flattenArrayOutput && Array.isArray(res) ? res : [res],
+          async r => {
+            if (r === END) {
+              isSettled = true // will be checked later
+              return false
+            }
+            return r !== SKIP && (!predicate || (await predicate(r, currentIndex)))
+          },
+        )
+
+        passedResults.forEach(r => this.push(r))
+
+        if (isSettled) {
+          logger.log(`transformMap END received at index ${currentIndex}`)
+          pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
+        }
 
-      logErrorStats(logger, true)
+        cb() // done processing
+      } catch (err) {
+        logger.error(err)
+        errors++
+        logErrorStats()
 
-      await beforeFinal?.() // call beforeFinal if defined
+        if (onError) {
+          try {
+            onError(err, chunk)
+          } catch {}
+        }
 
-      if (collectedErrors.length) {
-        // emit Aggregated error
-        // For the same reason, magically, let's not call `cb`, but emit an error event instead
-        // this.emit('error', new AggregatedError(collectedErrors))
-        cb(new AggregatedError(collectedErrors))
-      } else {
-        // emit no error
-        // It is truly a mistery, but calling cb() here was causing ERR_MULTIPLE_CALLBACK ?!
-        // Commenting it out seems to work ?!
-        // ?!
-        // cb()
-      }
-    },
+        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+          isSettled = true
+          return cb(err) // Emit error immediately
+        }
 
-    async transform(this: Transform, chunk: IN, _encoding, cb) {
-      index++
-      // console.log('transform', {index})
-
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
-
-      // It resolves when it is successfully STARTED execution.
-      // If it's queued instead - it'll wait and resolve only upon START.
-      await q.push(async () => {
-        try {
-          const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-          const res = await mapper(chunk, currentIndex)
-          const passedResults = await pFilter(
-            flattenArrayOutput && Array.isArray(res) ? res : [res],
-            async r => await predicate(r, currentIndex),
-          )
-
-          passedResults.forEach(r => this.push(r))
-        } catch (err) {
-          logger.error(err)
-
-          errors++
-
-          logErrorStats(logger)
-
-          if (onError) {
-            try {
-              onError(err, chunk)
-            } catch {}
-          }
-
-          if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-            isRejected = true
-            // Emit error immediately
-            // return cb(err as Error)
-            return this.emit('error', err as Error)
-          }
-
-          if (errorMode === ErrorMode.THROW_AGGREGATED) {
-            collectedErrors.push(err as Error)
-          }
+        if (errorMode === ErrorMode.THROW_AGGREGATED) {
+          collectedErrors.push(err as Error)
         }
-      })
 
-      // Resolved, which means it STARTED processing
-      // This means we can take more load
-      cb()
+        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+        cb()
+      }
     },
-  })
+  )
 
-  function logErrorStats(logger: CommonLogger, final = false): void {
+  function logErrorStats(final = false): void {
     if (!errors) return
-
     logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
 }

package/src/stream/transform/transformMapSimple.ts

@@ -1,5 +1,5 @@
 import { Transform } from 'stream'
-import { ErrorMode, Mapper } from '@naturalcycles/js-lib'
+import { CommonLogger, ErrorMode, Mapper } from '@naturalcycles/js-lib'
 import { TransformTyped } from '../stream.model'
 
 export interface TransformMapSimpleOptions {
@@ -9,6 +9,8 @@ export interface TransformMapSimpleOptions {
    * @default ErrorMode.THROW_IMMEDIATELY
    */
   errorMode?: ErrorMode.THROW_IMMEDIATELY | ErrorMode.SUPPRESS
+
+  logger?: CommonLogger
 }
 
 /**
@@ -25,15 +27,15 @@ export function transformMapSimple<IN = any, OUT = IN>(
   opt: TransformMapSimpleOptions = {},
 ): TransformTyped<IN, OUT> {
   let index = -1
-  const { errorMode = ErrorMode.THROW_IMMEDIATELY } = opt
+  const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console } = opt
 
   return new Transform({
     objectMode: true,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         cb(null, mapper(chunk, ++index))
       } catch (err) {
-        console.error(err)
+        logger.error(err)
 
         if (errorMode === ErrorMode.SUPPRESS) {
           cb() // suppress the error