@naturalcycles/nodejs-lib 12.57.0 → 12.61.0

package/src/stream/transform/transformLogProgress.ts

@@ -4,6 +4,7 @@ import { SimpleMovingAverage, _mb, _since, AnyObject, CommonLogger } from '@natu
 import { dayjs } from '@naturalcycles/time-lib'
 import { boldWhite, dimGrey, white, yellow } from '../../colors'
 import { hasColors } from '../../colors/colors'
+import { SizeStack } from '../sizeStack'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
@@ -103,6 +104,41 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    * Defaults to 1.
    */
   batchSize?: number
+
+  /**
+   * Experimental logging of item (shunk) sizes, when json-stringified.
+   *
+   * Defaults to false.
+   *
+   * @experimental
+   */
+  logSizes?: boolean
+
+  /**
+   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+   * Defaults to 100_000.
+   * Cannot be Infinity.
+   */
+  logSizesBuffer?: number
+
+  /**
+   * Works in addition to `logSizes`. Adds "zipped sizes".
+   *
+   * @experimental
+   */
+  logZippedSizes?: boolean
+}
+
+interface LogItem extends AnyObject {
+  heapUsed?: number
+  heapTotal?: number
+  rss?: number
+  peakRSS?: number
+  rssMinusHeap?: number
+  external?: number
+  arrayBuffers?: number
+  rps10?: number
+  rpsTotal?: number
 }
 
 const inspectOpt: InspectOptions = {
@@ -124,6 +160,9 @@ export function transformLogProgress<IN = any>(
     peakRSS: logPeakRSS = true,
     logRPS = true,
     logEvery = 1000,
+    logSizes = false,
+    logSizesBuffer = 100_000,
+    logZippedSizes = false,
     batchSize = 1,
     extra,
     logger = console,
@@ -138,15 +177,23 @@ export function transformLogProgress<IN = any>(
   let progress = 0
   let peakRSS = 0
 
+  const sizes = logSizes ? new SizeStack('json', logSizesBuffer) : undefined
+  const sizesZipped = logZippedSizes ? new SizeStack('json.gz', logSizesBuffer) : undefined
+
   logStats() // initial
 
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       progress++
       processedLastSecond++
 
+      if (sizes) {
+        // Check it, cause gzipping might be delayed here..
+        void SizeStack.countItem(chunk, logger, sizes, sizesZipped)
+      }
+
       if (logProgress && progress % logEvery === 0) {
         logStats(chunk, false, progress % logEvery10 === 0)
       }
@@ -175,28 +222,30 @@ export function transformLogProgress<IN = any>(
     const rps10 = Math.round(sma.push(lastRPS))
     if (mem.rss > peakRSS) peakRSS = mem.rss
 
-    logger.log(
-      inspect(
-        {
-          [final ? `${metric}_final` : metric]: batchedProgress,
-          ...(extra ? extra(chunk, progress) : {}),
-          ...(logHeapUsed ? { heapUsed: _mb(mem.heapUsed) } : {}),
-          ...(logHeapTotal ? { heapTotal: _mb(mem.heapTotal) } : {}),
-          ...(logRss ? { rss: _mb(mem.rss) } : {}),
-          ...(logPeakRSS ? { peakRSS: _mb(peakRSS) } : {}),
-          ...(opt.rssMinusHeap ? { rssMinusHeap: _mb(mem.rss - mem.heapTotal) } : {}),
-          ...(opt.external ? { external: _mb(mem.external) } : {}),
-          ...(opt.arrayBuffers ? { arrayBuffers: _mb(mem.arrayBuffers || 0) } : {}),
-          ...(logRPS
-            ? {
-                rps10,
-                rpsTotal,
-              }
-            : {}),
-        },
-        inspectOpt,
-      ),
-    )
+    const o: LogItem = {
+      [final ? `${metric}_final` : metric]: batchedProgress,
+    }
+
+    if (extra) Object.assign(o, extra(chunk, progress))
+    if (logHeapUsed) o.heapUsed = _mb(mem.heapUsed)
+    if (logHeapTotal) o.heapTotal = _mb(mem.heapTotal)
+    if (logRss) o.rss = _mb(mem.rss)
+    if (logPeakRSS) o.peakRSS = _mb(peakRSS)
+    if (opt.rssMinusHeap) o.rssMinusHeap = _mb(mem.rss - mem.heapTotal)
+    if (opt.external) o.external = _mb(mem.external)
+    if (opt.arrayBuffers) o.arrayBuffers = _mb(mem.arrayBuffers || 0)
+
+    if (logRPS) Object.assign(o, { rps10, rpsTotal })
+
+    logger.log(inspect(o, inspectOpt))
+
+    if (sizes?.items.length) {
+      logger.log(sizes.getStats())
+
+      if (sizesZipped?.items.length) {
+        logger.log(sizesZipped.getStats())
+      }
+    }
 
     if (tenx) {
       let perHour: number | string =

package/src/stream/transform/transformMap.ts

@@ -1,15 +1,18 @@
-import { Transform } from 'stream'
 import {
+  AbortableAsyncMapper,
   AggregatedError,
-  AsyncMapper,
   AsyncPredicate,
   CommonLogger,
+  END,
   ErrorMode,
   pFilter,
-  PQueue,
+  SKIP,
 } from '@naturalcycles/js-lib'
+import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors'
+import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
+import { pipelineClose } from '../stream.util'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
@@ -23,9 +26,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
    *
-   * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
-   *
-   * @default to filter out undefined/null values, but pass anything else
+   * Defaults to "pass everything" (including null, undefined, etc).
+   * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
    */
   predicate?: AsyncPredicate<OUT>
 
@@ -54,17 +56,11 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    */
   metric?: string
 
-  /**
-   * If defined - called BEFORE `final()` callback is called.
-   */
-  beforeFinal?: () => any
-
   logger?: CommonLogger
 }
 
-export function notNullishPredicate(item: any): boolean {
-  return item !== undefined && item !== null
-}
+// doesn't work, cause here we don't construct our Transform instance ourselves
+// export class TransformMap extends AbortableTransform {}
 
 /**
  * Like pMap, but for streams.
@@ -79,112 +75,96 @@ export function notNullishPredicate(item: any): boolean {
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export function transformMap<IN = any, OUT = IN>(
-  mapper: AsyncMapper<IN, OUT>,
+  mapper: AbortableAsyncMapper<IN, OUT>,
   opt: TransformMapOptions<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
   const {
     concurrency = 16,
-    predicate = notNullishPredicate,
+    predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,
     onError,
-    beforeFinal,
     metric = 'stream',
     logger = console,
   } = opt
 
   let index = -1
-  let isRejected = false
+  let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  const q = new PQueue({
-    concurrency,
-    resolveOn: 'start',
-    // debug: true,
-  })
-
-  return new Transform({
-    objectMode: true,
+  return through2Concurrent.obj(
+    {
+      maxConcurrency: concurrency,
+      async final(cb) {
+        // console.log('transformMap final')
 
-    async final(cb) {
-      // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
+        logErrorStats(true)
 
-      // wait for the current inFlight jobs to complete and push their results
-      await q.onIdle()
+        if (collectedErrors.length) {
+          // emit Aggregated error
+          cb(new AggregatedError(collectedErrors))
+        } else {
+          // emit no error
+          cb()
+        }
+      },
+    },
+    async function transformMapFn(this: AbortableTransform, chunk: IN, _, cb) {
+      // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
+      if (isSettled) return cb()
+
+      const currentIndex = ++index
+
+      try {
+        const res = await mapper(chunk, currentIndex)
+        const passedResults = await pFilter(
+          flattenArrayOutput && Array.isArray(res) ? res : [res],
+          async r => {
+            if (r === END) {
+              isSettled = true // will be checked later
+              return false
+            }
+            return r !== SKIP && (!predicate || (await predicate(r, currentIndex)))
+          },
+        )
+
+        passedResults.forEach(r => this.push(r))
+
+        if (isSettled) {
+          logger.log(`transformMap END received at index ${currentIndex}`)
+          pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
+        }
 
-      logErrorStats(logger, true)
+        cb() // done processing
+      } catch (err) {
+        logger.error(err)
+        errors++
+        logErrorStats()
 
-      await beforeFinal?.() // call beforeFinal if defined
+        if (onError) {
+          try {
+            onError(err, chunk)
+          } catch {}
+        }
 
-      if (collectedErrors.length) {
-        // emit Aggregated error
-        // For the same reason, magically, let's not call `cb`, but emit an error event instead
-        // this.emit('error', new AggregatedError(collectedErrors))
-        cb(new AggregatedError(collectedErrors))
-      } else {
-        // emit no error
-        // It is truly a mistery, but calling cb() here was causing ERR_MULTIPLE_CALLBACK ?!
-        // Commenting it out seems to work ?!
-        // ?!
-        // cb()
-      }
-    },
+        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+          isSettled = true
+          return cb(err) // Emit error immediately
+        }
 
-    async transform(this: Transform, chunk: IN, _encoding, cb) {
-      index++
-      // console.log('transform', {index})
-
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
-
-      // It resolves when it is successfully STARTED execution.
-      // If it's queued instead - it'll wait and resolve only upon START.
-      await q.push(async () => {
-        try {
-          const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-          const res = await mapper(chunk, currentIndex)
-          const passedResults = await pFilter(
-            flattenArrayOutput && Array.isArray(res) ? res : [res],
-            async r => await predicate(r, currentIndex),
-          )
-
-          passedResults.forEach(r => this.push(r))
-        } catch (err) {
-          logger.error(err)
-
-          errors++
-
-          logErrorStats(logger)
-
-          if (onError) {
-            try {
-              onError(err, chunk)
-            } catch {}
-          }
-
-          if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-            isRejected = true
-            // Emit error immediately
-            // return cb(err as Error)
-            return this.emit('error', err as Error)
-          }
-
-          if (errorMode === ErrorMode.THROW_AGGREGATED) {
-            collectedErrors.push(err as Error)
-          }
+        if (errorMode === ErrorMode.THROW_AGGREGATED) {
+          collectedErrors.push(err as Error)
         }
-      })
 
-      // Resolved, which means it STARTED processing
-      // This means we can take more load
-      cb()
+        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+        cb()
+      }
     },
-  })
+  )
 
-  function logErrorStats(logger: CommonLogger, final = false): void {
+  function logErrorStats(final = false): void {
     if (!errors) return
-
     logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
 }

package/src/stream/transform/transformMapSimple.ts

@@ -31,7 +31,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
 
   return new Transform({
     objectMode: true,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         cb(null, mapper(chunk, ++index))
       } catch (err) {

package/src/stream/transform/transformMapSync.ts

@@ -1,8 +1,16 @@
-import { Transform } from 'stream'
-import { AggregatedError, CommonLogger, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib'
+import {
+  AggregatedError,
+  CommonLogger,
+  END,
+  ErrorMode,
+  Mapper,
+  Predicate,
+  SKIP,
+} from '@naturalcycles/js-lib'
 import { yellow } from '../../colors'
+import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
-import { notNullishPredicate } from './transformMap'
+import { pipelineClose } from '../stream.util'
 
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   /**
@@ -20,9 +28,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
    *
-   * @default to filter out undefined/null values, but pass anything else
-   *
-   * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
+   * Defaults to "pass everything".
+   * Simpler way to skip individual entries is to return SKIP symbol.
    */
   predicate?: Predicate<OUT>
 
@@ -47,6 +54,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   logger?: CommonLogger
 }
 
+export class TransformMapSync extends AbortableTransform {}
+
 /**
  * Sync (not async) version of transformMap.
  * Supposedly faster, for cases when async is not needed.
@@ -58,7 +67,7 @@ export function transformMapSync<IN = any, OUT = IN>(
   let index = -1
 
   const {
-    predicate = notNullishPredicate,
+    predicate, // defaults to "no predicate" (pass everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput = false,
     onError,
@@ -66,34 +75,39 @@ export function transformMapSync<IN = any, OUT = IN>(
     objectMode = true,
     logger = console,
   } = opt
-  let isRejected = false
+  let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  return new Transform({
+  return new TransformMapSync({
     objectMode,
     ...opt,
-    transform(this: Transform, chunk: IN, _encoding, cb) {
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) {
-        return cb()
-      }
+    transform(this: AbortableTransform, chunk: IN, _, cb) {
+      // Stop processing if isSettled
+      if (isSettled) return cb()
 
-      try {
-        if (!predicate(chunk, ++index)) {
-          cb() // signal that we've finished processing, but emit no output here
-          return
-        }
+      const currentIndex = ++index
 
+      try {
         // map and pass through
-        const v = mapper(chunk, index)
+        const v = mapper(chunk, currentIndex)
+
+        const passedResults = (flattenArrayOutput && Array.isArray(v) ? v : [v]).filter(r => {
+          if (r === END) {
+            isSettled = true // will be checked later
+            return false
+          }
+          return r !== SKIP && (!predicate || predicate(r, currentIndex))
+        })
 
-        if (flattenArrayOutput && Array.isArray(v)) {
-          // Pass each item individually
-          v.forEach(item => this.push(item))
-        } else {
-          cb(null, v)
+        passedResults.forEach(r => this.push(r))
+
+        if (isSettled) {
+          logger.log(`transformMapSync END received at index ${currentIndex}`)
+          pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger)
         }
+
+        cb() // done processing
       } catch (err) {
         logger.error(err)
         errors++
@@ -107,7 +121,7 @@ export function transformMapSync<IN = any, OUT = IN>(
         }
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-          isRejected = true
+          isSettled = true
           // Emit error immediately
           return cb(err as Error)
         }

package/src/stream/transform/transformNoOp.ts

@@ -9,7 +9,7 @@ import { TransformTyped } from '../stream.model'
 export function transformNoOp<T = any>(): TransformTyped<T, T> {
   return new Transform({
     objectMode: true,
-    transform(chunk: T, _encoding, cb) {
+    transform(chunk: T, _, cb) {
       cb(null, chunk)
     },
   })

package/src/stream/transform/transformTap.ts

@@ -17,16 +17,16 @@ export function transformTap<IN>(
   opt: TransformTapOptions = {},
 ): TransformTyped<IN, IN> {
   const { logger = console } = opt
-  let index = 0
+  let index = -1
 
   return new Transform({
     objectMode: true,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
+    async transform(chunk: IN, _, cb) {
       // console.log('tap', chunk)
 
       try {
-        await fn(chunk, index++)
+        await fn(chunk, ++index)
       } catch (err) {
         logger.error(err)
         // suppressed error

package/src/stream/transform/transformToArray.ts

@@ -10,7 +10,7 @@ export function transformToArray<IN>(opt: TransformOptions = {}): TransformTyped
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       res.push(chunk)
       // callback to signal that we processed input, but not emitting any output
       cb()

package/src/stream/transform/transformToString.ts

@@ -14,8 +14,8 @@ export function transformToString(): TransformTyped<Buffer, string> {
   return new Transform({
     objectMode: false,
     readableObjectMode: true,
-    transform(chunk: Buffer, _encoding, cb) {
-      // console.log(`enc: ${_encoding}`, chunk.toString())
+    transform(chunk: Buffer, _, cb) {
+      // console.log(`enc: ${_}`, chunk.toString())
       cb(null, chunk.toString())
     },
   })

package/src/stream/transform/worker/transformMultiThreaded.ts

@@ -115,7 +115,7 @@ export function transformMultiThreaded<IN, OUT>(
         }
       },
     },
-    async function transformMapFn(chunk: IN, _encoding, cb) {
+    async function transformMapFn(chunk: IN, _, cb) {
       // Freezing the index, because it may change due to concurrency
       const currentIndex = ++index
 

package/src/stream/transform/worker/workerClassProxy.js

@@ -10,6 +10,7 @@ if (!workerFile) {
 // console.log(`worker#${workerIndex} created`)
 
 try {
+  // require('esbuild-register') // alternative
   require('ts-node/register/transpile-only')
   require('tsconfig-paths/register')
 } catch {} // require if exists

package/src/stream/writable/writableFork.ts

@@ -31,7 +31,7 @@ export function writableFork<T>(
   return new Writable({
     objectMode: true,
     ...opt,
-    write(chunk: T, _encoding, cb) {
+    write(chunk: T, _, cb) {
       // Push/fork to all sub-streams
       // No backpressure is ensured here, it'll push regardless of the
       readables.forEach(readable => readable.push(chunk))

package/src/stream/writable/writableLimit.ts

@@ -0,0 +1,28 @@
+import { Readable, Writable } from 'stream'
+import { WritableTyped } from '../stream.model'
+
+/**
+ * Allows to stop the Readable stream after the pipeline has processed X number of rows.
+ * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
+ * It ensures that everything has been processed before issuing a STOP on the readable.
+ */
+export function writableLimit<T>(readable: Readable, limit: number): WritableTyped<T> {
+  let i = 0
+
+  return new Writable({
+    objectMode: true,
+    write(chunk, _, cb) {
+      if (limit === 0) return cb() // no limit, just passthrough
+
+      i++
+
+      if (i === limit) {
+        console.log(`writableLimit of ${limit} reached`)
+        readable.destroy()
+        cb() // do we need it?
+      } else {
+        cb() // passthrough
+      }
+    },
+  })
+}

package/src/stream/writable/writablePushToArray.ts

@@ -8,7 +8,7 @@ export function writablePushToArray<IN>(arr: IN[], opt: TransformOptions = {}):
   return new Writable({
     objectMode: true,
     ...opt,
-    write(chunk: IN, _encoding, cb) {
+    write(chunk: IN, _, cb) {
       arr.push(chunk)
       // callback to signal that we processed input, but not emitting any output
       cb()

package/src/stream/writable/writableVoid.ts

@@ -1,17 +1,29 @@
 import { Writable } from 'stream'
+import { DeferredPromise } from '@naturalcycles/js-lib'
 import { TransformOptions } from '../stream.model'
 
+export interface WritableVoidOptions extends TransformOptions {
+  /**
+   * If set - it will be Resolved when the Stream is done (after final.cb)
+   */
+  streamDone?: DeferredPromise
+}
+
 /**
  * Use as a "null-terminator" of stream.pipeline.
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-export function writableVoid(opt?: TransformOptions): Writable {
+export function writableVoid(opt: WritableVoidOptions = {}): Writable {
   return new Writable({
     objectMode: true,
     ...opt,
-    write(chunk, _encoding, cb) {
+    write(chunk, _, cb) {
+      cb()
+    },
+    final(cb) {
       cb()
+      opt.streamDone?.resolve()
     },
   })
 }