@naturalcycles/nodejs-lib 12.59.0 → 12.60.0

package/src/stream/ndjson/transformJsonParse.ts

@@ -33,17 +33,17 @@ export function transformJsonParse<OUT = any>(
   const { strict = true, reviver } = opt
 
   return new Transform({
-    objectMode: false,
+    writableObjectMode: false,
     readableObjectMode: true,
-    transform(chunk: string, _encoding, cb) {
+    transform(chunk: string, _, cb) {
       try {
         const data = JSON.parse(chunk, reviver)
         cb(null, data)
       } catch (err) {
-        // console.error(err)
         if (strict) {
           cb(err as Error) // emit error
         } else {
+          console.error(err)
           cb() // emit no error, but no result neither
         }
       }

package/src/stream/ndjson/transformToNDJson.ts

@@ -37,9 +37,9 @@ export function transformToNDJson<IN = any>(
   const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt
 
   return new Transform({
-    objectMode: true,
+    writableObjectMode: true,
     readableObjectMode: false,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         if (sortObjects) {
           chunk = _sortObjectDeep(chunk as any)

package/src/stream/sizeStack.ts

@@ -0,0 +1,56 @@
+import { _hb, CommonLogger, NumberStack } from '@naturalcycles/js-lib'
+import { yellow } from '../colors'
+import { gzipBuffer } from '../util/zip.util'
+
+export class SizeStack extends NumberStack {
+  constructor(public name: string, size: number) {
+    super(size)
+  }
+
+  total = 0
+
+  override push(item: any): this {
+    this.total += item
+    return super.push(item)
+  }
+
+  getStats(): string {
+    const pcs = this.percentiles([50, 90])
+
+    return [
+      this.name,
+      'avg',
+      yellow(_hb(this.avg())),
+      'p50',
+      yellow(_hb(pcs[50])),
+      'p90',
+      yellow(_hb(pcs[90])),
+      'total',
+      yellow(_hb(this.total)),
+    ].join(' ')
+  }
+
+  static async countItem(
+    item: any,
+    logger: CommonLogger,
+    sizes?: SizeStack,
+    sizesZipped?: SizeStack,
+  ): Promise<void> {
+    if (!sizes) return
+
+    // try-catch, because we don't want to fail the pipeline on logProgress
+    try {
+      const buf = Buffer.from(JSON.stringify(item))
+      sizes.push(buf.byteLength)
+
+      if (sizesZipped) {
+        const { byteLength } = await gzipBuffer(buf)
+        sizesZipped.push(byteLength)
+      }
+    } catch (err) {
+      logger.warn(
+        `transformLogProgress failed to JSON.stringify the chunk: ${(err as Error).message}`,
+      )
+    }
+  }
+}

package/src/stream/transform/transformBuffer.ts

@@ -18,7 +18,7 @@ export function transformBuffer<IN = any>(opt: TransformBufferOptions): Transfor
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk, _encoding, cb) {
+    transform(chunk, _, cb) {
       buf.push(chunk)
 
       if (buf.length >= batchSize) {

package/src/stream/transform/transformFilter.ts

@@ -1,32 +1,18 @@
 import { Transform } from 'stream'
 import { AsyncPredicate, Predicate } from '@naturalcycles/js-lib'
 import { TransformOptions, TransformTyped } from '../stream.model'
+import { transformMap, TransformMapOptions } from './transformMap'
 
 /**
- * Note, that currently it's NOT concurrent! (concurrency = 1)
- * So, it's recommended to use transformMap instead, that is both concurrent and has
- * filtering feature by default.
+ * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 export function transformFilter<IN = any>(
   predicate: AsyncPredicate<IN>,
-  opt: TransformOptions = {},
+  opt: TransformMapOptions = {},
 ): TransformTyped<IN, IN> {
-  let index = 0
-
-  return new Transform({
-    objectMode: true,
+  return transformMap(v => v, {
+    predicate,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
-      try {
-        if (await predicate(chunk, index++)) {
-          cb(null, chunk) // pass through
-        } else {
-          cb() // signal that we've finished processing, but emit no output here
-        }
-      } catch (err) {
-        cb(err as Error)
-      }
-    },
   })
 }
 
@@ -42,7 +28,7 @@ export function transformFilterSync<IN = any>(
   return new Transform({
     objectMode: true,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         if (predicate(chunk, index++)) {
           cb(null, chunk) // pass through

package/src/stream/transform/transformLogProgress.ts

@@ -4,6 +4,7 @@ import { SimpleMovingAverage, _mb, _since, AnyObject, CommonLogger } from '@natu
 import { dayjs } from '@naturalcycles/time-lib'
 import { boldWhite, dimGrey, white, yellow } from '../../colors'
 import { hasColors } from '../../colors/colors'
+import { SizeStack } from '../sizeStack'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
@@ -103,6 +104,41 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    * Defaults to 1.
    */
   batchSize?: number
+
+  /**
+   * Experimental logging of item (shunk) sizes, when json-stringified.
+   *
+   * Defaults to false.
+   *
+   * @experimental
+   */
+  logSizes?: boolean
+
+  /**
+   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+   * Defaults to 100_000.
+   * Cannot be Infinity.
+   */
+  logSizesBuffer?: number
+
+  /**
+   * Works in addition to `logSizes`. Adds "zipped sizes".
+   *
+   * @experimental
+   */
+  logZippedSizes?: boolean
+}
+
+interface LogItem extends AnyObject {
+  heapUsed?: number
+  heapTotal?: number
+  rss?: number
+  peakRSS?: number
+  rssMinusHeap?: number
+  external?: number
+  arrayBuffers?: number
+  rps10?: number
+  rpsTotal?: number
 }
 
 const inspectOpt: InspectOptions = {
@@ -124,6 +160,9 @@ export function transformLogProgress<IN = any>(
     peakRSS: logPeakRSS = true,
     logRPS = true,
     logEvery = 1000,
+    logSizes = false,
+    logSizesBuffer = 100_000,
+    logZippedSizes = false,
     batchSize = 1,
     extra,
     logger = console,
@@ -138,15 +177,23 @@ export function transformLogProgress<IN = any>(
   let progress = 0
   let peakRSS = 0
 
+  const sizes = logSizes ? new SizeStack('json', logSizesBuffer) : undefined
+  const sizesZipped = logZippedSizes ? new SizeStack('json.gz', logSizesBuffer) : undefined
+
   logStats() // initial
 
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       progress++
       processedLastSecond++
 
+      if (sizes) {
+        // Check it, cause gzipping might be delayed here..
+        void SizeStack.countItem(chunk, logger, sizes, sizesZipped)
+      }
+
       if (logProgress && progress % logEvery === 0) {
         logStats(chunk, false, progress % logEvery10 === 0)
       }
@@ -175,28 +222,30 @@ export function transformLogProgress<IN = any>(
     const rps10 = Math.round(sma.push(lastRPS))
     if (mem.rss > peakRSS) peakRSS = mem.rss
 
-    logger.log(
-      inspect(
-        {
-          [final ? `${metric}_final` : metric]: batchedProgress,
-          ...(extra ? extra(chunk, progress) : {}),
-          ...(logHeapUsed ? { heapUsed: _mb(mem.heapUsed) } : {}),
-          ...(logHeapTotal ? { heapTotal: _mb(mem.heapTotal) } : {}),
-          ...(logRss ? { rss: _mb(mem.rss) } : {}),
-          ...(logPeakRSS ? { peakRSS: _mb(peakRSS) } : {}),
-          ...(opt.rssMinusHeap ? { rssMinusHeap: _mb(mem.rss - mem.heapTotal) } : {}),
-          ...(opt.external ? { external: _mb(mem.external) } : {}),
-          ...(opt.arrayBuffers ? { arrayBuffers: _mb(mem.arrayBuffers || 0) } : {}),
-          ...(logRPS
-            ? {
-                rps10,
-                rpsTotal,
-              }
-            : {}),
-        },
-        inspectOpt,
-      ),
-    )
+    const o: LogItem = {
+      [final ? `${metric}_final` : metric]: batchedProgress,
+    }
+
+    if (extra) Object.assign(o, extra(chunk, progress))
+    if (logHeapUsed) o.heapUsed = _mb(mem.heapUsed)
+    if (logHeapTotal) o.heapTotal = _mb(mem.heapTotal)
+    if (logRss) o.rss = _mb(mem.rss)
+    if (logPeakRSS) o.peakRSS = _mb(peakRSS)
+    if (opt.rssMinusHeap) o.rssMinusHeap = _mb(mem.rss - mem.heapTotal)
+    if (opt.external) o.external = _mb(mem.external)
+    if (opt.arrayBuffers) o.arrayBuffers = _mb(mem.arrayBuffers || 0)
+
+    if (logRPS) Object.assign(o, { rps10, rpsTotal })
+
+    logger.log(inspect(o, inspectOpt))
+
+    if (sizes?.items.length) {
+      logger.log(sizes.getStats())
+
+      if (sizesZipped?.items.length) {
+        logger.log(sizesZipped.getStats())
+      }
+    }
 
     if (tenx) {
       let perHour: number | string =

package/src/stream/transform/transformMap.ts

@@ -26,9 +26,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
    *
-   * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
-   *
-   * @default to filter out undefined/null values, but pass anything else
+   * Defaults to "pass everything" (including null, undefined, etc).
+   * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
    */
   predicate?: AsyncPredicate<OUT>
 
@@ -60,10 +59,6 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   logger?: CommonLogger
 }
 
-export function notNullishPredicate(item: any): boolean {
-  return item !== undefined && item !== null
-}
-
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
 
@@ -85,7 +80,7 @@ export function transformMap<IN = any, OUT = IN>(
 ): TransformTyped<IN, OUT> {
   const {
     concurrency = 16,
-    predicate = notNullishPredicate,
+    predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,
     onError,
@@ -116,14 +111,12 @@ export function transformMap<IN = any, OUT = IN>(
       },
     },
     async function transformMapFn(this: AbortableTransform, chunk: IN, _, cb) {
-      index++
-      // console.log({chunk, _encoding})
-
       // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
       if (isSettled) return cb()
 
+      const currentIndex = ++index
+
       try {
-        const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
         const res = await mapper(chunk, currentIndex)
         const passedResults = await pFilter(
           flattenArrayOutput && Array.isArray(res) ? res : [res],
@@ -132,14 +125,14 @@ export function transformMap<IN = any, OUT = IN>(
               isSettled = true // will be checked later
               return false
             }
-            return r !== SKIP && (await predicate(r, currentIndex))
+            return r !== SKIP && (!predicate || (await predicate(r, currentIndex)))
           },
         )
 
         passedResults.forEach(r => this.push(r))
 
         if (isSettled) {
-          logger.log(`transformMap END received at index ${index}`)
+          logger.log(`transformMap END received at index ${currentIndex}`)
           pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
         }
 

package/src/stream/transform/transformMapSimple.ts

@@ -31,7 +31,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
 
   return new Transform({
     objectMode: true,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         cb(null, mapper(chunk, ++index))
       } catch (err) {

package/src/stream/transform/transformMapSync.ts

@@ -1,8 +1,16 @@
-import { Transform } from 'stream'
-import { AggregatedError, CommonLogger, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib'
+import {
+  AggregatedError,
+  CommonLogger,
+  END,
+  ErrorMode,
+  Mapper,
+  Predicate,
+  SKIP,
+} from '@naturalcycles/js-lib'
 import { yellow } from '../../colors'
+import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
-import { notNullishPredicate } from './transformMap'
+import { pipelineClose } from '../stream.util'
 
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   /**
@@ -20,9 +28,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
    *
-   * @default to filter out undefined/null values, but pass anything else
-   *
-   * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
+   * Defaults to "pass everything".
+   * Simpler way to skip individual entries is to return SKIP symbol.
    */
   predicate?: Predicate<OUT>
 
@@ -47,6 +54,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   logger?: CommonLogger
 }
 
+export class TransformMapSync extends AbortableTransform {}
+
 /**
  * Sync (not async) version of transformMap.
  * Supposedly faster, for cases when async is not needed.
@@ -58,7 +67,7 @@ export function transformMapSync<IN = any, OUT = IN>(
   let index = -1
 
   const {
-    predicate = notNullishPredicate,
+    predicate, // defaults to "no predicate" (pass everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput = false,
     onError,
@@ -66,34 +75,39 @@ export function transformMapSync<IN = any, OUT = IN>(
     objectMode = true,
     logger = console,
   } = opt
-  let isRejected = false
+  let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  return new Transform({
+  return new TransformMapSync({
     objectMode,
     ...opt,
-    transform(this: Transform, chunk: IN, _encoding, cb) {
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) {
-        return cb()
-      }
+    transform(this: AbortableTransform, chunk: IN, _, cb) {
+      // Stop processing if isSettled
+      if (isSettled) return cb()
 
-      try {
-        if (!predicate(chunk, ++index)) {
-          cb() // signal that we've finished processing, but emit no output here
-          return
-        }
+      const currentIndex = ++index
 
+      try {
         // map and pass through
-        const v = mapper(chunk, index)
+        const v = mapper(chunk, currentIndex)
+
+        const passedResults = (flattenArrayOutput && Array.isArray(v) ? v : [v]).filter(r => {
+          if (r === END) {
+            isSettled = true // will be checked later
+            return false
+          }
+          return r !== SKIP && (!predicate || predicate(r, currentIndex))
+        })
 
-        if (flattenArrayOutput && Array.isArray(v)) {
-          // Pass each item individually
-          v.forEach(item => this.push(item))
-        } else {
-          cb(null, v)
+        passedResults.forEach(r => this.push(r))
+
+        if (isSettled) {
+          logger.log(`transformMapSync END received at index ${currentIndex}`)
+          pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger)
         }
+
+        cb() // done processing
       } catch (err) {
         logger.error(err)
         errors++
@@ -107,7 +121,7 @@ export function transformMapSync<IN = any, OUT = IN>(
         }
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-          isRejected = true
+          isSettled = true
           // Emit error immediately
           return cb(err as Error)
         }

package/src/stream/transform/transformNoOp.ts

@@ -9,7 +9,7 @@ import { TransformTyped } from '../stream.model'
 export function transformNoOp<T = any>(): TransformTyped<T, T> {
   return new Transform({
     objectMode: true,
-    transform(chunk: T, _encoding, cb) {
+    transform(chunk: T, _, cb) {
       cb(null, chunk)
     },
   })

package/src/stream/transform/transformTap.ts

@@ -17,16 +17,16 @@ export function transformTap<IN>(
   opt: TransformTapOptions = {},
 ): TransformTyped<IN, IN> {
   const { logger = console } = opt
-  let index = 0
+  let index = -1
 
   return new Transform({
     objectMode: true,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
+    async transform(chunk: IN, _, cb) {
       // console.log('tap', chunk)
 
       try {
-        await fn(chunk, index++)
+        await fn(chunk, ++index)
       } catch (err) {
         logger.error(err)
         // suppressed error

package/src/stream/transform/transformToArray.ts

@@ -10,7 +10,7 @@ export function transformToArray<IN>(opt: TransformOptions = {}): TransformTyped
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       res.push(chunk)
       // callback to signal that we processed input, but not emitting any output
       cb()

package/src/stream/transform/transformToString.ts

@@ -14,8 +14,8 @@ export function transformToString(): TransformTyped<Buffer, string> {
   return new Transform({
     objectMode: false,
     readableObjectMode: true,
-    transform(chunk: Buffer, _encoding, cb) {
-      // console.log(`enc: ${_encoding}`, chunk.toString())
+    transform(chunk: Buffer, _, cb) {
+      // console.log(`enc: ${_}`, chunk.toString())
       cb(null, chunk.toString())
     },
   })

package/src/stream/transform/worker/transformMultiThreaded.ts

@@ -115,7 +115,7 @@ export function transformMultiThreaded<IN, OUT>(
         }
       },
     },
-    async function transformMapFn(chunk: IN, _encoding, cb) {
+    async function transformMapFn(chunk: IN, _, cb) {
       // Freezing the index, because it may change due to concurrency
       const currentIndex = ++index
 

package/src/stream/writable/writableFork.ts

@@ -31,7 +31,7 @@ export function writableFork<T>(
   return new Writable({
     objectMode: true,
     ...opt,
-    write(chunk: T, _encoding, cb) {
+    write(chunk: T, _, cb) {
       // Push/fork to all sub-streams
       // No backpressure is ensured here, it'll push regardless of the
       readables.forEach(readable => readable.push(chunk))

package/src/stream/writable/writablePushToArray.ts

@@ -8,7 +8,7 @@ export function writablePushToArray<IN>(arr: IN[], opt: TransformOptions = {}):
   return new Writable({
     objectMode: true,
     ...opt,
-    write(chunk: IN, _encoding, cb) {
+    write(chunk: IN, _, cb) {
       arr.push(chunk)
       // callback to signal that we processed input, but not emitting any output
       cb()

package/src/stream/writable/writableVoid.ts

@@ -18,7 +18,7 @@ export function writableVoid(opt: WritableVoidOptions = {}): Writable {
   return new Writable({
     objectMode: true,
     ...opt,
-    write(chunk, _encoding, cb) {
+    write(chunk, _, cb) {
       cb()
     },
     final(cb) {

package/src/util/zip.util.ts

@@ -8,9 +8,10 @@ const gzip = promisify(zlib.gzip.bind(zlib))
 const gunzip = promisify(zlib.gunzip.bind(zlib))
 
 // string > zip
+
 /**
  * zipBuffer uses `deflate`.
- * It's 9 bytes shorter than gzip.
+ * It's 9 bytes shorter than `gzip`.
  */
 export async function zipBuffer(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer> {
   return await deflate(buf, options)
@@ -18,7 +19,7 @@ export async function zipBuffer(buf: Buffer, options: ZlibOptions = {}): Promise
 
 /**
  * gzipBuffer uses `gzip`
- * It's 9 bytes longer than deflate.
+ * It's 9 bytes longer than `deflate`.
  */
 export async function gzipBuffer(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer> {
   return await gzip(buf, options)
@@ -33,11 +34,18 @@ export async function gunzipBuffer(buf: Buffer, options: ZlibOptions = {}): Prom
   return await gunzip(buf, options)
 }
 
-// convenience
+/**
+ * zipString uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
+ */
 export async function zipString(s: string, options?: ZlibOptions): Promise<Buffer> {
   return await zipBuffer(Buffer.from(s), options)
 }
 
+/**
+ * gzipString uses `gzip`.
+ * It's 9 bytes longer than `deflate`.
+ */
 export async function gzipString(s: string, options?: ZlibOptions): Promise<Buffer> {
   return await gzipBuffer(Buffer.from(s), options)
 }