@naturalcycles/nodejs-lib 13.9.1 → 13.11.0

package/src/stream/transform/transformLogProgress.ts

@@ -1,155 +1,10 @@
 import { Transform } from 'node:stream'
-import { inspect, InspectOptions } from 'node:util'
-import {
-  SimpleMovingAverage,
-  _mb,
-  _since,
-  AnyObject,
-  CommonLogger,
-  localTimeNow,
-} from '@naturalcycles/js-lib'
-import { hasColors, boldWhite, dimGrey, white, yellow } from '../../colors/colors'
-import { SizeStack } from '../sizeStack'
+import { progressLogger, ProgressLoggerCfg } from '../progressLogger'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
-export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
-  /**
-   * Progress metric
-   *
-   * @default `progress`
-   */
-  metric?: string
-
-  /**
-   * Include `heapUsed` in log.
-   *
-   * @default false
-   */
-  heapUsed?: boolean
-
-  /**
-   * Include `heapTotal` in log.
-   *
-   * @default false
-   */
-  heapTotal?: boolean
-
-  /**
-   * Include `rss` in log.
-   *
-   * @default true
-   */
-  rss?: boolean
-
-  /**
-   * Incude Peak RSS in log.
-   *
-   * @default true
-   */
-  peakRSS?: boolean
-
-  /**
-   * Include `external` in log.
-   *
-   * @default false
-   */
-  external?: boolean
-
-  /**
-   * Include `arrayBuffers` in log.
-   *
-   * @default false
-   */
-  arrayBuffers?: boolean
-
-  /**
-   * Log (rss - heapTotal)
-   * For convenience of debugging "out-of-heap" memory size.
-   *
-   * @default false
-   */
-  rssMinusHeap?: boolean
-
-  /**
-   * Log "rows per second"
-   *
-   * @default true
-   */
-  logRPS?: boolean
-
-  /**
-   * Set to false to disable logging progress
-   *
-   * @default true
-   */
-  logProgress?: boolean
-
-  /**
-   * Log progress event Nth record that is _processed_ (went through mapper).
-   * Set to 0 to disable logging.
-   *
-   * @default 1000
-   */
-  logEvery?: number
-
-  logger?: CommonLogger
-
-  /**
-   * Function to return extra properties to the "progress object".
-   *
-   * chunk is undefined for "final" stats, otherwise is defined.
-   */
-  extra?: (chunk: IN | undefined, index: number) => AnyObject
-
-  /**
-   * If specified - will multiply the counter by this number.
-   * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
-   * it'll accurately represent the number of processed entries (not batches).
-   *
-   * Defaults to 1.
-   */
-  batchSize?: number
-
-  /**
-   * Experimental logging of item (shunk) sizes, when json-stringified.
-   *
-   * Defaults to false.
-   *
-   * @experimental
-   */
-  logSizes?: boolean
-
-  /**
-   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
-   * Defaults to 100_000.
-   * Cannot be Infinity.
-   */
-  logSizesBuffer?: number
-
-  /**
-   * Works in addition to `logSizes`. Adds "zipped sizes".
-   *
-   * @experimental
-   */
-  logZippedSizes?: boolean
-}
-
-interface LogItem extends AnyObject {
-  heapUsed?: number
-  heapTotal?: number
-  rss?: number
-  peakRSS?: number
-  rssMinusHeap?: number
-  external?: number
-  arrayBuffers?: number
-  rps10?: number
-  rpsTotal?: number
-}
-
-const inspectOpt: InspectOptions = {
-  colors: hasColors,
-  breakLength: 300,
-}
+export interface TransformLogProgressOptions<IN = any>
+  extends ProgressLoggerCfg<IN>,
+    TransformOptions {}
 
 /**
  * Pass-through transform that optionally logs progress.
@@ -157,119 +12,18 @@ const inspectOpt: InspectOptions = {
 export function transformLogProgress<IN = any>(
   opt: TransformLogProgressOptions = {},
 ): TransformTyped<IN, IN> {
-  const {
-    metric = 'progress',
-    heapTotal: logHeapTotal = false,
-    heapUsed: logHeapUsed = false,
-    rss: logRss = true,
-    peakRSS: logPeakRSS = true,
-    logRPS = true,
-    logEvery = 1000,
-    logSizes = false,
-    logSizesBuffer = 100_000,
-    logZippedSizes = false,
-    batchSize = 1,
-    extra,
-    logger = console,
-  } = opt
-  const logProgress = opt.logProgress !== false && logEvery !== 0 // true by default
-  const logEvery10 = logEvery * 10
-
-  const started = Date.now()
-  let lastSecondStarted = Date.now()
-  const sma = new SimpleMovingAverage(10) // over last 10 seconds
-  let processedLastSecond = 0
-  let progress = 0
-  let peakRSS = 0
-
-  const sizes = logSizes ? new SizeStack('json', logSizesBuffer) : undefined
-  const sizesZipped = logZippedSizes ? new SizeStack('json.gz', logSizesBuffer) : undefined
-
-  logStats() // initial
+  const progress = progressLogger(opt)
 
   return new Transform({
     objectMode: true,
     ...opt,
     transform(chunk: IN, _, cb) {
-      progress++
-      processedLastSecond++
-
-      if (sizes) {
-        // Check it, cause gzipping might be delayed here..
-        void SizeStack.countItem(chunk, logger, sizes, sizesZipped)
-      }
-
-      if (logProgress && progress % logEvery === 0) {
-        logStats(chunk, false, progress % logEvery10 === 0)
-      }
-
+      progress.log(chunk)
       cb(null, chunk) // pass-through
     },
     final(cb) {
-      logStats(undefined, true)
-
+      progress.done()
       cb()
     },
   })
-
-  function logStats(chunk?: IN, final = false, tenx = false): void {
-    if (!logProgress) return
-
-    const mem = process.memoryUsage()
-
-    const now = Date.now()
-    const batchedProgress = progress * batchSize
-    const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0
-    const rpsTotal = Math.round(batchedProgress / ((now - started) / 1000)) || 0
-    lastSecondStarted = now
-    processedLastSecond = 0
-
-    const rps10 = Math.round(sma.pushGetAvg(lastRPS))
-    if (mem.rss > peakRSS) peakRSS = mem.rss
-
-    const o: LogItem = {
-      [final ? `${metric}_final` : metric]: batchedProgress,
-    }
-
-    if (extra) Object.assign(o, extra(chunk, progress))
-    if (logHeapUsed) o.heapUsed = _mb(mem.heapUsed)
-    if (logHeapTotal) o.heapTotal = _mb(mem.heapTotal)
-    if (logRss) o.rss = _mb(mem.rss)
-    if (logPeakRSS) o.peakRSS = _mb(peakRSS)
-    if (opt.rssMinusHeap) o.rssMinusHeap = _mb(mem.rss - mem.heapTotal)
-    if (opt.external) o.external = _mb(mem.external)
-    if (opt.arrayBuffers) o.arrayBuffers = _mb(mem.arrayBuffers || 0)
-
-    if (logRPS) Object.assign(o, { rps10, rpsTotal })
-
-    logger.log(inspect(o, inspectOpt))
-
-    if (sizes?.items.length) {
-      logger.log(sizes.getStats())
-
-      if (sizesZipped?.items.length) {
-        logger.log(sizesZipped.getStats())
-      }
-    }
-
-    if (tenx) {
-      let perHour: number | string =
-        Math.round((batchedProgress * 1000 * 60 * 60) / (now - started)) || 0
-      if (perHour > 900) {
-        perHour = Math.round(perHour / 1000) + 'K'
-      }
-
-      logger.log(
-        `${dimGrey(localTimeNow().toPretty())} ${white(metric)} took ${yellow(
-          _since(started),
-        )} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`,
-      )
-    } else if (final) {
-      logger.log(
-        `${boldWhite(metric)} took ${yellow(_since(started))} to process ${yellow(
-          batchedProgress,
-        )} rows with total RPS of ${yellow(rpsTotal)}`,
-      )
-    }
-  }
 }

package/src/stream/transform/transformTee.ts

@@ -14,7 +14,7 @@ type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream
  *
  * @experimental
  */
-export function transformTee<T>(streams: AnyStream[]): TransformTyped<T> {
+export function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T> {
   const readable = readableCreate<T>()
 
   const secondPipelinePromise = _pipeline([readable, ...streams])

package/dist/stream/readable/readableMap.d.ts

@@ -1,3 +0,0 @@
-import { AbortableAsyncMapper } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '../stream.model';
-export declare function readableMap<IN, OUT>(readable: ReadableTyped<IN>, mapper: AbortableAsyncMapper<IN, OUT>): ReadableTyped<OUT>;

package/dist/stream/readable/readableMap.js

@@ -1,31 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readableMap = void 0;
-const node_stream_1 = require("node:stream");
-const js_lib_1 = require("@naturalcycles/js-lib");
-function readableMap(readable, mapper) {
-    let i = -1;
-    const stream = readable
-        .on('error', err => stream.emit('error', err))
-        .pipe(new node_stream_1.Transform({
-        objectMode: true,
-        async transform(chunk, _enc, cb) {
-            try {
-                const r = await mapper(chunk, ++i);
-                if (r === js_lib_1.SKIP) {
-                    cb();
-                }
-                else {
-                    // _assert(r !== END, `readableMap END not supported`)
-                    cb(null, r);
-                }
-            }
-            catch (err) {
-                console.error(err);
-                cb(err);
-            }
-        },
-    }));
-    return stream;
-}
-exports.readableMap = readableMap;

package/dist/stream/readable/readableMapToArray.d.ts

@@ -1,10 +0,0 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '../stream.model';
-import { TransformMapOptions } from '../transform/transformMap';
-/**
- * Map Readable items to array of results (in memory),
- * passing each result via `transformMap`.
- *
- * Warning! All results are stored in memory (no backpressure).
- */
-export declare function readableMapToArray<IN, OUT = IN>(stream: ReadableTyped<IN>, mapper?: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): Promise<OUT[]>;

package/dist/stream/readable/readableMapToArray.js

@@ -1,16 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readableMapToArray = void 0;
-const index_1 = require("../../index");
-/**
- * Map Readable items to array of results (in memory),
- * passing each result via `transformMap`.
- *
- * Warning! All results are stored in memory (no backpressure).
- */
-async function readableMapToArray(stream, mapper = item => item, opt) {
-    const res = [];
-    await (0, index_1._pipeline)([stream, (0, index_1.transformMap)(mapper, opt), (0, index_1.writablePushToArray)(res)]);
-    return res;
-}
-exports.readableMapToArray = readableMapToArray;

package/src/stream/readable/readableMap.ts

@@ -1,34 +0,0 @@
-import { Transform } from 'node:stream'
-import { AbortableAsyncMapper, SKIP } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '../stream.model'
-
-export function readableMap<IN, OUT>(
-  readable: ReadableTyped<IN>,
-  mapper: AbortableAsyncMapper<IN, OUT>,
-): ReadableTyped<OUT> {
-  let i = -1
-
-  const stream: ReadableTyped<OUT> = readable
-    .on('error', err => stream.emit('error', err))
-    .pipe(
-      new Transform({
-        objectMode: true,
-        async transform(chunk, _enc, cb) {
-          try {
-            const r = await mapper(chunk, ++i)
-            if (r === SKIP) {
-              cb()
-            } else {
-              // _assert(r !== END, `readableMap END not supported`)
-              cb(null, r)
-            }
-          } catch (err) {
-            console.error(err)
-            cb(err as Error)
-          }
-        },
-      }),
-    )
-
-  return stream
-}

package/src/stream/readable/readableMapToArray.ts

@@ -1,22 +0,0 @@
-import { AsyncMapper } from '@naturalcycles/js-lib'
-import { transformMap, writablePushToArray, _pipeline } from '../../index'
-import { ReadableTyped } from '../stream.model'
-import { TransformMapOptions } from '../transform/transformMap'
-
-/**
- * Map Readable items to array of results (in memory),
- * passing each result via `transformMap`.
- *
- * Warning! All results are stored in memory (no backpressure).
- */
-export async function readableMapToArray<IN, OUT = IN>(
-  stream: ReadableTyped<IN>,
-  mapper: AsyncMapper<IN, OUT> = item => item as any,
-  opt?: TransformMapOptions<IN, OUT>,
-): Promise<OUT[]> {
-  const res: OUT[] = []
-
-  await _pipeline([stream, transformMap(mapper, opt), writablePushToArray(res)])
-
-  return res
-}