@naturalcycles/nodejs-lib 13.9.0 → 13.10.0

package/dist/csv/csvReader.js

@@ -35,7 +35,7 @@ function csvStringParse(str, cfg = {}) {
 exports.csvStringParse = csvStringParse;
 function csvStringToArray(str) {
     const objPattern = new RegExp('(,|\\r?\\n|\\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\\r\\n]*))', 'gi');
-    let matches = null;
+    let matches;
     const arr = [[]];
     while ((matches = objPattern.exec(str))) {
         if (matches[1].length && matches[1] !== ',') {

package/dist/index.d.ts

@@ -35,6 +35,7 @@ export * from './stream/readable/readableMap';
 export * from './stream/readable/readableMapToArray';
 export * from './stream/readable/readableToArray';
 export * from './stream/stream.model';
+export * from './stream/progressLogger';
 export * from './stream/transform/transformBuffer';
 export * from './stream/transform/transformFilter';
 export * from './stream/transform/transformLimit';

package/dist/index.js

@@ -39,6 +39,7 @@ tslib_1.__exportStar(require("./stream/readable/readableMap"), exports);
 tslib_1.__exportStar(require("./stream/readable/readableMapToArray"), exports);
 tslib_1.__exportStar(require("./stream/readable/readableToArray"), exports);
 tslib_1.__exportStar(require("./stream/stream.model"), exports);
+tslib_1.__exportStar(require("./stream/progressLogger"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformBuffer"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformFilter"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformLimit"), exports);

package/dist/stream/progressLogger.d.ts

@@ -0,0 +1,145 @@
+import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
+export interface ProgressLoggerCfg<IN = any> {
+    /**
+     * Progress metric
+     *
+     * @default `progress`
+     */
+    metric?: string;
+    /**
+     * Include `heapUsed` in log.
+     *
+     * @default false
+     */
+    heapUsed?: boolean;
+    /**
+     * Include `heapTotal` in log.
+     *
+     * @default false
+     */
+    heapTotal?: boolean;
+    /**
+     * Include `rss` in log.
+     *
+     * @default true
+     */
+    rss?: boolean;
+    /**
+     * Incude Peak RSS in log.
+     *
+     * @default true
+     */
+    peakRSS?: boolean;
+    /**
+     * Include `external` in log.
+     *
+     * @default false
+     */
+    external?: boolean;
+    /**
+     * Include `arrayBuffers` in log.
+     *
+     * @default false
+     */
+    arrayBuffers?: boolean;
+    /**
+     * Log (rss - heapTotal)
+     * For convenience of debugging "out-of-heap" memory size.
+     *
+     * @default false
+     */
+    rssMinusHeap?: boolean;
+    /**
+     * Log "rows per second"
+     *
+     * @default true
+     */
+    logRPS?: boolean;
+    /**
+     * Set to false to disable logging progress
+     *
+     * @default true
+     */
+    logProgress?: boolean;
+    /**
+     * Log progress event Nth record that is _processed_ (went through mapper).
+     * Set to 0 to disable logging.
+     *
+     * @default 1000
+     */
+    logEvery?: number;
+    logger?: CommonLogger;
+    /**
+     * Function to return extra properties to the "progress object".
+     *
+     * chunk is undefined for "final" stats, otherwise is defined.
+     */
+    extra?: (chunk: IN | undefined, index: number) => AnyObject;
+    /**
+     * If specified - will multiply the counter by this number.
+     * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
+     * it'll accurately represent the number of processed entries (not batches).
+     *
+     * Defaults to 1.
+     */
+    batchSize?: number;
+    /**
+     * Experimental logging of item (shunk) sizes, when json-stringified.
+     *
+     * Defaults to false.
+     *
+     * @experimental
+     */
+    logSizes?: boolean;
+    /**
+     * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+     * Defaults to 100_000.
+     * Cannot be Infinity.
+     */
+    logSizesBuffer?: number;
+    /**
+     * Works in addition to `logSizes`. Adds "zipped sizes".
+     *
+     * @experimental
+     */
+    logZippedSizes?: boolean;
+}
+export interface ProgressLogItem extends AnyObject {
+    heapUsed?: number;
+    heapTotal?: number;
+    rss?: number;
+    peakRSS?: number;
+    rssMinusHeap?: number;
+    external?: number;
+    arrayBuffers?: number;
+    rps10?: number;
+    rpsTotal?: number;
+}
+export declare class ProgressLogger<IN> implements Disposable {
+    constructor(cfg?: ProgressLoggerCfg<IN>);
+    cfg: ProgressLoggerCfg<IN> & {
+        logEvery: number;
+        logSizesBuffer: number;
+        batchSize: number;
+        metric: string;
+        logger: CommonLogger;
+    };
+    private started;
+    private lastSecondStarted;
+    private sma;
+    private logEvery10;
+    private processedLastSecond;
+    private progress;
+    private peakRSS;
+    private sizes?;
+    private sizesZipped?;
+    private start;
+    log(chunk?: IN): void;
+    done(): void;
+    [Symbol.dispose](): void;
+    private logStats;
+}
+/**
+ * Create new ProgressLogger.
+ */
+export declare function progressLogger<IN>(cfg?: ProgressLoggerCfg<IN>): ProgressLogger<IN>;

package/dist/stream/progressLogger.js

@@ -0,0 +1,120 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.progressLogger = exports.ProgressLogger = void 0;
+const node_util_1 = require("node:util");
+const js_lib_1 = require("@naturalcycles/js-lib");
+const colors_1 = require("../colors/colors");
+const sizeStack_1 = require("./sizeStack");
+const inspectOpt = {
+    colors: colors_1.hasColors,
+    breakLength: 300,
+};
+class ProgressLogger {
+    constructor(cfg = {}) {
+        this.cfg = {
+            metric: 'progress',
+            rss: true,
+            peakRSS: true,
+            logRPS: true,
+            logEvery: 1000,
+            logSizesBuffer: 100_000,
+            batchSize: 1,
+            logger: console,
+            logProgress: cfg.logProgress !== false && cfg.logEvery !== 0,
+            ...cfg,
+        };
+        this.logEvery10 = this.cfg.logEvery * 10;
+        this.start();
+        this.logStats(); // initial
+    }
+    start() {
+        this.started = Date.now();
+        this.lastSecondStarted = Date.now();
+        this.sma = new js_lib_1.SimpleMovingAverage(10);
+        this.processedLastSecond = 0;
+        this.progress = 0;
+        this.peakRSS = 0;
+        this.sizes = this.cfg.logSizes ? new sizeStack_1.SizeStack('json', this.cfg.logSizesBuffer) : undefined;
+        this.sizesZipped = this.cfg.logZippedSizes
+            ? new sizeStack_1.SizeStack('json.gz', this.cfg.logSizesBuffer)
+            : undefined;
+    }
+    log(chunk) {
+        this.progress++;
+        this.processedLastSecond++;
+        if (this.sizes) {
+            // Check it, cause gzipping might be delayed here..
+            void sizeStack_1.SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
+        }
+        if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
+            this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
+        }
+    }
+    done() {
+        this.logStats(undefined, true);
+    }
+    [Symbol.dispose]() {
+        this.done();
+    }
+    logStats(chunk, final = false, tenx = false) {
+        if (!this.cfg.logProgress)
+            return;
+        const { metric, extra, batchSize, heapUsed: logHeapUsed, heapTotal: logHeapTotal, rss: logRss, peakRSS: logPeakRss, rssMinusHeap, external, arrayBuffers, logRPS, logger, } = this.cfg;
+        const mem = process.memoryUsage();
+        const now = Date.now();
+        const batchedProgress = this.progress * batchSize;
+        const lastRPS = (this.processedLastSecond * batchSize) / ((now - this.lastSecondStarted) / 1000) || 0;
+        const rpsTotal = Math.round(batchedProgress / ((now - this.started) / 1000)) || 0;
+        this.lastSecondStarted = now;
+        this.processedLastSecond = 0;
+        const rps10 = Math.round(this.sma.pushGetAvg(lastRPS));
+        if (mem.rss > this.peakRSS)
+            this.peakRSS = mem.rss;
+        const o = {
+            [final ? `${this.cfg.metric}_final` : this.cfg.metric]: batchedProgress,
+        };
+        if (extra)
+            Object.assign(o, extra(chunk, this.progress));
+        if (logHeapUsed)
+            o.heapUsed = (0, js_lib_1._mb)(mem.heapUsed);
+        if (logHeapTotal)
+            o.heapTotal = (0, js_lib_1._mb)(mem.heapTotal);
+        if (logRss)
+            o.rss = (0, js_lib_1._mb)(mem.rss);
+        if (logPeakRss)
+            o.peakRSS = (0, js_lib_1._mb)(this.peakRSS);
+        if (rssMinusHeap)
+            o.rssMinusHeap = (0, js_lib_1._mb)(mem.rss - mem.heapTotal);
+        if (external)
+            o.external = (0, js_lib_1._mb)(mem.external);
+        if (arrayBuffers)
+            o.arrayBuffers = (0, js_lib_1._mb)(mem.arrayBuffers || 0);
+        if (logRPS)
+            Object.assign(o, { rps10, rpsTotal });
+        logger.log((0, node_util_1.inspect)(o, inspectOpt));
+        if (this.sizes?.items.length) {
+            logger.log(this.sizes.getStats());
+            if (this.sizesZipped?.items.length) {
+                logger.log(this.sizesZipped.getStats());
+            }
+        }
+        if (tenx) {
+            let perHour = Math.round((batchedProgress * 1000 * 60 * 60) / (now - this.started)) || 0;
+            if (perHour > 900) {
+                perHour = Math.round(perHour / 1000) + 'K';
+            }
+            logger.log(`${(0, colors_1.dimGrey)((0, js_lib_1.localTimeNow)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
+        }
+        else if (final) {
+            logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
+        }
+    }
+}
+exports.ProgressLogger = ProgressLogger;
+/**
+ * Create new ProgressLogger.
+ */
+function progressLogger(cfg = {}) {
+    return new ProgressLogger(cfg);
+}
+exports.progressLogger = progressLogger;

package/dist/stream/readable/readableForEach.d.ts

@@ -5,6 +5,8 @@ import { TransformMapOptions } from '../transform/transformMap';
  * Convenience function to do `.forEach` over a Readable.
  * Typed! (unlike default Readable).
  *
+ * Try native readable.forEach() instead!
+ *
  * @experimental
  */
 export declare function readableForEach<T>(readable: ReadableTyped<T>, mapper: AbortableAsyncMapper<T, void>, opt?: TransformMapOptions<T, void>): Promise<void>;

package/dist/stream/readable/readableForEach.js

@@ -8,6 +8,8 @@ const transformMap_1 = require("../transform/transformMap");
  * Convenience function to do `.forEach` over a Readable.
  * Typed! (unlike default Readable).
  *
+ * Try native readable.forEach() instead!
+ *
  * @experimental
  */
 async function readableForEach(readable, mapper, opt = {}) {

package/dist/stream/readable/readableMapToArray.d.ts

@@ -6,5 +6,7 @@ import { TransformMapOptions } from '../transform/transformMap';
  * passing each result via `transformMap`.
  *
  * Warning! All results are stored in memory (no backpressure).
+ *
+ * Try native readable.toArray instead!
  */
 export declare function readableMapToArray<IN, OUT = IN>(stream: ReadableTyped<IN>, mapper?: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): Promise<OUT[]>;

package/dist/stream/readable/readableMapToArray.js

@@ -7,6 +7,8 @@ const index_1 = require("../../index");
  * passing each result via `transformMap`.
  *
  * Warning! All results are stored in memory (no backpressure).
+ *
+ * Try native readable.toArray instead!
  */
 async function readableMapToArray(stream, mapper = item => item, opt) {
     const res = [];

package/dist/stream/transform/transformLogProgress.d.ts

@@ -1,109 +1,6 @@
-import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
+import { ProgressLoggerCfg } from '../progressLogger';
 import { TransformOptions, TransformTyped } from '../stream.model';
-export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
-    /**
-     * Progress metric
-     *
-     * @default `progress`
-     */
-    metric?: string;
-    /**
-     * Include `heapUsed` in log.
-     *
-     * @default false
-     */
-    heapUsed?: boolean;
-    /**
-     * Include `heapTotal` in log.
-     *
-     * @default false
-     */
-    heapTotal?: boolean;
-    /**
-     * Include `rss` in log.
-     *
-     * @default true
-     */
-    rss?: boolean;
-    /**
-     * Incude Peak RSS in log.
-     *
-     * @default true
-     */
-    peakRSS?: boolean;
-    /**
-     * Include `external` in log.
-     *
-     * @default false
-     */
-    external?: boolean;
-    /**
-     * Include `arrayBuffers` in log.
-     *
-     * @default false
-     */
-    arrayBuffers?: boolean;
-    /**
-     * Log (rss - heapTotal)
-     * For convenience of debugging "out-of-heap" memory size.
-     *
-     * @default false
-     */
-    rssMinusHeap?: boolean;
-    /**
-     * Log "rows per second"
-     *
-     * @default true
-     */
-    logRPS?: boolean;
-    /**
-     * Set to false to disable logging progress
-     *
-     * @default true
-     */
-    logProgress?: boolean;
-    /**
-     * Log progress event Nth record that is _processed_ (went through mapper).
-     * Set to 0 to disable logging.
-     *
-     * @default 1000
-     */
-    logEvery?: number;
-    logger?: CommonLogger;
-    /**
-     * Function to return extra properties to the "progress object".
-     *
-     * chunk is undefined for "final" stats, otherwise is defined.
-     */
-    extra?: (chunk: IN | undefined, index: number) => AnyObject;
-    /**
-     * If specified - will multiply the counter by this number.
-     * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
-     * it'll accurately represent the number of processed entries (not batches).
-     *
-     * Defaults to 1.
-     */
-    batchSize?: number;
-    /**
-     * Experimental logging of item (shunk) sizes, when json-stringified.
-     *
-     * Defaults to false.
-     *
-     * @experimental
-     */
-    logSizes?: boolean;
-    /**
-     * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
-     * Defaults to 100_000.
-     * Cannot be Infinity.
-     */
-    logSizesBuffer?: number;
-    /**
-     * Works in addition to `logSizes`. Adds "zipped sizes".
-     *
-     * @experimental
-     */
-    logZippedSizes?: boolean;
+export interface TransformLogProgressOptions<IN = any> extends ProgressLoggerCfg<IN>, TransformOptions {
 }
 /**
  * Pass-through transform that optionally logs progress.

package/dist/stream/transform/transformLogProgress.js

@@ -2,101 +2,23 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transformLogProgress = void 0;
 const node_stream_1 = require("node:stream");
-const node_util_1 = require("node:util");
-const js_lib_1 = require("@naturalcycles/js-lib");
-const colors_1 = require("../../colors/colors");
-const sizeStack_1 = require("../sizeStack");
-const inspectOpt = {
-    colors: colors_1.hasColors,
-    breakLength: 300,
-};
+const progressLogger_1 = require("../progressLogger");
 /**
  * Pass-through transform that optionally logs progress.
  */
 function transformLogProgress(opt = {}) {
-    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, logSizes = false, logSizesBuffer = 100_000, logZippedSizes = false, batchSize = 1, extra, logger = console, } = opt;
-    const logProgress = opt.logProgress !== false && logEvery !== 0; // true by default
-    const logEvery10 = logEvery * 10;
-    const started = Date.now();
-    let lastSecondStarted = Date.now();
-    const sma = new js_lib_1.SimpleMovingAverage(10); // over last 10 seconds
-    let processedLastSecond = 0;
-    let progress = 0;
-    let peakRSS = 0;
-    const sizes = logSizes ? new sizeStack_1.SizeStack('json', logSizesBuffer) : undefined;
-    const sizesZipped = logZippedSizes ? new sizeStack_1.SizeStack('json.gz', logSizesBuffer) : undefined;
-    logStats(); // initial
+    const progress = (0, progressLogger_1.progressLogger)(opt);
     return new node_stream_1.Transform({
         objectMode: true,
         ...opt,
         transform(chunk, _, cb) {
-            progress++;
-            processedLastSecond++;
-            if (sizes) {
-                // Check it, cause gzipping might be delayed here..
-                void sizeStack_1.SizeStack.countItem(chunk, logger, sizes, sizesZipped);
-            }
-            if (logProgress && progress % logEvery === 0) {
-                logStats(chunk, false, progress % logEvery10 === 0);
-            }
+            progress.log(chunk);
             cb(null, chunk); // pass-through
         },
         final(cb) {
-            logStats(undefined, true);
+            progress.done();
             cb();
         },
     });
-    function logStats(chunk, final = false, tenx = false) {
-        if (!logProgress)
-            return;
-        const mem = process.memoryUsage();
-        const now = Date.now();
-        const batchedProgress = progress * batchSize;
-        const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0;
-        const rpsTotal = Math.round(batchedProgress / ((now - started) / 1000)) || 0;
-        lastSecondStarted = now;
-        processedLastSecond = 0;
-        const rps10 = Math.round(sma.pushGetAvg(lastRPS));
-        if (mem.rss > peakRSS)
-            peakRSS = mem.rss;
-        const o = {
-            [final ? `${metric}_final` : metric]: batchedProgress,
-        };
-        if (extra)
-            Object.assign(o, extra(chunk, progress));
-        if (logHeapUsed)
-            o.heapUsed = (0, js_lib_1._mb)(mem.heapUsed);
-        if (logHeapTotal)
-            o.heapTotal = (0, js_lib_1._mb)(mem.heapTotal);
-        if (logRss)
-            o.rss = (0, js_lib_1._mb)(mem.rss);
-        if (logPeakRSS)
-            o.peakRSS = (0, js_lib_1._mb)(peakRSS);
-        if (opt.rssMinusHeap)
-            o.rssMinusHeap = (0, js_lib_1._mb)(mem.rss - mem.heapTotal);
-        if (opt.external)
-            o.external = (0, js_lib_1._mb)(mem.external);
-        if (opt.arrayBuffers)
-            o.arrayBuffers = (0, js_lib_1._mb)(mem.arrayBuffers || 0);
-        if (logRPS)
-            Object.assign(o, { rps10, rpsTotal });
-        logger.log((0, node_util_1.inspect)(o, inspectOpt));
-        if (sizes?.items.length) {
-            logger.log(sizes.getStats());
-            if (sizesZipped?.items.length) {
-                logger.log(sizesZipped.getStats());
-            }
-        }
-        if (tenx) {
-            let perHour = Math.round((batchedProgress * 1000 * 60 * 60) / (now - started)) || 0;
-            if (perHour > 900) {
-                perHour = Math.round(perHour / 1000) + 'K';
-            }
-            logger.log(`${(0, colors_1.dimGrey)((0, js_lib_1.localTimeNow)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
-        }
-        else if (final) {
-            logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
-        }
-    }
 }
 exports.transformLogProgress = transformLogProgress;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.9.0",
+  "version": "13.10.0",
   "scripts": {
-    "prepare": "husky install",
+    "prepare": "husky",
     "docs-serve": "vuepress dev docs",
     "docs-build": "vuepress build docs",
     "slack-this-debug": "tsn ./src/bin/slack-this.ts --channel test --msg 'Hello slack!'",
@@ -19,7 +19,7 @@
     "@types/jsonwebtoken": "^9.0.0",
     "@types/through2-concurrent": "^2.0.0",
     "ajv": "^8.6.2",
-    "ajv-formats": "^2.1.0",
+    "ajv-formats": "^3.0.1",
     "ajv-keywords": "^5.0.0",
     "binary-split": "^1.0.5",
     "chalk": "^4.0.0",

package/src/csv/csvReader.ts

@@ -57,7 +57,7 @@ export function csvStringParse<T extends AnyObject = any>(
 
 export function csvStringToArray(str: string): string[][] {
   const objPattern = new RegExp('(,|\\r?\\n|\\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\\r\\n]*))', 'gi')
-  let matches = null
+  let matches: RegExpExecArray | null
   const arr: any[][] = [[]]
 
   while ((matches = objPattern.exec(str))) {

package/src/index.ts

@@ -45,6 +45,7 @@ export * from './stream/readable/readableMap'
 export * from './stream/readable/readableMapToArray'
 export * from './stream/readable/readableToArray'
 export * from './stream/stream.model'
+export * from './stream/progressLogger'
 export * from './stream/transform/transformBuffer'
 export * from './stream/transform/transformFilter'
 export * from './stream/transform/transformLimit'

package/src/stream/progressLogger.ts

@@ -0,0 +1,309 @@
+import { inspect, InspectOptions } from 'node:util'
+import {
+  _mb,
+  _since,
+  AnyObject,
+  CommonLogger,
+  localTimeNow,
+  SimpleMovingAverage,
+  UnixTimestampMillisNumber,
+} from '@naturalcycles/js-lib'
+import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors'
+import { SizeStack } from './sizeStack'
+
+export interface ProgressLoggerCfg<IN = any> {
+  /**
+   * Progress metric
+   *
+   * @default `progress`
+   */
+  metric?: string
+
+  /**
+   * Include `heapUsed` in log.
+   *
+   * @default false
+   */
+  heapUsed?: boolean
+
+  /**
+   * Include `heapTotal` in log.
+   *
+   * @default false
+   */
+  heapTotal?: boolean
+
+  /**
+   * Include `rss` in log.
+   *
+   * @default true
+   */
+  rss?: boolean
+
+  /**
+   * Incude Peak RSS in log.
+   *
+   * @default true
+   */
+  peakRSS?: boolean
+
+  /**
+   * Include `external` in log.
+   *
+   * @default false
+   */
+  external?: boolean
+
+  /**
+   * Include `arrayBuffers` in log.
+   *
+   * @default false
+   */
+  arrayBuffers?: boolean
+
+  /**
+   * Log (rss - heapTotal)
+   * For convenience of debugging "out-of-heap" memory size.
+   *
+   * @default false
+   */
+  rssMinusHeap?: boolean
+
+  /**
+   * Log "rows per second"
+   *
+   * @default true
+   */
+  logRPS?: boolean
+
+  /**
+   * Set to false to disable logging progress
+   *
+   * @default true
+   */
+  logProgress?: boolean
+
+  /**
+   * Log progress event Nth record that is _processed_ (went through mapper).
+   * Set to 0 to disable logging.
+   *
+   * @default 1000
+   */
+  logEvery?: number
+
+  logger?: CommonLogger
+
+  /**
+   * Function to return extra properties to the "progress object".
+   *
+   * chunk is undefined for "final" stats, otherwise is defined.
+   */
+  extra?: (chunk: IN | undefined, index: number) => AnyObject
+
+  /**
+   * If specified - will multiply the counter by this number.
+   * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
+   * it'll accurately represent the number of processed entries (not batches).
+   *
+   * Defaults to 1.
+   */
+  batchSize?: number
+
+  /**
+   * Experimental logging of item (shunk) sizes, when json-stringified.
+   *
+   * Defaults to false.
+   *
+   * @experimental
+   */
+  logSizes?: boolean
+
+  /**
+   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+   * Defaults to 100_000.
+   * Cannot be Infinity.
+   */
+  logSizesBuffer?: number
+
+  /**
+   * Works in addition to `logSizes`. Adds "zipped sizes".
+   *
+   * @experimental
+   */
+  logZippedSizes?: boolean
+}
+
+export interface ProgressLogItem extends AnyObject {
+  heapUsed?: number
+  heapTotal?: number
+  rss?: number
+  peakRSS?: number
+  rssMinusHeap?: number
+  external?: number
+  arrayBuffers?: number
+  rps10?: number
+  rpsTotal?: number
+}
+
+const inspectOpt: InspectOptions = {
+  colors: hasColors,
+  breakLength: 300,
+}
+
+export class ProgressLogger<IN> implements Disposable {
+  constructor(cfg: ProgressLoggerCfg<IN> = {}) {
+    this.cfg = {
+      metric: 'progress',
+      rss: true,
+      peakRSS: true,
+      logRPS: true,
+      logEvery: 1000,
+      logSizesBuffer: 100_000,
+      batchSize: 1,
+      logger: console,
+      logProgress: cfg.logProgress !== false && cfg.logEvery !== 0,
+      ...cfg,
+    }
+    this.logEvery10 = this.cfg.logEvery * 10
+
+    this.start()
+    this.logStats() // initial
+  }
+
+  cfg!: ProgressLoggerCfg<IN> & {
+    logEvery: number
+    logSizesBuffer: number
+    batchSize: number
+    metric: string
+    logger: CommonLogger
+  }
+
+  private started!: UnixTimestampMillisNumber
+  private lastSecondStarted!: UnixTimestampMillisNumber
+  private sma!: SimpleMovingAverage
+  private logEvery10!: number
+  private processedLastSecond!: number
+  private progress!: number
+  private peakRSS!: number
+  private sizes?: SizeStack
+  private sizesZipped?: SizeStack
+
+  private start(): void {
+    this.started = Date.now()
+    this.lastSecondStarted = Date.now()
+    this.sma = new SimpleMovingAverage(10)
+    this.processedLastSecond = 0
+    this.progress = 0
+    this.peakRSS = 0
+    this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined
+    this.sizesZipped = this.cfg.logZippedSizes
+      ? new SizeStack('json.gz', this.cfg.logSizesBuffer)
+      : undefined
+  }
+
+  log(chunk?: IN): void {
+    this.progress++
+    this.processedLastSecond++
+
+    if (this.sizes) {
+      // Check it, cause gzipping might be delayed here..
+      void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped)
+    }
+
+    if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
+      this.logStats(chunk, false, this.progress % this.logEvery10 === 0)
+    }
+  }
+
+  done(): void {
+    this.logStats(undefined, true)
+  }
+
+  [Symbol.dispose](): void {
+    this.done()
+  }
+
+  private logStats(chunk?: IN, final = false, tenx = false): void {
+    if (!this.cfg.logProgress) return
+
+    const {
+      metric,
+      extra,
+      batchSize,
+      heapUsed: logHeapUsed,
+      heapTotal: logHeapTotal,
+      rss: logRss,
+      peakRSS: logPeakRss,
+      rssMinusHeap,
+      external,
+      arrayBuffers,
+      logRPS,
+      logger,
+    } = this.cfg
+
+    const mem = process.memoryUsage()
+
+    const now = Date.now()
+    const batchedProgress = this.progress * batchSize
+    const lastRPS =
+      (this.processedLastSecond * batchSize) / ((now - this.lastSecondStarted) / 1000) || 0
+    const rpsTotal = Math.round(batchedProgress / ((now - this.started) / 1000)) || 0
+    this.lastSecondStarted = now
+    this.processedLastSecond = 0
+
+    const rps10 = Math.round(this.sma.pushGetAvg(lastRPS))
+    if (mem.rss > this.peakRSS) this.peakRSS = mem.rss
+
+    const o: ProgressLogItem = {
+      [final ? `${this.cfg.metric}_final` : this.cfg.metric]: batchedProgress,
+    }
+
+    if (extra) Object.assign(o, extra(chunk, this.progress))
+    if (logHeapUsed) o.heapUsed = _mb(mem.heapUsed)
+    if (logHeapTotal) o.heapTotal = _mb(mem.heapTotal)
+    if (logRss) o.rss = _mb(mem.rss)
+    if (logPeakRss) o.peakRSS = _mb(this.peakRSS)
+    if (rssMinusHeap) o.rssMinusHeap = _mb(mem.rss - mem.heapTotal)
+    if (external) o.external = _mb(mem.external)
+    if (arrayBuffers) o.arrayBuffers = _mb(mem.arrayBuffers || 0)
+
+    if (logRPS) Object.assign(o, { rps10, rpsTotal })
+
+    logger.log(inspect(o, inspectOpt))
+
+    if (this.sizes?.items.length) {
+      logger.log(this.sizes.getStats())
+
+      if (this.sizesZipped?.items.length) {
+        logger.log(this.sizesZipped.getStats())
+      }
+    }
+
+    if (tenx) {
+      let perHour: number | string =
+        Math.round((batchedProgress * 1000 * 60 * 60) / (now - this.started)) || 0
+      if (perHour > 900) {
+        perHour = Math.round(perHour / 1000) + 'K'
+      }
+
+      logger.log(
+        `${dimGrey(localTimeNow().toPretty())} ${white(metric)} took ${yellow(
+          _since(this.started),
+        )} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`,
+      )
+    } else if (final) {
+      logger.log(
+        `${boldWhite(metric)} took ${yellow(_since(this.started))} to process ${yellow(
+          batchedProgress,
+        )} rows with total RPS of ${yellow(rpsTotal)}`,
+      )
+    }
+  }
+}
+
+/**
+ * Create new ProgressLogger.
+ */
+export function progressLogger<IN>(cfg: ProgressLoggerCfg<IN> = {}): ProgressLogger<IN> {
+  return new ProgressLogger(cfg)
+}

package/src/stream/readable/readableForEach.ts

@@ -6,6 +6,8 @@ import { transformMap, TransformMapOptions } from '../transform/transformMap'
  * Convenience function to do `.forEach` over a Readable.
  * Typed! (unlike default Readable).
  *
+ * Try native readable.forEach() instead!
+ *
  * @experimental
  */
 export async function readableForEach<T>(

package/src/stream/readable/readableMapToArray.ts

@@ -8,6 +8,8 @@ import { TransformMapOptions } from '../transform/transformMap'
  * passing each result via `transformMap`.
  *
  * Warning! All results are stored in memory (no backpressure).
+ *
+ * Try native readable.toArray instead!
  */
 export async function readableMapToArray<IN, OUT = IN>(
   stream: ReadableTyped<IN>,

package/src/stream/transform/transformLogProgress.ts

@@ -1,155 +1,10 @@
 import { Transform } from 'node:stream'
-import { inspect, InspectOptions } from 'node:util'
-import {
-  SimpleMovingAverage,
-  _mb,
-  _since,
-  AnyObject,
-  CommonLogger,
-  localTimeNow,
-} from '@naturalcycles/js-lib'
-import { hasColors, boldWhite, dimGrey, white, yellow } from '../../colors/colors'
-import { SizeStack } from '../sizeStack'
+import { progressLogger, ProgressLoggerCfg } from '../progressLogger'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
-export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
-  /**
-   * Progress metric
-   *
-   * @default `progress`
-   */
-  metric?: string
-
-  /**
-   * Include `heapUsed` in log.
-   *
-   * @default false
-   */
-  heapUsed?: boolean
-
-  /**
-   * Include `heapTotal` in log.
-   *
-   * @default false
-   */
-  heapTotal?: boolean
-
-  /**
-   * Include `rss` in log.
-   *
-   * @default true
-   */
-  rss?: boolean
-
-  /**
-   * Incude Peak RSS in log.
-   *
-   * @default true
-   */
-  peakRSS?: boolean
-
-  /**
-   * Include `external` in log.
-   *
-   * @default false
-   */
-  external?: boolean
-
-  /**
-   * Include `arrayBuffers` in log.
-   *
-   * @default false
-   */
-  arrayBuffers?: boolean
-
-  /**
-   * Log (rss - heapTotal)
-   * For convenience of debugging "out-of-heap" memory size.
-   *
-   * @default false
-   */
-  rssMinusHeap?: boolean
-
-  /**
-   * Log "rows per second"
-   *
-   * @default true
-   */
-  logRPS?: boolean
-
-  /**
-   * Set to false to disable logging progress
-   *
-   * @default true
-   */
-  logProgress?: boolean
-
-  /**
-   * Log progress event Nth record that is _processed_ (went through mapper).
-   * Set to 0 to disable logging.
-   *
-   * @default 1000
-   */
-  logEvery?: number
-
-  logger?: CommonLogger
-
-  /**
-   * Function to return extra properties to the "progress object".
-   *
-   * chunk is undefined for "final" stats, otherwise is defined.
-   */
-  extra?: (chunk: IN | undefined, index: number) => AnyObject
-
-  /**
-   * If specified - will multiply the counter by this number.
-   * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
-   * it'll accurately represent the number of processed entries (not batches).
-   *
-   * Defaults to 1.
-   */
-  batchSize?: number
-
-  /**
-   * Experimental logging of item (shunk) sizes, when json-stringified.
-   *
-   * Defaults to false.
-   *
-   * @experimental
-   */
-  logSizes?: boolean
-
-  /**
-   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
-   * Defaults to 100_000.
-   * Cannot be Infinity.
-   */
-  logSizesBuffer?: number
-
-  /**
-   * Works in addition to `logSizes`. Adds "zipped sizes".
-   *
-   * @experimental
-   */
-  logZippedSizes?: boolean
-}
-
-interface LogItem extends AnyObject {
-  heapUsed?: number
-  heapTotal?: number
-  rss?: number
-  peakRSS?: number
-  rssMinusHeap?: number
-  external?: number
-  arrayBuffers?: number
-  rps10?: number
-  rpsTotal?: number
-}
-
-const inspectOpt: InspectOptions = {
-  colors: hasColors,
-  breakLength: 300,
-}
+export interface TransformLogProgressOptions<IN = any>
+  extends ProgressLoggerCfg<IN>,
+    TransformOptions {}
 
 /**
  * Pass-through transform that optionally logs progress.
@@ -157,119 +12,18 @@ const inspectOpt: InspectOptions = {
 export function transformLogProgress<IN = any>(
   opt: TransformLogProgressOptions = {},
 ): TransformTyped<IN, IN> {
-  const {
-    metric = 'progress',
-    heapTotal: logHeapTotal = false,
-    heapUsed: logHeapUsed = false,
-    rss: logRss = true,
-    peakRSS: logPeakRSS = true,
-    logRPS = true,
-    logEvery = 1000,
-    logSizes = false,
-    logSizesBuffer = 100_000,
-    logZippedSizes = false,
-    batchSize = 1,
-    extra,
-    logger = console,
-  } = opt
-  const logProgress = opt.logProgress !== false && logEvery !== 0 // true by default
-  const logEvery10 = logEvery * 10
-
-  const started = Date.now()
-  let lastSecondStarted = Date.now()
-  const sma = new SimpleMovingAverage(10) // over last 10 seconds
-  let processedLastSecond = 0
-  let progress = 0
-  let peakRSS = 0
-
-  const sizes = logSizes ? new SizeStack('json', logSizesBuffer) : undefined
-  const sizesZipped = logZippedSizes ? new SizeStack('json.gz', logSizesBuffer) : undefined
-
-  logStats() // initial
+  const progress = progressLogger(opt)
 
   return new Transform({
     objectMode: true,
     ...opt,
     transform(chunk: IN, _, cb) {
-      progress++
-      processedLastSecond++
-
-      if (sizes) {
-        // Check it, cause gzipping might be delayed here..
-        void SizeStack.countItem(chunk, logger, sizes, sizesZipped)
-      }
-
-      if (logProgress && progress % logEvery === 0) {
-        logStats(chunk, false, progress % logEvery10 === 0)
-      }
-
+      progress.log(chunk)
       cb(null, chunk) // pass-through
     },
     final(cb) {
-      logStats(undefined, true)
-
+      progress.done()
       cb()
     },
   })
-
-  function logStats(chunk?: IN, final = false, tenx = false): void {
-    if (!logProgress) return
-
-    const mem = process.memoryUsage()
-
-    const now = Date.now()
-    const batchedProgress = progress * batchSize
-    const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0
-    const rpsTotal = Math.round(batchedProgress / ((now - started) / 1000)) || 0
-    lastSecondStarted = now
-    processedLastSecond = 0
-
-    const rps10 = Math.round(sma.pushGetAvg(lastRPS))
-    if (mem.rss > peakRSS) peakRSS = mem.rss
-
-    const o: LogItem = {
-      [final ? `${metric}_final` : metric]: batchedProgress,
-    }
-
-    if (extra) Object.assign(o, extra(chunk, progress))
-    if (logHeapUsed) o.heapUsed = _mb(mem.heapUsed)
-    if (logHeapTotal) o.heapTotal = _mb(mem.heapTotal)
-    if (logRss) o.rss = _mb(mem.rss)
-    if (logPeakRSS) o.peakRSS = _mb(peakRSS)
-    if (opt.rssMinusHeap) o.rssMinusHeap = _mb(mem.rss - mem.heapTotal)
-    if (opt.external) o.external = _mb(mem.external)
-    if (opt.arrayBuffers) o.arrayBuffers = _mb(mem.arrayBuffers || 0)
-
-    if (logRPS) Object.assign(o, { rps10, rpsTotal })
-
-    logger.log(inspect(o, inspectOpt))
-
-    if (sizes?.items.length) {
-      logger.log(sizes.getStats())
-
-      if (sizesZipped?.items.length) {
-        logger.log(sizesZipped.getStats())
-      }
-    }
-
-    if (tenx) {
-      let perHour: number | string =
-        Math.round((batchedProgress * 1000 * 60 * 60) / (now - started)) || 0
-      if (perHour > 900) {
-        perHour = Math.round(perHour / 1000) + 'K'
-      }
-
-      logger.log(
-        `${dimGrey(localTimeNow().toPretty())} ${white(metric)} took ${yellow(
-          _since(started),
-        )} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`,
-      )
-    } else if (final) {
-      logger.log(
-        `${boldWhite(metric)} took ${yellow(_since(started))} to process ${yellow(
-          batchedProgress,
-        )} rows with total RPS of ${yellow(rpsTotal)}`,
-      )
-    }
-  }
 }