@naturalcycles/nodejs-lib 12.47.3 → 12.48.2

package/dist/stream/pipeline/pipeline.js

@@ -9,7 +9,7 @@ const util_1 = require("util");
 exports._pipeline = (0, util_1.promisify)(stream_1.pipeline);
 // Workaround https://github.com/nodejs/node/issues/40191
 // todo: remove it when fix is released in 16.x and in AppEngine 16.x
-if (process.version > 'v16.13') {
+if (process.version >= 'v16.10') {
     const { pipeline } = require('stream/promises');
     exports._pipeline = ((streams) => pipeline(...streams));
 }

package/dist/stream/transform/transformLogProgress.d.ts

@@ -75,6 +75,14 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
      * chunk is undefined for "final" stats, otherwise is defined.
      */
     extra?: (chunk: IN | undefined, index: number) => AnyObject;
+    /**
+     * If specified - will multiply the counter by this number.
+     * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
+     * it'll accurately represent the number of processed entries (not batches).
+     *
+     * Defaults to 1.
+     */
+    batchSize?: number;
 }
 /**
  * Pass-through transform that optionally logs progress.

package/dist/stream/transform/transformLogProgress.js

@@ -15,7 +15,7 @@ const inspectOpt = {
  * Pass-through transform that optionally logs progress.
  */
 function transformLogProgress(opt = {}) {
-    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, extra, } = opt;
+    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, batchSize = 1, extra, } = opt;
     const logProgress = opt.logProgress !== false && logEvery !== 0; // true by default
     const logEvery10 = logEvery * 10;
     const started = Date.now();
@@ -46,15 +46,16 @@ function transformLogProgress(opt = {}) {
             return;
         const mem = process.memoryUsage();
         const now = Date.now();
-        const lastRPS = processedLastSecond / ((now - lastSecondStarted) / 1000) || 0;
-        const rpsTotal = Math.round(progress / ((now - started) / 1000)) || 0;
+        const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0;
+        const rpsTotal = Math.round((progress * batchSize) / ((now - started) / 1000)) || 0;
         lastSecondStarted = now;
         processedLastSecond = 0;
         const rps10 = Math.round(sma.push(lastRPS));
         if (mem.rss > peakRSS)
             peakRSS = mem.rss;
+        const batchedProgress = progress * batchSize;
         console.log((0, util_1.inspect)({
-            [final ? `${metric}_final` : metric]: progress,
+            [final ? `${metric}_final` : metric]: batchedProgress,
             ...(extra ? extra(chunk, progress) : {}),
             ...(logHeapUsed ? { heapUsed: (0, js_lib_1._mb)(mem.heapUsed) } : {}),
             ...(logHeapTotal ? { heapTotal: (0, js_lib_1._mb)(mem.heapTotal) } : {}),
@@ -71,14 +72,14 @@ function transformLogProgress(opt = {}) {
                 : {}),
         }, inspectOpt));
         if (tenx) {
-            let perHour = Math.round((progress * 1000 * 60 * 60) / (now - started)) || 0;
+            let perHour = Math.round((batchedProgress * 1000 * 60 * 60) / (now - started)) || 0;
             if (perHour > 900) {
                 perHour = Math.round(perHour / 1000) + 'K';
             }
-            console.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(progress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
+            console.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
         }
         else if (final) {
-            console.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(progress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
+            console.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.47.3",
+  "version": "12.48.2",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",

package/src/stream/pipeline/pipeline.ts

@@ -10,7 +10,7 @@ export let _pipeline = promisify(pipeline)
 
 // Workaround https://github.com/nodejs/node/issues/40191
 // todo: remove it when fix is released in 16.x and in AppEngine 16.x
-if (process.version > 'v16.13') {
+if (process.version >= 'v16.10') {
   const { pipeline } = require('stream/promises')
   _pipeline = ((streams: AnyStream[]) => pipeline(...streams)) as any
 }

package/src/stream/transform/transformLogProgress.ts

@@ -92,6 +92,15 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    * chunk is undefined for "final" stats, otherwise is defined.
    */
   extra?: (chunk: IN | undefined, index: number) => AnyObject
+
+  /**
+   * If specified - will multiply the counter by this number.
+   * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
+   * it'll accurately represent the number of processed entries (not batches).
+   *
+   * Defaults to 1.
+   */
+  batchSize?: number
 }
 
 const inspectOpt: InspectOptions = {
@@ -113,6 +122,7 @@ export function transformLogProgress<IN = any>(
     peakRSS: logPeakRSS = true,
     logRPS = true,
     logEvery = 1000,
+    batchSize = 1,
     extra,
   } = opt
   const logProgress = opt.logProgress !== false && logEvery !== 0 // true by default
@@ -153,18 +163,20 @@ export function transformLogProgress<IN = any>(
     const mem = process.memoryUsage()
 
     const now = Date.now()
-    const lastRPS = processedLastSecond / ((now - lastSecondStarted) / 1000) || 0
-    const rpsTotal = Math.round(progress / ((now - started) / 1000)) || 0
+    const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0
+    const rpsTotal = Math.round((progress * batchSize) / ((now - started) / 1000)) || 0
     lastSecondStarted = now
     processedLastSecond = 0
 
     const rps10 = Math.round(sma.push(lastRPS))
     if (mem.rss > peakRSS) peakRSS = mem.rss
 
+    const batchedProgress = progress * batchSize
+
     console.log(
       inspect(
         {
-          [final ? `${metric}_final` : metric]: progress,
+          [final ? `${metric}_final` : metric]: batchedProgress,
           ...(extra ? extra(chunk, progress) : {}),
           ...(logHeapUsed ? { heapUsed: _mb(mem.heapUsed) } : {}),
           ...(logHeapTotal ? { heapTotal: _mb(mem.heapTotal) } : {}),
@@ -185,7 +197,8 @@ export function transformLogProgress<IN = any>(
     )
 
     if (tenx) {
-      let perHour: number | string = Math.round((progress * 1000 * 60 * 60) / (now - started)) || 0
+      let perHour: number | string =
+        Math.round((batchedProgress * 1000 * 60 * 60) / (now - started)) || 0
       if (perHour > 900) {
         perHour = Math.round(perHour / 1000) + 'K'
       }
@@ -193,12 +206,12 @@ export function transformLogProgress<IN = any>(
       console.log(
         `${dimGrey(dayjs().toPretty())} ${white(metric)} took ${yellow(
           _since(started),
-        )} so far to process ${yellow(progress)} rows, ~${yellow(perHour)}/hour`,
+        )} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`,
       )
     } else if (final) {
       console.log(
         `${boldWhite(metric)} took ${yellow(_since(started))} to process ${yellow(
-          progress,
+          batchedProgress,
         )} rows with total RPS of ${yellow(rpsTotal)}`,
       )
     }