@naturalcycles/nodejs-lib 13.15.0 → 13.16.0

package/dist/stream/progressLogger.d.ts

@@ -76,6 +76,11 @@ export interface ProgressLoggerCfg<T = any> {
      * chunk is undefined for "final" stats, otherwise is defined.
      */
     extra?: (chunk: T | undefined, index: number) => AnyObject;
+    /**
+     * Hook that is called when the last item is passed through.
+     * Passes the final stats as `ProgressLogItem`.
+     */
+    onProgressDone?: (stats: ProgressLogItem) => any;
     /**
      * If specified - will multiply the counter by this number.
      * Useful e.g when using `transformChunk({ chunkSize: 500 })`, so

package/dist/stream/progressLogger.js

@@ -107,6 +107,7 @@ class ProgressLogger {
         }
         else if (final) {
             logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
+            this.cfg.onProgressDone?.(o);
         }
     }
 }

package/dist/stream/transform/transformMap.d.ts

@@ -30,6 +30,17 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * Called BEFORE observable will emit error (unless skipErrors is set to true).
      */
     onError?: (err: Error, input: IN) => any;
+    /**
+     * A hook that is called when the last item is finished processing.
+     * stats object is passed, containing countIn and countOut -
+     * number of items that entered the transform and number of items that left it.
+     *
+     * Callback is called **before** [possible] Aggregated error is thrown,
+     * and before [possible] THROW_IMMEDIATELY error.
+     *
+     * onDone callback will be called before Error is thrown.
+     */
+    onDone?: (stats: TransformMapStats) => any;
     /**
      * Progress metric
      *
@@ -38,6 +49,19 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
     metric?: string;
     logger?: CommonLogger;
 }
+export interface TransformMapStats {
+    /**
+     * True if transform was successful (didn't throw Immediate or Aggregated error).
+     */
+    ok: boolean;
+    /**
+     * Only used (and returned) for ErrorMode.Aggregated
+     */
+    collectedErrors: Error[];
+    countErrors: number;
+    countIn: number;
+    countOut: number;
+}
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.

package/dist/stream/transform/transformMap.js

@@ -21,8 +21,9 @@ const stream_util_1 = require("../stream.util");
  */
 function transformMap(mapper, opt = {}) {
     const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
-    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, metric = 'stream', logger = console, } = opt;
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, onDone, metric = 'stream', logger = console, } = opt;
     let index = -1;
+    let countOut = 0;
     let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
@@ -32,11 +33,25 @@ function transformMap(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
+                onDone?.({
+                    ok: false,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                });
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMap resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
+                onDone?.({
+                    ok: true,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                });
                 cb();
             }
         },
@@ -54,6 +69,7 @@ function transformMap(mapper, opt = {}) {
                 }
                 return r !== js_lib_1.SKIP && (!predicate || (await predicate(r, currentIndex)));
             });
+            countOut += passedResults.length;
             passedResults.forEach(r => this.push(r));
             if (isSettled) {
                 logger.log(`transformMap END received at index ${currentIndex}`);
@@ -73,6 +89,13 @@ function transformMap(mapper, opt = {}) {
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                 isSettled = true;
+                onDone?.({
+                    ok: false,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                });
                 return cb(err); // Emit error immediately
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,6 +1,7 @@
 import { CommonLogger, END, ErrorMode, Mapper, Predicate, SKIP } from '@naturalcycles/js-lib';
 import { AbortableTransform } from '../pipeline/pipeline';
 import { TransformTyped } from '../stream.model';
+import { TransformMapStats } from './transformMap';
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
     /**
      * @default true
@@ -28,6 +29,17 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * Called BEFORE observable will emit error (unless skipErrors is set to true).
      */
     onError?: (err: Error, input: IN) => any;
+    /**
+     * A hook that is called when the last item is finished processing.
+     * stats object is passed, containing countIn and countOut -
+     * number of items that entered the transform and number of items that left it.
+     *
+     * Callback is called **before** [possible] Aggregated error is thrown,
+     * and before [possible] THROW_IMMEDIATELY error.
+     *
+     * onDone callback will be called before Error is thrown.
+     */
+    onDone?: (stats: TransformMapStats) => any;
     /**
      * Progress metric
      *

package/dist/stream/transform/transformMapSync.js

@@ -13,9 +13,10 @@ exports.TransformMapSync = TransformMapSync;
  * Supposedly faster, for cases when async is not needed.
  */
 function transformMapSync(mapper, opt = {}) {
-    let index = -1;
     const { predicate, // defaults to "no predicate" (pass everything)
-    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, logger = console, } = opt;
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, onDone, metric = 'stream', objectMode = true, logger = console, } = opt;
+    let index = -1;
+    let countOut = 0;
     let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
@@ -37,6 +38,7 @@ function transformMapSync(mapper, opt = {}) {
                     }
                     return r !== js_lib_1.SKIP && (!predicate || predicate(r, currentIndex));
                 });
+                countOut += passedResults.length;
                 passedResults.forEach(r => this.push(r));
                 if (isSettled) {
                     logger.log(`transformMapSync END received at index ${currentIndex}`);
@@ -56,6 +58,13 @@ function transformMapSync(mapper, opt = {}) {
                 }
                 if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                     isSettled = true;
+                    onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                    });
                     // Emit error immediately
                     return cb(err);
                 }
@@ -69,11 +78,25 @@ function transformMapSync(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
+                onDone?.({
+                    ok: false,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                });
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMapSync resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
+                onDone?.({
+                    ok: true,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                });
                 cb();
             }
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.15.0",
+  "version": "13.16.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/stream/progressLogger.ts

@@ -101,6 +101,12 @@ export interface ProgressLoggerCfg<T = any> {
    */
   extra?: (chunk: T | undefined, index: number) => AnyObject
 
+  /**
+   * Hook that is called when the last item is passed through.
+   * Passes the final stats as `ProgressLogItem`.
+   */
+  onProgressDone?: (stats: ProgressLogItem) => any
+
   /**
    * If specified - will multiply the counter by this number.
    * Useful e.g when using `transformChunk({ chunkSize: 500 })`, so
@@ -298,6 +304,8 @@ export class ProgressLogger<T> implements Disposable {
           batchedProgress,
         )} rows with total RPS of ${yellow(rpsTotal)}`,
       )
+
+      this.cfg.onProgressDone?.(o)
     }
   }
 }

package/src/stream/transform/transformMap.ts

@@ -49,6 +49,18 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    */
   onError?: (err: Error, input: IN) => any
 
+  /**
+   * A hook that is called when the last item is finished processing.
+   * stats object is passed, containing countIn and countOut -
+   * number of items that entered the transform and number of items that left it.
+   *
+   * Callback is called **before** [possible] Aggregated error is thrown,
+   * and before [possible] THROW_IMMEDIATELY error.
+   *
+   * onDone callback will be called before Error is thrown.
+   */
+  onDone?: (stats: TransformMapStats) => any
+
   /**
    * Progress metric
    *
@@ -59,6 +71,20 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   logger?: CommonLogger
 }
 
+export interface TransformMapStats {
+  /**
+   * True if transform was successful (didn't throw Immediate or Aggregated error).
+   */
+  ok: boolean
+  /**
+   * Only used (and returned) for ErrorMode.Aggregated
+   */
+  collectedErrors: Error[]
+  countErrors: number
+  countIn: number
+  countOut: number
+}
+
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
 
@@ -84,11 +110,13 @@ export function transformMap<IN = any, OUT = IN>(
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,
     onError,
+    onDone,
     metric = 'stream',
     logger = console,
   } = opt
 
   let index = -1
+  let countOut = 0
   let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
@@ -102,6 +130,14 @@ export function transformMap<IN = any, OUT = IN>(
         logErrorStats(true)
 
         if (collectedErrors.length) {
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+          })
+
           // emit Aggregated error
           cb(
             new AggregateError(
@@ -111,6 +147,15 @@ export function transformMap<IN = any, OUT = IN>(
           )
         } else {
           // emit no error
+
+          onDone?.({
+            ok: true,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+          })
+
           cb()
         }
       },
@@ -134,6 +179,7 @@ export function transformMap<IN = any, OUT = IN>(
           },
         )
 
+        countOut += passedResults.length
         passedResults.forEach(r => this.push(r))
 
         if (isSettled) {
@@ -155,6 +201,13 @@ export function transformMap<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+          })
           return cb(err) // Emit error immediately
         }
 

package/src/stream/transform/transformMapSync.ts

@@ -11,6 +11,7 @@ import { yellow } from '../../colors/colors'
 import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
 import { pipelineClose } from '../stream.util'
+import { TransformMapStats } from './transformMap'
 
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   /**
@@ -44,6 +45,18 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
    */
   onError?: (err: Error, input: IN) => any
 
+  /**
+   * A hook that is called when the last item is finished processing.
+   * stats object is passed, containing countIn and countOut -
+   * number of items that entered the transform and number of items that left it.
+   *
+   * Callback is called **before** [possible] Aggregated error is thrown,
+   * and before [possible] THROW_IMMEDIATELY error.
+   *
+   * onDone callback will be called before Error is thrown.
+   */
+  onDone?: (stats: TransformMapStats) => any
+
   /**
    * Progress metric
    *
@@ -64,17 +77,19 @@ export function transformMapSync<IN = any, OUT = IN>(
   mapper: Mapper<IN, OUT | typeof SKIP | typeof END>,
   opt: TransformMapSyncOptions = {},
 ): TransformTyped<IN, OUT> {
-  let index = -1
-
   const {
     predicate, // defaults to "no predicate" (pass everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput = false,
     onError,
+    onDone,
     metric = 'stream',
     objectMode = true,
     logger = console,
   } = opt
+
+  let index = -1
+  let countOut = 0
   let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
@@ -100,6 +115,7 @@ export function transformMapSync<IN = any, OUT = IN>(
           return r !== SKIP && (!predicate || predicate(r, currentIndex))
         })
 
+        countOut += passedResults.length
         passedResults.forEach(r => this.push(r))
 
         if (isSettled) {
@@ -122,6 +138,13 @@ export function transformMapSync<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+          })
           // Emit error immediately
           return cb(err as Error)
         }
@@ -139,6 +162,14 @@ export function transformMapSync<IN = any, OUT = IN>(
       logErrorStats(true)
 
       if (collectedErrors.length) {
+        onDone?.({
+          ok: false,
+          collectedErrors,
+          countErrors: errors,
+          countIn: index + 1,
+          countOut,
+        })
+
         // emit Aggregated error
         cb(
           new AggregateError(
@@ -148,6 +179,15 @@ export function transformMapSync<IN = any, OUT = IN>(
         )
       } else {
         // emit no error
+
+        onDone?.({
+          ok: true,
+          collectedErrors,
+          countErrors: errors,
+          countIn: index + 1,
+          countOut,
+        })
+
         cb()
       }
     },