@naturalcycles/nodejs-lib 13.15.0 → 13.17.0

package/dist/fs/json2env.d.ts

@@ -30,7 +30,7 @@ export declare function appendToGithubOutput(obj: AnyObject, prefix?: string): v
 /**
  * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
  */
-export declare function appendToGithubSummary(str: string): void;
+export declare function appendToGithubSummary(...lines: string[]): void;
 /**
  * Turns Object with keys/values into a *.sh script that exports all keys as values.
  *

package/dist/fs/json2env.js

@@ -81,10 +81,11 @@ exports.appendToGithubOutput = appendToGithubOutput;
 /**
  * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
  */
-function appendToGithubSummary(str) {
+function appendToGithubSummary(...lines) {
     const { GITHUB_STEP_SUMMARY } = process.env;
     if (GITHUB_STEP_SUMMARY) {
-        node_fs_1.default.appendFileSync(GITHUB_STEP_SUMMARY, str + '\n');
+        const str = lines.join('\n') + '\n';
+        node_fs_1.default.appendFileSync(GITHUB_STEP_SUMMARY, str);
         console.log(`GITHUB_STEP_SUMMARY appended:\n${str}`);
     }
 }

package/dist/stream/progressLogger.d.ts

@@ -76,6 +76,11 @@ export interface ProgressLoggerCfg<T = any> {
      * chunk is undefined for "final" stats, otherwise is defined.
      */
     extra?: (chunk: T | undefined, index: number) => AnyObject;
+    /**
+     * Hook that is called when the last item is passed through.
+     * Passes the final stats as `ProgressLogItem`.
+     */
+    onProgressDone?: (stats: ProgressLogItem) => any;
     /**
      * If specified - will multiply the counter by this number.
      * Useful e.g when using `transformChunk({ chunkSize: 500 })`, so

package/dist/stream/progressLogger.js

@@ -107,6 +107,7 @@ class ProgressLogger {
         }
         else if (final) {
             logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
+            this.cfg.onProgressDone?.(o);
         }
     }
 }

package/dist/stream/transform/transformMap.d.ts

@@ -1,4 +1,4 @@
-import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, END, ErrorMode, SKIP } from '@naturalcycles/js-lib';
+import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, END, ErrorMode, SKIP, StringMap, UnixTimestampMillisNumber } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -30,6 +30,17 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * Called BEFORE observable will emit error (unless skipErrors is set to true).
      */
     onError?: (err: Error, input: IN) => any;
+    /**
+     * A hook that is called when the last item is finished processing.
+     * stats object is passed, containing countIn and countOut -
+     * number of items that entered the transform and number of items that left it.
+     *
+     * Callback is called **before** [possible] Aggregated error is thrown,
+     * and before [possible] THROW_IMMEDIATELY error.
+     *
+     * onDone callback will be called before Error is thrown.
+     */
+    onDone?: (stats: TransformMapStats) => any;
     /**
      * Progress metric
      *
@@ -38,6 +49,20 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
     metric?: string;
     logger?: CommonLogger;
 }
+export interface TransformMapStats {
+    /**
+     * True if transform was successful (didn't throw Immediate or Aggregated error).
+     */
+    ok: boolean;
+    /**
+     * Only used (and returned) for ErrorMode.Aggregated
+     */
+    collectedErrors: Error[];
+    countErrors: number;
+    countIn: number;
+    countOut: number;
+    started: UnixTimestampMillisNumber;
+}
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -51,3 +76,7 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export declare function transformMap<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
+export declare function appendTransformMapStatsToGithubSummary(stats: TransformMapStats & {
+    name?: string;
+    extra?: StringMap<any>;
+}): void;

package/dist/stream/transform/transformMap.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMap = void 0;
+exports.appendTransformMapStatsToGithubSummary = exports.transformMap = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors/colors");
+const json2env_1 = require("../../fs/json2env");
 const stream_util_1 = require("../stream.util");
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
@@ -21,8 +22,10 @@ const stream_util_1 = require("../stream.util");
  */
 function transformMap(mapper, opt = {}) {
     const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
-    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, metric = 'stream', logger = console, } = opt;
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, onDone, metric = 'stream', logger = console, } = opt;
+    const started = Date.now();
     let index = -1;
+    let countOut = 0;
     let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
@@ -32,11 +35,27 @@ function transformMap(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
+                onDone?.({
+                    ok: false,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                    started,
+                });
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMap resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
+                onDone?.({
+                    ok: true,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                    started,
+                });
                 cb();
             }
         },
@@ -54,6 +73,7 @@ function transformMap(mapper, opt = {}) {
                 }
                 return r !== js_lib_1.SKIP && (!predicate || (await predicate(r, currentIndex)));
             });
+            countOut += passedResults.length;
             passedResults.forEach(r => this.push(r));
             if (isSettled) {
                 logger.log(`transformMap END received at index ${currentIndex}`);
@@ -73,6 +93,14 @@ function transformMap(mapper, opt = {}) {
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                 isSettled = true;
+                onDone?.({
+                    ok: false,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                    started,
+                });
                 return cb(err); // Emit error immediately
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
@@ -89,3 +117,14 @@ function transformMap(mapper, opt = {}) {
     }
 }
 exports.transformMap = transformMap;
+function appendTransformMapStatsToGithubSummary(stats) {
+    const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = stats;
+    (0, json2env_1.appendToGithubSummary)(...[
+        `### ${name} summary\n`,
+        `${(0, js_lib_1._since)(started)} spent`,
+        `${(0, js_lib_1._hc)(countIn)} / ${(0, js_lib_1._hc)(countOut)} rows in / out`,
+        countErrors ? `${countErrors} errors` : '',
+        ...Object.entries(extra).map(([k, v]) => `${k}: ${v}`),
+    ].filter(Boolean));
+}
+exports.appendTransformMapStatsToGithubSummary = appendTransformMapStatsToGithubSummary;

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,6 +1,7 @@
 import { CommonLogger, END, ErrorMode, Mapper, Predicate, SKIP } from '@naturalcycles/js-lib';
 import { AbortableTransform } from '../pipeline/pipeline';
 import { TransformTyped } from '../stream.model';
+import { TransformMapStats } from './transformMap';
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
     /**
      * @default true
@@ -28,6 +29,17 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * Called BEFORE observable will emit error (unless skipErrors is set to true).
      */
     onError?: (err: Error, input: IN) => any;
+    /**
+     * A hook that is called when the last item is finished processing.
+     * stats object is passed, containing countIn and countOut -
+     * number of items that entered the transform and number of items that left it.
+     *
+     * Callback is called **before** [possible] Aggregated error is thrown,
+     * and before [possible] THROW_IMMEDIATELY error.
+     *
+     * onDone callback will be called before Error is thrown.
+     */
+    onDone?: (stats: TransformMapStats) => any;
     /**
      * Progress metric
      *

package/dist/stream/transform/transformMapSync.js

@@ -13,9 +13,11 @@ exports.TransformMapSync = TransformMapSync;
  * Supposedly faster, for cases when async is not needed.
  */
 function transformMapSync(mapper, opt = {}) {
-    let index = -1;
     const { predicate, // defaults to "no predicate" (pass everything)
-    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, logger = console, } = opt;
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, onDone, metric = 'stream', objectMode = true, logger = console, } = opt;
+    const started = Date.now();
+    let index = -1;
+    let countOut = 0;
     let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
@@ -37,6 +39,7 @@ function transformMapSync(mapper, opt = {}) {
                     }
                     return r !== js_lib_1.SKIP && (!predicate || predicate(r, currentIndex));
                 });
+                countOut += passedResults.length;
                 passedResults.forEach(r => this.push(r));
                 if (isSettled) {
                     logger.log(`transformMapSync END received at index ${currentIndex}`);
@@ -56,6 +59,14 @@ function transformMapSync(mapper, opt = {}) {
                 }
                 if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                     isSettled = true;
+                    onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
                     // Emit error immediately
                     return cb(err);
                 }
@@ -69,11 +80,27 @@ function transformMapSync(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
+                onDone?.({
+                    ok: false,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                    started,
+                });
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMapSync resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
+                onDone?.({
+                    ok: true,
+                    collectedErrors,
+                    countErrors: errors,
+                    countIn: index + 1,
+                    countOut,
+                    started,
+                });
                 cb();
             }
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.15.0",
+  "version": "13.17.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/fs/json2env.ts

@@ -120,10 +120,11 @@ export function appendToGithubOutput(obj: AnyObject, prefix = ''): void {
 /**
  * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
  */
-export function appendToGithubSummary(str: string): void {
+export function appendToGithubSummary(...lines: string[]): void {
   const { GITHUB_STEP_SUMMARY } = process.env
   if (GITHUB_STEP_SUMMARY) {
-    fs.appendFileSync(GITHUB_STEP_SUMMARY, str + '\n')
+    const str = lines.join('\n') + '\n'
+    fs.appendFileSync(GITHUB_STEP_SUMMARY, str)
     console.log(`GITHUB_STEP_SUMMARY appended:\n${str}`)
   }
 }

package/src/stream/progressLogger.ts

@@ -101,6 +101,12 @@ export interface ProgressLoggerCfg<T = any> {
    */
   extra?: (chunk: T | undefined, index: number) => AnyObject
 
+  /**
+   * Hook that is called when the last item is passed through.
+   * Passes the final stats as `ProgressLogItem`.
+   */
+  onProgressDone?: (stats: ProgressLogItem) => any
+
   /**
    * If specified - will multiply the counter by this number.
    * Useful e.g when using `transformChunk({ chunkSize: 500 })`, so
@@ -298,6 +304,8 @@ export class ProgressLogger<T> implements Disposable {
           batchedProgress,
         )} rows with total RPS of ${yellow(rpsTotal)}`,
       )
+
+      this.cfg.onProgressDone?.(o)
     }
   }
 }

package/src/stream/transform/transformMap.ts

@@ -1,5 +1,7 @@
 import {
   _anyToError,
+  _hc,
+  _since,
   AbortableAsyncMapper,
   AsyncPredicate,
   CommonLogger,
@@ -7,9 +9,12 @@ import {
   ErrorMode,
   pFilter,
   SKIP,
+  StringMap,
+  UnixTimestampMillisNumber,
 } from '@naturalcycles/js-lib'
 import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors/colors'
+import { appendToGithubSummary } from '../../fs/json2env'
 import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
 import { pipelineClose } from '../stream.util'
@@ -49,6 +54,18 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    */
   onError?: (err: Error, input: IN) => any
 
+  /**
+   * A hook that is called when the last item is finished processing.
+   * stats object is passed, containing countIn and countOut -
+   * number of items that entered the transform and number of items that left it.
+   *
+   * Callback is called **before** [possible] Aggregated error is thrown,
+   * and before [possible] THROW_IMMEDIATELY error.
+   *
+   * onDone callback will be called before Error is thrown.
+   */
+  onDone?: (stats: TransformMapStats) => any
+
   /**
    * Progress metric
    *
@@ -59,6 +76,21 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   logger?: CommonLogger
 }
 
+export interface TransformMapStats {
+  /**
+   * True if transform was successful (didn't throw Immediate or Aggregated error).
+   */
+  ok: boolean
+  /**
+   * Only used (and returned) for ErrorMode.Aggregated
+   */
+  collectedErrors: Error[]
+  countErrors: number
+  countIn: number
+  countOut: number
+  started: UnixTimestampMillisNumber
+}
+
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
 
@@ -84,11 +116,14 @@ export function transformMap<IN = any, OUT = IN>(
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,
     onError,
+    onDone,
     metric = 'stream',
     logger = console,
   } = opt
 
+  const started = Date.now()
   let index = -1
+  let countOut = 0
   let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
@@ -102,6 +137,15 @@ export function transformMap<IN = any, OUT = IN>(
         logErrorStats(true)
 
         if (collectedErrors.length) {
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
+
           // emit Aggregated error
           cb(
             new AggregateError(
@@ -111,6 +155,16 @@ export function transformMap<IN = any, OUT = IN>(
           )
         } else {
           // emit no error
+
+          onDone?.({
+            ok: true,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
+
           cb()
         }
       },
@@ -134,6 +188,7 @@ export function transformMap<IN = any, OUT = IN>(
           },
         )
 
+        countOut += passedResults.length
         passedResults.forEach(r => this.push(r))
 
         if (isSettled) {
@@ -155,6 +210,14 @@ export function transformMap<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
           return cb(err) // Emit error immediately
         }
 
@@ -173,3 +236,19 @@ export function transformMap<IN = any, OUT = IN>(
     logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
 }
+
+export function appendTransformMapStatsToGithubSummary(
+  stats: TransformMapStats & { name?: string; extra?: StringMap<any> },
+): void {
+  const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = stats
+
+  appendToGithubSummary(
+    ...[
+      `### ${name} summary\n`,
+      `${_since(started)} spent`,
+      `${_hc(countIn)} / ${_hc(countOut)} rows in / out`,
+      countErrors ? `${countErrors} errors` : '',
+      ...Object.entries(extra).map(([k, v]) => `${k}: ${v}`),
+    ].filter(Boolean),
+  )
+}

package/src/stream/transform/transformMapSync.ts

@@ -11,6 +11,7 @@ import { yellow } from '../../colors/colors'
 import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
 import { pipelineClose } from '../stream.util'
+import { TransformMapStats } from './transformMap'
 
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   /**
@@ -44,6 +45,18 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
    */
   onError?: (err: Error, input: IN) => any
 
+  /**
+   * A hook that is called when the last item is finished processing.
+   * stats object is passed, containing countIn and countOut -
+   * number of items that entered the transform and number of items that left it.
+   *
+   * Callback is called **before** [possible] Aggregated error is thrown,
+   * and before [possible] THROW_IMMEDIATELY error.
+   *
+   * onDone callback will be called before Error is thrown.
+   */
+  onDone?: (stats: TransformMapStats) => any
+
   /**
    * Progress metric
    *
@@ -64,17 +77,20 @@ export function transformMapSync<IN = any, OUT = IN>(
   mapper: Mapper<IN, OUT | typeof SKIP | typeof END>,
   opt: TransformMapSyncOptions = {},
 ): TransformTyped<IN, OUT> {
-  let index = -1
-
   const {
     predicate, // defaults to "no predicate" (pass everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput = false,
     onError,
+    onDone,
     metric = 'stream',
     objectMode = true,
     logger = console,
   } = opt
+
+  const started = Date.now()
+  let index = -1
+  let countOut = 0
   let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
@@ -100,6 +116,7 @@ export function transformMapSync<IN = any, OUT = IN>(
           return r !== SKIP && (!predicate || predicate(r, currentIndex))
         })
 
+        countOut += passedResults.length
         passedResults.forEach(r => this.push(r))
 
         if (isSettled) {
@@ -122,6 +139,14 @@ export function transformMapSync<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
           // Emit error immediately
           return cb(err as Error)
         }
@@ -139,6 +164,15 @@ export function transformMapSync<IN = any, OUT = IN>(
       logErrorStats(true)
 
       if (collectedErrors.length) {
+        onDone?.({
+          ok: false,
+          collectedErrors,
+          countErrors: errors,
+          countIn: index + 1,
+          countOut,
+          started,
+        })
+
         // emit Aggregated error
         cb(
           new AggregateError(
@@ -148,6 +182,16 @@ export function transformMapSync<IN = any, OUT = IN>(
         )
       } else {
         // emit no error
+
+        onDone?.({
+          ok: true,
+          collectedErrors,
+          countErrors: errors,
+          countIn: index + 1,
+          countOut,
+          started,
+        })
+
         cb()
       }
     },