@naturalcycles/nodejs-lib 13.16.0 → 13.18.0

package/dist/fs/json2env.d.ts

@@ -30,7 +30,7 @@ export declare function appendToGithubOutput(obj: AnyObject, prefix?: string): v
 /**
  * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
  */
-export declare function appendToGithubSummary(str: string): void;
+export declare function appendToGithubSummary(...lines: string[]): void;
 /**
  * Turns Object with keys/values into a *.sh script that exports all keys as values.
  *

package/dist/fs/json2env.js

@@ -81,10 +81,11 @@ exports.appendToGithubOutput = appendToGithubOutput;
 /**
  * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
  */
-function appendToGithubSummary(str) {
+function appendToGithubSummary(...lines) {
     const { GITHUB_STEP_SUMMARY } = process.env;
     if (GITHUB_STEP_SUMMARY) {
-        node_fs_1.default.appendFileSync(GITHUB_STEP_SUMMARY, str + '\n');
+        const str = lines.join('\n') + '\n';
+        node_fs_1.default.appendFileSync(GITHUB_STEP_SUMMARY, str);
         console.log(`GITHUB_STEP_SUMMARY appended:\n${str}`);
     }
 }

package/dist/stream/progressLogger.js

@@ -107,7 +107,12 @@ class ProgressLogger {
         }
         else if (final) {
             logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
-            this.cfg.onProgressDone?.(o);
+            try {
+                this.cfg.onProgressDone?.(o);
+            }
+            catch (err) {
+                logger.error(err);
+            }
         }
     }
 }

package/dist/stream/transform/transformMap.d.ts

@@ -1,4 +1,4 @@
-import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, END, ErrorMode, SKIP } from '@naturalcycles/js-lib';
+import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, END, ErrorMode, Promisable, SKIP, StringMap, UnixTimestampMillisNumber } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -38,9 +38,9 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * Callback is called **before** [possible] Aggregated error is thrown,
      * and before [possible] THROW_IMMEDIATELY error.
      *
-     * onDone callback will be called before Error is thrown.
+     * onDone callback will be awaited before Error is thrown.
      */
-    onDone?: (stats: TransformMapStats) => any;
+    onDone?: (stats: TransformMapStats) => Promisable<any>;
     /**
      * Progress metric
      *
@@ -61,6 +61,19 @@ export interface TransformMapStats {
     countErrors: number;
     countIn: number;
     countOut: number;
+    started: UnixTimestampMillisNumber;
+}
+export interface TransformMapStatsSummary extends TransformMapStats {
+    /**
+     * Name of the summary, defaults to `Transform`
+     */
+    name?: string;
+    /**
+     * Allows to pass extra key-value object, which will be rendered as:
+     * key: value
+     * key2: value2
+     */
+    extra?: StringMap<any>;
 }
 /**
  * Like pMap, but for streams.
@@ -75,3 +88,8 @@ export interface TransformMapStats {
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export declare function transformMap<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
+/**
+ * Renders TransformMapStatsSummary into a friendly string,
+ * to be used e.g in Github Actions summary or Slack.
+ */
+export declare function transformMapStatsSummary(summary: TransformMapStatsSummary): string;

package/dist/stream/transform/transformMap.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMap = void 0;
+exports.transformMapStatsSummary = exports.transformMap = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors/colors");
@@ -22,6 +22,7 @@ const stream_util_1 = require("../stream.util");
 function transformMap(mapper, opt = {}) {
     const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, onDone, metric = 'stream', logger = console, } = opt;
+    const started = Date.now();
     let index = -1;
     let countOut = 0;
     let isSettled = false;
@@ -33,25 +34,37 @@ function transformMap(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
-                onDone?.({
-                    ok: false,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                });
+                try {
+                    await onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMap resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
-                onDone?.({
-                    ok: true,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                });
+                try {
+                    await onDone?.({
+                        ok: true,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 cb();
             }
         },
@@ -89,13 +102,19 @@ function transformMap(mapper, opt = {}) {
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                 isSettled = true;
-                onDone?.({
-                    ok: false,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                });
+                try {
+                    await onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 return cb(err); // Emit error immediately
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
@@ -112,3 +131,20 @@ function transformMap(mapper, opt = {}) {
     }
 }
 exports.transformMap = transformMap;
+/**
+ * Renders TransformMapStatsSummary into a friendly string,
+ * to be used e.g in Github Actions summary or Slack.
+ */
+function transformMapStatsSummary(summary) {
+    const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = summary;
+    return [
+        `### ${name} summary\n`,
+        `${(0, js_lib_1._since)(started)} spent`,
+        `${(0, js_lib_1._hc)(countIn)} / ${(0, js_lib_1._hc)(countOut)} row(s) in / out`,
+        countErrors ? `${countErrors} error(s)` : '',
+        ...Object.entries(extra).map(([k, v]) => `${k}: ${(0, js_lib_1._stringify)(v)}`),
+    ]
+        .filter(Boolean)
+        .join('\n');
+}
+exports.transformMapStatsSummary = transformMapStatsSummary;

package/dist/stream/transform/transformMapSync.js

@@ -15,6 +15,7 @@ exports.TransformMapSync = TransformMapSync;
 function transformMapSync(mapper, opt = {}) {
     const { predicate, // defaults to "no predicate" (pass everything)
     errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, onDone, metric = 'stream', objectMode = true, logger = console, } = opt;
+    const started = Date.now();
     let index = -1;
     let countOut = 0;
     let isSettled = false;
@@ -58,13 +59,19 @@ function transformMapSync(mapper, opt = {}) {
                 }
                 if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                     isSettled = true;
-                    onDone?.({
-                        ok: false,
-                        collectedErrors,
-                        countErrors: errors,
-                        countIn: index + 1,
-                        countOut,
-                    });
+                    try {
+                        onDone?.({
+                            ok: false,
+                            collectedErrors,
+                            countErrors: errors,
+                            countIn: index + 1,
+                            countOut,
+                            started,
+                        });
+                    }
+                    catch (err) {
+                        logger.error(err);
+                    }
                     // Emit error immediately
                     return cb(err);
                 }
@@ -78,25 +85,37 @@ function transformMapSync(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
-                onDone?.({
-                    ok: false,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                });
+                try {
+                    onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMapSync resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
-                onDone?.({
-                    ok: true,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                });
+                try {
+                    onDone?.({
+                        ok: true,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 cb();
             }
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.16.0",
+  "version": "13.18.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/fs/json2env.ts

@@ -120,10 +120,11 @@ export function appendToGithubOutput(obj: AnyObject, prefix = ''): void {
 /**
  * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
  */
-export function appendToGithubSummary(str: string): void {
+export function appendToGithubSummary(...lines: string[]): void {
   const { GITHUB_STEP_SUMMARY } = process.env
   if (GITHUB_STEP_SUMMARY) {
-    fs.appendFileSync(GITHUB_STEP_SUMMARY, str + '\n')
+    const str = lines.join('\n') + '\n'
+    fs.appendFileSync(GITHUB_STEP_SUMMARY, str)
     console.log(`GITHUB_STEP_SUMMARY appended:\n${str}`)
   }
 }

package/src/stream/progressLogger.ts

@@ -305,7 +305,11 @@ export class ProgressLogger<T> implements Disposable {
         )} rows with total RPS of ${yellow(rpsTotal)}`,
       )
 
-      this.cfg.onProgressDone?.(o)
+      try {
+        this.cfg.onProgressDone?.(o)
+      } catch (err) {
+        logger.error(err)
+      }
     }
   }
 }

package/src/stream/transform/transformMap.ts

@@ -1,12 +1,18 @@
 import {
   _anyToError,
+  _hc,
+  _since,
+  _stringify,
   AbortableAsyncMapper,
   AsyncPredicate,
   CommonLogger,
   END,
   ErrorMode,
   pFilter,
+  Promisable,
   SKIP,
+  StringMap,
+  UnixTimestampMillisNumber,
 } from '@naturalcycles/js-lib'
 import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors/colors'
@@ -57,9 +63,9 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * Callback is called **before** [possible] Aggregated error is thrown,
    * and before [possible] THROW_IMMEDIATELY error.
    *
-   * onDone callback will be called before Error is thrown.
+   * onDone callback will be awaited before Error is thrown.
    */
-  onDone?: (stats: TransformMapStats) => any
+  onDone?: (stats: TransformMapStats) => Promisable<any>
 
   /**
    * Progress metric
@@ -83,6 +89,21 @@ export interface TransformMapStats {
   countErrors: number
   countIn: number
   countOut: number
+  started: UnixTimestampMillisNumber
+}
+
+export interface TransformMapStatsSummary extends TransformMapStats {
+  /**
+   * Name of the summary, defaults to `Transform`
+   */
+  name?: string
+
+  /**
+   * Allows to pass extra key-value object, which will be rendered as:
+   * key: value
+   * key2: value2
+   */
+  extra?: StringMap<any>
 }
 
 // doesn't work, cause here we don't construct our Transform instance ourselves
@@ -115,6 +136,7 @@ export function transformMap<IN = any, OUT = IN>(
     logger = console,
   } = opt
 
+  const started = Date.now()
   let index = -1
   let countOut = 0
   let isSettled = false
@@ -130,13 +152,18 @@ export function transformMap<IN = any, OUT = IN>(
         logErrorStats(true)
 
         if (collectedErrors.length) {
-          onDone?.({
-            ok: false,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-          })
+          try {
+            await onDone?.({
+              ok: false,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
 
           // emit Aggregated error
           cb(
@@ -148,13 +175,18 @@ export function transformMap<IN = any, OUT = IN>(
         } else {
           // emit no error
 
-          onDone?.({
-            ok: true,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-          })
+          try {
+            await onDone?.({
+              ok: true,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
 
           cb()
         }
@@ -201,13 +233,20 @@ export function transformMap<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
-          onDone?.({
-            ok: false,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-          })
+
+          try {
+            await onDone?.({
+              ok: false,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
+
           return cb(err) // Emit error immediately
         }
 
@@ -226,3 +265,21 @@ export function transformMap<IN = any, OUT = IN>(
     logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
 }
+
+/**
+ * Renders TransformMapStatsSummary into a friendly string,
+ * to be used e.g in Github Actions summary or Slack.
+ */
+export function transformMapStatsSummary(summary: TransformMapStatsSummary): string {
+  const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = summary
+
+  return [
+    `### ${name} summary\n`,
+    `${_since(started)} spent`,
+    `${_hc(countIn)} / ${_hc(countOut)} row(s) in / out`,
+    countErrors ? `${countErrors} error(s)` : '',
+    ...Object.entries(extra).map(([k, v]) => `${k}: ${_stringify(v)}`),
+  ]
+    .filter(Boolean)
+    .join('\n')
+}

package/src/stream/transform/transformMapSync.ts

@@ -88,6 +88,7 @@ export function transformMapSync<IN = any, OUT = IN>(
     logger = console,
   } = opt
 
+  const started = Date.now()
   let index = -1
   let countOut = 0
   let isSettled = false
@@ -138,13 +139,20 @@ export function transformMapSync<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
-          onDone?.({
-            ok: false,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-          })
+
+          try {
+            onDone?.({
+              ok: false,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
+
           // Emit error immediately
           return cb(err as Error)
         }
@@ -162,13 +170,18 @@ export function transformMapSync<IN = any, OUT = IN>(
       logErrorStats(true)
 
       if (collectedErrors.length) {
-        onDone?.({
-          ok: false,
-          collectedErrors,
-          countErrors: errors,
-          countIn: index + 1,
-          countOut,
-        })
+        try {
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
+        } catch (err) {
+          logger.error(err)
+        }
 
         // emit Aggregated error
         cb(
@@ -180,13 +193,18 @@ export function transformMapSync<IN = any, OUT = IN>(
       } else {
         // emit no error
 
-        onDone?.({
-          ok: true,
-          collectedErrors,
-          countErrors: errors,
-          countIn: index + 1,
-          countOut,
-        })
+        try {
+          onDone?.({
+            ok: true,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
+        } catch (err) {
+          logger.error(err)
+        }
 
         cb()
       }