@naturalcycles/nodejs-lib 13.17.0 → 13.19.0

package/dist/index.d.ts

@@ -37,6 +37,7 @@ export * from './stream/progressLogger';
 export * from './stream/transform/transformChunk';
 export * from './stream/transform/transformFilter';
 export * from './stream/transform/transformLimit';
+export * from './stream/transform/transformOffset';
 export * from './stream/transform/transformLogProgress';
 export * from './stream/transform/transformMap';
 export * from './stream/transform/transformMapSimple';

package/dist/index.js

@@ -41,6 +41,7 @@ tslib_1.__exportStar(require("./stream/progressLogger"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformChunk"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformFilter"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformLimit"), exports);
+tslib_1.__exportStar(require("./stream/transform/transformOffset"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformLogProgress"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformMap"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformMapSimple"), exports);

package/dist/stream/progressLogger.js

@@ -107,7 +107,12 @@ class ProgressLogger {
         }
         else if (final) {
             logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
-            this.cfg.onProgressDone?.(o);
+            try {
+                this.cfg.onProgressDone?.(o);
+            }
+            catch (err) {
+                logger.error(err);
+            }
         }
     }
 }

package/dist/stream/transform/transformLimit.d.ts

@@ -34,7 +34,4 @@ export interface TransformLimitOptions extends TransformOptions {
  */
 export declare class TransformLimit extends AbortableTransform {
 }
-/**
- * 0 or falsy value means "no limit"
- */
-export declare function transformLimit<IN>(opt?: TransformLimitOptions): TransformTyped<IN, IN>;
+export declare function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformLimit.js

@@ -10,10 +10,7 @@ const stream_util_1 = require("../stream.util");
 class TransformLimit extends index_1.AbortableTransform {
 }
 exports.TransformLimit = TransformLimit;
-/**
- * 0 or falsy value means "no limit"
- */
-function transformLimit(opt = {}) {
+function transformLimit(opt) {
     const { logger = console, limit, debug } = opt;
     if (!limit) {
         // No limit - returning pass-through transform

package/dist/stream/transform/transformMap.d.ts

@@ -1,4 +1,4 @@
-import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, END, ErrorMode, SKIP, StringMap, UnixTimestampMillisNumber } from '@naturalcycles/js-lib';
+import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, END, ErrorMode, Promisable, SKIP, StringMap, UnixTimestampMillisNumber } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -38,9 +38,9 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * Callback is called **before** [possible] Aggregated error is thrown,
      * and before [possible] THROW_IMMEDIATELY error.
      *
-     * onDone callback will be called before Error is thrown.
+     * onDone callback will be awaited before Error is thrown.
      */
-    onDone?: (stats: TransformMapStats) => any;
+    onDone?: (stats: TransformMapStats) => Promisable<any>;
     /**
      * Progress metric
      *
@@ -63,6 +63,18 @@ export interface TransformMapStats {
     countOut: number;
     started: UnixTimestampMillisNumber;
 }
+export interface TransformMapStatsSummary extends TransformMapStats {
+    /**
+     * Name of the summary, defaults to `Transform`
+     */
+    name?: string;
+    /**
+     * Allows to pass extra key-value object, which will be rendered as:
+     * key: value
+     * key2: value2
+     */
+    extra?: StringMap<any>;
+}
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -76,7 +88,8 @@ export interface TransformMapStats {
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export declare function transformMap<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
-export declare function appendTransformMapStatsToGithubSummary(stats: TransformMapStats & {
-    name?: string;
-    extra?: StringMap<any>;
-}): void;
+/**
+ * Renders TransformMapStatsSummary into a friendly string,
+ * to be used e.g in Github Actions summary or Slack.
+ */
+export declare function transformMapStatsSummary(summary: TransformMapStatsSummary): string;

package/dist/stream/transform/transformMap.js

@@ -1,10 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.appendTransformMapStatsToGithubSummary = exports.transformMap = void 0;
+exports.transformMapStatsSummary = exports.transformMap = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors/colors");
-const json2env_1 = require("../../fs/json2env");
 const stream_util_1 = require("../stream.util");
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
@@ -35,27 +34,37 @@ function transformMap(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
-                onDone?.({
-                    ok: false,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                    started,
-                });
+                try {
+                    await onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMap resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
-                onDone?.({
-                    ok: true,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                    started,
-                });
+                try {
+                    await onDone?.({
+                        ok: true,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 cb();
             }
         },
@@ -93,14 +102,19 @@ function transformMap(mapper, opt = {}) {
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                 isSettled = true;
-                onDone?.({
-                    ok: false,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                    started,
-                });
+                try {
+                    await onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 return cb(err); // Emit error immediately
             }
             if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
@@ -117,14 +131,20 @@ function transformMap(mapper, opt = {}) {
     }
 }
 exports.transformMap = transformMap;
-function appendTransformMapStatsToGithubSummary(stats) {
-    const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = stats;
-    (0, json2env_1.appendToGithubSummary)(...[
+/**
+ * Renders TransformMapStatsSummary into a friendly string,
+ * to be used e.g in Github Actions summary or Slack.
+ */
+function transformMapStatsSummary(summary) {
+    const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = summary;
+    return [
         `### ${name} summary\n`,
         `${(0, js_lib_1._since)(started)} spent`,
-        `${(0, js_lib_1._hc)(countIn)} / ${(0, js_lib_1._hc)(countOut)} rows in / out`,
-        countErrors ? `${countErrors} errors` : '',
-        ...Object.entries(extra).map(([k, v]) => `${k}: ${v}`),
-    ].filter(Boolean));
+        `${(0, js_lib_1._hc)(countIn)} / ${(0, js_lib_1._hc)(countOut)} row(s) in / out`,
+        countErrors ? `${countErrors} error(s)` : '',
+        ...Object.entries(extra).map(([k, v]) => `${k}: ${(0, js_lib_1._stringify)(v)}`),
+    ]
+        .filter(Boolean)
+        .join('\n');
 }
-exports.appendTransformMapStatsToGithubSummary = appendTransformMapStatsToGithubSummary;
+exports.transformMapStatsSummary = transformMapStatsSummary;

package/dist/stream/transform/transformMapSync.js

@@ -59,14 +59,19 @@ function transformMapSync(mapper, opt = {}) {
                 }
                 if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
                     isSettled = true;
-                    onDone?.({
-                        ok: false,
-                        collectedErrors,
-                        countErrors: errors,
-                        countIn: index + 1,
-                        countOut,
-                        started,
-                    });
+                    try {
+                        onDone?.({
+                            ok: false,
+                            collectedErrors,
+                            countErrors: errors,
+                            countIn: index + 1,
+                            countOut,
+                            started,
+                        });
+                    }
+                    catch (err) {
+                        logger.error(err);
+                    }
                     // Emit error immediately
                     return cb(err);
                 }
@@ -80,27 +85,37 @@ function transformMapSync(mapper, opt = {}) {
             // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
-                onDone?.({
-                    ok: false,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                    started,
-                });
+                try {
+                    onDone?.({
+                        ok: false,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMapSync resulted in ${collectedErrors.length} error(s)`));
             }
             else {
                 // emit no error
-                onDone?.({
-                    ok: true,
-                    collectedErrors,
-                    countErrors: errors,
-                    countIn: index + 1,
-                    countOut,
-                    started,
-                });
+                try {
+                    onDone?.({
+                        ok: true,
+                        collectedErrors,
+                        countErrors: errors,
+                        countIn: index + 1,
+                        countOut,
+                        started,
+                    });
+                }
+                catch (err) {
+                    logger.error(err);
+                }
                 cb();
             }
         },

package/dist/stream/transform/transformOffset.d.ts

@@ -0,0 +1,10 @@
+import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformOffsetOptions extends TransformOptions {
+    /**
+     * How many items to skip (offset) in the stream.
+     *
+     * Nullish value (e.g 0 or undefined) would mean "no offset".
+     */
+    offset?: number;
+}
+export declare function transformOffset<IN>(opt: TransformOffsetOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformOffset.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transformOffset = void 0;
+const node_stream_1 = require("node:stream");
+const index_1 = require("../../index");
+function transformOffset(opt) {
+    const { offset } = opt;
+    if (!offset) {
+        // No offset - returning pass-through transform
+        return (0, index_1.transformNoOp)();
+    }
+    let i = 0; // so we start first chunk with 1
+    return new node_stream_1.Transform({
+        objectMode: true,
+        ...opt,
+        transform(chunk, _, cb) {
+            if (++i <= offset) {
+                return cb(); // skip
+            }
+            cb(null, chunk);
+        },
+    });
+}
+exports.transformOffset = transformOffset;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.17.0",
+  "version": "13.19.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/index.ts

@@ -47,6 +47,7 @@ export * from './stream/progressLogger'
 export * from './stream/transform/transformChunk'
 export * from './stream/transform/transformFilter'
 export * from './stream/transform/transformLimit'
+export * from './stream/transform/transformOffset'
 export * from './stream/transform/transformLogProgress'
 export * from './stream/transform/transformMap'
 export * from './stream/transform/transformMapSimple'

package/src/stream/progressLogger.ts

@@ -305,7 +305,11 @@ export class ProgressLogger<T> implements Disposable {
         )} rows with total RPS of ${yellow(rpsTotal)}`,
       )
 
-      this.cfg.onProgressDone?.(o)
+      try {
+        this.cfg.onProgressDone?.(o)
+      } catch (err) {
+        logger.error(err)
+      }
     }
   }
 }

package/src/stream/transform/transformLimit.ts

@@ -40,10 +40,7 @@ export interface TransformLimitOptions extends TransformOptions {
  */
 export class TransformLimit extends AbortableTransform {}
 
-/**
- * 0 or falsy value means "no limit"
- */
-export function transformLimit<IN>(opt: TransformLimitOptions = {}): TransformTyped<IN, IN> {
+export function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN> {
   const { logger = console, limit, debug } = opt
 
   if (!limit) {

package/src/stream/transform/transformMap.ts

@@ -2,19 +2,20 @@ import {
   _anyToError,
   _hc,
   _since,
+  _stringify,
   AbortableAsyncMapper,
   AsyncPredicate,
   CommonLogger,
   END,
   ErrorMode,
   pFilter,
+  Promisable,
   SKIP,
   StringMap,
   UnixTimestampMillisNumber,
 } from '@naturalcycles/js-lib'
 import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors/colors'
-import { appendToGithubSummary } from '../../fs/json2env'
 import { AbortableTransform } from '../pipeline/pipeline'
 import { TransformTyped } from '../stream.model'
 import { pipelineClose } from '../stream.util'
@@ -62,9 +63,9 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * Callback is called **before** [possible] Aggregated error is thrown,
    * and before [possible] THROW_IMMEDIATELY error.
    *
-   * onDone callback will be called before Error is thrown.
+   * onDone callback will be awaited before Error is thrown.
    */
-  onDone?: (stats: TransformMapStats) => any
+  onDone?: (stats: TransformMapStats) => Promisable<any>
 
   /**
    * Progress metric
@@ -91,6 +92,20 @@ export interface TransformMapStats {
   started: UnixTimestampMillisNumber
 }
 
+export interface TransformMapStatsSummary extends TransformMapStats {
+  /**
+   * Name of the summary, defaults to `Transform`
+   */
+  name?: string
+
+  /**
+   * Allows to pass extra key-value object, which will be rendered as:
+   * key: value
+   * key2: value2
+   */
+  extra?: StringMap<any>
+}
+
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
 
@@ -137,14 +152,18 @@ export function transformMap<IN = any, OUT = IN>(
         logErrorStats(true)
 
         if (collectedErrors.length) {
-          onDone?.({
-            ok: false,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-            started,
-          })
+          try {
+            await onDone?.({
+              ok: false,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
 
           // emit Aggregated error
           cb(
@@ -156,14 +175,18 @@ export function transformMap<IN = any, OUT = IN>(
         } else {
           // emit no error
 
-          onDone?.({
-            ok: true,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-            started,
-          })
+          try {
+            await onDone?.({
+              ok: true,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
 
           cb()
         }
@@ -210,14 +233,20 @@ export function transformMap<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
-          onDone?.({
-            ok: false,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-            started,
-          })
+
+          try {
+            await onDone?.({
+              ok: false,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
+
           return cb(err) // Emit error immediately
         }
 
@@ -237,18 +266,20 @@ export function transformMap<IN = any, OUT = IN>(
   }
 }
 
-export function appendTransformMapStatsToGithubSummary(
-  stats: TransformMapStats & { name?: string; extra?: StringMap<any> },
-): void {
-  const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = stats
-
-  appendToGithubSummary(
-    ...[
-      `### ${name} summary\n`,
-      `${_since(started)} spent`,
-      `${_hc(countIn)} / ${_hc(countOut)} rows in / out`,
-      countErrors ? `${countErrors} errors` : '',
-      ...Object.entries(extra).map(([k, v]) => `${k}: ${v}`),
-    ].filter(Boolean),
-  )
+/**
+ * Renders TransformMapStatsSummary into a friendly string,
+ * to be used e.g in Github Actions summary or Slack.
+ */
+export function transformMapStatsSummary(summary: TransformMapStatsSummary): string {
+  const { countIn, countOut, countErrors, started, name = 'Transform', extra = {} } = summary
+
+  return [
+    `### ${name} summary\n`,
+    `${_since(started)} spent`,
+    `${_hc(countIn)} / ${_hc(countOut)} row(s) in / out`,
+    countErrors ? `${countErrors} error(s)` : '',
+    ...Object.entries(extra).map(([k, v]) => `${k}: ${_stringify(v)}`),
+  ]
+    .filter(Boolean)
+    .join('\n')
 }

package/src/stream/transform/transformMapSync.ts

@@ -139,14 +139,20 @@ export function transformMapSync<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
-          onDone?.({
-            ok: false,
-            collectedErrors,
-            countErrors: errors,
-            countIn: index + 1,
-            countOut,
-            started,
-          })
+
+          try {
+            onDone?.({
+              ok: false,
+              collectedErrors,
+              countErrors: errors,
+              countIn: index + 1,
+              countOut,
+              started,
+            })
+          } catch (err) {
+            logger.error(err)
+          }
+
           // Emit error immediately
           return cb(err as Error)
         }
@@ -164,14 +170,18 @@ export function transformMapSync<IN = any, OUT = IN>(
       logErrorStats(true)
 
       if (collectedErrors.length) {
-        onDone?.({
-          ok: false,
-          collectedErrors,
-          countErrors: errors,
-          countIn: index + 1,
-          countOut,
-          started,
-        })
+        try {
+          onDone?.({
+            ok: false,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
+        } catch (err) {
+          logger.error(err)
+        }
 
         // emit Aggregated error
         cb(
@@ -183,14 +193,18 @@ export function transformMapSync<IN = any, OUT = IN>(
       } else {
         // emit no error
 
-        onDone?.({
-          ok: true,
-          collectedErrors,
-          countErrors: errors,
-          countIn: index + 1,
-          countOut,
-          started,
-        })
+        try {
+          onDone?.({
+            ok: true,
+            collectedErrors,
+            countErrors: errors,
+            countIn: index + 1,
+            countOut,
+            started,
+          })
+        } catch (err) {
+          logger.error(err)
+        }
 
         cb()
       }

package/src/stream/transform/transformOffset.ts

@@ -0,0 +1,34 @@
+import { Transform } from 'node:stream'
+import { transformNoOp } from '../../index'
+import { TransformOptions, TransformTyped } from '../stream.model'
+
+export interface TransformOffsetOptions extends TransformOptions {
+  /**
+   * How many items to skip (offset) in the stream.
+   *
+   * Nullish value (e.g 0 or undefined) would mean "no offset".
+   */
+  offset?: number
+}
+
+export function transformOffset<IN>(opt: TransformOffsetOptions): TransformTyped<IN, IN> {
+  const { offset } = opt
+
+  if (!offset) {
+    // No offset - returning pass-through transform
+    return transformNoOp()
+  }
+
+  let i = 0 // so we start first chunk with 1
+  return new Transform({
+    objectMode: true,
+    ...opt,
+    transform(chunk: IN, _, cb) {
+      if (++i <= offset) {
+        return cb() // skip
+      }
+
+      cb(null, chunk)
+    },
+  })
+}