@naturalcycles/nodejs-lib 15.18.0 → 15.19.0

package/dist/stream/index.d.ts

@@ -14,6 +14,7 @@ export * from './readable/readableToArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';
 export * from './transform/transformFilter.js';
+export * from './transform/transformFlatten.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';

package/dist/stream/index.js

@@ -14,6 +14,7 @@ export * from './readable/readableToArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';
 export * from './transform/transformFilter.js';
+export * from './transform/transformFlatten.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';

package/dist/stream/ndjson/ndjsonMap.d.ts

@@ -9,12 +9,6 @@ export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOption
      * @default 100_000
      */
     logEveryOutput?: number;
-    /**
-     * Defaults to `true` for ndjsonMap
-     *
-     * @default true
-     */
-    flattenArrayOutput?: boolean;
 }
 /**
  * Unzips input file automatically, if it ends with `.gz`.

package/dist/stream/ndjson/ndjsonMap.js

@@ -1,5 +1,5 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
-import { createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '../index.js';
+import { createReadStreamAsNDJSON, createWriteStreamAsNDJSON, transformFlatten, } from '../index.js';
 import { _pipeline, transformLimit, transformLogProgress, transformMap } from '../index.js';
 /**
  * Unzips input file automatically, if it ends with `.gz`.
@@ -16,10 +16,10 @@ export async function ndjsonMap(mapper, opt) {
         readable,
         transformLogProgress({ metric: 'read', ...opt }),
         transformMap(mapper, {
-            flattenArrayOutput: true,
             errorMode: ErrorMode.SUPPRESS,
             ...opt,
         }),
+        transformFlatten(),
         transformLimit({ limit: limitOutput, sourceReadable: readable }),
         transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
         ...createWriteStreamAsNDJSON(outputFilePath),

package/dist/stream/progressLogger.js

@@ -1,5 +1,5 @@
 import { inspect } from 'node:util';
-import { _mb } from '@naturalcycles/js-lib';
+import { _hc, _mb } from '@naturalcycles/js-lib';
 import { _since, localTime } from '@naturalcycles/js-lib/datetime';
 import { SimpleMovingAverage } from '@naturalcycles/js-lib/math';
 import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
@@ -108,11 +108,8 @@ export class ProgressLogger {
             }
         }
         if (tenx) {
-            let perHour = Math.round((batchedProgress * 1000 * 60 * 60) / (now - this.started)) || 0;
-            if (perHour > 900) {
-                perHour = Math.round(perHour / 1000) + 'K';
-            }
-            logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`);
+            const perHour = _hc((batchedProgress * 1000 * 60 * 60) / (now - this.started));
+            logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(_hc(batchedProgress))} rows, ~${yellow(perHour)}/hour`);
         }
         else if (final) {
             logger.log(`${boldWhite(metric)} took ${yellow(_since(this.started))} to process ${yellow(batchedProgress)} rows with total RPS of ${yellow(rpsTotal)}`);

package/dist/stream/transform/transformFlatten.d.ts

@@ -0,0 +1,2 @@
+import type { TransformTyped } from '../stream.model.js';
+export declare function transformFlatten<T>(): TransformTyped<T[], T>;

package/dist/stream/transform/transformFlatten.js

@@ -0,0 +1,18 @@
+import { Transform } from 'node:stream';
+export function transformFlatten() {
+    return new Transform({
+        objectMode: true,
+        transform(chunk, _, cb) {
+            if (!Array.isArray(chunk)) {
+                // As a safety precaution, to not crash the pipeline - push as is
+                this.push(chunk);
+            }
+            else {
+                for (const item of chunk) {
+                    this.push(item);
+                }
+            }
+            cb(); // acknowledge
+        },
+    });
+}

package/dist/stream/transform/transformMap.d.ts

@@ -3,12 +3,6 @@ import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import { type AbortableAsyncMapper, type AsyncPredicate, END, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
 import type { TransformTyped } from '../stream.model.js';
 export interface TransformMapOptions<IN = any, OUT = IN> {
-    /**
-     * Set true to support "multiMap" - possibility to return [] and emit 1 result for each item in the array.
-     *
-     * @default false
-     */
-    flattenArrayOutput?: boolean;
     /**
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.

package/dist/stream/transform/transformMap.js

@@ -1,7 +1,6 @@
 import { _hc } from '@naturalcycles/js-lib';
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error';
-import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _stringify } from '@naturalcycles/js-lib/string/stringify.js';
 import { END, SKIP, } from '@naturalcycles/js-lib/types';
 import through2Concurrent from 'through2-concurrent';
@@ -23,7 +22,7 @@ import { pipelineClose } from '../stream.util.js';
  */
 export function transformMap(mapper, opt = {}) {
     const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, onDone, metric = 'stream', logger = console, } = opt;
+    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', logger = console, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
@@ -77,38 +76,23 @@ export function transformMap(mapper, opt = {}) {
         const currentIndex = ++index;
         try {
             const res = await mapper(chunk, currentIndex);
-            // todo: consider retiring flattenArrayOutput from here
-            // and implementing it as a separate .flat transform/operator
-            const resInput = (flattenArrayOutput && Array.isArray(res) ? res : [res]);
-            if (predicate) {
-                await pMap(resInput, async (r) => {
-                    if (r === END) {
-                        isSettled = true; // will be checked later
-                        return END;
-                    }
-                    if (r === SKIP)
-                        return;
-                    if (await predicate(r, currentIndex)) {
-                        countOut++;
-                        this.push(r);
-                    }
-                });
-            }
-            else {
-                for (const r of resInput) {
-                    if (r === END) {
-                        isSettled = true; // will be checked later
-                        break;
-                    }
-                    if (r === SKIP)
-                        continue;
-                    countOut++;
-                    this.push(r);
-                }
-            }
-            if (isSettled) {
+            // Check for isSettled again, as it may happen while mapper was running
+            if (isSettled)
+                return cb();
+            if (res === END) {
+                isSettled = true;
                 logger.log(`transformMap END received at index ${currentIndex}`);
                 pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger);
+                return cb();
+            }
+            if (res === SKIP) {
+                // do nothing, don't push
+                return cb();
+            }
+            if (!predicate || ((await predicate(res, currentIndex)) && !isSettled)) {
+                // isSettled could have happened in parallel, hence the extra check
+                countOut++;
+                this.push(res);
             }
             cb(); // done processing
         }

package/dist/stream/transform/transformMapSync.d.ts

@@ -10,11 +10,6 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * @default true
      */
     objectMode?: boolean;
-    /**
-     * @default false
-     * Set true to support "multiMap" - possibility to return [] and emit 1 result for each item in the array.
-     */
-    flattenArrayOutput?: boolean;
     /**
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.

package/dist/stream/transform/transformMapSync.js

@@ -11,7 +11,7 @@ export class TransformMapSync extends AbortableTransform {
  */
 export function transformMapSync(mapper, opt = {}) {
     const { predicate, // defaults to "no predicate" (pass everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, onDone, metric = 'stream', objectMode = true, logger = console, } = opt;
+    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', objectMode = true, logger = console, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
@@ -29,21 +29,19 @@ export function transformMapSync(mapper, opt = {}) {
             try {
                 // map and pass through
                 const v = mapper(chunk, currentIndex);
-                // todo: consider retiring flattenArrayOutput option
-                const vInput = (flattenArrayOutput && Array.isArray(v) ? v : [v]);
-                for (const r of vInput) {
-                    if (r === END) {
-                        isSettled = true; // will be checked later
-                        break;
-                    }
-                    if (r !== SKIP && (!predicate || predicate(r, currentIndex))) {
-                        countOut++;
-                        this.push(r);
-                    }
-                }
-                if (isSettled) {
+                if (v === END) {
+                    isSettled = true; // will be checked later
                     logger.log(`transformMapSync END received at index ${currentIndex}`);
                     pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger);
+                    return cb();
+                }
+                if (v === SKIP) {
+                    // do nothing, don't push
+                    return cb();
+                }
+                if (!predicate || predicate(v, currentIndex)) {
+                    countOut++;
+                    this.push(v);
                 }
                 cb(); // done processing
             }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.18.0",
+  "version": "15.19.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -14,6 +14,7 @@ export * from './readable/readableToArray.js'
 export * from './stream.model.js'
 export * from './transform/transformChunk.js'
 export * from './transform/transformFilter.js'
+export * from './transform/transformFlatten.js'
 export * from './transform/transformLimit.js'
 export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'

package/src/stream/ndjson/ndjsonMap.ts

@@ -3,6 +3,7 @@ import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
 import {
   createReadStreamAsNDJSON,
   createWriteStreamAsNDJSON,
+  transformFlatten,
   type TransformLogProgressOptions,
   type TransformMapOptions,
 } from '../index.js'
@@ -21,13 +22,6 @@ export interface NDJSONMapOptions<IN = any, OUT = IN>
    * @default 100_000
    */
   logEveryOutput?: number
-
-  /**
-   * Defaults to `true` for ndjsonMap
-   *
-   * @default true
-   */
-  flattenArrayOutput?: boolean
 }
 
 /**
@@ -53,10 +47,10 @@ export async function ndjsonMap<IN = any, OUT = any>(
     readable,
     transformLogProgress({ metric: 'read', ...opt }),
     transformMap(mapper, {
-      flattenArrayOutput: true,
       errorMode: ErrorMode.SUPPRESS,
       ...opt,
     }),
+    transformFlatten(),
     transformLimit({ limit: limitOutput, sourceReadable: readable }),
     transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
     ...createWriteStreamAsNDJSON(outputFilePath),

package/src/stream/progressLogger.ts

@@ -1,6 +1,6 @@
 import type { InspectOptions } from 'node:util'
 import { inspect } from 'node:util'
-import { _mb } from '@naturalcycles/js-lib'
+import { _hc, _mb } from '@naturalcycles/js-lib'
 import { _since, localTime } from '@naturalcycles/js-lib/datetime'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
@@ -284,16 +284,12 @@ export class ProgressLogger<T> implements Disposable {
     }
 
     if (tenx) {
-      let perHour: number | string =
-        Math.round((batchedProgress * 1000 * 60 * 60) / (now - this.started)) || 0
-      if (perHour > 900) {
-        perHour = Math.round(perHour / 1000) + 'K'
-      }
+      const perHour = _hc((batchedProgress * 1000 * 60 * 60) / (now - this.started))
 
       logger.log(
         `${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(
           _since(this.started),
-        )} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`,
+        )} so far to process ${yellow(_hc(batchedProgress))} rows, ~${yellow(perHour)}/hour`,
       )
     } else if (final) {
       logger.log(

package/src/stream/transform/transformFlatten.ts

@@ -0,0 +1,19 @@
+import { Transform } from 'node:stream'
+import type { TransformTyped } from '../stream.model.js'
+
+export function transformFlatten<T>(): TransformTyped<T[], T> {
+  return new Transform({
+    objectMode: true,
+    transform(chunk: T[], _, cb) {
+      if (!Array.isArray(chunk)) {
+        // As a safety precaution, to not crash the pipeline - push as is
+        this.push(chunk)
+      } else {
+        for (const item of chunk) {
+          this.push(item)
+        }
+      }
+      cb() // acknowledge
+    },
+  })
+}

package/src/stream/transform/transformMap.ts

@@ -2,7 +2,6 @@ import { _hc } from '@naturalcycles/js-lib'
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
 import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import { _stringify } from '@naturalcycles/js-lib/string/stringify.js'
 import {
   type AbortableAsyncMapper,
@@ -20,13 +19,6 @@ import type { TransformTyped } from '../stream.model.js'
 import { pipelineClose } from '../stream.util.js'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
-  /**
-   * Set true to support "multiMap" - possibility to return [] and emit 1 result for each item in the array.
-   *
-   * @default false
-   */
-  flattenArrayOutput?: boolean
-
   /**
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
@@ -133,7 +125,6 @@ export function transformMap<IN = any, OUT = IN>(
     concurrency = 16,
     predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
-    flattenArrayOutput,
     onError,
     onDone,
     metric = 'stream',
@@ -204,41 +195,25 @@ export function transformMap<IN = any, OUT = IN>(
 
       try {
         const res: OUT | typeof SKIP | typeof END = await mapper(chunk, currentIndex)
-        // todo: consider retiring flattenArrayOutput from here
-        // and implementing it as a separate .flat transform/operator
-        const resInput = (flattenArrayOutput && Array.isArray(res) ? res : [res]) as (
-          | OUT
-          | typeof SKIP
-          | typeof END
-        )[]
-
-        if (predicate) {
-          await pMap(resInput, async r => {
-            if (r === END) {
-              isSettled = true // will be checked later
-              return END
-            }
-            if (r === SKIP) return
-            if (await predicate(r, currentIndex)) {
-              countOut++
-              this.push(r)
-            }
-          })
-        } else {
-          for (const r of resInput) {
-            if (r === END) {
-              isSettled = true // will be checked later
-              break
-            }
-            if (r === SKIP) continue
-            countOut++
-            this.push(r)
-          }
-        }
+        // Check for isSettled again, as it may happen while mapper was running
+        if (isSettled) return cb()
 
-        if (isSettled) {
+        if (res === END) {
+          isSettled = true
           logger.log(`transformMap END received at index ${currentIndex}`)
           pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
+          return cb()
+        }
+
+        if (res === SKIP) {
+          // do nothing, don't push
+          return cb()
+        }
+
+        if (!predicate || ((await predicate(res, currentIndex)) && !isSettled)) {
+          // isSettled could have happened in parallel, hence the extra check
+          countOut++
+          this.push(res)
         }
 
         cb() // done processing

package/src/stream/transform/transformMapSync.ts

@@ -14,12 +14,6 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
    */
   objectMode?: boolean
 
-  /**
-   * @default false
-   * Set true to support "multiMap" - possibility to return [] and emit 1 result for each item in the array.
-   */
-  flattenArrayOutput?: boolean
-
   /**
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
@@ -75,7 +69,6 @@ export function transformMapSync<IN = any, OUT = IN>(
   const {
     predicate, // defaults to "no predicate" (pass everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
-    flattenArrayOutput = false,
     onError,
     onDone,
     metric = 'stream',
@@ -102,27 +95,22 @@ export function transformMapSync<IN = any, OUT = IN>(
       try {
         // map and pass through
         const v = mapper(chunk, currentIndex)
-        // todo: consider retiring flattenArrayOutput option
-        const vInput = (flattenArrayOutput && Array.isArray(v) ? v : [v]) as (
-          | OUT
-          | typeof SKIP
-          | typeof END
-        )[]
-
-        for (const r of vInput) {
-          if (r === END) {
-            isSettled = true // will be checked later
-            break
-          }
-          if (r !== SKIP && (!predicate || predicate(r, currentIndex))) {
-            countOut++
-            this.push(r)
-          }
-        }
 
-        if (isSettled) {
+        if (v === END) {
+          isSettled = true // will be checked later
           logger.log(`transformMapSync END received at index ${currentIndex}`)
           pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger)
+          return cb()
+        }
+
+        if (v === SKIP) {
+          // do nothing, don't push
+          return cb()
+        }
+
+        if (!predicate || predicate(v, currentIndex)) {
+          countOut++
+          this.push(v)
         }
 
         cb() // done processing