@naturalcycles/nodejs-lib 12.56.1 → 12.60.0

package/dist/stream/readable/readableMap.js

@@ -1,24 +1,29 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.readableMap = void 0;
-const index_1 = require("../../index");
+const stream_1 = require("stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
 function readableMap(readable, mapper) {
-    const out = (0, index_1.readableCreate)();
-    void (async () => {
-        try {
-            let index = 0;
-            for await (const item of readable) {
-                const v = await mapper(item, index++);
-                out.push(v);
+    let i = -1;
+    // todo: check if we need to handle errors somehow specifically
+    return readable.pipe(new stream_1.Transform({
+        objectMode: true,
+        async transform(chunk, _enc, cb) {
+            try {
+                const r = await mapper(chunk, ++i);
+                if (r === js_lib_1.SKIP) {
+                    cb();
+                }
+                else {
+                    // _assert(r !== END, `readableMap END not supported`)
+                    cb(null, r);
+                }
             }
-            // We're done
-            out.push(null);
-        }
-        catch (err) {
-            console.error(err);
-            out.emit('error', err);
-        }
-    })();
-    return out;
+            catch (err) {
+                console.error(err);
+                cb(err);
+            }
+        },
+    }));
 }
 exports.readableMap = readableMap;

package/dist/stream/sizeStack.d.ts

@@ -0,0 +1,9 @@
+import { CommonLogger, NumberStack } from '@naturalcycles/js-lib';
+export declare class SizeStack extends NumberStack {
+    name: string;
+    constructor(name: string, size: number);
+    total: number;
+    push(item: any): this;
+    getStats(): string;
+    static countItem(item: any, logger: CommonLogger, sizes?: SizeStack, sizesZipped?: SizeStack): Promise<void>;
+}

package/dist/stream/sizeStack.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SizeStack = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+const colors_1 = require("../colors");
+const zip_util_1 = require("../util/zip.util");
+class SizeStack extends js_lib_1.NumberStack {
+    constructor(name, size) {
+        super(size);
+        this.name = name;
+        this.total = 0;
+    }
+    push(item) {
+        this.total += item;
+        return super.push(item);
+    }
+    getStats() {
+        const pcs = this.percentiles([50, 90]);
+        return [
+            this.name,
+            'avg',
+            (0, colors_1.yellow)((0, js_lib_1._hb)(this.avg())),
+            'p50',
+            (0, colors_1.yellow)((0, js_lib_1._hb)(pcs[50])),
+            'p90',
+            (0, colors_1.yellow)((0, js_lib_1._hb)(pcs[90])),
+            'total',
+            (0, colors_1.yellow)((0, js_lib_1._hb)(this.total)),
+        ].join(' ');
+    }
+    static async countItem(item, logger, sizes, sizesZipped) {
+        if (!sizes)
+            return;
+        // try-catch, because we don't want to fail the pipeline on logProgress
+        try {
+            const buf = Buffer.from(JSON.stringify(item));
+            sizes.push(buf.byteLength);
+            if (sizesZipped) {
+                const { byteLength } = await (0, zip_util_1.gzipBuffer)(buf);
+                sizesZipped.push(byteLength);
+            }
+        }
+        catch (err) {
+            logger.warn(`transformLogProgress failed to JSON.stringify the chunk: ${err.message}`);
+        }
+    }
+}
+exports.SizeStack = SizeStack;

package/dist/stream/stream.util.d.ts

@@ -0,0 +1,4 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { CommonLogger } from '@naturalcycles/js-lib';
+export declare function pipelineClose(name: string, readableDownstream: Readable, sourceReadable: Readable | undefined, streamDone: Promise<void> | undefined, logger: CommonLogger): void;

package/dist/stream/stream.util.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pipelineClose = void 0;
+function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
+    readableDownstream.push(null); // this closes the stream, so downstream Readable will receive `end` and won't write anything
+    if (!sourceReadable) {
+        logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`);
+    }
+    else {
+        logger.log(`${name} is calling readable.unpipe() to pause the stream`);
+        sourceReadable.unpipe(); // it is expected to pause the stream
+        if (!streamDone) {
+            logger.log(`${name} streamDone is not provided, will do readable.destroy right away`);
+            sourceReadable.destroy();
+        }
+        else {
+            void streamDone.then(() => {
+                logger.log(`${name} streamDone, calling readable.destroy()`);
+                sourceReadable.destroy(); // this throws ERR_STREAM_PREMATURE_CLOSE
+            });
+        }
+    }
+}
+exports.pipelineClose = pipelineClose;

package/dist/stream/transform/transformBuffer.js

@@ -13,7 +13,7 @@ function transformBuffer(opt) {
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             buf.push(chunk);
             if (buf.length >= batchSize) {
                 cb(null, buf);

package/dist/stream/transform/transformFilter.d.ts

@@ -1,11 +1,10 @@
 import { AsyncPredicate, Predicate } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
+import { TransformMapOptions } from './transformMap';
 /**
- * Note, that currently it's NOT concurrent! (concurrency = 1)
- * So, it's recommended to use transformMap instead, that is both concurrent and has
- * filtering feature by default.
+ * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
-export declare function transformFilter<IN = any>(predicate: AsyncPredicate<IN>, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformFilter<IN = any>(predicate: AsyncPredicate<IN>, opt?: TransformMapOptions): TransformTyped<IN, IN>;
 /**
  * Sync version of `transformFilter`
  */

package/dist/stream/transform/transformFilter.js

@@ -2,29 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transformFilterSync = exports.transformFilter = void 0;
 const stream_1 = require("stream");
+const transformMap_1 = require("./transformMap");
 /**
- * Note, that currently it's NOT concurrent! (concurrency = 1)
- * So, it's recommended to use transformMap instead, that is both concurrent and has
- * filtering feature by default.
+ * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 function transformFilter(predicate, opt = {}) {
-    let index = 0;
-    return new stream_1.Transform({
-        objectMode: true,
+    return (0, transformMap_1.transformMap)(v => v, {
+        predicate,
         ...opt,
-        async transform(chunk, _encoding, cb) {
-            try {
-                if (await predicate(chunk, index++)) {
-                    cb(null, chunk); // pass through
-                }
-                else {
-                    cb(); // signal that we've finished processing, but emit no output here
-                }
-            }
-            catch (err) {
-                cb(err);
-            }
-        },
     });
 }
 exports.transformFilter = transformFilter;
@@ -36,7 +21,7 @@ function transformFilterSync(predicate, opt = {}) {
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        async transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             try {
                 if (predicate(chunk, index++)) {
                     cb(null, chunk); // pass through

package/dist/stream/transform/transformLimit.d.ts

@@ -1,5 +1,40 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { CommonLogger } from '@naturalcycles/js-lib';
+import { AbortableTransform } from '../../index';
 import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformLimitOptions extends TransformOptions {
+    /**
+     * Nullish value (e.g 0 or undefined) would mean "no limit"
+     */
+    limit?: number;
+    /**
+     * If provided (recommended!) - it will call readable.destroy() on limit.
+     * Without it - it will only stop the downstream consumers, but won't stop
+     * the Readable ("source" of the stream).
+     * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+     */
+    sourceReadable?: Readable;
+    /**
+     * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
+     * to be able to wait for Consumers before calling `readable.destroy`.
+     * Has no effect if `readable` is not provided.
+     */
+    streamDone?: Promise<void>;
+    logger?: CommonLogger;
+    /**
+     * Set to true to enable additional debug messages, e.g it'll log
+     * when readable still emits values after the limit is reached.
+     */
+    debug?: boolean;
+}
+/**
+ * Class only exists to be able to do `instanceof TransformLimit`
+ * and to set sourceReadable+streamDone to it in `_pipeline`.
+ */
+export declare class TransformLimit extends AbortableTransform {
+}
 /**
  * 0 or falsy value means "no limit"
  */
-export declare function transformLimit<IN>(limit?: number, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformLimit<IN>(opt?: TransformLimitOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformLimit.js

@@ -1,28 +1,46 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformLimit = void 0;
-const stream_1 = require("stream");
+exports.transformLimit = exports.TransformLimit = void 0;
+const index_1 = require("../../index");
+const stream_util_1 = require("../stream.util");
+/**
+ * Class only exists to be able to do `instanceof TransformLimit`
+ * and to set sourceReadable+streamDone to it in `_pipeline`.
+ */
+class TransformLimit extends index_1.AbortableTransform {
+}
+exports.TransformLimit = TransformLimit;
 /**
  * 0 or falsy value means "no limit"
  */
-function transformLimit(limit, opt = {}) {
-    let index = 0;
+function transformLimit(opt = {}) {
+    const { logger = console, limit, debug } = opt;
+    if (!limit) {
+        // No limit - returning pass-through transform
+        return (0, index_1.transformNoOp)();
+    }
+    let i = 0; // so we start first chunk with 1
     let ended = false;
-    return new stream_1.Transform({
+    return new TransformLimit({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
-            index++;
-            if (!ended) {
-                cb(null, chunk); // pass through the item
+        transform(chunk, _, cb) {
+            i++;
+            if (i === limit) {
+                ended = true;
+                logger.log(`transformLimit of ${limit} reached`);
+                this.push(chunk);
+                (0, stream_util_1.pipelineClose)('transformLimit', this, opt.sourceReadable || this.sourceReadable, opt.streamDone || this.streamDone, logger);
+                cb(); // after pause
             }
-            else {
-                cb(null); // pass-through empty
+            else if (!ended) {
+                cb(null, chunk);
             }
-            if (limit && index === limit) {
-                ended = true;
-                console.log(`transformLimit: limit of ${limit} reached`);
-                // this.emit('end') // this makes it "halt" on Node 14 lts
+            else {
+                if (debug)
+                    logger.log(`transformLimit.transform after limit`, i);
+                // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
+                cb(); // ended, don't emit anything
             }
         },
     });

package/dist/stream/transform/transformLogProgress.d.ts

@@ -1,4 +1,4 @@
-import { AnyObject } from '@naturalcycles/js-lib';
+import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
     /**
@@ -69,6 +69,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
      * @default 1000
      */
     logEvery?: number;
+    logger?: CommonLogger;
     /**
      * Function to return extra properties to the "progress object".
      *
@@ -83,6 +84,26 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
      * Defaults to 1.
      */
     batchSize?: number;
+    /**
+     * Experimental logging of item (shunk) sizes, when json-stringified.
+     *
+     * Defaults to false.
+     *
+     * @experimental
+     */
+    logSizes?: boolean;
+    /**
+     * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+     * Defaults to 100_000.
+     * Cannot be Infinity.
+     */
+    logSizesBuffer?: number;
+    /**
+     * Works in addition to `logSizes`. Adds "zipped sizes".
+     *
+     * @experimental
+     */
+    logZippedSizes?: boolean;
 }
 /**
  * Pass-through transform that optionally logs progress.

package/dist/stream/transform/transformLogProgress.js

@@ -7,6 +7,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
 const time_lib_1 = require("@naturalcycles/time-lib");
 const colors_1 = require("../../colors");
 const colors_2 = require("../../colors/colors");
+const sizeStack_1 = require("../sizeStack");
 const inspectOpt = {
     colors: colors_2.hasColors,
     breakLength: 300,
@@ -15,7 +16,7 @@ const inspectOpt = {
  * Pass-through transform that optionally logs progress.
  */
 function transformLogProgress(opt = {}) {
-    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, batchSize = 1, extra, } = opt;
+    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, logSizes = false, logSizesBuffer = 100000, logZippedSizes = false, batchSize = 1, extra, logger = console, } = opt;
     const logProgress = opt.logProgress !== false && logEvery !== 0; // true by default
     const logEvery10 = logEvery * 10;
     const started = Date.now();
@@ -24,13 +25,19 @@ function transformLogProgress(opt = {}) {
     let processedLastSecond = 0;
     let progress = 0;
     let peakRSS = 0;
+    const sizes = logSizes ? new sizeStack_1.SizeStack('json', logSizesBuffer) : undefined;
+    const sizesZipped = logZippedSizes ? new sizeStack_1.SizeStack('json.gz', logSizesBuffer) : undefined;
     logStats(); // initial
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             progress++;
             processedLastSecond++;
+            if (sizes) {
+                // Check it, cause gzipping might be delayed here..
+                void sizeStack_1.SizeStack.countItem(chunk, logger, sizes, sizesZipped);
+            }
             if (logProgress && progress % logEvery === 0) {
                 logStats(chunk, false, progress % logEvery10 === 0);
             }
@@ -54,32 +61,43 @@ function transformLogProgress(opt = {}) {
         const rps10 = Math.round(sma.push(lastRPS));
         if (mem.rss > peakRSS)
             peakRSS = mem.rss;
-        console.log((0, util_1.inspect)({
+        const o = {
             [final ? `${metric}_final` : metric]: batchedProgress,
-            ...(extra ? extra(chunk, progress) : {}),
-            ...(logHeapUsed ? { heapUsed: (0, js_lib_1._mb)(mem.heapUsed) } : {}),
-            ...(logHeapTotal ? { heapTotal: (0, js_lib_1._mb)(mem.heapTotal) } : {}),
-            ...(logRss ? { rss: (0, js_lib_1._mb)(mem.rss) } : {}),
-            ...(logPeakRSS ? { peakRSS: (0, js_lib_1._mb)(peakRSS) } : {}),
-            ...(opt.rssMinusHeap ? { rssMinusHeap: (0, js_lib_1._mb)(mem.rss - mem.heapTotal) } : {}),
-            ...(opt.external ? { external: (0, js_lib_1._mb)(mem.external) } : {}),
-            ...(opt.arrayBuffers ? { arrayBuffers: (0, js_lib_1._mb)(mem.arrayBuffers || 0) } : {}),
-            ...(logRPS
-                ? {
-                    rps10,
-                    rpsTotal,
-                }
-                : {}),
-        }, inspectOpt));
+        };
+        if (extra)
+            Object.assign(o, extra(chunk, progress));
+        if (logHeapUsed)
+            o.heapUsed = (0, js_lib_1._mb)(mem.heapUsed);
+        if (logHeapTotal)
+            o.heapTotal = (0, js_lib_1._mb)(mem.heapTotal);
+        if (logRss)
+            o.rss = (0, js_lib_1._mb)(mem.rss);
+        if (logPeakRSS)
+            o.peakRSS = (0, js_lib_1._mb)(peakRSS);
+        if (opt.rssMinusHeap)
+            o.rssMinusHeap = (0, js_lib_1._mb)(mem.rss - mem.heapTotal);
+        if (opt.external)
+            o.external = (0, js_lib_1._mb)(mem.external);
+        if (opt.arrayBuffers)
+            o.arrayBuffers = (0, js_lib_1._mb)(mem.arrayBuffers || 0);
+        if (logRPS)
+            Object.assign(o, { rps10, rpsTotal });
+        logger.log((0, util_1.inspect)(o, inspectOpt));
+        if (sizes?.items.length) {
+            logger.log(sizes.getStats());
+            if (sizesZipped?.items.length) {
+                logger.log(sizesZipped.getStats());
+            }
+        }
         if (tenx) {
             let perHour = Math.round((batchedProgress * 1000 * 60 * 60) / (now - started)) || 0;
             if (perHour > 900) {
                 perHour = Math.round(perHour / 1000) + 'K';
             }
-            console.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
+            logger.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
         }
         else if (final) {
-            console.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
+            logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
         }
     }
 }

package/dist/stream/transform/transformMap.d.ts

@@ -1,4 +1,4 @@
-import { AsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
+import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -11,9 +11,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.
      *
-     * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
-     *
-     * @default to filter out undefined/null values, but pass anything else
+     * Defaults to "pass everything" (including null, undefined, etc).
+     * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
      */
     predicate?: AsyncPredicate<OUT>;
     /**
@@ -37,13 +36,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * @default `stream`
      */
     metric?: string;
-    /**
-     * If defined - called BEFORE `final()` callback is called.
-     */
-    beforeFinal?: () => any;
     logger?: CommonLogger;
 }
-export declare function notNullishPredicate(item: any): boolean;
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -56,4 +50,4 @@ export declare function notNullishPredicate(item: any): boolean;
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-export declare function transformMap<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
+export declare function transformMap<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/transformMap.js

@@ -1,13 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMap = exports.notNullishPredicate = void 0;
-const stream_1 = require("stream");
+exports.transformMap = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
+const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors");
-function notNullishPredicate(item) {
-    return item !== undefined && item !== null;
-}
-exports.notNullishPredicate = notNullishPredicate;
+const stream_util_1 = require("../stream.util");
+// doesn't work, cause here we don't construct our Transform instance ourselves
+// export class TransformMap extends AbortableTransform {}
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -21,80 +20,69 @@ exports.notNullishPredicate = notNullishPredicate;
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
+    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, metric = 'stream', logger = console, } = opt;
     let index = -1;
-    let isRejected = false;
+    let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    const q = new js_lib_1.PQueue({
-        concurrency,
-        resolveOn: 'start',
-        // debug: true,
-    });
-    return new stream_1.Transform({
-        objectMode: true,
+    return through2Concurrent.obj({
+        maxConcurrency: concurrency,
         async final(cb) {
-            // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
-            // wait for the current inFlight jobs to complete and push their results
-            await q.onIdle();
-            logErrorStats(logger, true);
-            await beforeFinal?.(); // call beforeFinal if defined
+            // console.log('transformMap final')
+            logErrorStats(true);
             if (collectedErrors.length) {
                 // emit Aggregated error
-                // For the same reason, magically, let's not call `cb`, but emit an error event instead
-                // this.emit('error', new AggregatedError(collectedErrors))
                 cb(new js_lib_1.AggregatedError(collectedErrors));
             }
             else {
                 // emit no error
-                // It is truly a mistery, but calling cb() here was causing ERR_MULTIPLE_CALLBACK ?!
-                // Commenting it out seems to work ?!
-                // ?!
-                // cb()
+                cb();
             }
         },
-        async transform(chunk, _encoding, cb) {
-            index++;
-            // console.log('transform', {index})
-            // Stop processing if THROW_IMMEDIATELY mode is used
-            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
-                return cb();
-            // It resolves when it is successfully STARTED execution.
-            // If it's queued instead - it'll wait and resolve only upon START.
-            await q.push(async () => {
-                try {
-                    const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-                    const res = await mapper(chunk, currentIndex);
-                    const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
-                    passedResults.forEach(r => this.push(r));
-                }
-                catch (err) {
-                    logger.error(err);
-                    errors++;
-                    logErrorStats(logger);
-                    if (onError) {
-                        try {
-                            onError(err, chunk);
-                        }
-                        catch { }
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                        isRejected = true;
-                        // Emit error immediately
-                        // return cb(err as Error)
-                        return this.emit('error', err);
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
-                        collectedErrors.push(err);
-                    }
+    }, async function transformMapFn(chunk, _, cb) {
+        // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
+        if (isSettled)
+            return cb();
+        const currentIndex = ++index;
+        try {
+            const res = await mapper(chunk, currentIndex);
+            const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => {
+                if (r === js_lib_1.END) {
+                    isSettled = true; // will be checked later
+                    return false;
                 }
+                return r !== js_lib_1.SKIP && (!predicate || (await predicate(r, currentIndex)));
             });
-            // Resolved, which means it STARTED processing
-            // This means we can take more load
+            passedResults.forEach(r => this.push(r));
+            if (isSettled) {
+                logger.log(`transformMap END received at index ${currentIndex}`);
+                (0, stream_util_1.pipelineClose)('transformMap', this, this.sourceReadable, this.streamDone, logger);
+            }
+            cb(); // done processing
+        }
+        catch (err) {
+            logger.error(err);
+            errors++;
+            logErrorStats();
+            if (onError) {
+                try {
+                    onError(err, chunk);
+                }
+                catch { }
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+                isSettled = true;
+                return cb(err); // Emit error immediately
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
+                collectedErrors.push(err);
+            }
+            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
             cb();
-        },
+        }
     });
-    function logErrorStats(logger, final = false) {
+    function logErrorStats(final = false) {
         if (!errors)
             return;
         logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);

package/dist/stream/transform/transformMapSimple.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode, Mapper } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, Mapper } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapSimpleOptions {
     /**
@@ -7,6 +7,7 @@ export interface TransformMapSimpleOptions {
      * @default ErrorMode.THROW_IMMEDIATELY
      */
     errorMode?: ErrorMode.THROW_IMMEDIATELY | ErrorMode.SUPPRESS;
+    logger?: CommonLogger;
 }
 /**
  * Simplest version of `transformMap`.